mirror of
https://github.com/OffchainLabs/prysm.git
synced 2026-01-10 05:47:59 -05:00
Compare commits
67 Commits
gas-limit-
...
otel-traci
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6c5147215d | ||
|
|
d5596589ec | ||
|
|
15025837bb | ||
|
|
0229a2055e | ||
|
|
eb9af15c7a | ||
|
|
0584746815 | ||
|
|
8c4ea850ba | ||
|
|
4b43f13e65 | ||
|
|
26d35474e9 | ||
|
|
9fbe3564df | ||
|
|
bed5547890 | ||
|
|
47922fe7d8 | ||
|
|
dcd25d1d97 | ||
|
|
81a2a17c5f | ||
|
|
6b3f1de19d | ||
|
|
7c17af2a41 | ||
|
|
ecf5a368d7 | ||
|
|
557c5be433 | ||
|
|
49405c3afd | ||
|
|
3439122629 | ||
|
|
25e5b7a1a8 | ||
|
|
f6e5da6723 | ||
|
|
842f241cb9 | ||
|
|
41daac1b04 | ||
|
|
2a7fc84044 | ||
|
|
44ff0b1a14 | ||
|
|
91cdd318a8 | ||
|
|
3dc00816fb | ||
|
|
e331d5b371 | ||
|
|
8d5090ce54 | ||
|
|
25244d906d | ||
|
|
aa445713ac | ||
|
|
177769a1ce | ||
|
|
967e9255a2 | ||
|
|
910609a75f | ||
|
|
f9c202190a | ||
|
|
4a63a194b1 | ||
|
|
d887536eb7 | ||
|
|
1069da1cd2 | ||
|
|
4a487ba3bc | ||
|
|
bf81cd4449 | ||
|
|
00337fe005 | ||
|
|
bb3fba4d8e | ||
|
|
89967fe209 | ||
|
|
56712b5e49 | ||
|
|
0be9391e62 | ||
|
|
4a9c60f75f | ||
|
|
9cf6b93356 | ||
|
|
b4220e35c4 | ||
|
|
536cded4cc | ||
|
|
86fc64c917 | ||
|
|
2504a637e4 | ||
|
|
ad1d124a4a | ||
|
|
6a90a29b93 | ||
|
|
5d6a406829 | ||
|
|
2c78e501b3 | ||
|
|
c8cb0f37b2 | ||
|
|
78722239da | ||
|
|
3ffef024c7 | ||
|
|
a1eef44492 | ||
|
|
2845ab9365 | ||
|
|
4f43c15ebb | ||
|
|
e473d7cc4d | ||
|
|
794a05af26 | ||
|
|
15df13c7e6 | ||
|
|
b76f7fed2f | ||
|
|
e263687ea5 |
@@ -1 +1 @@
|
||||
7.1.0
|
||||
7.4.1
|
||||
|
||||
2
.github/actions/gomodtidy/Dockerfile
vendored
2
.github/actions/gomodtidy/Dockerfile
vendored
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.22-alpine
|
||||
FROM golang:1.23-alpine
|
||||
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
|
||||
|
||||
21
.github/workflows/clang-format.yml
vendored
Normal file
21
.github/workflows/clang-format.yml
vendored
Normal file
@@ -0,0 +1,21 @@
|
||||
name: Protobuf Format
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ '*' ]
|
||||
pull_request:
|
||||
branches: [ '*' ]
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
jobs:
|
||||
clang-format-checking:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
# Is this step failing for you?
|
||||
# Run: clang-format -i proto/**/*.proto
|
||||
# See: https://clang.llvm.org/docs/ClangFormat.html
|
||||
- uses: RafikFarhad/clang-format-github-action@v3
|
||||
with:
|
||||
sources: "proto/**/*.proto"
|
||||
4
.github/workflows/fuzz.yml
vendored
4
.github/workflows/fuzz.yml
vendored
@@ -16,7 +16,7 @@ jobs:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.22.10'
|
||||
go-version: '1.23.5'
|
||||
- id: list
|
||||
uses: shogo82148/actions-go-fuzz/list@v0
|
||||
with:
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.22.10'
|
||||
go-version: '1.23.5'
|
||||
- uses: shogo82148/actions-go-fuzz/run@v0
|
||||
with:
|
||||
packages: ${{ matrix.package }}
|
||||
|
||||
12
.github/workflows/go.yml
vendored
12
.github/workflows/go.yml
vendored
@@ -28,10 +28,10 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Set up Go 1.22
|
||||
- name: Set up Go 1.23
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.22.10'
|
||||
go-version: '1.23.5'
|
||||
- name: Run Gosec Security Scanner
|
||||
run: | # https://github.com/securego/gosec/issues/469
|
||||
export PATH=$PATH:$(go env GOPATH)/bin
|
||||
@@ -45,16 +45,16 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go 1.22
|
||||
- name: Set up Go 1.23
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.22.10'
|
||||
go-version: '1.23.5'
|
||||
id: go
|
||||
|
||||
- name: Golangci-lint
|
||||
uses: golangci/golangci-lint-action@v5
|
||||
with:
|
||||
version: v1.56.1
|
||||
version: v1.63.4
|
||||
args: --config=.golangci.yml --out-${NO_FUTURE}format colored-line-number
|
||||
|
||||
build:
|
||||
@@ -64,7 +64,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.22.10'
|
||||
go-version: '1.23.5'
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
|
||||
@@ -1,28 +1,20 @@
|
||||
run:
|
||||
skip-files:
|
||||
timeout: 10m
|
||||
go: '1.23.5'
|
||||
|
||||
issues:
|
||||
exclude-files:
|
||||
- validator/web/site_data.go
|
||||
- .*_test.go
|
||||
skip-dirs:
|
||||
exclude-dirs:
|
||||
- proto
|
||||
- tools/analyzers
|
||||
timeout: 10m
|
||||
go: '1.22.10'
|
||||
|
||||
linters:
|
||||
enable-all: true
|
||||
disable:
|
||||
# Deprecated linters:
|
||||
- deadcode
|
||||
- exhaustivestruct
|
||||
- golint
|
||||
- govet
|
||||
- ifshort
|
||||
- interfacer
|
||||
- maligned
|
||||
- nosnakecase
|
||||
- scopelint
|
||||
- structcheck
|
||||
- varcheck
|
||||
|
||||
# Disabled for now:
|
||||
- asasalint
|
||||
@@ -34,6 +26,8 @@ linters:
|
||||
- dogsled
|
||||
- dupl
|
||||
- durationcheck
|
||||
- errname
|
||||
- err113
|
||||
- exhaustive
|
||||
- exhaustruct
|
||||
- forbidigo
|
||||
@@ -47,17 +41,17 @@ linters:
|
||||
- gocyclo
|
||||
- godot
|
||||
- godox
|
||||
- goerr113
|
||||
- gofumpt
|
||||
- gomnd
|
||||
- gomoddirectives
|
||||
- gosec
|
||||
- inamedparam
|
||||
- interfacebloat
|
||||
- intrange
|
||||
- ireturn
|
||||
- lll
|
||||
- maintidx
|
||||
- makezero
|
||||
- mnd
|
||||
- musttag
|
||||
- nakedret
|
||||
- nestif
|
||||
@@ -72,6 +66,7 @@ linters:
|
||||
- predeclared
|
||||
- promlinter
|
||||
- protogetter
|
||||
- recvcheck
|
||||
- revive
|
||||
- spancheck
|
||||
- staticcheck
|
||||
|
||||
80
CHANGELOG.md
80
CHANGELOG.md
@@ -60,7 +60,7 @@ Notable features:
|
||||
- Updated the default `scrape-interval` in `Client-stats` to 2 minutes to accommodate Beaconcha.in API rate limits.
|
||||
- Switch to compounding when consolidating with source==target.
|
||||
- Revert block db save when saving state fails.
|
||||
- Return false from HasBlock if the block is being synced.
|
||||
- Return false from HasBlock if the block is being synced.
|
||||
- Cleanup forkchoice on failed insertions.
|
||||
- Use read only validator for core processing to avoid unnecessary copying.
|
||||
- Use ROBlock across block processing pipeline.
|
||||
@@ -73,7 +73,7 @@ Notable features:
|
||||
- Simplified `EjectedValidatorIndices`.
|
||||
- `engine_newPayloadV4`,`engine_getPayloadV4` are changes due to new execution request serialization decisions, [PR](https://github.com/prysmaticlabs/prysm/pull/14580)
|
||||
- Fixed various small things in state-native code.
|
||||
- Use ROBlock earlier in block syncing pipeline.
|
||||
- Use ROBlock earlier in block syncing pipeline.
|
||||
- Changed the signature of `ProcessPayload`.
|
||||
- Only Build the Protobuf state once during serialization.
|
||||
- Capella blocks are execution.
|
||||
@@ -139,9 +139,9 @@ Notable features:
|
||||
|
||||
### Security
|
||||
|
||||
## [v5.1.2](https://github.com/prysmaticlabs/prysm/compare/v5.1.1...v5.1.2) - 2024-10-16
|
||||
## [v5.1.2](https://github.com/prysmaticlabs/prysm/compare/v5.1.1...v5.1.2) - 2024-10-16
|
||||
|
||||
This is a hotfix release with one change.
|
||||
This is a hotfix release with one change.
|
||||
|
||||
Prysm v5.1.1 contains an updated implementation of the beacon api streaming events endpoint. This
|
||||
new implementation contains a bug that can cause a panic in certain conditions. The issue is
|
||||
@@ -153,20 +153,20 @@ prysm REST mode validator (a feature which requires the validator to be configur
|
||||
api instead of prysm's stock grpc endpoints) or accessory software that connects to the events api,
|
||||
like https://github.com/ethpandaops/ethereum-metrics-exporter
|
||||
|
||||
### Fixed
|
||||
### Fixed
|
||||
|
||||
- Recover from panics when writing the event stream [#14545](https://github.com/prysmaticlabs/prysm/pull/14545)
|
||||
|
||||
## [v5.1.1](https://github.com/prysmaticlabs/prysm/compare/v5.1.0...v5.1.1) - 2024-10-15
|
||||
|
||||
This release has a number of features and improvements. Most notably, the feature flag
|
||||
`--enable-experimental-state` has been flipped to "opt out" via `--disable-experimental-state`.
|
||||
This release has a number of features and improvements. Most notably, the feature flag
|
||||
`--enable-experimental-state` has been flipped to "opt out" via `--disable-experimental-state`.
|
||||
The experimental state management design has shown significant improvements in memory usage at
|
||||
runtime. Updates to libp2p's gossipsub have some bandwidith stability improvements with support for
|
||||
IDONTWANT control messages.
|
||||
IDONTWANT control messages.
|
||||
|
||||
The gRPC gateway has been deprecated from Prysm in this release. If you need JSON data, consider the
|
||||
standardized beacon-APIs.
|
||||
standardized beacon-APIs.
|
||||
|
||||
Updating to this release is recommended at your convenience.
|
||||
|
||||
@@ -208,7 +208,7 @@ Updating to this release is recommended at your convenience.
|
||||
- `grpc-gateway-corsdomain` is renamed to http-cors-domain. The old name can still be used as an alias.
|
||||
- `api-timeout` is changed from int flag to duration flag, default value updated.
|
||||
- Light client support: abstracted out the light client headers with different versions.
|
||||
- `ApplyToEveryValidator` has been changed to prevent misuse bugs, it takes a closure that takes a `ReadOnlyValidator` and returns a raw pointer to a `Validator`.
|
||||
- `ApplyToEveryValidator` has been changed to prevent misuse bugs, it takes a closure that takes a `ReadOnlyValidator` and returns a raw pointer to a `Validator`.
|
||||
- Removed gorilla mux library and replaced it with net/http updates in go 1.22.
|
||||
- Clean up `ProposeBlock` for validator client to reduce cognitive scoring and enable further changes.
|
||||
- Updated k8s-io/client-go to v0.30.4 and k8s-io/apimachinery to v0.30.4
|
||||
@@ -219,7 +219,7 @@ Updating to this release is recommended at your convenience.
|
||||
- Updated Sepolia bootnodes.
|
||||
- Make committee aware packing the default by deprecating `--enable-committee-aware-packing`.
|
||||
- Moved `ConvertKzgCommitmentToVersionedHash` to the `primitives` package.
|
||||
- Updated correlation penalty for EIP-7251.
|
||||
- Updated correlation penalty for EIP-7251.
|
||||
|
||||
### Deprecated
|
||||
- `--disable-grpc-gateway` flag is deprecated due to grpc gateway removal.
|
||||
@@ -693,34 +693,34 @@ AVX support (eg Celeron) after the Deneb fork. This is not an issue for mainnet.
|
||||
|
||||
- Linter: Wastedassign linter enabled to improve code quality.
|
||||
- API Enhancements:
|
||||
- Added payload return in Wei for /eth/v3/validator/blocks.
|
||||
- Added Holesky Deneb Epoch for better epoch management.
|
||||
- Added payload return in Wei for /eth/v3/validator/blocks.
|
||||
- Added Holesky Deneb Epoch for better epoch management.
|
||||
- Testing Enhancements:
|
||||
- Clear cache in tests of core helpers to ensure test reliability.
|
||||
- Added Debug State Transition Method for improved debugging.
|
||||
- Backfilling test: Enabled backfill in E2E tests for more comprehensive coverage.
|
||||
- Clear cache in tests of core helpers to ensure test reliability.
|
||||
- Added Debug State Transition Method for improved debugging.
|
||||
- Backfilling test: Enabled backfill in E2E tests for more comprehensive coverage.
|
||||
- API Updates: Re-enabled jwt on keymanager API for enhanced security.
|
||||
- Logging Improvements: Enhanced block by root log for better traceability.
|
||||
- Validator Client Improvements:
|
||||
- Added Spans to Core Validator Methods for enhanced monitoring.
|
||||
- Improved readability in validator client code for better maintenance (various commits).
|
||||
- Added Spans to Core Validator Methods for enhanced monitoring.
|
||||
- Improved readability in validator client code for better maintenance (various commits).
|
||||
|
||||
### Changed
|
||||
|
||||
- Optimizations and Refinements:
|
||||
- Lowered resource usage in certain processes for efficiency.
|
||||
- Moved blob rpc validation closer to peer read for optimized processing.
|
||||
- Cleaned up validate beacon block code for clarity and efficiency.
|
||||
- Updated Sepolia Deneb fork epoch for alignment with network changes.
|
||||
- Changed blob latency metrics to milliseconds for more precise measurement.
|
||||
- Altered getLegacyDatabaseLocation message for better clarity.
|
||||
- Improved wait for activation method for enhanced performance.
|
||||
- Capitalized Aggregated Unaggregated Attestations Log for consistency.
|
||||
- Modified HistoricalRoots usage for accuracy.
|
||||
- Adjusted checking of attribute emptiness for efficiency.
|
||||
- Lowered resource usage in certain processes for efficiency.
|
||||
- Moved blob rpc validation closer to peer read for optimized processing.
|
||||
- Cleaned up validate beacon block code for clarity and efficiency.
|
||||
- Updated Sepolia Deneb fork epoch for alignment with network changes.
|
||||
- Changed blob latency metrics to milliseconds for more precise measurement.
|
||||
- Altered getLegacyDatabaseLocation message for better clarity.
|
||||
- Improved wait for activation method for enhanced performance.
|
||||
- Capitalized Aggregated Unaggregated Attestations Log for consistency.
|
||||
- Modified HistoricalRoots usage for accuracy.
|
||||
- Adjusted checking of attribute emptiness for efficiency.
|
||||
- Database Management:
|
||||
- Moved --db-backup-output-dir as a deprecated flag for database management simplification.
|
||||
- Added the Ability to Defragment the Beacon State for improved database performance.
|
||||
- Moved --db-backup-output-dir as a deprecated flag for database management simplification.
|
||||
- Added the Ability to Defragment the Beacon State for improved database performance.
|
||||
- Dependency Update: Bumped quic-go version from 0.39.3 to 0.39.4 for up-to-date dependencies.
|
||||
|
||||
### Removed
|
||||
@@ -731,12 +731,12 @@ AVX support (eg Celeron) after the Deneb fork. This is not an issue for mainnet.
|
||||
### Fixed
|
||||
|
||||
- Bug Fixes:
|
||||
- Fixed off by one error for improved accuracy.
|
||||
- Resolved small typo in error messages for clarity.
|
||||
- Addressed minor issue in blsToExecChange validator for better validation.
|
||||
- Corrected blobsidecar json tag for commitment inclusion proof.
|
||||
- Fixed ssz post-requests content type check.
|
||||
- Resolved issue with port logging in bootnode.
|
||||
- Fixed off by one error for improved accuracy.
|
||||
- Resolved small typo in error messages for clarity.
|
||||
- Addressed minor issue in blsToExecChange validator for better validation.
|
||||
- Corrected blobsidecar json tag for commitment inclusion proof.
|
||||
- Fixed ssz post-requests content type check.
|
||||
- Resolved issue with port logging in bootnode.
|
||||
- Test Fixes: Re-enabled Slasher E2E Test for more comprehensive testing.
|
||||
|
||||
### Security
|
||||
@@ -1163,9 +1163,9 @@ No security issues in this release.
|
||||
now features runtime detection, automatically enabling optimized code paths if your CPU supports it.
|
||||
- **Multiarch Containers Preview Available**: multiarch (:wave: arm64 support :wave:) containers will be offered for
|
||||
preview at the following locations:
|
||||
- Beacon Chain: [gcr.io/prylabs-dev/prysm/beacon-chain:v4.1.0](gcr.io/prylabs-dev/prysm/beacon-chain:v4.1.0)
|
||||
- Validator: [gcr.io/prylabs-dev/prysm/validator:v4.1.0](gcr.io/prylabs-dev/prysm/validator:v4.1.0)
|
||||
- Please note that in the next cycle, we will exclusively use these containers at the canonical URLs.
|
||||
- Beacon Chain: [gcr.io/prylabs-dev/prysm/beacon-chain:v4.1.0](gcr.io/prylabs-dev/prysm/beacon-chain:v4.1.0)
|
||||
- Validator: [gcr.io/prylabs-dev/prysm/validator:v4.1.0](gcr.io/prylabs-dev/prysm/validator:v4.1.0)
|
||||
- Please note that in the next cycle, we will exclusively use these containers at the canonical URLs.
|
||||
|
||||
### Added
|
||||
|
||||
@@ -2987,4 +2987,4 @@ There are no security updates in this release.
|
||||
|
||||
# Older than v2.0.0
|
||||
|
||||
For changelog history for releases older than v2.0.0, please refer to https://github.com/prysmaticlabs/prysm/releases
|
||||
For changelog history for releases older than v2.0.0, please refer to https://github.com/prysmaticlabs/prysm/releases
|
||||
1689
MODULE.bazel.lock
generated
1689
MODULE.bazel.lock
generated
File diff suppressed because it is too large
Load Diff
46
WORKSPACE
46
WORKSPACE
@@ -16,6 +16,34 @@ load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
|
||||
|
||||
rules_pkg_dependencies()
|
||||
|
||||
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
|
||||
|
||||
http_archive(
|
||||
name = "toolchains_protoc",
|
||||
sha256 = "abb1540f8a9e045422730670ebb2f25b41fa56ca5a7cf795175a110a0a68f4ad",
|
||||
strip_prefix = "toolchains_protoc-0.3.6",
|
||||
url = "https://github.com/aspect-build/toolchains_protoc/releases/download/v0.3.6/toolchains_protoc-v0.3.6.tar.gz",
|
||||
)
|
||||
|
||||
load("@toolchains_protoc//protoc:repositories.bzl", "rules_protoc_dependencies")
|
||||
|
||||
rules_protoc_dependencies()
|
||||
|
||||
load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies")
|
||||
|
||||
rules_proto_dependencies()
|
||||
|
||||
load("@bazel_features//:deps.bzl", "bazel_features_deps")
|
||||
|
||||
bazel_features_deps()
|
||||
|
||||
load("@toolchains_protoc//protoc:toolchain.bzl", "protoc_toolchains")
|
||||
|
||||
protoc_toolchains(
|
||||
name = "protoc_toolchains",
|
||||
version = "v25.3",
|
||||
)
|
||||
|
||||
HERMETIC_CC_TOOLCHAIN_VERSION = "v3.0.1"
|
||||
|
||||
http_archive(
|
||||
@@ -137,10 +165,10 @@ http_archive(
|
||||
# Expose internals of go_test for custom build transitions.
|
||||
"//third_party:io_bazel_rules_go_test.patch",
|
||||
],
|
||||
sha256 = "80a98277ad1311dacd837f9b16db62887702e9f1d1c4c9f796d0121a46c8e184",
|
||||
sha256 = "b2038e2de2cace18f032249cb4bb0048abf583a36369fa98f687af1b3f880b26",
|
||||
urls = [
|
||||
"https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.46.0/rules_go-v0.46.0.zip",
|
||||
"https://github.com/bazelbuild/rules_go/releases/download/v0.46.0/rules_go-v0.46.0.zip",
|
||||
"https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.48.1/rules_go-v0.48.1.zip",
|
||||
"https://github.com/bazelbuild/rules_go/releases/download/v0.48.1/rules_go-v0.48.1.zip",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -182,7 +210,7 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_depe
|
||||
go_rules_dependencies()
|
||||
|
||||
go_register_toolchains(
|
||||
go_version = "1.22.10",
|
||||
go_version = "1.23.5",
|
||||
nogo = "@//:nogo",
|
||||
)
|
||||
|
||||
@@ -227,7 +255,7 @@ filegroup(
|
||||
url = "https://github.com/ethereum/EIPs/archive/5480440fe51742ed23342b68cf106cefd427e39d.tar.gz",
|
||||
)
|
||||
|
||||
consensus_spec_version = "v1.5.0-beta.0"
|
||||
consensus_spec_version = "v1.5.0-beta.2"
|
||||
|
||||
bls_test_version = "v0.1.1"
|
||||
|
||||
@@ -243,7 +271,7 @@ filegroup(
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
""",
|
||||
integrity = "sha256-HdMlTN3wv+hUMCkIRPk+EHcLixY1cSZlvkx3obEp4AM=",
|
||||
integrity = "sha256-X/bMxbKg1clo2aFEjBoeuFq/U+BF1eQopgRP/7nI3Qg=",
|
||||
url = "https://github.com/ethereum/consensus-spec-tests/releases/download/%s/general.tar.gz" % consensus_spec_version,
|
||||
)
|
||||
|
||||
@@ -259,7 +287,7 @@ filegroup(
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
""",
|
||||
integrity = "sha256-eX/ihmHQ+OvfoGJxSMgy22yAU3SZ3xjsX0FU0EaZrSs=",
|
||||
integrity = "sha256-WSxdri5OJGuNApW+odKle5UzToDyEOx+F3lMiqamJAg=",
|
||||
url = "https://github.com/ethereum/consensus-spec-tests/releases/download/%s/minimal.tar.gz" % consensus_spec_version,
|
||||
)
|
||||
|
||||
@@ -275,7 +303,7 @@ filegroup(
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
""",
|
||||
integrity = "sha256-k3Onf42vOzIqyddecR6G82sDy3mmDA+R8RN66QjB0GI=",
|
||||
integrity = "sha256-LYE8l3y/zSt4YVrehrJ3ralqtgeYNildiIp+HR6+xAI=",
|
||||
url = "https://github.com/ethereum/consensus-spec-tests/releases/download/%s/mainnet.tar.gz" % consensus_spec_version,
|
||||
)
|
||||
|
||||
@@ -290,7 +318,7 @@ filegroup(
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
""",
|
||||
integrity = "sha256-N/d4AwdOSlb70Dr+2l20dfXxNSzJDj/qKA9Rkn8Gb5w=",
|
||||
integrity = "sha256-jvZQ90qcJMTOqMsPO7sgeEVQmewZTHcz7LVDkNqwTFQ=",
|
||||
strip_prefix = "consensus-specs-" + consensus_spec_version[1:],
|
||||
url = "https://github.com/ethereum/consensus-specs/archive/refs/tags/%s.tar.gz" % consensus_spec_version,
|
||||
)
|
||||
|
||||
@@ -9,7 +9,10 @@ go_library(
|
||||
],
|
||||
importpath = "github.com/prysmaticlabs/prysm/v5/api/client",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["@com_github_pkg_errors//:go_default_library"],
|
||||
deps = [
|
||||
"@com_github_pkg_errors//:go_default_library",
|
||||
"@io_opentelemetry_go_contrib_instrumentation_net_http_otelhttp//:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
go_test(
|
||||
|
||||
@@ -3,7 +3,6 @@ load("@prysm//tools/go:def.bzl", "go_library", "go_test")
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = [
|
||||
"checkpoint.go",
|
||||
"client.go",
|
||||
"doc.go",
|
||||
"health.go",
|
||||
@@ -16,28 +15,19 @@ go_library(
|
||||
"//api/client/beacon/iface:go_default_library",
|
||||
"//api/server:go_default_library",
|
||||
"//api/server/structs:go_default_library",
|
||||
"//beacon-chain/core/helpers:go_default_library",
|
||||
"//beacon-chain/state:go_default_library",
|
||||
"//consensus-types/interfaces:go_default_library",
|
||||
"//consensus-types/primitives:go_default_library",
|
||||
"//encoding/bytesutil:go_default_library",
|
||||
"//encoding/ssz/detect:go_default_library",
|
||||
"//io/file:go_default_library",
|
||||
"//network/forks:go_default_library",
|
||||
"//proto/prysm/v1alpha1:go_default_library",
|
||||
"//runtime/version:go_default_library",
|
||||
"//time/slots:go_default_library",
|
||||
"@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
|
||||
"@com_github_pkg_errors//:go_default_library",
|
||||
"@com_github_sirupsen_logrus//:go_default_library",
|
||||
"@org_golang_x_mod//semver:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = [
|
||||
"checkpoint_test.go",
|
||||
"client_test.go",
|
||||
"health_test.go",
|
||||
],
|
||||
@@ -45,19 +35,7 @@ go_test(
|
||||
deps = [
|
||||
"//api/client:go_default_library",
|
||||
"//api/client/beacon/testing:go_default_library",
|
||||
"//beacon-chain/state:go_default_library",
|
||||
"//config/params:go_default_library",
|
||||
"//consensus-types/blocks:go_default_library",
|
||||
"//consensus-types/blocks/testing:go_default_library",
|
||||
"//consensus-types/primitives:go_default_library",
|
||||
"//encoding/ssz/detect:go_default_library",
|
||||
"//network/forks:go_default_library",
|
||||
"//proto/prysm/v1alpha1:go_default_library",
|
||||
"//runtime/version:go_default_library",
|
||||
"//testing/require:go_default_library",
|
||||
"//testing/util:go_default_library",
|
||||
"//time/slots:go_default_library",
|
||||
"@com_github_pkg_errors//:go_default_library",
|
||||
"@org_uber_go_mock//gomock:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -1,276 +0,0 @@
|
||||
package beacon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common/hexutil"
|
||||
"github.com/pkg/errors"
|
||||
base "github.com/prysmaticlabs/prysm/v5/api/client"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/helpers"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/state"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/interfaces"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
"github.com/prysmaticlabs/prysm/v5/encoding/bytesutil"
|
||||
"github.com/prysmaticlabs/prysm/v5/encoding/ssz/detect"
|
||||
"github.com/prysmaticlabs/prysm/v5/io/file"
|
||||
"github.com/prysmaticlabs/prysm/v5/runtime/version"
|
||||
"github.com/prysmaticlabs/prysm/v5/time/slots"
|
||||
"github.com/sirupsen/logrus"
|
||||
"golang.org/x/mod/semver"
|
||||
)
|
||||
|
||||
var errCheckpointBlockMismatch = errors.New("mismatch between checkpoint sync state and block")
|
||||
|
||||
// OriginData represents the BeaconState and ReadOnlySignedBeaconBlock necessary to start an empty Beacon Node
|
||||
// using Checkpoint Sync.
|
||||
type OriginData struct {
|
||||
sb []byte
|
||||
bb []byte
|
||||
st state.BeaconState
|
||||
b interfaces.ReadOnlySignedBeaconBlock
|
||||
vu *detect.VersionedUnmarshaler
|
||||
br [32]byte
|
||||
sr [32]byte
|
||||
}
|
||||
|
||||
// SaveBlock saves the downloaded block to a unique file in the given path.
|
||||
// For readability and collision avoidance, the file name includes: type, config name, slot and root
|
||||
func (o *OriginData) SaveBlock(dir string) (string, error) {
|
||||
blockPath := path.Join(dir, fname("block", o.vu, o.b.Block().Slot(), o.br))
|
||||
return blockPath, file.WriteFile(blockPath, o.BlockBytes())
|
||||
}
|
||||
|
||||
// SaveState saves the downloaded state to a unique file in the given path.
|
||||
// For readability and collision avoidance, the file name includes: type, config name, slot and root
|
||||
func (o *OriginData) SaveState(dir string) (string, error) {
|
||||
statePath := path.Join(dir, fname("state", o.vu, o.st.Slot(), o.sr))
|
||||
return statePath, file.WriteFile(statePath, o.StateBytes())
|
||||
}
|
||||
|
||||
// StateBytes returns the ssz-encoded bytes of the downloaded BeaconState value.
|
||||
func (o *OriginData) StateBytes() []byte {
|
||||
return o.sb
|
||||
}
|
||||
|
||||
// BlockBytes returns the ssz-encoded bytes of the downloaded ReadOnlySignedBeaconBlock value.
|
||||
func (o *OriginData) BlockBytes() []byte {
|
||||
return o.bb
|
||||
}
|
||||
|
||||
func fname(prefix string, vu *detect.VersionedUnmarshaler, slot primitives.Slot, root [32]byte) string {
|
||||
return fmt.Sprintf("%s_%s_%s_%d-%#x.ssz", prefix, vu.Config.ConfigName, version.String(vu.Fork), slot, root)
|
||||
}
|
||||
|
||||
// DownloadFinalizedData downloads the most recently finalized state, and the block most recently applied to that state.
|
||||
// This pair can be used to initialize a new beacon node via checkpoint sync.
|
||||
func DownloadFinalizedData(ctx context.Context, client *Client) (*OriginData, error) {
|
||||
sb, err := client.GetState(ctx, IdFinalized)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
vu, err := detect.FromState(sb)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "error detecting chain config for finalized state")
|
||||
}
|
||||
|
||||
log.WithFields(logrus.Fields{
|
||||
"name": vu.Config.ConfigName,
|
||||
"fork": version.String(vu.Fork),
|
||||
}).Info("Detected supported config in remote finalized state")
|
||||
|
||||
s, err := vu.UnmarshalBeaconState(sb)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "error unmarshaling finalized state to correct version")
|
||||
}
|
||||
|
||||
slot := s.LatestBlockHeader().Slot
|
||||
bb, err := client.GetBlock(ctx, IdFromSlot(slot))
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "error requesting block by slot = %d", slot)
|
||||
}
|
||||
b, err := vu.UnmarshalBeaconBlock(bb)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "unable to unmarshal block to a supported type using the detected fork schedule")
|
||||
}
|
||||
br, err := b.Block().HashTreeRoot()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "error computing hash_tree_root of retrieved block")
|
||||
}
|
||||
bodyRoot, err := b.Block().Body().HashTreeRoot()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "error computing hash_tree_root of retrieved block body")
|
||||
}
|
||||
|
||||
sbr := bytesutil.ToBytes32(s.LatestBlockHeader().BodyRoot)
|
||||
if sbr != bodyRoot {
|
||||
return nil, errors.Wrapf(errCheckpointBlockMismatch, "state body root = %#x, block body root = %#x", sbr, bodyRoot)
|
||||
}
|
||||
sr, err := s.HashTreeRoot(ctx)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to compute htr for finalized state at slot=%d", s.Slot())
|
||||
}
|
||||
|
||||
log.
|
||||
WithField("blockSlot", b.Block().Slot()).
|
||||
WithField("stateSlot", s.Slot()).
|
||||
WithField("stateRoot", hexutil.Encode(sr[:])).
|
||||
WithField("blockRoot", hexutil.Encode(br[:])).
|
||||
Info("Downloaded checkpoint sync state and block.")
|
||||
return &OriginData{
|
||||
st: s,
|
||||
b: b,
|
||||
sb: sb,
|
||||
bb: bb,
|
||||
vu: vu,
|
||||
br: br,
|
||||
sr: sr,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// WeakSubjectivityData represents the state root, block root and epoch of the BeaconState + ReadOnlySignedBeaconBlock
|
||||
// that falls at the beginning of the current weak subjectivity period. These values can be used to construct
|
||||
// a weak subjectivity checkpoint beacon node flag to be used for validation.
|
||||
type WeakSubjectivityData struct {
|
||||
BlockRoot [32]byte
|
||||
StateRoot [32]byte
|
||||
Epoch primitives.Epoch
|
||||
}
|
||||
|
||||
// CheckpointString returns the standard string representation of a Checkpoint.
|
||||
// The format is a hex-encoded block root, followed by the epoch of the block, separated by a colon. For example:
|
||||
// "0x1c35540cac127315fabb6bf29181f2ae0de1a3fc909d2e76ba771e61312cc49a:74888"
|
||||
func (wsd *WeakSubjectivityData) CheckpointString() string {
|
||||
return fmt.Sprintf("%#x:%d", wsd.BlockRoot, wsd.Epoch)
|
||||
}
|
||||
|
||||
// ComputeWeakSubjectivityCheckpoint attempts to use the prysm weak_subjectivity api
|
||||
// to obtain the current weak_subjectivity checkpoint.
|
||||
// For non-prysm nodes, the same computation will be performed with extra steps,
|
||||
// using the head state downloaded from the beacon node api.
|
||||
func ComputeWeakSubjectivityCheckpoint(ctx context.Context, client *Client) (*WeakSubjectivityData, error) {
|
||||
ws, err := client.GetWeakSubjectivity(ctx)
|
||||
if err != nil {
|
||||
// a 404/405 is expected if querying an endpoint that doesn't support the weak subjectivity checkpoint api
|
||||
if !errors.Is(err, base.ErrNotOK) {
|
||||
return nil, errors.Wrap(err, "unexpected API response for prysm-only weak subjectivity checkpoint API")
|
||||
}
|
||||
// fall back to vanilla Beacon Node API method
|
||||
return computeBackwardsCompatible(ctx, client)
|
||||
}
|
||||
log.Printf("server weak subjectivity checkpoint response - epoch=%d, block_root=%#x, state_root=%#x", ws.Epoch, ws.BlockRoot, ws.StateRoot)
|
||||
return ws, nil
|
||||
}
|
||||
|
||||
const (
|
||||
prysmMinimumVersion = "v2.0.7"
|
||||
prysmImplementationName = "Prysm"
|
||||
)
|
||||
|
||||
// errUnsupportedPrysmCheckpointVersion indicates remote beacon node can't be used for checkpoint retrieval.
|
||||
var errUnsupportedPrysmCheckpointVersion = errors.New("node does not meet minimum version requirements for checkpoint retrieval")
|
||||
|
||||
// for older endpoints or clients that do not support the weak_subjectivity api method
|
||||
// we gather the necessary data for a checkpoint sync by:
|
||||
// - inspecting the remote server's head state and computing the weak subjectivity epoch locally
|
||||
// - requesting the state at the first slot of the epoch
|
||||
// - using hash_tree_root(state.latest_block_header) to compute the block the state integrates
|
||||
// - requesting that block by its root
|
||||
func computeBackwardsCompatible(ctx context.Context, client *Client) (*WeakSubjectivityData, error) {
|
||||
log.Print("falling back to generic checkpoint derivation, weak_subjectivity API not supported by server")
|
||||
nv, err := client.GetNodeVersion(ctx)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "unable to proceed with fallback method without confirming node version")
|
||||
}
|
||||
if nv.implementation == prysmImplementationName && semver.Compare(nv.semver, prysmMinimumVersion) < 0 {
|
||||
return nil, errors.Wrapf(errUnsupportedPrysmCheckpointVersion, "%s < minimum (%s)", nv.semver, prysmMinimumVersion)
|
||||
}
|
||||
epoch, err := getWeakSubjectivityEpochFromHead(ctx, client)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "error computing weak subjectivity epoch via head state inspection")
|
||||
}
|
||||
|
||||
// use first slot of the epoch for the state slot
|
||||
slot, err := slots.EpochStart(epoch)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "error computing first slot of epoch=%d", epoch)
|
||||
}
|
||||
|
||||
log.Printf("requesting checkpoint state at slot %d", slot)
|
||||
// get the state at the first slot of the epoch
|
||||
sb, err := client.GetState(ctx, IdFromSlot(slot))
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to request state by slot from api, slot=%d", slot)
|
||||
}
|
||||
|
||||
// ConfigFork is used to unmarshal the BeaconState so we can read the block root in latest_block_header
|
||||
vu, err := detect.FromState(sb)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "error detecting chain config for beacon state")
|
||||
}
|
||||
log.Printf("detected supported config in checkpoint state, name=%s, fork=%s", vu.Config.ConfigName, version.String(vu.Fork))
|
||||
|
||||
s, err := vu.UnmarshalBeaconState(sb)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "error using detected config fork to unmarshal state bytes")
|
||||
}
|
||||
|
||||
// compute state and block roots
|
||||
sr, err := s.HashTreeRoot(ctx)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "error computing hash_tree_root of state")
|
||||
}
|
||||
|
||||
h := s.LatestBlockHeader()
|
||||
h.StateRoot = sr[:]
|
||||
br, err := h.HashTreeRoot()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "error while computing block root using state data")
|
||||
}
|
||||
|
||||
bb, err := client.GetBlock(ctx, IdFromRoot(br))
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "error requesting block by root = %d", br)
|
||||
}
|
||||
b, err := vu.UnmarshalBeaconBlock(bb)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "unable to unmarshal block to a supported type using the detected fork schedule")
|
||||
}
|
||||
br, err = b.Block().HashTreeRoot()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "error computing hash_tree_root for block obtained via root")
|
||||
}
|
||||
|
||||
return &WeakSubjectivityData{
|
||||
Epoch: epoch,
|
||||
BlockRoot: br,
|
||||
StateRoot: sr,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// this method downloads the head state, which can be used to find the correct chain config
|
||||
// and use prysm's helper methods to compute the latest weak subjectivity epoch.
|
||||
func getWeakSubjectivityEpochFromHead(ctx context.Context, client *Client) (primitives.Epoch, error) {
|
||||
headBytes, err := client.GetState(ctx, IdHead)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
vu, err := detect.FromState(headBytes)
|
||||
if err != nil {
|
||||
return 0, errors.Wrap(err, "error detecting chain config for beacon state")
|
||||
}
|
||||
log.Printf("detected supported config in remote head state, name=%s, fork=%s", vu.Config.ConfigName, version.String(vu.Fork))
|
||||
headState, err := vu.UnmarshalBeaconState(headBytes)
|
||||
if err != nil {
|
||||
return 0, errors.Wrap(err, "error unmarshaling state to correct version")
|
||||
}
|
||||
|
||||
epoch, err := helpers.LatestWeakSubjectivityEpoch(ctx, headState, vu.Config)
|
||||
if err != nil {
|
||||
return 0, errors.Wrap(err, "error computing the weak subjectivity epoch from head state")
|
||||
}
|
||||
|
||||
log.Printf("(computed client-side) weak subjectivity epoch = %d", epoch)
|
||||
return epoch, nil
|
||||
}
|
||||
@@ -29,12 +29,13 @@ const (
|
||||
getSignedBlockPath = "/eth/v2/beacon/blocks"
|
||||
getBlockRootPath = "/eth/v1/beacon/blocks/{{.Id}}/root"
|
||||
getForkForStatePath = "/eth/v1/beacon/states/{{.Id}}/fork"
|
||||
getWeakSubjectivityPath = "/prysm/v1/beacon/weak_subjectivity"
|
||||
getForkSchedulePath = "/eth/v1/config/fork_schedule"
|
||||
getConfigSpecPath = "/eth/v1/config/spec"
|
||||
getStatePath = "/eth/v2/debug/beacon/states"
|
||||
getNodeVersionPath = "/eth/v1/node/version"
|
||||
changeBLStoExecutionPath = "/eth/v1/beacon/pool/bls_to_execution_changes"
|
||||
|
||||
GetNodeVersionPath = "/eth/v1/node/version"
|
||||
GetWeakSubjectivityPath = "/prysm/v1/beacon/weak_subjectivity"
|
||||
)
|
||||
|
||||
// StateOrBlockId represents the block_id / state_id parameters that several of the Eth Beacon API methods accept.
|
||||
@@ -80,7 +81,8 @@ func idTemplate(ts string) func(StateOrBlockId) string {
|
||||
return f
|
||||
}
|
||||
|
||||
func renderGetBlockPath(id StateOrBlockId) string {
|
||||
// RenderGetBlockPath formats a block id into a path for the GetBlock API endpoint.
|
||||
func RenderGetBlockPath(id StateOrBlockId) string {
|
||||
return path.Join(getSignedBlockPath, string(id))
|
||||
}
|
||||
|
||||
@@ -104,7 +106,7 @@ func NewClient(host string, opts ...client.ClientOpt) (*Client, error) {
|
||||
// for the named identifiers.
|
||||
// The return value contains the ssz-encoded bytes.
|
||||
func (c *Client) GetBlock(ctx context.Context, blockId StateOrBlockId) ([]byte, error) {
|
||||
blockPath := renderGetBlockPath(blockId)
|
||||
blockPath := RenderGetBlockPath(blockId)
|
||||
b, err := c.Get(ctx, blockPath, client.WithSSZEncoding())
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "error requesting state by id = %s", blockId)
|
||||
@@ -195,6 +197,10 @@ type NodeVersion struct {
|
||||
systemInfo string
|
||||
}
|
||||
|
||||
func (nv *NodeVersion) SetImplementation(impl string) {
|
||||
nv.implementation = impl
|
||||
}
|
||||
|
||||
var versionRE = regexp.MustCompile(`^(\w+)/(v\d+\.\d+\.\d+[-a-zA-Z0-9]*)\s*/?(.*)$`)
|
||||
|
||||
func parseNodeVersion(v string) (*NodeVersion, error) {
|
||||
@@ -212,7 +218,7 @@ func parseNodeVersion(v string) (*NodeVersion, error) {
|
||||
// GetNodeVersion requests that the beacon node identify information about its implementation in a format
|
||||
// similar to a HTTP User-Agent field. ex: Lighthouse/v0.1.5 (Linux x86_64)
|
||||
func (c *Client) GetNodeVersion(ctx context.Context) (*NodeVersion, error) {
|
||||
b, err := c.Get(ctx, getNodeVersionPath)
|
||||
b, err := c.Get(ctx, GetNodeVersionPath)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "error requesting node version")
|
||||
}
|
||||
@@ -228,7 +234,8 @@ func (c *Client) GetNodeVersion(ctx context.Context) (*NodeVersion, error) {
|
||||
return parseNodeVersion(d.Data.Version)
|
||||
}
|
||||
|
||||
func renderGetStatePath(id StateOrBlockId) string {
|
||||
// RenderGetStatePath formats a state id into a path for the GetState API endpoint.
|
||||
func RenderGetStatePath(id StateOrBlockId) string {
|
||||
return path.Join(getStatePath, string(id))
|
||||
}
|
||||
|
||||
@@ -246,13 +253,29 @@ func (c *Client) GetState(ctx context.Context, stateId StateOrBlockId) ([]byte,
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// WeakSubjectivityData represents the state root, block root and epoch of the BeaconState + ReadOnlySignedBeaconBlock
|
||||
// that falls at the beginning of the current weak subjectivity period. These values can be used to construct
|
||||
// a weak subjectivity checkpoint beacon node flag to be used for validation.
|
||||
type WeakSubjectivityData struct {
|
||||
BlockRoot [32]byte
|
||||
StateRoot [32]byte
|
||||
Epoch primitives.Epoch
|
||||
}
|
||||
|
||||
// CheckpointString returns the standard string representation of a Checkpoint.
|
||||
// The format is a hex-encoded block root, followed by the epoch of the block, separated by a colon. For example:
|
||||
// "0x1c35540cac127315fabb6bf29181f2ae0de1a3fc909d2e76ba771e61312cc49a:74888"
|
||||
func (wsd *WeakSubjectivityData) CheckpointString() string {
|
||||
return fmt.Sprintf("%#x:%d", wsd.BlockRoot, wsd.Epoch)
|
||||
}
|
||||
|
||||
// GetWeakSubjectivity calls a proposed API endpoint that is unique to prysm
|
||||
// This api method does the following:
|
||||
// - computes weak subjectivity epoch
|
||||
// - finds the highest non-skipped block preceding the epoch
|
||||
// - returns the htr of the found block and returns this + the value of state_root from the block
|
||||
func (c *Client) GetWeakSubjectivity(ctx context.Context) (*WeakSubjectivityData, error) {
|
||||
body, err := c.Get(ctx, getWeakSubjectivityPath)
|
||||
body, err := c.Get(ctx, GetWeakSubjectivityPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -97,31 +97,31 @@ func TestValidHostname(t *testing.T) {
|
||||
{
|
||||
name: "hostname with port",
|
||||
hostArg: "mydomain.org:3500",
|
||||
path: getNodeVersionPath,
|
||||
path: GetNodeVersionPath,
|
||||
joined: "http://mydomain.org:3500/eth/v1/node/version",
|
||||
},
|
||||
{
|
||||
name: "https scheme, hostname with port",
|
||||
hostArg: "https://mydomain.org:3500",
|
||||
path: getNodeVersionPath,
|
||||
path: GetNodeVersionPath,
|
||||
joined: "https://mydomain.org:3500/eth/v1/node/version",
|
||||
},
|
||||
{
|
||||
name: "http scheme, hostname without port",
|
||||
hostArg: "http://mydomain.org",
|
||||
path: getNodeVersionPath,
|
||||
path: GetNodeVersionPath,
|
||||
joined: "http://mydomain.org/eth/v1/node/version",
|
||||
},
|
||||
{
|
||||
name: "http scheme, trailing slash, hostname without port",
|
||||
hostArg: "http://mydomain.org/",
|
||||
path: getNodeVersionPath,
|
||||
path: GetNodeVersionPath,
|
||||
joined: "http://mydomain.org/eth/v1/node/version",
|
||||
},
|
||||
{
|
||||
name: "http scheme, hostname with basic auth creds and no port",
|
||||
hostArg: "http://username:pass@mydomain.org/",
|
||||
path: getNodeVersionPath,
|
||||
path: GetNodeVersionPath,
|
||||
joined: "http://username:pass@mydomain.org/eth/v1/node/version",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ go_library(
|
||||
deps = [
|
||||
"//api:go_default_library",
|
||||
"//api/client:go_default_library",
|
||||
"//api/server:go_default_library",
|
||||
"//api/server/structs:go_default_library",
|
||||
"//config/fieldparams:go_default_library",
|
||||
"//config/params:go_default_library",
|
||||
@@ -27,6 +28,7 @@ go_library(
|
||||
"//proto/engine/v1:go_default_library",
|
||||
"//proto/prysm/v1alpha1:go_default_library",
|
||||
"//runtime/version:go_default_library",
|
||||
"@com_github_ethereum_go_ethereum//common:go_default_library",
|
||||
"@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
|
||||
"@com_github_pkg_errors//:go_default_library",
|
||||
"@com_github_prysmaticlabs_fastssz//:go_default_library",
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
package builder
|
||||
|
||||
import (
|
||||
"github.com/pkg/errors"
|
||||
ssz "github.com/prysmaticlabs/fastssz"
|
||||
consensus_types "github.com/prysmaticlabs/prysm/v5/consensus-types"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/blocks"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/interfaces"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
v1 "github.com/prysmaticlabs/prysm/v5/proto/engine/v1"
|
||||
ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"
|
||||
"github.com/prysmaticlabs/prysm/v5/runtime/version"
|
||||
)
|
||||
@@ -22,7 +22,6 @@ type SignedBid interface {
|
||||
// Bid is an interface describing the method set of a builder bid.
|
||||
type Bid interface {
|
||||
Header() (interfaces.ExecutionData, error)
|
||||
BlobKzgCommitments() ([][]byte, error)
|
||||
Value() primitives.Wei
|
||||
Pubkey() []byte
|
||||
Version() int
|
||||
@@ -31,6 +30,18 @@ type Bid interface {
|
||||
HashTreeRootWith(hh *ssz.Hasher) error
|
||||
}
|
||||
|
||||
// BidDeneb is an interface that exposes newly added kzg commitments on top of builder bid
|
||||
type BidDeneb interface {
|
||||
Bid
|
||||
BlobKzgCommitments() [][]byte
|
||||
}
|
||||
|
||||
// BidElectra is an interface that exposes the newly added execution requests on top of the builder bid
|
||||
type BidElectra interface {
|
||||
BidDeneb
|
||||
ExecutionRequests() *v1.ExecutionRequests
|
||||
}
|
||||
|
||||
type signedBuilderBid struct {
|
||||
p *ethpb.SignedBuilderBid
|
||||
}
|
||||
@@ -115,11 +126,6 @@ func (b builderBid) Header() (interfaces.ExecutionData, error) {
|
||||
return blocks.WrappedExecutionPayloadHeader(b.p.Header)
|
||||
}
|
||||
|
||||
// BlobKzgCommitments --
|
||||
func (b builderBid) BlobKzgCommitments() ([][]byte, error) {
|
||||
return [][]byte{}, errors.New("blob kzg commitments not available before Deneb")
|
||||
}
|
||||
|
||||
// Version --
|
||||
func (b builderBid) Version() int {
|
||||
return version.Bellatrix
|
||||
@@ -169,11 +175,6 @@ func (b builderBidCapella) Header() (interfaces.ExecutionData, error) {
|
||||
return blocks.WrappedExecutionPayloadHeaderCapella(b.p.Header)
|
||||
}
|
||||
|
||||
// BlobKzgCommitments --
|
||||
func (b builderBidCapella) BlobKzgCommitments() ([][]byte, error) {
|
||||
return [][]byte{}, errors.New("blob kzg commitments not available before Deneb")
|
||||
}
|
||||
|
||||
// Version --
|
||||
func (b builderBidCapella) Version() int {
|
||||
return version.Capella
|
||||
@@ -254,8 +255,8 @@ func (b builderBidDeneb) Header() (interfaces.ExecutionData, error) {
|
||||
}
|
||||
|
||||
// BlobKzgCommitments --
|
||||
func (b builderBidDeneb) BlobKzgCommitments() ([][]byte, error) {
|
||||
return b.p.BlobKzgCommitments, nil
|
||||
func (b builderBidDeneb) BlobKzgCommitments() [][]byte {
|
||||
return b.p.BlobKzgCommitments
|
||||
}
|
||||
|
||||
type signedBuilderBidDeneb struct {
|
||||
@@ -290,3 +291,95 @@ func (b signedBuilderBidDeneb) Version() int {
|
||||
func (b signedBuilderBidDeneb) IsNil() bool {
|
||||
return b.p == nil
|
||||
}
|
||||
|
||||
type builderBidElectra struct {
|
||||
p *ethpb.BuilderBidElectra
|
||||
}
|
||||
|
||||
// WrappedBuilderBidElectra is a constructor which wraps a protobuf bid into an interface.
|
||||
func WrappedBuilderBidElectra(p *ethpb.BuilderBidElectra) (Bid, error) {
|
||||
w := builderBidElectra{p: p}
|
||||
if w.IsNil() {
|
||||
return nil, consensus_types.ErrNilObjectWrapped
|
||||
}
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// Version --
|
||||
func (b builderBidElectra) Version() int {
|
||||
return version.Electra
|
||||
}
|
||||
|
||||
// Value --
|
||||
func (b builderBidElectra) Value() primitives.Wei {
|
||||
return primitives.LittleEndianBytesToWei(b.p.Value)
|
||||
}
|
||||
|
||||
// Pubkey --
|
||||
func (b builderBidElectra) Pubkey() []byte {
|
||||
return b.p.Pubkey
|
||||
}
|
||||
|
||||
// IsNil --
|
||||
func (b builderBidElectra) IsNil() bool {
|
||||
return b.p == nil
|
||||
}
|
||||
|
||||
// HashTreeRoot --
|
||||
func (b builderBidElectra) HashTreeRoot() ([32]byte, error) {
|
||||
return b.p.HashTreeRoot()
|
||||
}
|
||||
|
||||
// HashTreeRootWith --
|
||||
func (b builderBidElectra) HashTreeRootWith(hh *ssz.Hasher) error {
|
||||
return b.p.HashTreeRootWith(hh)
|
||||
}
|
||||
|
||||
// Header --
|
||||
func (b builderBidElectra) Header() (interfaces.ExecutionData, error) {
|
||||
// We have to convert big endian to little endian because the value is coming from the execution layer.
|
||||
return blocks.WrappedExecutionPayloadHeaderDeneb(b.p.Header)
|
||||
}
|
||||
|
||||
// ExecutionRequests --
|
||||
func (b builderBidElectra) ExecutionRequests() *v1.ExecutionRequests {
|
||||
return b.p.ExecutionRequests // does not copy
|
||||
}
|
||||
|
||||
// BlobKzgCommitments --
|
||||
func (b builderBidElectra) BlobKzgCommitments() [][]byte {
|
||||
return b.p.BlobKzgCommitments
|
||||
}
|
||||
|
||||
type signedBuilderBidElectra struct {
|
||||
p *ethpb.SignedBuilderBidElectra
|
||||
}
|
||||
|
||||
// WrappedSignedBuilderBidElectra is a constructor which wraps a protobuf signed bit into an interface.
|
||||
func WrappedSignedBuilderBidElectra(p *ethpb.SignedBuilderBidElectra) (SignedBid, error) {
|
||||
w := signedBuilderBidElectra{p: p}
|
||||
if w.IsNil() {
|
||||
return nil, consensus_types.ErrNilObjectWrapped
|
||||
}
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// Message --
|
||||
func (b signedBuilderBidElectra) Message() (Bid, error) {
|
||||
return WrappedBuilderBidElectra(b.p.Message)
|
||||
}
|
||||
|
||||
// Signature --
|
||||
func (b signedBuilderBidElectra) Signature() []byte {
|
||||
return b.p.Signature
|
||||
}
|
||||
|
||||
// Version --
|
||||
func (b signedBuilderBidElectra) Version() int {
|
||||
return version.Electra
|
||||
}
|
||||
|
||||
// IsNil --
|
||||
func (b signedBuilderBidElectra) IsNil() bool {
|
||||
return b.p == nil
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"
|
||||
"github.com/prysmaticlabs/prysm/v5/runtime/version"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -108,7 +109,9 @@ func NewClient(host string, opts ...ClientOpt) (*Client, error) {
|
||||
return nil, err
|
||||
}
|
||||
c := &Client{
|
||||
hc: &http.Client{},
|
||||
hc: &http.Client{
|
||||
Transport: otelhttp.NewTransport(http.DefaultTransport),
|
||||
},
|
||||
baseURL: u,
|
||||
}
|
||||
for _, o := range opts {
|
||||
@@ -219,8 +222,23 @@ func (c *Client) GetHeader(ctx context.Context, slot primitives.Slot, parentHash
|
||||
if err := json.Unmarshal(hb, v); err != nil {
|
||||
return nil, errors.Wrapf(err, "error unmarshaling the builder GetHeader response, using slot=%d, parentHash=%#x, pubkey=%#x", slot, parentHash, pubkey)
|
||||
}
|
||||
switch strings.ToLower(v.Version) {
|
||||
case strings.ToLower(version.String(version.Deneb)):
|
||||
|
||||
ver, err := version.FromString(strings.ToLower(v.Version))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, fmt.Sprintf("unsupported header version %s", strings.ToLower(v.Version)))
|
||||
}
|
||||
if ver >= version.Electra {
|
||||
hr := &ExecHeaderResponseElectra{}
|
||||
if err := json.Unmarshal(hb, hr); err != nil {
|
||||
return nil, errors.Wrapf(err, "error unmarshaling the builder GetHeader response, using slot=%d, parentHash=%#x, pubkey=%#x", slot, parentHash, pubkey)
|
||||
}
|
||||
p, err := hr.ToProto()
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "could not extract proto message from header")
|
||||
}
|
||||
return WrappedSignedBuilderBidElectra(p)
|
||||
}
|
||||
if ver >= version.Deneb {
|
||||
hr := &ExecHeaderResponseDeneb{}
|
||||
if err := json.Unmarshal(hb, hr); err != nil {
|
||||
return nil, errors.Wrapf(err, "error unmarshaling the builder GetHeader response, using slot=%d, parentHash=%#x, pubkey=%#x", slot, parentHash, pubkey)
|
||||
@@ -230,7 +248,8 @@ func (c *Client) GetHeader(ctx context.Context, slot primitives.Slot, parentHash
|
||||
return nil, errors.Wrapf(err, "could not extract proto message from header")
|
||||
}
|
||||
return WrappedSignedBuilderBidDeneb(p)
|
||||
case strings.ToLower(version.String(version.Capella)):
|
||||
}
|
||||
if ver >= version.Capella {
|
||||
hr := &ExecHeaderResponseCapella{}
|
||||
if err := json.Unmarshal(hb, hr); err != nil {
|
||||
return nil, errors.Wrapf(err, "error unmarshaling the builder GetHeader response, using slot=%d, parentHash=%#x, pubkey=%#x", slot, parentHash, pubkey)
|
||||
@@ -240,7 +259,8 @@ func (c *Client) GetHeader(ctx context.Context, slot primitives.Slot, parentHash
|
||||
return nil, errors.Wrapf(err, "could not extract proto message from header")
|
||||
}
|
||||
return WrappedSignedBuilderBidCapella(p)
|
||||
case strings.ToLower(version.String(version.Bellatrix)):
|
||||
}
|
||||
if ver >= version.Bellatrix {
|
||||
hr := &ExecHeaderResponse{}
|
||||
if err := json.Unmarshal(hb, hr); err != nil {
|
||||
return nil, errors.Wrapf(err, "error unmarshaling the builder GetHeader response, using slot=%d, parentHash=%#x, pubkey=%#x", slot, parentHash, pubkey)
|
||||
@@ -250,9 +270,8 @@ func (c *Client) GetHeader(ctx context.Context, slot primitives.Slot, parentHash
|
||||
return nil, errors.Wrap(err, "could not extract proto message from header")
|
||||
}
|
||||
return WrappedSignedBuilderBid(p)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported header version %s", strings.ToLower(v.Version))
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported header version %s", strings.ToLower(v.Version))
|
||||
}
|
||||
|
||||
// RegisterValidator encodes the SignedValidatorRegistrationV1 message to json (including hex-encoding the byte
|
||||
|
||||
@@ -266,9 +266,9 @@ func TestClient_GetHeader(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, value.Int.Cmp(primitives.WeiToBigInt(bid.Value())))
|
||||
require.Equal(t, bidStr, primitives.WeiToBigInt(bid.Value()).String())
|
||||
|
||||
kcgCommitments, err := bid.BlobKzgCommitments()
|
||||
require.NoError(t, err)
|
||||
dbid, ok := bid.(builderBidDeneb)
|
||||
require.Equal(t, true, ok)
|
||||
kcgCommitments := dbid.BlobKzgCommitments()
|
||||
require.Equal(t, len(kcgCommitments) > 0, true)
|
||||
for i := range kcgCommitments {
|
||||
require.Equal(t, len(kcgCommitments[i]) == 48, true)
|
||||
@@ -292,6 +292,50 @@ func TestClient_GetHeader(t *testing.T) {
|
||||
_, err := c.GetHeader(ctx, slot, bytesutil.ToBytes32(parentHash), bytesutil.ToBytes48(pubkey))
|
||||
require.ErrorContains(t, "could not extract proto message from header: too many blob commitments: 7", err)
|
||||
})
|
||||
t.Run("electra", func(t *testing.T) {
|
||||
hc := &http.Client{
|
||||
Transport: roundtrip(func(r *http.Request) (*http.Response, error) {
|
||||
require.Equal(t, expectedPath, r.URL.Path)
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Body: io.NopCloser(bytes.NewBufferString(testExampleHeaderResponseElectra)),
|
||||
Request: r.Clone(ctx),
|
||||
}, nil
|
||||
}),
|
||||
}
|
||||
c := &Client{
|
||||
hc: hc,
|
||||
baseURL: &url.URL{Host: "localhost:3500", Scheme: "http"},
|
||||
}
|
||||
h, err := c.GetHeader(ctx, slot, bytesutil.ToBytes32(parentHash), bytesutil.ToBytes48(pubkey))
|
||||
require.NoError(t, err)
|
||||
expectedWithdrawalsRoot := ezDecode(t, "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2")
|
||||
bid, err := h.Message()
|
||||
require.NoError(t, err)
|
||||
bidHeader, err := bid.Header()
|
||||
require.NoError(t, err)
|
||||
withdrawalsRoot, err := bidHeader.WithdrawalsRoot()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, true, bytes.Equal(expectedWithdrawalsRoot, withdrawalsRoot))
|
||||
|
||||
bidStr := "652312848583266388373324160190187140051835877600158453279131187530910662656"
|
||||
value, err := stringToUint256(bidStr)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, value.Int.Cmp(primitives.WeiToBigInt(bid.Value())))
|
||||
require.Equal(t, bidStr, primitives.WeiToBigInt(bid.Value()).String())
|
||||
ebid, ok := bid.(builderBidElectra)
|
||||
require.Equal(t, true, ok)
|
||||
kcgCommitments := ebid.BlobKzgCommitments()
|
||||
require.Equal(t, len(kcgCommitments) > 0, true)
|
||||
for i := range kcgCommitments {
|
||||
require.Equal(t, len(kcgCommitments[i]) == 48, true)
|
||||
}
|
||||
requests := ebid.ExecutionRequests()
|
||||
require.Equal(t, 1, len(requests.Deposits))
|
||||
require.Equal(t, 1, len(requests.Withdrawals))
|
||||
require.Equal(t, 1, len(requests.Consolidations))
|
||||
|
||||
})
|
||||
t.Run("unsupported version", func(t *testing.T) {
|
||||
hc := &http.Client{
|
||||
Transport: roundtrip(func(r *http.Request) (*http.Response, error) {
|
||||
|
||||
@@ -5,13 +5,15 @@ import (
|
||||
"fmt"
|
||||
"math/big"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/common/hexutil"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prysmaticlabs/prysm/v5/api/server"
|
||||
fieldparams "github.com/prysmaticlabs/prysm/v5/config/fieldparams"
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
consensusblocks "github.com/prysmaticlabs/prysm/v5/consensus-types/blocks"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/interfaces"
|
||||
types "github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
"github.com/prysmaticlabs/prysm/v5/encoding/bytesutil"
|
||||
"github.com/prysmaticlabs/prysm/v5/math"
|
||||
@@ -414,54 +416,10 @@ func FromProtoDeneb(payload *v1.ExecutionPayloadDeneb) (ExecutionPayloadDeneb, e
|
||||
}, nil
|
||||
}
|
||||
|
||||
var errInvalidTypeConversion = errors.New("unable to translate between api and foreign type")
|
||||
|
||||
// ExecutionPayloadResponseFromData converts an ExecutionData interface value to a payload response.
|
||||
// This involves serializing the execution payload value so that the abstract payload envelope can be used.
|
||||
func ExecutionPayloadResponseFromData(ed interfaces.ExecutionData, bundle *v1.BlobsBundle) (*ExecutionPayloadResponse, error) {
|
||||
pb := ed.Proto()
|
||||
var data interface{}
|
||||
var err error
|
||||
var ver string
|
||||
switch pbStruct := pb.(type) {
|
||||
case *v1.ExecutionPayload:
|
||||
ver = version.String(version.Bellatrix)
|
||||
data, err = FromProto(pbStruct)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to convert a Bellatrix ExecutionPayload to an API response")
|
||||
}
|
||||
case *v1.ExecutionPayloadCapella:
|
||||
ver = version.String(version.Capella)
|
||||
data, err = FromProtoCapella(pbStruct)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to convert a Capella ExecutionPayload to an API response")
|
||||
}
|
||||
case *v1.ExecutionPayloadDeneb:
|
||||
ver = version.String(version.Deneb)
|
||||
payloadStruct, err := FromProtoDeneb(pbStruct)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to convert a Deneb ExecutionPayload to an API response")
|
||||
}
|
||||
data = &ExecutionPayloadDenebAndBlobsBundle{
|
||||
ExecutionPayload: &payloadStruct,
|
||||
BlobsBundle: FromBundleProto(bundle),
|
||||
}
|
||||
default:
|
||||
return nil, errInvalidTypeConversion
|
||||
}
|
||||
encoded, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to marshal execution payload version=%s", ver)
|
||||
}
|
||||
return &ExecutionPayloadResponse{
|
||||
Version: ver,
|
||||
Data: encoded,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ExecHeaderResponseCapella is the response of builder API /eth/v1/builder/header/{slot}/{parent_hash}/{pubkey} for Capella.
|
||||
type ExecHeaderResponseCapella struct {
|
||||
Data struct {
|
||||
Version string `json:"version"`
|
||||
Data struct {
|
||||
Signature hexutil.Bytes `json:"signature"`
|
||||
Message *BuilderBidCapella `json:"message"`
|
||||
} `json:"data"`
|
||||
@@ -605,17 +563,25 @@ type BlobBundler interface {
|
||||
BundleProto() (*v1.BlobsBundle, error)
|
||||
}
|
||||
|
||||
// ParsedExecutionRequests can retrieve the underlying execution requests for the given execution payload response.
|
||||
type ParsedExecutionRequests interface {
|
||||
ExecutionRequestsProto() (*v1.ExecutionRequests, error)
|
||||
}
|
||||
|
||||
func (r *ExecutionPayloadResponse) ParsePayload() (ParsedPayload, error) {
|
||||
var toProto ParsedPayload
|
||||
switch r.Version {
|
||||
case version.String(version.Bellatrix):
|
||||
toProto = &ExecutionPayload{}
|
||||
case version.String(version.Capella):
|
||||
toProto = &ExecutionPayloadCapella{}
|
||||
case version.String(version.Deneb):
|
||||
v, err := version.FromString(strings.ToLower(r.Version))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, fmt.Sprintf("unsupported version %s", strings.ToLower(r.Version)))
|
||||
}
|
||||
if v >= version.Deneb {
|
||||
toProto = &ExecutionPayloadDenebAndBlobsBundle{}
|
||||
default:
|
||||
return nil, consensusblocks.ErrUnsupportedVersion
|
||||
} else if v >= version.Capella {
|
||||
toProto = &ExecutionPayloadCapella{}
|
||||
} else if v >= version.Bellatrix {
|
||||
toProto = &ExecutionPayload{}
|
||||
} else {
|
||||
return nil, fmt.Errorf("unsupported version %s", strings.ToLower(r.Version))
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(r.Data, toProto); err != nil {
|
||||
@@ -990,7 +956,8 @@ func (ch *BLSToExecutionChange) MarshalJSON() ([]byte, error) {
|
||||
|
||||
// ExecHeaderResponseDeneb is the header response for builder API /eth/v1/builder/header/{slot}/{parent_hash}/{pubkey}.
|
||||
type ExecHeaderResponseDeneb struct {
|
||||
Data struct {
|
||||
Version string `json:"version"`
|
||||
Data struct {
|
||||
Signature hexutil.Bytes `json:"signature"`
|
||||
Message *BuilderBidDeneb `json:"message"`
|
||||
} `json:"data"`
|
||||
@@ -1307,6 +1274,208 @@ func (p *ExecutionPayloadDeneb) ToProto() (*v1.ExecutionPayloadDeneb, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ExecHeaderResponseElectra is the header response for builder API /eth/v1/builder/header/{slot}/{parent_hash}/{pubkey}.
|
||||
type ExecHeaderResponseElectra struct {
|
||||
Version string `json:"version"`
|
||||
Data struct {
|
||||
Signature hexutil.Bytes `json:"signature"`
|
||||
Message *BuilderBidElectra `json:"message"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// ToProto creates a SignedBuilderBidElectra Proto from ExecHeaderResponseElectra.
|
||||
func (ehr *ExecHeaderResponseElectra) ToProto() (*eth.SignedBuilderBidElectra, error) {
|
||||
bb, err := ehr.Data.Message.ToProto()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ð.SignedBuilderBidElectra{
|
||||
Message: bb,
|
||||
Signature: bytesutil.SafeCopyBytes(ehr.Data.Signature),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ToProto creates a BuilderBidElectra Proto from BuilderBidElectra.
|
||||
func (bb *BuilderBidElectra) ToProto() (*eth.BuilderBidElectra, error) {
|
||||
header, err := bb.Header.ToProto()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(bb.BlobKzgCommitments) > params.BeaconConfig().MaxBlobsPerBlockByVersion(version.Electra) {
|
||||
return nil, fmt.Errorf("blob commitment count %d exceeds the maximum %d", len(bb.BlobKzgCommitments), params.BeaconConfig().MaxBlobsPerBlockByVersion(version.Electra))
|
||||
}
|
||||
kzgCommitments := make([][]byte, len(bb.BlobKzgCommitments))
|
||||
for i, commit := range bb.BlobKzgCommitments {
|
||||
if len(commit) != fieldparams.BLSPubkeyLength {
|
||||
return nil, fmt.Errorf("commitment length %d is not %d", len(commit), fieldparams.BLSPubkeyLength)
|
||||
}
|
||||
kzgCommitments[i] = bytesutil.SafeCopyBytes(commit)
|
||||
}
|
||||
// post electra execution requests should not be nil, if no requests exist use an empty request
|
||||
if bb.ExecutionRequests == nil {
|
||||
return nil, errors.New("bid contains nil execution requests")
|
||||
}
|
||||
executionRequests, err := bb.ExecutionRequests.ToProto()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to convert ExecutionRequests")
|
||||
}
|
||||
return ð.BuilderBidElectra{
|
||||
Header: header,
|
||||
BlobKzgCommitments: kzgCommitments,
|
||||
ExecutionRequests: executionRequests,
|
||||
// Note that SSZBytes() reverses byte order for the little-endian representation.
|
||||
// Uint256.Bytes() is big-endian, SSZBytes takes this value and reverses it.
|
||||
Value: bytesutil.SafeCopyBytes(bb.Value.SSZBytes()),
|
||||
Pubkey: bytesutil.SafeCopyBytes(bb.Pubkey),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ExecutionRequestsV1 is a wrapper for different execution requests
|
||||
type ExecutionRequestsV1 struct {
|
||||
Deposits []*DepositRequestV1 `json:"deposits"`
|
||||
Withdrawals []*WithdrawalRequestV1 `json:"withdrawals"`
|
||||
Consolidations []*ConsolidationRequestV1 `json:"consolidations"`
|
||||
}
|
||||
|
||||
func (er *ExecutionRequestsV1) ToProto() (*v1.ExecutionRequests, error) {
|
||||
if uint64(len(er.Deposits)) > params.BeaconConfig().MaxDepositRequestsPerPayload {
|
||||
return nil, fmt.Errorf("deposit requests count %d exceeds the maximum %d", len(er.Deposits), params.BeaconConfig().MaxDepositRequestsPerPayload)
|
||||
}
|
||||
deposits := make([]*v1.DepositRequest, len(er.Deposits))
|
||||
for i, dep := range er.Deposits {
|
||||
d, err := dep.ToProto()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
deposits[i] = d
|
||||
}
|
||||
if uint64(len(er.Withdrawals)) > params.BeaconConfig().MaxWithdrawalRequestsPerPayload {
|
||||
return nil, fmt.Errorf("withdrawal requests count %d exceeds the maximum %d", len(er.Withdrawals), params.BeaconConfig().MaxWithdrawalRequestsPerPayload)
|
||||
}
|
||||
withdrawals := make([]*v1.WithdrawalRequest, len(er.Withdrawals))
|
||||
for i, wr := range er.Withdrawals {
|
||||
w, err := wr.ToProto()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
withdrawals[i] = w
|
||||
}
|
||||
if uint64(len(er.Consolidations)) > params.BeaconConfig().MaxConsolidationsRequestsPerPayload {
|
||||
return nil, fmt.Errorf("consolidation requests count %d exceeds the maximum %d", len(er.Consolidations), params.BeaconConfig().MaxConsolidationsRequestsPerPayload)
|
||||
}
|
||||
consolidations := make([]*v1.ConsolidationRequest, len(er.Consolidations))
|
||||
for i, con := range er.Consolidations {
|
||||
c, err := con.ToProto()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
consolidations[i] = c
|
||||
}
|
||||
return &v1.ExecutionRequests{
|
||||
Deposits: deposits,
|
||||
Withdrawals: withdrawals,
|
||||
Consolidations: consolidations,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// BuilderBidElectra is a field of ExecHeaderResponseElectra.
|
||||
type BuilderBidElectra struct {
|
||||
Header *ExecutionPayloadHeaderDeneb `json:"header"`
|
||||
BlobKzgCommitments []hexutil.Bytes `json:"blob_kzg_commitments"`
|
||||
ExecutionRequests *ExecutionRequestsV1 `json:"execution_requests"`
|
||||
Value Uint256 `json:"value"`
|
||||
Pubkey hexutil.Bytes `json:"pubkey"`
|
||||
}
|
||||
|
||||
// WithdrawalRequestV1 is a field of ExecutionRequestsV1.
|
||||
type WithdrawalRequestV1 struct {
|
||||
SourceAddress hexutil.Bytes `json:"source_address"`
|
||||
ValidatorPubkey hexutil.Bytes `json:"validator_pubkey"`
|
||||
Amount Uint256 `json:"amount"`
|
||||
}
|
||||
|
||||
func (wr *WithdrawalRequestV1) ToProto() (*v1.WithdrawalRequest, error) {
|
||||
srcAddress, err := bytesutil.DecodeHexWithLength(wr.SourceAddress.String(), common.AddressLength)
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "source_address")
|
||||
}
|
||||
pubkey, err := bytesutil.DecodeHexWithLength(wr.ValidatorPubkey.String(), fieldparams.BLSPubkeyLength)
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "validator_pubkey")
|
||||
}
|
||||
|
||||
return &v1.WithdrawalRequest{
|
||||
SourceAddress: srcAddress,
|
||||
ValidatorPubkey: pubkey,
|
||||
Amount: wr.Amount.Uint64(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DepositRequestV1 is a field of ExecutionRequestsV1.
|
||||
type DepositRequestV1 struct {
|
||||
PubKey hexutil.Bytes `json:"pubkey"`
|
||||
// withdrawalCredentials: DATA, 32 Bytes
|
||||
WithdrawalCredentials hexutil.Bytes `json:"withdrawal_credentials"`
|
||||
// amount: QUANTITY, 64 Bits
|
||||
Amount Uint256 `json:"amount"`
|
||||
// signature: DATA, 96 Bytes
|
||||
Signature hexutil.Bytes `json:"signature"`
|
||||
// index: QUANTITY, 64 Bits
|
||||
Index Uint256 `json:"index"`
|
||||
}
|
||||
|
||||
func (dr *DepositRequestV1) ToProto() (*v1.DepositRequest, error) {
|
||||
pubkey, err := bytesutil.DecodeHexWithLength(dr.PubKey.String(), fieldparams.BLSPubkeyLength)
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "pubkey")
|
||||
}
|
||||
wc, err := bytesutil.DecodeHexWithLength(dr.WithdrawalCredentials.String(), fieldparams.RootLength)
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "withdrawal_credentials")
|
||||
}
|
||||
sig, err := bytesutil.DecodeHexWithLength(dr.Signature.String(), fieldparams.BLSSignatureLength)
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "signature")
|
||||
}
|
||||
return &v1.DepositRequest{
|
||||
Pubkey: pubkey,
|
||||
WithdrawalCredentials: wc,
|
||||
Amount: dr.Amount.Uint64(),
|
||||
Signature: sig,
|
||||
Index: dr.Index.Uint64(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ConsolidationRequestV1 is a field of ExecutionRequestsV1.
|
||||
type ConsolidationRequestV1 struct {
|
||||
// sourceAddress: DATA, 20 Bytes
|
||||
SourceAddress hexutil.Bytes `json:"source_address"`
|
||||
// sourcePubkey: DATA, 48 Bytes
|
||||
SourcePubkey hexutil.Bytes `json:"source_pubkey"`
|
||||
// targetPubkey: DATA, 48 Bytes
|
||||
TargetPubkey hexutil.Bytes `json:"target_pubkey"`
|
||||
}
|
||||
|
||||
func (cr *ConsolidationRequestV1) ToProto() (*v1.ConsolidationRequest, error) {
|
||||
srcAddress, err := bytesutil.DecodeHexWithLength(cr.SourceAddress.String(), common.AddressLength)
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "source_address")
|
||||
}
|
||||
sourcePubkey, err := bytesutil.DecodeHexWithLength(cr.SourcePubkey.String(), fieldparams.BLSPubkeyLength)
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "source_pubkey")
|
||||
}
|
||||
targetPubkey, err := bytesutil.DecodeHexWithLength(cr.TargetPubkey.String(), fieldparams.BLSPubkeyLength)
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "target_pubkey")
|
||||
}
|
||||
return &v1.ConsolidationRequest{
|
||||
SourceAddress: srcAddress,
|
||||
SourcePubkey: sourcePubkey,
|
||||
TargetPubkey: targetPubkey,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ErrorMessage is a JSON representation of the builder API's returned error message.
|
||||
type ErrorMessage struct {
|
||||
Code int `json:"code"`
|
||||
|
||||
@@ -154,6 +154,64 @@ var testExampleHeaderResponseDeneb = `{
|
||||
}
|
||||
}`
|
||||
|
||||
var testExampleHeaderResponseElectra = `{
|
||||
"version": "electra",
|
||||
"data": {
|
||||
"message": {
|
||||
"header": {
|
||||
"parent_hash": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
|
||||
"fee_recipient": "0xabcf8e0d4e9587369b2301d0790347320302cc09",
|
||||
"state_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
|
||||
"receipts_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
|
||||
"logs_bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
|
||||
"prev_randao": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
|
||||
"block_number": "1",
|
||||
"gas_limit": "1",
|
||||
"gas_used": "1",
|
||||
"timestamp": "1",
|
||||
"extra_data": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
|
||||
"base_fee_per_gas": "1",
|
||||
"blob_gas_used": "1",
|
||||
"excess_blob_gas": "1",
|
||||
"block_hash": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
|
||||
"transactions_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
|
||||
"withdrawals_root": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"
|
||||
},
|
||||
"blob_kzg_commitments": [
|
||||
"0xa94170080872584e54a1cf092d845703b13907f2e6b3b1c0ad573b910530499e3bcd48c6378846b80d2bfa58c81cf3d5"
|
||||
],
|
||||
"execution_requests": {
|
||||
"deposits": [
|
||||
{
|
||||
"pubkey": "0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a",
|
||||
"withdrawal_credentials": "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2",
|
||||
"amount": "1",
|
||||
"signature": "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505",
|
||||
"index": "1"
|
||||
}
|
||||
],
|
||||
"withdrawals": [
|
||||
{
|
||||
"source_address": "0xabcf8e0d4e9587369b2301d0790347320302cc09",
|
||||
"validator_pubkey": "0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a",
|
||||
"amount": "1"
|
||||
}
|
||||
],
|
||||
"consolidations": [
|
||||
{
|
||||
"source_address": "0xabcf8e0d4e9587369b2301d0790347320302cc09",
|
||||
"source_pubkey": "0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a",
|
||||
"target_pubkey": "0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a"
|
||||
}
|
||||
]
|
||||
},
|
||||
"value": "652312848583266388373324160190187140051835877600158453279131187530910662656",
|
||||
"pubkey": "0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a"
|
||||
},
|
||||
"signature": "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505"
|
||||
}
|
||||
}`
|
||||
|
||||
var testExampleHeaderResponseDenebNoBundle = `{
|
||||
"version": "deneb",
|
||||
"data": {
|
||||
@@ -1924,9 +1982,9 @@ func TestEmptyResponseBody(t *testing.T) {
|
||||
emptyResponse := &ExecutionPayloadResponse{}
|
||||
require.NoError(t, json.Unmarshal(empty, emptyResponse))
|
||||
_, err := emptyResponse.ParsePayload()
|
||||
require.ErrorIs(t, err, consensusblocks.ErrUnsupportedVersion)
|
||||
require.ErrorContains(t, "unsupported version", err)
|
||||
})
|
||||
versions := []int{version.Bellatrix, version.Capella, version.Deneb}
|
||||
versions := []int{version.Bellatrix, version.Capella, version.Deneb, version.Electra}
|
||||
for i := range versions {
|
||||
vstr := version.String(versions[i])
|
||||
t.Run("populated version without payload"+vstr, func(t *testing.T) {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"net/url"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -32,8 +33,10 @@ func NewClient(host string, opts ...ClientOpt) (*Client, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c := &Client{
|
||||
hc: &http.Client{},
|
||||
var c = &Client{
|
||||
hc: &http.Client{
|
||||
Transport: otelhttp.NewTransport(http.DefaultTransport),
|
||||
},
|
||||
baseURL: u,
|
||||
maxBodySize: MaxBodySize,
|
||||
}
|
||||
|
||||
@@ -74,7 +74,7 @@ func AppendHeaders(parent context.Context, headers []string) context.Context {
|
||||
logrus.Warnf("Incorrect gRPC header flag format. Skipping %v", keyValue[0])
|
||||
continue
|
||||
}
|
||||
parent = metadata.AppendToOutgoingContext(parent, keyValue[0], strings.Join(keyValue[1:], "="))
|
||||
parent = metadata.AppendToOutgoingContext(parent, keyValue[0], strings.Join(keyValue[1:], "=")) // nolint:fatcontext
|
||||
}
|
||||
}
|
||||
return parent
|
||||
|
||||
@@ -28,6 +28,7 @@ go_library(
|
||||
"//api/server:go_default_library",
|
||||
"//beacon-chain/state:go_default_library",
|
||||
"//config/fieldparams:go_default_library",
|
||||
"//config/params:go_default_library",
|
||||
"//consensus-types/interfaces:go_default_library",
|
||||
"//consensus-types/primitives:go_default_library",
|
||||
"//consensus-types/validator:go_default_library",
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prysmaticlabs/prysm/v5/api/server"
|
||||
fieldparams "github.com/prysmaticlabs/prysm/v5/config/fieldparams"
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/validator"
|
||||
"github.com/prysmaticlabs/prysm/v5/container/slice"
|
||||
@@ -51,6 +52,9 @@ func HistoricalSummaryFromConsensus(s *eth.HistoricalSummary) *HistoricalSummary
|
||||
}
|
||||
|
||||
func (s *SignedBLSToExecutionChange) ToConsensus() (*eth.SignedBLSToExecutionChange, error) {
|
||||
if s.Message == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "Message")
|
||||
}
|
||||
change, err := s.Message.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Message")
|
||||
@@ -102,14 +106,17 @@ func SignedBLSChangeFromConsensus(ch *eth.SignedBLSToExecutionChange) *SignedBLS
|
||||
|
||||
func SignedBLSChangesToConsensus(src []*SignedBLSToExecutionChange) ([]*eth.SignedBLSToExecutionChange, error) {
|
||||
if src == nil {
|
||||
return nil, errNilValue
|
||||
return nil, server.NewDecodeError(errNilValue, "SignedBLSToExecutionChanges")
|
||||
}
|
||||
err := slice.VerifyMaxLength(src, 16)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, server.NewDecodeError(err, "SignedBLSToExecutionChanges")
|
||||
}
|
||||
changes := make([]*eth.SignedBLSToExecutionChange, len(src))
|
||||
for i, ch := range src {
|
||||
if ch == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, fmt.Sprintf("[%d]", i))
|
||||
}
|
||||
changes[i], err = ch.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, fmt.Sprintf("[%d]", i))
|
||||
@@ -155,6 +162,9 @@ func ForkFromConsensus(f *eth.Fork) *Fork {
|
||||
}
|
||||
|
||||
func (s *SignedValidatorRegistration) ToConsensus() (*eth.SignedValidatorRegistrationV1, error) {
|
||||
if s.Message == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "Message")
|
||||
}
|
||||
msg, err := s.Message.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Message")
|
||||
@@ -211,6 +221,9 @@ func SignedValidatorRegistrationFromConsensus(vr *eth.SignedValidatorRegistratio
|
||||
}
|
||||
|
||||
func (s *SignedContributionAndProof) ToConsensus() (*eth.SignedContributionAndProof, error) {
|
||||
if s.Message == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "Message")
|
||||
}
|
||||
msg, err := s.Message.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Message")
|
||||
@@ -235,6 +248,9 @@ func SignedContributionAndProofFromConsensus(c *eth.SignedContributionAndProof)
|
||||
}
|
||||
|
||||
func (c *ContributionAndProof) ToConsensus() (*eth.ContributionAndProof, error) {
|
||||
if c.Contribution == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "Contribution")
|
||||
}
|
||||
contribution, err := c.Contribution.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Contribution")
|
||||
@@ -243,7 +259,7 @@ func (c *ContributionAndProof) ToConsensus() (*eth.ContributionAndProof, error)
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "AggregatorIndex")
|
||||
}
|
||||
selectionProof, err := bytesutil.DecodeHexWithLength(c.SelectionProof, 96)
|
||||
selectionProof, err := bytesutil.DecodeHexWithLength(c.SelectionProof, fieldparams.BLSSignatureLength)
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "SelectionProof")
|
||||
}
|
||||
@@ -306,6 +322,9 @@ func SyncCommitteeContributionFromConsensus(c *eth.SyncCommitteeContribution) *S
|
||||
}
|
||||
|
||||
func (s *SignedAggregateAttestationAndProof) ToConsensus() (*eth.SignedAggregateAttestationAndProof, error) {
|
||||
if s.Message == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "Message")
|
||||
}
|
||||
msg, err := s.Message.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Message")
|
||||
@@ -326,11 +345,14 @@ func (a *AggregateAttestationAndProof) ToConsensus() (*eth.AggregateAttestationA
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "AggregatorIndex")
|
||||
}
|
||||
if a.Aggregate == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "Aggregate")
|
||||
}
|
||||
agg, err := a.Aggregate.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Aggregate")
|
||||
}
|
||||
proof, err := bytesutil.DecodeHexWithLength(a.SelectionProof, 96)
|
||||
proof, err := bytesutil.DecodeHexWithLength(a.SelectionProof, fieldparams.BLSSignatureLength)
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "SelectionProof")
|
||||
}
|
||||
@@ -342,6 +364,9 @@ func (a *AggregateAttestationAndProof) ToConsensus() (*eth.AggregateAttestationA
|
||||
}
|
||||
|
||||
func (s *SignedAggregateAttestationAndProofElectra) ToConsensus() (*eth.SignedAggregateAttestationAndProofElectra, error) {
|
||||
if s.Message == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "Message")
|
||||
}
|
||||
msg, err := s.Message.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Message")
|
||||
@@ -362,11 +387,14 @@ func (a *AggregateAttestationAndProofElectra) ToConsensus() (*eth.AggregateAttes
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "AggregatorIndex")
|
||||
}
|
||||
if a.Aggregate == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "Aggregate")
|
||||
}
|
||||
agg, err := a.Aggregate.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Aggregate")
|
||||
}
|
||||
proof, err := bytesutil.DecodeHexWithLength(a.SelectionProof, 96)
|
||||
proof, err := bytesutil.DecodeHexWithLength(a.SelectionProof, fieldparams.BLSSignatureLength)
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "SelectionProof")
|
||||
}
|
||||
@@ -382,6 +410,9 @@ func (a *Attestation) ToConsensus() (*eth.Attestation, error) {
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "AggregationBits")
|
||||
}
|
||||
if a.Data == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "Data")
|
||||
}
|
||||
data, err := a.Data.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Data")
|
||||
@@ -411,6 +442,9 @@ func (a *AttestationElectra) ToConsensus() (*eth.AttestationElectra, error) {
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "AggregationBits")
|
||||
}
|
||||
if a.Data == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "Data")
|
||||
}
|
||||
data, err := a.Data.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Data")
|
||||
@@ -432,6 +466,15 @@ func (a *AttestationElectra) ToConsensus() (*eth.AttestationElectra, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func SingleAttFromConsensus(a *eth.SingleAttestation) *SingleAttestation {
|
||||
return &SingleAttestation{
|
||||
CommitteeIndex: fmt.Sprintf("%d", a.CommitteeId),
|
||||
AttesterIndex: fmt.Sprintf("%d", a.AttesterIndex),
|
||||
Data: AttDataFromConsensus(a.Data),
|
||||
Signature: hexutil.Encode(a.Signature),
|
||||
}
|
||||
}
|
||||
|
||||
func (a *SingleAttestation) ToConsensus() (*eth.SingleAttestation, error) {
|
||||
ci, err := strconv.ParseUint(a.CommitteeIndex, 10, 64)
|
||||
if err != nil {
|
||||
@@ -441,6 +484,9 @@ func (a *SingleAttestation) ToConsensus() (*eth.SingleAttestation, error) {
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "AttesterIndex")
|
||||
}
|
||||
if a.Data == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "Data")
|
||||
}
|
||||
data, err := a.Data.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Data")
|
||||
@@ -480,10 +526,16 @@ func (a *AttestationData) ToConsensus() (*eth.AttestationData, error) {
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "BeaconBlockRoot")
|
||||
}
|
||||
if a.Source == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "Source")
|
||||
}
|
||||
source, err := a.Source.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Source")
|
||||
}
|
||||
if a.Target == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "Target")
|
||||
}
|
||||
target, err := a.Target.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Target")
|
||||
@@ -583,15 +635,17 @@ func (b *BeaconCommitteeSubscription) ToConsensus() (*validator.BeaconCommitteeS
|
||||
}
|
||||
|
||||
func (e *SignedVoluntaryExit) ToConsensus() (*eth.SignedVoluntaryExit, error) {
|
||||
sig, err := bytesutil.DecodeHexWithLength(e.Signature, fieldparams.BLSSignatureLength)
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Signature")
|
||||
if e.Message == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "Message")
|
||||
}
|
||||
exit, err := e.Message.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Message")
|
||||
}
|
||||
|
||||
sig, err := bytesutil.DecodeHexWithLength(e.Signature, fieldparams.BLSSignatureLength)
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Signature")
|
||||
}
|
||||
return ð.SignedVoluntaryExit{
|
||||
Exit: exit,
|
||||
Signature: sig,
|
||||
@@ -694,10 +748,16 @@ func Eth1DataFromConsensus(e1d *eth.Eth1Data) *Eth1Data {
|
||||
}
|
||||
|
||||
func (s *ProposerSlashing) ToConsensus() (*eth.ProposerSlashing, error) {
|
||||
if s.SignedHeader1 == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "SignedHeader1")
|
||||
}
|
||||
h1, err := s.SignedHeader1.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "SignedHeader1")
|
||||
}
|
||||
if s.SignedHeader2 == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "SignedHeader2")
|
||||
}
|
||||
h2, err := s.SignedHeader2.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "SignedHeader2")
|
||||
@@ -710,10 +770,16 @@ func (s *ProposerSlashing) ToConsensus() (*eth.ProposerSlashing, error) {
|
||||
}
|
||||
|
||||
func (s *AttesterSlashing) ToConsensus() (*eth.AttesterSlashing, error) {
|
||||
if s.Attestation1 == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "Attestation1")
|
||||
}
|
||||
att1, err := s.Attestation1.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Attestation1")
|
||||
}
|
||||
if s.Attestation2 == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "Attestation2")
|
||||
}
|
||||
att2, err := s.Attestation2.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Attestation2")
|
||||
@@ -722,10 +788,16 @@ func (s *AttesterSlashing) ToConsensus() (*eth.AttesterSlashing, error) {
|
||||
}
|
||||
|
||||
func (s *AttesterSlashingElectra) ToConsensus() (*eth.AttesterSlashingElectra, error) {
|
||||
if s.Attestation1 == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "Attestation1")
|
||||
}
|
||||
att1, err := s.Attestation1.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Attestation1")
|
||||
}
|
||||
if s.Attestation2 == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "Attestation2")
|
||||
}
|
||||
att2, err := s.Attestation2.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Attestation2")
|
||||
@@ -734,6 +806,10 @@ func (s *AttesterSlashingElectra) ToConsensus() (*eth.AttesterSlashingElectra, e
|
||||
}
|
||||
|
||||
func (a *IndexedAttestation) ToConsensus() (*eth.IndexedAttestation, error) {
|
||||
if err := slice.VerifyMaxLength(a.AttestingIndices, params.BeaconConfig().MaxValidatorsPerCommittee); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
indices := make([]uint64, len(a.AttestingIndices))
|
||||
var err error
|
||||
for i, ix := range a.AttestingIndices {
|
||||
@@ -742,6 +818,9 @@ func (a *IndexedAttestation) ToConsensus() (*eth.IndexedAttestation, error) {
|
||||
return nil, server.NewDecodeError(err, fmt.Sprintf("AttestingIndices[%d]", i))
|
||||
}
|
||||
}
|
||||
if a.Data == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "Data")
|
||||
}
|
||||
data, err := a.Data.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Data")
|
||||
@@ -759,6 +838,13 @@ func (a *IndexedAttestation) ToConsensus() (*eth.IndexedAttestation, error) {
|
||||
}
|
||||
|
||||
func (a *IndexedAttestationElectra) ToConsensus() (*eth.IndexedAttestationElectra, error) {
|
||||
if err := slice.VerifyMaxLength(
|
||||
a.AttestingIndices,
|
||||
params.BeaconConfig().MaxValidatorsPerCommittee*params.BeaconConfig().MaxCommitteesPerSlot,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
indices := make([]uint64, len(a.AttestingIndices))
|
||||
var err error
|
||||
for i, ix := range a.AttestingIndices {
|
||||
@@ -767,6 +853,9 @@ func (a *IndexedAttestationElectra) ToConsensus() (*eth.IndexedAttestationElectr
|
||||
return nil, server.NewDecodeError(err, fmt.Sprintf("AttestingIndices[%d]", i))
|
||||
}
|
||||
}
|
||||
if a.Data == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, "Data")
|
||||
}
|
||||
data, err := a.Data.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Data")
|
||||
@@ -922,11 +1011,11 @@ func (d *DepositRequest) ToConsensus() (*enginev1.DepositRequest, error) {
|
||||
|
||||
func ProposerSlashingsToConsensus(src []*ProposerSlashing) ([]*eth.ProposerSlashing, error) {
|
||||
if src == nil {
|
||||
return nil, errNilValue
|
||||
return nil, server.NewDecodeError(errNilValue, "ProposerSlashings")
|
||||
}
|
||||
err := slice.VerifyMaxLength(src, 16)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, server.NewDecodeError(err, "ProposerSlashings")
|
||||
}
|
||||
proposerSlashings := make([]*eth.ProposerSlashing, len(src))
|
||||
for i, s := range src {
|
||||
@@ -1055,11 +1144,11 @@ func ProposerSlashingFromConsensus(src *eth.ProposerSlashing) *ProposerSlashing
|
||||
|
||||
func AttesterSlashingsToConsensus(src []*AttesterSlashing) ([]*eth.AttesterSlashing, error) {
|
||||
if src == nil {
|
||||
return nil, errNilValue
|
||||
return nil, server.NewDecodeError(errNilValue, "AttesterSlashings")
|
||||
}
|
||||
err := slice.VerifyMaxLength(src, 2)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, server.NewDecodeError(err, "AttesterSlashings")
|
||||
}
|
||||
|
||||
attesterSlashings := make([]*eth.AttesterSlashing, len(src))
|
||||
@@ -1070,10 +1159,19 @@ func AttesterSlashingsToConsensus(src []*AttesterSlashing) ([]*eth.AttesterSlash
|
||||
if s.Attestation1 == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, fmt.Sprintf("[%d].Attestation1", i))
|
||||
}
|
||||
|
||||
if s.Attestation1.Data == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, fmt.Sprintf("[%d].Attestation1.Data", i))
|
||||
}
|
||||
|
||||
if s.Attestation2 == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, fmt.Sprintf("[%d].Attestation2", i))
|
||||
}
|
||||
|
||||
if s.Attestation2.Data == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, fmt.Sprintf("[%d].Attestation2.Data", i))
|
||||
}
|
||||
|
||||
a1Sig, err := bytesutil.DecodeHexWithLength(s.Attestation1.Signature, fieldparams.BLSSignatureLength)
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, fmt.Sprintf("[%d].Attestation1.Signature", i))
|
||||
@@ -1090,6 +1188,7 @@ func AttesterSlashingsToConsensus(src []*AttesterSlashing) ([]*eth.AttesterSlash
|
||||
}
|
||||
a1AttestingIndices[j] = attestingIndex
|
||||
}
|
||||
|
||||
a1Data, err := s.Attestation1.Data.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, fmt.Sprintf("[%d].Attestation1.Data", i))
|
||||
@@ -1187,11 +1286,11 @@ func AttesterSlashingFromConsensus(src *eth.AttesterSlashing) *AttesterSlashing
|
||||
|
||||
func AttesterSlashingsElectraToConsensus(src []*AttesterSlashingElectra) ([]*eth.AttesterSlashingElectra, error) {
|
||||
if src == nil {
|
||||
return nil, errNilValue
|
||||
return nil, server.NewDecodeError(errNilValue, "AttesterSlashingsElectra")
|
||||
}
|
||||
err := slice.VerifyMaxLength(src, 2)
|
||||
err := slice.VerifyMaxLength(src, fieldparams.MaxAttesterSlashingsElectra)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, server.NewDecodeError(err, "AttesterSlashingsElectra")
|
||||
}
|
||||
|
||||
attesterSlashings := make([]*eth.AttesterSlashingElectra, len(src))
|
||||
@@ -1199,18 +1298,28 @@ func AttesterSlashingsElectraToConsensus(src []*AttesterSlashingElectra) ([]*eth
|
||||
if s == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, fmt.Sprintf("[%d]", i))
|
||||
}
|
||||
|
||||
if s.Attestation1 == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, fmt.Sprintf("[%d].Attestation1", i))
|
||||
}
|
||||
|
||||
if s.Attestation1.Data == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, fmt.Sprintf("[%d].Attestation1.Data", i))
|
||||
}
|
||||
|
||||
if s.Attestation2 == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, fmt.Sprintf("[%d].Attestation2", i))
|
||||
}
|
||||
|
||||
if s.Attestation2.Data == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, fmt.Sprintf("[%d].Attestation2.Data", i))
|
||||
}
|
||||
|
||||
a1Sig, err := bytesutil.DecodeHexWithLength(s.Attestation1.Signature, fieldparams.BLSSignatureLength)
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, fmt.Sprintf("[%d].Attestation1.Signature", i))
|
||||
}
|
||||
err = slice.VerifyMaxLength(s.Attestation1.AttestingIndices, 2048)
|
||||
err = slice.VerifyMaxLength(s.Attestation1.AttestingIndices, params.BeaconConfig().MaxValidatorsPerCommittee*params.BeaconConfig().MaxCommitteesPerSlot)
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, fmt.Sprintf("[%d].Attestation1.AttestingIndices", i))
|
||||
}
|
||||
@@ -1230,7 +1339,7 @@ func AttesterSlashingsElectraToConsensus(src []*AttesterSlashingElectra) ([]*eth
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, fmt.Sprintf("[%d].Attestation2.Signature", i))
|
||||
}
|
||||
err = slice.VerifyMaxLength(s.Attestation2.AttestingIndices, 2048)
|
||||
err = slice.VerifyMaxLength(s.Attestation2.AttestingIndices, params.BeaconConfig().MaxValidatorsPerCommittee*params.BeaconConfig().MaxCommitteesPerSlot)
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, fmt.Sprintf("[%d].Attestation2.AttestingIndices", i))
|
||||
}
|
||||
@@ -1319,15 +1428,18 @@ func AttesterSlashingElectraFromConsensus(src *eth.AttesterSlashingElectra) *Att
|
||||
|
||||
func AttsToConsensus(src []*Attestation) ([]*eth.Attestation, error) {
|
||||
if src == nil {
|
||||
return nil, errNilValue
|
||||
return nil, server.NewDecodeError(errNilValue, "Attestations")
|
||||
}
|
||||
err := slice.VerifyMaxLength(src, 128)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, server.NewDecodeError(err, "Attestations")
|
||||
}
|
||||
|
||||
atts := make([]*eth.Attestation, len(src))
|
||||
for i, a := range src {
|
||||
if a == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, fmt.Sprintf("[%d]", i))
|
||||
}
|
||||
atts[i], err = a.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, fmt.Sprintf("[%d]", i))
|
||||
@@ -1346,15 +1458,18 @@ func AttsFromConsensus(src []*eth.Attestation) []*Attestation {
|
||||
|
||||
func AttsElectraToConsensus(src []*AttestationElectra) ([]*eth.AttestationElectra, error) {
|
||||
if src == nil {
|
||||
return nil, errNilValue
|
||||
return nil, server.NewDecodeError(errNilValue, "AttestationsElectra")
|
||||
}
|
||||
err := slice.VerifyMaxLength(src, 8)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, server.NewDecodeError(err, "AttestationsElectra")
|
||||
}
|
||||
|
||||
atts := make([]*eth.AttestationElectra, len(src))
|
||||
for i, a := range src {
|
||||
if a == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, fmt.Sprintf("[%d]", i))
|
||||
}
|
||||
atts[i], err = a.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, fmt.Sprintf("[%d]", i))
|
||||
@@ -1373,11 +1488,11 @@ func AttsElectraFromConsensus(src []*eth.AttestationElectra) []*AttestationElect
|
||||
|
||||
func DepositsToConsensus(src []*Deposit) ([]*eth.Deposit, error) {
|
||||
if src == nil {
|
||||
return nil, errNilValue
|
||||
return nil, server.NewDecodeError(errNilValue, "Deposits")
|
||||
}
|
||||
err := slice.VerifyMaxLength(src, 16)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, server.NewDecodeError(err, "Deposits")
|
||||
}
|
||||
|
||||
deposits := make([]*eth.Deposit, len(src))
|
||||
@@ -1449,15 +1564,18 @@ func DepositsFromConsensus(src []*eth.Deposit) []*Deposit {
|
||||
|
||||
func SignedExitsToConsensus(src []*SignedVoluntaryExit) ([]*eth.SignedVoluntaryExit, error) {
|
||||
if src == nil {
|
||||
return nil, errNilValue
|
||||
return nil, server.NewDecodeError(errNilValue, "SignedVoluntaryExits")
|
||||
}
|
||||
err := slice.VerifyMaxLength(src, 16)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, server.NewDecodeError(err, "SignedVoluntaryExits")
|
||||
}
|
||||
|
||||
exits := make([]*eth.SignedVoluntaryExit, len(src))
|
||||
for i, e := range src {
|
||||
if e == nil {
|
||||
return nil, server.NewDecodeError(errNilValue, fmt.Sprintf("[%d]", i))
|
||||
}
|
||||
exits[i], err = e.ToConsensus()
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, fmt.Sprintf("[%d]", i))
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prysmaticlabs/prysm/v5/api/server"
|
||||
fieldparams "github.com/prysmaticlabs/prysm/v5/config/fieldparams"
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/interfaces"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
"github.com/prysmaticlabs/prysm/v5/container/slice"
|
||||
@@ -2519,6 +2520,7 @@ func (b *BeaconBlockContentsElectra) ToConsensus() (*eth.BeaconBlockContentsElec
|
||||
}, nil
|
||||
}
|
||||
|
||||
// nolint:gocognit
|
||||
func (b *BeaconBlockElectra) ToConsensus() (*eth.BeaconBlockElectra, error) {
|
||||
if b == nil {
|
||||
return nil, errNilValue
|
||||
@@ -2706,6 +2708,9 @@ func (b *BeaconBlockElectra) ToConsensus() (*eth.BeaconBlockElectra, error) {
|
||||
return nil, server.NewDecodeError(errors.New("nil execution requests"), "Body.ExequtionRequests")
|
||||
}
|
||||
|
||||
if err = slice.VerifyMaxLength(b.Body.ExecutionRequests.Deposits, params.BeaconConfig().MaxDepositRequestsPerPayload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
depositRequests := make([]*enginev1.DepositRequest, len(b.Body.ExecutionRequests.Deposits))
|
||||
for i, d := range b.Body.ExecutionRequests.Deposits {
|
||||
depositRequests[i], err = d.ToConsensus()
|
||||
@@ -2714,6 +2719,9 @@ func (b *BeaconBlockElectra) ToConsensus() (*eth.BeaconBlockElectra, error) {
|
||||
}
|
||||
}
|
||||
|
||||
if err = slice.VerifyMaxLength(b.Body.ExecutionRequests.Withdrawals, params.BeaconConfig().MaxWithdrawalRequestsPerPayload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
withdrawalRequests := make([]*enginev1.WithdrawalRequest, len(b.Body.ExecutionRequests.Withdrawals))
|
||||
for i, w := range b.Body.ExecutionRequests.Withdrawals {
|
||||
withdrawalRequests[i], err = w.ToConsensus()
|
||||
@@ -2722,6 +2730,9 @@ func (b *BeaconBlockElectra) ToConsensus() (*eth.BeaconBlockElectra, error) {
|
||||
}
|
||||
}
|
||||
|
||||
if err = slice.VerifyMaxLength(b.Body.ExecutionRequests.Consolidations, params.BeaconConfig().MaxConsolidationsRequestsPerPayload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
consolidationRequests := make([]*enginev1.ConsolidationRequest, len(b.Body.ExecutionRequests.Consolidations))
|
||||
for i, c := range b.Body.ExecutionRequests.Consolidations {
|
||||
consolidationRequests[i], err = c.ToConsensus()
|
||||
@@ -3003,9 +3014,14 @@ func (b *BlindedBeaconBlockElectra) ToConsensus() (*eth.BlindedBeaconBlockElectr
|
||||
if err != nil {
|
||||
return nil, server.NewDecodeError(err, "Body.ExecutionPayload.ExcessBlobGas")
|
||||
}
|
||||
|
||||
if b.Body.ExecutionRequests == nil {
|
||||
return nil, server.NewDecodeError(errors.New("nil execution requests"), "Body.ExecutionRequests")
|
||||
}
|
||||
|
||||
if err = slice.VerifyMaxLength(b.Body.ExecutionRequests.Deposits, params.BeaconConfig().MaxDepositRequestsPerPayload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
depositRequests := make([]*enginev1.DepositRequest, len(b.Body.ExecutionRequests.Deposits))
|
||||
for i, d := range b.Body.ExecutionRequests.Deposits {
|
||||
depositRequests[i], err = d.ToConsensus()
|
||||
@@ -3014,6 +3030,9 @@ func (b *BlindedBeaconBlockElectra) ToConsensus() (*eth.BlindedBeaconBlockElectr
|
||||
}
|
||||
}
|
||||
|
||||
if err = slice.VerifyMaxLength(b.Body.ExecutionRequests.Withdrawals, params.BeaconConfig().MaxWithdrawalRequestsPerPayload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
withdrawalRequests := make([]*enginev1.WithdrawalRequest, len(b.Body.ExecutionRequests.Withdrawals))
|
||||
for i, w := range b.Body.ExecutionRequests.Withdrawals {
|
||||
withdrawalRequests[i], err = w.ToConsensus()
|
||||
@@ -3022,6 +3041,9 @@ func (b *BlindedBeaconBlockElectra) ToConsensus() (*eth.BlindedBeaconBlockElectr
|
||||
}
|
||||
}
|
||||
|
||||
if err = slice.VerifyMaxLength(b.Body.ExecutionRequests.Consolidations, params.BeaconConfig().MaxConsolidationsRequestsPerPayload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
consolidationRequests := make([]*enginev1.ConsolidationRequest, len(b.Body.ExecutionRequests.Consolidations))
|
||||
for i, c := range b.Body.ExecutionRequests.Consolidations {
|
||||
consolidationRequests[i], err = c.ToConsensus()
|
||||
|
||||
@@ -24,3 +24,96 @@ func TestDepositSnapshotFromConsensus(t *testing.T) {
|
||||
require.Equal(t, "0x1234", res.ExecutionBlockHash)
|
||||
require.Equal(t, "67890", res.ExecutionBlockHeight)
|
||||
}
|
||||
|
||||
func TestSignedBLSToExecutionChange_ToConsensus(t *testing.T) {
|
||||
s := &SignedBLSToExecutionChange{Message: nil, Signature: ""}
|
||||
_, err := s.ToConsensus()
|
||||
require.ErrorContains(t, errNilValue.Error(), err)
|
||||
}
|
||||
|
||||
func TestSignedValidatorRegistration_ToConsensus(t *testing.T) {
|
||||
s := &SignedValidatorRegistration{Message: nil, Signature: ""}
|
||||
_, err := s.ToConsensus()
|
||||
require.ErrorContains(t, errNilValue.Error(), err)
|
||||
}
|
||||
|
||||
func TestSignedContributionAndProof_ToConsensus(t *testing.T) {
|
||||
s := &SignedContributionAndProof{Message: nil, Signature: ""}
|
||||
_, err := s.ToConsensus()
|
||||
require.ErrorContains(t, errNilValue.Error(), err)
|
||||
}
|
||||
|
||||
func TestContributionAndProof_ToConsensus(t *testing.T) {
|
||||
c := &ContributionAndProof{
|
||||
Contribution: nil,
|
||||
AggregatorIndex: "invalid",
|
||||
SelectionProof: "",
|
||||
}
|
||||
_, err := c.ToConsensus()
|
||||
require.ErrorContains(t, errNilValue.Error(), err)
|
||||
}
|
||||
|
||||
func TestSignedAggregateAttestationAndProof_ToConsensus(t *testing.T) {
|
||||
s := &SignedAggregateAttestationAndProof{Message: nil, Signature: ""}
|
||||
_, err := s.ToConsensus()
|
||||
require.ErrorContains(t, errNilValue.Error(), err)
|
||||
}
|
||||
|
||||
func TestAggregateAttestationAndProof_ToConsensus(t *testing.T) {
|
||||
a := &AggregateAttestationAndProof{
|
||||
AggregatorIndex: "1",
|
||||
Aggregate: nil,
|
||||
SelectionProof: "",
|
||||
}
|
||||
_, err := a.ToConsensus()
|
||||
require.ErrorContains(t, errNilValue.Error(), err)
|
||||
}
|
||||
|
||||
func TestAttestation_ToConsensus(t *testing.T) {
|
||||
a := &Attestation{
|
||||
AggregationBits: "0x10",
|
||||
Data: nil,
|
||||
Signature: "",
|
||||
}
|
||||
_, err := a.ToConsensus()
|
||||
require.ErrorContains(t, errNilValue.Error(), err)
|
||||
}
|
||||
|
||||
func TestSingleAttestation_ToConsensus(t *testing.T) {
|
||||
s := &SingleAttestation{
|
||||
CommitteeIndex: "1",
|
||||
AttesterIndex: "1",
|
||||
Data: nil,
|
||||
Signature: "",
|
||||
}
|
||||
_, err := s.ToConsensus()
|
||||
require.ErrorContains(t, errNilValue.Error(), err)
|
||||
}
|
||||
|
||||
func TestSignedVoluntaryExit_ToConsensus(t *testing.T) {
|
||||
s := &SignedVoluntaryExit{Message: nil, Signature: ""}
|
||||
_, err := s.ToConsensus()
|
||||
require.ErrorContains(t, errNilValue.Error(), err)
|
||||
}
|
||||
|
||||
func TestProposerSlashing_ToConsensus(t *testing.T) {
|
||||
p := &ProposerSlashing{SignedHeader1: nil, SignedHeader2: nil}
|
||||
_, err := p.ToConsensus()
|
||||
require.ErrorContains(t, errNilValue.Error(), err)
|
||||
}
|
||||
|
||||
func TestAttesterSlashing_ToConsensus(t *testing.T) {
|
||||
a := &AttesterSlashing{Attestation1: nil, Attestation2: nil}
|
||||
_, err := a.ToConsensus()
|
||||
require.ErrorContains(t, errNilValue.Error(), err)
|
||||
}
|
||||
|
||||
func TestIndexedAttestation_ToConsensus(t *testing.T) {
|
||||
a := &IndexedAttestation{
|
||||
AttestingIndices: []string{"1"},
|
||||
Data: nil,
|
||||
Signature: "invalid",
|
||||
}
|
||||
_, err := a.ToConsensus()
|
||||
require.ErrorContains(t, errNilValue.Error(), err)
|
||||
}
|
||||
|
||||
@@ -43,6 +43,7 @@ go_library(
|
||||
"//beacon-chain/cache:go_default_library",
|
||||
"//beacon-chain/core/altair:go_default_library",
|
||||
"//beacon-chain/core/blocks:go_default_library",
|
||||
"//beacon-chain/core/electra:go_default_library",
|
||||
"//beacon-chain/core/epoch/precompute:go_default_library",
|
||||
"//beacon-chain/core/feed:go_default_library",
|
||||
"//beacon-chain/core/feed/state:go_default_library",
|
||||
|
||||
@@ -2,6 +2,7 @@ package blockchain
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
@@ -11,5 +12,5 @@ func TestMain(m *testing.M) {
|
||||
logrus.SetLevel(logrus.DebugLevel)
|
||||
logrus.SetOutput(io.Discard)
|
||||
|
||||
m.Run()
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
@@ -268,6 +268,9 @@ func (s *Service) notifyNewPayload(ctx context.Context, preStateVersion int,
|
||||
if err != nil {
|
||||
return false, errors.Wrap(err, "could not get execution requests")
|
||||
}
|
||||
if requests == nil {
|
||||
return false, errors.New("nil execution requests")
|
||||
}
|
||||
}
|
||||
lastValidHash, err = s.cfg.ExecutionEngineCaller.NewPayload(ctx, payload, versionedHashes, parentRoot, requests)
|
||||
|
||||
|
||||
@@ -512,17 +512,11 @@ func missingIndices(bs *filesystem.BlobStorage, root [32]byte, expected [][]byte
|
||||
if len(expected) > maxBlobsPerBlock {
|
||||
return nil, errMaxBlobsExceeded
|
||||
}
|
||||
indices, err := bs.Indices(root, slot)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
indices := bs.Summary(root)
|
||||
missing := make(map[uint64]struct{}, len(expected))
|
||||
for i := range expected {
|
||||
ui := uint64(i)
|
||||
if len(expected[i]) > 0 {
|
||||
if !indices[i] {
|
||||
missing[ui] = struct{}{}
|
||||
}
|
||||
if len(expected[i]) > 0 && !indices.HasIndex(uint64(i)) {
|
||||
missing[uint64(i)] = struct{}{}
|
||||
}
|
||||
}
|
||||
return missing, nil
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/helpers"
|
||||
lightclient "github.com/prysmaticlabs/prysm/v5/beacon-chain/core/light-client"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
@@ -552,7 +553,8 @@ func (s *Service) fillInForkChoiceMissingBlocks(ctx context.Context, signed inte
|
||||
|
||||
// inserts finalized deposits into our finalized deposit trie, needs to be
|
||||
// called in the background
|
||||
func (s *Service) insertFinalizedDeposits(ctx context.Context, fRoot [32]byte) {
|
||||
// Post-Electra: prunes all proofs and pending deposits in the cache
|
||||
func (s *Service) insertFinalizedDepositsAndPrune(ctx context.Context, fRoot [32]byte) {
|
||||
ctx, span := trace.StartSpan(ctx, "blockChain.insertFinalizedDeposits")
|
||||
defer span.End()
|
||||
startTime := time.Now()
|
||||
@@ -563,6 +565,16 @@ func (s *Service) insertFinalizedDeposits(ctx context.Context, fRoot [32]byte) {
|
||||
log.WithError(err).Error("could not fetch finalized state")
|
||||
return
|
||||
}
|
||||
|
||||
// Check if we should prune all pending deposits.
|
||||
// In post-Electra(after the legacy deposit mechanism is deprecated),
|
||||
// we can prune all pending deposits in the deposit cache.
|
||||
// See: https://eips.ethereum.org/EIPS/eip-6110#eth1data-poll-deprecation
|
||||
if helpers.DepositRequestsStarted(finalizedState) {
|
||||
s.pruneAllPendingDepositsAndProofs(ctx)
|
||||
return
|
||||
}
|
||||
|
||||
// We update the cache up to the last deposit index in the finalized block's state.
|
||||
// We can be confident that these deposits will be included in some block
|
||||
// because the Eth1 follow distance makes such long-range reorgs extremely unlikely.
|
||||
@@ -591,6 +603,12 @@ func (s *Service) insertFinalizedDeposits(ctx context.Context, fRoot [32]byte) {
|
||||
log.WithField("duration", time.Since(startTime).String()).Debugf("Finalized deposit insertion completed at index %d", finalizedEth1DepIdx)
|
||||
}
|
||||
|
||||
// pruneAllPendingDepositsAndProofs prunes all proofs and pending deposits in the cache.
|
||||
func (s *Service) pruneAllPendingDepositsAndProofs(ctx context.Context) {
|
||||
s.cfg.DepositCache.PruneAllPendingDeposits(ctx)
|
||||
s.cfg.DepositCache.PruneAllProofs(ctx)
|
||||
}
|
||||
|
||||
// This ensures that the input root defaults to using genesis root instead of zero hashes. This is needed for handling
|
||||
// fork choice justification routine.
|
||||
func (s *Service) ensureRootNotZeros(root [32]byte) [32]byte {
|
||||
|
||||
@@ -723,7 +723,7 @@ func TestInsertFinalizedDeposits(t *testing.T) {
|
||||
Signature: zeroSig[:],
|
||||
}, Proof: [][]byte{root}}, 100+i, int64(i), bytesutil.ToBytes32(root)))
|
||||
}
|
||||
service.insertFinalizedDeposits(ctx, [32]byte{'m', 'o', 'c', 'k'})
|
||||
service.insertFinalizedDepositsAndPrune(ctx, [32]byte{'m', 'o', 'c', 'k'})
|
||||
fDeposits, err := depositCache.FinalizedDeposits(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 7, int(fDeposits.MerkleTrieIndex()), "Finalized deposits not inserted correctly")
|
||||
@@ -759,7 +759,7 @@ func TestInsertFinalizedDeposits_PrunePendingDeposits(t *testing.T) {
|
||||
Signature: zeroSig[:],
|
||||
}, Proof: [][]byte{root}}, 100+i, int64(i), bytesutil.ToBytes32(root))
|
||||
}
|
||||
service.insertFinalizedDeposits(ctx, [32]byte{'m', 'o', 'c', 'k'})
|
||||
service.insertFinalizedDepositsAndPrune(ctx, [32]byte{'m', 'o', 'c', 'k'})
|
||||
fDeposits, err := depositCache.FinalizedDeposits(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 7, int(fDeposits.MerkleTrieIndex()), "Finalized deposits not inserted correctly")
|
||||
@@ -799,7 +799,7 @@ func TestInsertFinalizedDeposits_MultipleFinalizedRoutines(t *testing.T) {
|
||||
}
|
||||
// Insert 3 deposits before hand.
|
||||
require.NoError(t, depositCache.InsertFinalizedDeposits(ctx, 2, [32]byte{}, 0))
|
||||
service.insertFinalizedDeposits(ctx, [32]byte{'m', 'o', 'c', 'k'})
|
||||
service.insertFinalizedDepositsAndPrune(ctx, [32]byte{'m', 'o', 'c', 'k'})
|
||||
fDeposits, err := depositCache.FinalizedDeposits(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 5, int(fDeposits.MerkleTrieIndex()), "Finalized deposits not inserted correctly")
|
||||
@@ -810,7 +810,7 @@ func TestInsertFinalizedDeposits_MultipleFinalizedRoutines(t *testing.T) {
|
||||
}
|
||||
|
||||
// Insert New Finalized State with higher deposit count.
|
||||
service.insertFinalizedDeposits(ctx, [32]byte{'m', 'o', 'c', 'k', '2'})
|
||||
service.insertFinalizedDepositsAndPrune(ctx, [32]byte{'m', 'o', 'c', 'k', '2'})
|
||||
fDeposits, err = depositCache.FinalizedDeposits(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 12, int(fDeposits.MerkleTrieIndex()), "Finalized deposits not inserted correctly")
|
||||
@@ -1076,6 +1076,48 @@ func TestService_insertSlashingsToForkChoiceStore(t *testing.T) {
|
||||
service.InsertSlashingsToForkChoiceStore(ctx, wb.Block().Body().AttesterSlashings())
|
||||
}
|
||||
|
||||
func TestService_insertSlashingsToForkChoiceStoreElectra(t *testing.T) {
|
||||
service, tr := minimalTestService(t)
|
||||
ctx := tr.ctx
|
||||
|
||||
beaconState, privKeys := util.DeterministicGenesisStateElectra(t, 100)
|
||||
att1 := util.HydrateIndexedAttestationElectra(ðpb.IndexedAttestationElectra{
|
||||
Data: ðpb.AttestationData{
|
||||
Source: ðpb.Checkpoint{Epoch: 1},
|
||||
},
|
||||
AttestingIndices: []uint64{0, 1},
|
||||
})
|
||||
domain, err := signing.Domain(beaconState.Fork(), 0, params.BeaconConfig().DomainBeaconAttester, beaconState.GenesisValidatorsRoot())
|
||||
require.NoError(t, err)
|
||||
signingRoot, err := signing.ComputeSigningRoot(att1.Data, domain)
|
||||
assert.NoError(t, err, "Could not get signing root of beacon block header")
|
||||
sig0 := privKeys[0].Sign(signingRoot[:])
|
||||
sig1 := privKeys[1].Sign(signingRoot[:])
|
||||
aggregateSig := bls.AggregateSignatures([]bls.Signature{sig0, sig1})
|
||||
att1.Signature = aggregateSig.Marshal()
|
||||
|
||||
att2 := util.HydrateIndexedAttestationElectra(ðpb.IndexedAttestationElectra{
|
||||
AttestingIndices: []uint64{0, 1},
|
||||
})
|
||||
signingRoot, err = signing.ComputeSigningRoot(att2.Data, domain)
|
||||
assert.NoError(t, err, "Could not get signing root of beacon block header")
|
||||
sig0 = privKeys[0].Sign(signingRoot[:])
|
||||
sig1 = privKeys[1].Sign(signingRoot[:])
|
||||
aggregateSig = bls.AggregateSignatures([]bls.Signature{sig0, sig1})
|
||||
att2.Signature = aggregateSig.Marshal()
|
||||
slashings := []*ethpb.AttesterSlashingElectra{
|
||||
{
|
||||
Attestation_1: att1,
|
||||
Attestation_2: att2,
|
||||
},
|
||||
}
|
||||
b := util.NewBeaconBlockElectra()
|
||||
b.Block.Body.AttesterSlashings = slashings
|
||||
wb, err := consensusblocks.NewSignedBeaconBlock(b)
|
||||
require.NoError(t, err)
|
||||
service.InsertSlashingsToForkChoiceStore(ctx, wb.Block().Body().AttesterSlashings())
|
||||
}
|
||||
|
||||
func TestOnBlock_ProcessBlocksParallel(t *testing.T) {
|
||||
service, tr := minimalTestService(t)
|
||||
ctx := tr.ctx
|
||||
@@ -2255,7 +2297,7 @@ func TestMissingIndices(t *testing.T) {
|
||||
for _, c := range cases {
|
||||
bm, bs := filesystem.NewEphemeralBlobStorageWithMocker(t)
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
require.NoError(t, bm.CreateFakeIndices(c.root, c.present...))
|
||||
require.NoError(t, bm.CreateFakeIndices(c.root, 0, c.present...))
|
||||
missing, err := missingIndices(bs, c.root, c.expected, 0)
|
||||
if c.err != nil {
|
||||
require.ErrorIs(t, err, c.err)
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/electra"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/feed"
|
||||
statefeed "github.com/prysmaticlabs/prysm/v5/beacon-chain/core/feed/state"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/helpers"
|
||||
@@ -278,9 +279,10 @@ func (s *Service) executePostFinalizationTasks(ctx context.Context, finalizedSta
|
||||
go func() {
|
||||
s.sendNewFinalizedEvent(ctx, finalizedState)
|
||||
}()
|
||||
|
||||
depCtx, cancel := context.WithTimeout(context.Background(), depositDeadline)
|
||||
go func() {
|
||||
s.insertFinalizedDeposits(depCtx, finalized.Root)
|
||||
s.insertFinalizedDepositsAndPrune(depCtx, finalized.Root)
|
||||
cancel()
|
||||
}()
|
||||
}
|
||||
@@ -468,6 +470,9 @@ func (s *Service) validateStateTransition(ctx context.Context, preState state.Be
|
||||
stateTransitionStartTime := time.Now()
|
||||
postState, err := transition.ExecuteStateTransition(ctx, preState, signed)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil || electra.IsExecutionRequestError(err) {
|
||||
return nil, err
|
||||
}
|
||||
return nil, invalidBlock{error: err}
|
||||
}
|
||||
stateTransitionProcessingTime.Observe(float64(time.Since(stateTransitionStartTime).Milliseconds()))
|
||||
|
||||
@@ -455,41 +455,81 @@ func Test_executePostFinalizationTasks(t *testing.T) {
|
||||
Root: headRoot[:],
|
||||
}))
|
||||
require.NoError(t, headState.SetGenesisValidatorsRoot(params.BeaconConfig().ZeroHash[:]))
|
||||
t.Run("pre deposit request", func(t *testing.T) {
|
||||
require.NoError(t, headState.SetEth1DepositIndex(1))
|
||||
s, tr := minimalTestService(t, WithFinalizedStateAtStartUp(headState))
|
||||
ctx, beaconDB, stateGen := tr.ctx, tr.db, tr.sg
|
||||
|
||||
s, tr := minimalTestService(t, WithFinalizedStateAtStartUp(headState))
|
||||
ctx, beaconDB, stateGen := tr.ctx, tr.db, tr.sg
|
||||
require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, genesisRoot))
|
||||
util.SaveBlock(t, ctx, beaconDB, genesis)
|
||||
require.NoError(t, beaconDB.SaveState(ctx, headState, headRoot))
|
||||
require.NoError(t, beaconDB.SaveState(ctx, headState, genesisRoot))
|
||||
util.SaveBlock(t, ctx, beaconDB, headBlock)
|
||||
require.NoError(t, beaconDB.SaveFinalizedCheckpoint(ctx, ðpb.Checkpoint{Epoch: slots.ToEpoch(finalizedSlot), Root: headRoot[:]}))
|
||||
|
||||
require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, genesisRoot))
|
||||
util.SaveBlock(t, ctx, beaconDB, genesis)
|
||||
require.NoError(t, beaconDB.SaveState(ctx, headState, headRoot))
|
||||
require.NoError(t, beaconDB.SaveState(ctx, headState, genesisRoot))
|
||||
util.SaveBlock(t, ctx, beaconDB, headBlock)
|
||||
require.NoError(t, beaconDB.SaveFinalizedCheckpoint(ctx, ðpb.Checkpoint{Epoch: slots.ToEpoch(finalizedSlot), Root: headRoot[:]}))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, stateGen.SaveState(ctx, headRoot, headState))
|
||||
require.NoError(t, beaconDB.SaveLastValidatedCheckpoint(ctx, ðpb.Checkpoint{Epoch: slots.ToEpoch(finalizedSlot), Root: headRoot[:]}))
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, stateGen.SaveState(ctx, headRoot, headState))
|
||||
require.NoError(t, beaconDB.SaveLastValidatedCheckpoint(ctx, ðpb.Checkpoint{Epoch: slots.ToEpoch(finalizedSlot), Root: headRoot[:]}))
|
||||
notifier := &blockchainTesting.MockStateNotifier{RecordEvents: true}
|
||||
s.cfg.StateNotifier = notifier
|
||||
s.executePostFinalizationTasks(s.ctx, headState)
|
||||
|
||||
notifier := &blockchainTesting.MockStateNotifier{RecordEvents: true}
|
||||
s.cfg.StateNotifier = notifier
|
||||
s.executePostFinalizationTasks(s.ctx, headState)
|
||||
time.Sleep(1 * time.Second) // sleep for a second because event is in a separate go routine
|
||||
require.Equal(t, 1, len(notifier.ReceivedEvents()))
|
||||
e := notifier.ReceivedEvents()[0]
|
||||
assert.Equal(t, statefeed.FinalizedCheckpoint, int(e.Type))
|
||||
fc, ok := e.Data.(*ethpbv1.EventFinalizedCheckpoint)
|
||||
require.Equal(t, true, ok, "event has wrong data type")
|
||||
assert.Equal(t, primitives.Epoch(123), fc.Epoch)
|
||||
assert.DeepEqual(t, headRoot[:], fc.Block)
|
||||
assert.DeepEqual(t, finalizedStRoot[:], fc.State)
|
||||
assert.Equal(t, false, fc.ExecutionOptimistic)
|
||||
|
||||
time.Sleep(1 * time.Second) // sleep for a second because event is in a separate go routine
|
||||
require.Equal(t, 1, len(notifier.ReceivedEvents()))
|
||||
e := notifier.ReceivedEvents()[0]
|
||||
assert.Equal(t, statefeed.FinalizedCheckpoint, int(e.Type))
|
||||
fc, ok := e.Data.(*ethpbv1.EventFinalizedCheckpoint)
|
||||
require.Equal(t, true, ok, "event has wrong data type")
|
||||
assert.Equal(t, primitives.Epoch(123), fc.Epoch)
|
||||
assert.DeepEqual(t, headRoot[:], fc.Block)
|
||||
assert.DeepEqual(t, finalizedStRoot[:], fc.State)
|
||||
assert.Equal(t, false, fc.ExecutionOptimistic)
|
||||
// check the cache
|
||||
index, ok := headState.ValidatorIndexByPubkey(bytesutil.ToBytes48(key))
|
||||
require.Equal(t, true, ok)
|
||||
require.Equal(t, primitives.ValidatorIndex(0), index) // first index
|
||||
|
||||
// check the cache
|
||||
index, ok := headState.ValidatorIndexByPubkey(bytesutil.ToBytes48(key))
|
||||
require.Equal(t, true, ok)
|
||||
require.Equal(t, primitives.ValidatorIndex(0), index) // first index
|
||||
// check deposit
|
||||
require.LogsContain(t, logHook, "Finalized deposit insertion completed at index")
|
||||
})
|
||||
t.Run("deposit requests started", func(t *testing.T) {
|
||||
require.NoError(t, headState.SetEth1DepositIndex(1))
|
||||
require.NoError(t, headState.SetDepositRequestsStartIndex(1))
|
||||
s, tr := minimalTestService(t, WithFinalizedStateAtStartUp(headState))
|
||||
ctx, beaconDB, stateGen := tr.ctx, tr.db, tr.sg
|
||||
|
||||
require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, genesisRoot))
|
||||
util.SaveBlock(t, ctx, beaconDB, genesis)
|
||||
require.NoError(t, beaconDB.SaveState(ctx, headState, headRoot))
|
||||
require.NoError(t, beaconDB.SaveState(ctx, headState, genesisRoot))
|
||||
util.SaveBlock(t, ctx, beaconDB, headBlock)
|
||||
require.NoError(t, beaconDB.SaveFinalizedCheckpoint(ctx, ðpb.Checkpoint{Epoch: slots.ToEpoch(finalizedSlot), Root: headRoot[:]}))
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, stateGen.SaveState(ctx, headRoot, headState))
|
||||
require.NoError(t, beaconDB.SaveLastValidatedCheckpoint(ctx, ðpb.Checkpoint{Epoch: slots.ToEpoch(finalizedSlot), Root: headRoot[:]}))
|
||||
|
||||
notifier := &blockchainTesting.MockStateNotifier{RecordEvents: true}
|
||||
s.cfg.StateNotifier = notifier
|
||||
s.executePostFinalizationTasks(s.ctx, headState)
|
||||
|
||||
time.Sleep(1 * time.Second) // sleep for a second because event is in a separate go routine
|
||||
require.Equal(t, 1, len(notifier.ReceivedEvents()))
|
||||
e := notifier.ReceivedEvents()[0]
|
||||
assert.Equal(t, statefeed.FinalizedCheckpoint, int(e.Type))
|
||||
fc, ok := e.Data.(*ethpbv1.EventFinalizedCheckpoint)
|
||||
require.Equal(t, true, ok, "event has wrong data type")
|
||||
assert.Equal(t, primitives.Epoch(123), fc.Epoch)
|
||||
assert.DeepEqual(t, headRoot[:], fc.Block)
|
||||
assert.DeepEqual(t, finalizedStRoot[:], fc.State)
|
||||
assert.Equal(t, false, fc.ExecutionOptimistic)
|
||||
|
||||
// check the cache
|
||||
index, ok := headState.ValidatorIndexByPubkey(bytesutil.ToBytes48(key))
|
||||
require.Equal(t, true, ok)
|
||||
require.Equal(t, primitives.ValidatorIndex(0), index) // first index
|
||||
})
|
||||
|
||||
// check deposit
|
||||
require.LogsContain(t, logHook, "Finalized deposit insertion completed at index")
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ type MockBuilderService struct {
|
||||
Bid *ethpb.SignedBuilderBid
|
||||
BidCapella *ethpb.SignedBuilderBidCapella
|
||||
BidDeneb *ethpb.SignedBuilderBidDeneb
|
||||
BidElectra *ethpb.SignedBuilderBidElectra
|
||||
RegistrationCache *cache.RegistrationCache
|
||||
ErrGetHeader error
|
||||
ErrRegisterValidator error
|
||||
@@ -59,7 +60,7 @@ func (s *MockBuilderService) SubmitBlindedBlock(_ context.Context, b interfaces.
|
||||
return nil, nil, errors.Wrap(err, "could not wrap capella payload")
|
||||
}
|
||||
return w, nil, s.ErrSubmitBlindedBlock
|
||||
case version.Deneb:
|
||||
case version.Deneb, version.Electra:
|
||||
w, err := blocks.WrappedExecutionPayloadDeneb(s.PayloadDeneb)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Wrap(err, "could not wrap deneb payload")
|
||||
@@ -72,6 +73,9 @@ func (s *MockBuilderService) SubmitBlindedBlock(_ context.Context, b interfaces.
|
||||
|
||||
// GetHeader for mocking.
|
||||
func (s *MockBuilderService) GetHeader(_ context.Context, slot primitives.Slot, _ [32]byte, _ [48]byte) (builder.SignedBid, error) {
|
||||
if slots.ToEpoch(slot) >= params.BeaconConfig().ElectraForkEpoch || s.BidElectra != nil {
|
||||
return builder.WrappedSignedBuilderBidElectra(s.BidElectra)
|
||||
}
|
||||
if slots.ToEpoch(slot) >= params.BeaconConfig().DenebForkEpoch || s.BidDeneb != nil {
|
||||
return builder.WrappedSignedBuilderBidDeneb(s.BidDeneb)
|
||||
}
|
||||
|
||||
3
beacon-chain/cache/cache_test.go
vendored
3
beacon-chain/cache/cache_test.go
vendored
@@ -1,9 +1,10 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
m.Run()
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ go_library(
|
||||
srcs = [
|
||||
"deposit_fetcher.go",
|
||||
"deposit_inserter.go",
|
||||
"deposit_pruner.go",
|
||||
"deposit_tree.go",
|
||||
"deposit_tree_snapshot.go",
|
||||
"merkle_tree.go",
|
||||
@@ -35,6 +36,7 @@ go_test(
|
||||
srcs = [
|
||||
"deposit_cache_test.go",
|
||||
"deposit_fetcher_test.go",
|
||||
"deposit_pruner_test.go",
|
||||
"deposit_tree_snapshot_test.go",
|
||||
"merkle_tree_test.go",
|
||||
"spec_test.go",
|
||||
|
||||
@@ -903,189 +903,6 @@ func TestMin(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
func TestPruneProofs_Ok(t *testing.T) {
|
||||
dc, err := New()
|
||||
require.NoError(t, err)
|
||||
|
||||
deposits := []struct {
|
||||
blkNum uint64
|
||||
deposit *ethpb.Deposit
|
||||
index int64
|
||||
}{
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk0"), 48)}},
|
||||
index: 0,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk1"), 48)}},
|
||||
index: 1,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk2"), 48)}},
|
||||
index: 2,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk3"), 48)}},
|
||||
index: 3,
|
||||
},
|
||||
}
|
||||
|
||||
for _, ins := range deposits {
|
||||
assert.NoError(t, dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{}))
|
||||
}
|
||||
|
||||
require.NoError(t, dc.PruneProofs(context.Background(), 1))
|
||||
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[0].Deposit.Proof)
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[1].Deposit.Proof)
|
||||
assert.NotNil(t, dc.deposits[2].Deposit.Proof)
|
||||
assert.NotNil(t, dc.deposits[3].Deposit.Proof)
|
||||
}
|
||||
|
||||
func TestPruneProofs_SomeAlreadyPruned(t *testing.T) {
|
||||
dc, err := New()
|
||||
require.NoError(t, err)
|
||||
|
||||
deposits := []struct {
|
||||
blkNum uint64
|
||||
deposit *ethpb.Deposit
|
||||
index int64
|
||||
}{
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: nil, Data: ðpb.Deposit_Data{
|
||||
PublicKey: bytesutil.PadTo([]byte("pk0"), 48)}},
|
||||
index: 0,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: nil, Data: ðpb.Deposit_Data{
|
||||
PublicKey: bytesutil.PadTo([]byte("pk1"), 48)}}, index: 1,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(), Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk2"), 48)}},
|
||||
index: 2,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk3"), 48)}},
|
||||
index: 3,
|
||||
},
|
||||
}
|
||||
|
||||
for _, ins := range deposits {
|
||||
assert.NoError(t, dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{}))
|
||||
}
|
||||
|
||||
require.NoError(t, dc.PruneProofs(context.Background(), 2))
|
||||
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[2].Deposit.Proof)
|
||||
}
|
||||
|
||||
func TestPruneProofs_PruneAllWhenDepositIndexTooBig(t *testing.T) {
|
||||
dc, err := New()
|
||||
require.NoError(t, err)
|
||||
|
||||
deposits := []struct {
|
||||
blkNum uint64
|
||||
deposit *ethpb.Deposit
|
||||
index int64
|
||||
}{
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk0"), 48)}},
|
||||
index: 0,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk1"), 48)}},
|
||||
index: 1,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk2"), 48)}},
|
||||
index: 2,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk3"), 48)}},
|
||||
index: 3,
|
||||
},
|
||||
}
|
||||
|
||||
for _, ins := range deposits {
|
||||
assert.NoError(t, dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{}))
|
||||
}
|
||||
|
||||
require.NoError(t, dc.PruneProofs(context.Background(), 99))
|
||||
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[0].Deposit.Proof)
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[1].Deposit.Proof)
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[2].Deposit.Proof)
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[3].Deposit.Proof)
|
||||
}
|
||||
|
||||
func TestPruneProofs_CorrectlyHandleLastIndex(t *testing.T) {
|
||||
dc, err := New()
|
||||
require.NoError(t, err)
|
||||
|
||||
deposits := []struct {
|
||||
blkNum uint64
|
||||
deposit *ethpb.Deposit
|
||||
index int64
|
||||
}{
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk0"), 48)}},
|
||||
index: 0,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk1"), 48)}},
|
||||
index: 1,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk2"), 48)}},
|
||||
index: 2,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk3"), 48)}},
|
||||
index: 3,
|
||||
},
|
||||
}
|
||||
|
||||
for _, ins := range deposits {
|
||||
assert.NoError(t, dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{}))
|
||||
}
|
||||
|
||||
require.NoError(t, dc.PruneProofs(context.Background(), 4))
|
||||
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[0].Deposit.Proof)
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[1].Deposit.Proof)
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[2].Deposit.Proof)
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[3].Deposit.Proof)
|
||||
}
|
||||
|
||||
func TestDepositMap_WorksCorrectly(t *testing.T) {
|
||||
dc, err := New()
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -178,52 +178,6 @@ func (c *Cache) NonFinalizedDeposits(ctx context.Context, lastFinalizedIndex int
|
||||
return deposits
|
||||
}
|
||||
|
||||
// PruneProofs removes proofs from all deposits whose index is equal or less than untilDepositIndex.
|
||||
func (c *Cache) PruneProofs(ctx context.Context, untilDepositIndex int64) error {
|
||||
_, span := trace.StartSpan(ctx, "Cache.PruneProofs")
|
||||
defer span.End()
|
||||
c.depositsLock.Lock()
|
||||
defer c.depositsLock.Unlock()
|
||||
|
||||
if untilDepositIndex >= int64(len(c.deposits)) {
|
||||
untilDepositIndex = int64(len(c.deposits) - 1)
|
||||
}
|
||||
|
||||
for i := untilDepositIndex; i >= 0; i-- {
|
||||
// Finding a nil proof means that all proofs up to this deposit have been already pruned.
|
||||
if c.deposits[i].Deposit.Proof == nil {
|
||||
break
|
||||
}
|
||||
c.deposits[i].Deposit.Proof = nil
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// PrunePendingDeposits removes any deposit which is older than the given deposit merkle tree index.
|
||||
func (c *Cache) PrunePendingDeposits(ctx context.Context, merkleTreeIndex int64) {
|
||||
_, span := trace.StartSpan(ctx, "Cache.PrunePendingDeposits")
|
||||
defer span.End()
|
||||
|
||||
if merkleTreeIndex == 0 {
|
||||
log.Debug("Ignoring 0 deposit removal")
|
||||
return
|
||||
}
|
||||
|
||||
c.depositsLock.Lock()
|
||||
defer c.depositsLock.Unlock()
|
||||
|
||||
cleanDeposits := make([]*ethpb.DepositContainer, 0, len(c.pendingDeposits))
|
||||
for _, dp := range c.pendingDeposits {
|
||||
if dp.Index >= merkleTreeIndex {
|
||||
cleanDeposits = append(cleanDeposits, dp)
|
||||
}
|
||||
}
|
||||
|
||||
c.pendingDeposits = cleanDeposits
|
||||
pendingDepositsCount.Set(float64(len(c.pendingDeposits)))
|
||||
}
|
||||
|
||||
// InsertPendingDeposit into the database. If deposit or block number are nil
|
||||
// then this method does nothing.
|
||||
func (c *Cache) InsertPendingDeposit(ctx context.Context, d *ethpb.Deposit, blockNum uint64, index int64, depositRoot [32]byte) {
|
||||
|
||||
@@ -44,67 +44,3 @@ func TestPendingDeposits_OK(t *testing.T) {
|
||||
all := dc.PendingDeposits(context.Background(), nil)
|
||||
assert.Equal(t, len(dc.pendingDeposits), len(all), "PendingDeposits(ctx, nil) did not return all deposits")
|
||||
}
|
||||
|
||||
func TestPrunePendingDeposits_ZeroMerkleIndex(t *testing.T) {
|
||||
dc := Cache{}
|
||||
|
||||
dc.pendingDeposits = []*ethpb.DepositContainer{
|
||||
{Eth1BlockHeight: 2, Index: 2},
|
||||
{Eth1BlockHeight: 4, Index: 4},
|
||||
{Eth1BlockHeight: 6, Index: 6},
|
||||
{Eth1BlockHeight: 8, Index: 8},
|
||||
{Eth1BlockHeight: 10, Index: 10},
|
||||
{Eth1BlockHeight: 12, Index: 12},
|
||||
}
|
||||
|
||||
dc.PrunePendingDeposits(context.Background(), 0)
|
||||
expected := []*ethpb.DepositContainer{
|
||||
{Eth1BlockHeight: 2, Index: 2},
|
||||
{Eth1BlockHeight: 4, Index: 4},
|
||||
{Eth1BlockHeight: 6, Index: 6},
|
||||
{Eth1BlockHeight: 8, Index: 8},
|
||||
{Eth1BlockHeight: 10, Index: 10},
|
||||
{Eth1BlockHeight: 12, Index: 12},
|
||||
}
|
||||
assert.DeepEqual(t, expected, dc.pendingDeposits)
|
||||
}
|
||||
|
||||
func TestPrunePendingDeposits_OK(t *testing.T) {
|
||||
dc := Cache{}
|
||||
|
||||
dc.pendingDeposits = []*ethpb.DepositContainer{
|
||||
{Eth1BlockHeight: 2, Index: 2},
|
||||
{Eth1BlockHeight: 4, Index: 4},
|
||||
{Eth1BlockHeight: 6, Index: 6},
|
||||
{Eth1BlockHeight: 8, Index: 8},
|
||||
{Eth1BlockHeight: 10, Index: 10},
|
||||
{Eth1BlockHeight: 12, Index: 12},
|
||||
}
|
||||
|
||||
dc.PrunePendingDeposits(context.Background(), 6)
|
||||
expected := []*ethpb.DepositContainer{
|
||||
{Eth1BlockHeight: 6, Index: 6},
|
||||
{Eth1BlockHeight: 8, Index: 8},
|
||||
{Eth1BlockHeight: 10, Index: 10},
|
||||
{Eth1BlockHeight: 12, Index: 12},
|
||||
}
|
||||
|
||||
assert.DeepEqual(t, expected, dc.pendingDeposits)
|
||||
|
||||
dc.pendingDeposits = []*ethpb.DepositContainer{
|
||||
{Eth1BlockHeight: 2, Index: 2},
|
||||
{Eth1BlockHeight: 4, Index: 4},
|
||||
{Eth1BlockHeight: 6, Index: 6},
|
||||
{Eth1BlockHeight: 8, Index: 8},
|
||||
{Eth1BlockHeight: 10, Index: 10},
|
||||
{Eth1BlockHeight: 12, Index: 12},
|
||||
}
|
||||
|
||||
dc.PrunePendingDeposits(context.Background(), 10)
|
||||
expected = []*ethpb.DepositContainer{
|
||||
{Eth1BlockHeight: 10, Index: 10},
|
||||
{Eth1BlockHeight: 12, Index: 12},
|
||||
}
|
||||
|
||||
assert.DeepEqual(t, expected, dc.pendingDeposits)
|
||||
}
|
||||
|
||||
88
beacon-chain/cache/depositsnapshot/deposit_pruner.go
vendored
Normal file
88
beacon-chain/cache/depositsnapshot/deposit_pruner.go
vendored
Normal file
@@ -0,0 +1,88 @@
|
||||
package depositsnapshot
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/prysmaticlabs/prysm/v5/monitoring/tracing/trace"
|
||||
ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"
|
||||
)
|
||||
|
||||
// PruneProofs removes proofs from all deposits whose index is equal or less than untilDepositIndex.
|
||||
func (c *Cache) PruneProofs(ctx context.Context, untilDepositIndex int64) error {
|
||||
_, span := trace.StartSpan(ctx, "Cache.PruneProofs")
|
||||
defer span.End()
|
||||
c.depositsLock.Lock()
|
||||
defer c.depositsLock.Unlock()
|
||||
|
||||
if untilDepositIndex >= int64(len(c.deposits)) {
|
||||
untilDepositIndex = int64(len(c.deposits) - 1)
|
||||
}
|
||||
|
||||
for i := untilDepositIndex; i >= 0; i-- {
|
||||
// Finding a nil proof means that all proofs up to this deposit have been already pruned.
|
||||
if c.deposits[i].Deposit.Proof == nil {
|
||||
break
|
||||
}
|
||||
c.deposits[i].Deposit.Proof = nil
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// PruneAllProofs removes proofs from all deposits.
|
||||
// As EIP-6110 applies and the legacy deposit mechanism is deprecated,
|
||||
// proofs in deposit snapshot are no longer needed.
|
||||
// See: https://eips.ethereum.org/EIPS/eip-6110#eth1data-poll-deprecation
|
||||
func (c *Cache) PruneAllProofs(ctx context.Context) {
|
||||
_, span := trace.StartSpan(ctx, "Cache.PruneAllProofs")
|
||||
defer span.End()
|
||||
|
||||
c.depositsLock.Lock()
|
||||
defer c.depositsLock.Unlock()
|
||||
|
||||
for i := len(c.deposits) - 1; i >= 0; i-- {
|
||||
if c.deposits[i].Deposit.Proof == nil {
|
||||
break
|
||||
}
|
||||
c.deposits[i].Deposit.Proof = nil
|
||||
}
|
||||
}
|
||||
|
||||
// PrunePendingDeposits removes any deposit which is older than the given deposit merkle tree index.
|
||||
func (c *Cache) PrunePendingDeposits(ctx context.Context, merkleTreeIndex int64) {
|
||||
_, span := trace.StartSpan(ctx, "Cache.PrunePendingDeposits")
|
||||
defer span.End()
|
||||
|
||||
if merkleTreeIndex == 0 {
|
||||
log.Debug("Ignoring 0 deposit removal")
|
||||
return
|
||||
}
|
||||
|
||||
c.depositsLock.Lock()
|
||||
defer c.depositsLock.Unlock()
|
||||
|
||||
cleanDeposits := make([]*ethpb.DepositContainer, 0, len(c.pendingDeposits))
|
||||
for _, dp := range c.pendingDeposits {
|
||||
if dp.Index >= merkleTreeIndex {
|
||||
cleanDeposits = append(cleanDeposits, dp)
|
||||
}
|
||||
}
|
||||
|
||||
c.pendingDeposits = cleanDeposits
|
||||
pendingDepositsCount.Set(float64(len(c.pendingDeposits)))
|
||||
}
|
||||
|
||||
// PruneAllPendingDeposits removes all pending deposits from the cache.
|
||||
// As EIP-6110 applies and the legacy deposit mechanism is deprecated,
|
||||
// pending deposits in deposit snapshot are no longer needed.
|
||||
// See: https://eips.ethereum.org/EIPS/eip-6110#eth1data-poll-deprecation
|
||||
func (c *Cache) PruneAllPendingDeposits(ctx context.Context) {
|
||||
_, span := trace.StartSpan(ctx, "Cache.PruneAllPendingDeposits")
|
||||
defer span.End()
|
||||
|
||||
c.depositsLock.Lock()
|
||||
defer c.depositsLock.Unlock()
|
||||
|
||||
c.pendingDeposits = make([]*ethpb.DepositContainer, 0)
|
||||
pendingDepositsCount.Set(float64(0))
|
||||
}
|
||||
323
beacon-chain/cache/depositsnapshot/deposit_pruner_test.go
vendored
Normal file
323
beacon-chain/cache/depositsnapshot/deposit_pruner_test.go
vendored
Normal file
@@ -0,0 +1,323 @@
|
||||
package depositsnapshot
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/prysmaticlabs/prysm/v5/encoding/bytesutil"
|
||||
ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/assert"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/require"
|
||||
)
|
||||
|
||||
func TestPrunePendingDeposits_ZeroMerkleIndex(t *testing.T) {
|
||||
dc := Cache{}
|
||||
|
||||
dc.pendingDeposits = []*ethpb.DepositContainer{
|
||||
{Eth1BlockHeight: 2, Index: 2},
|
||||
{Eth1BlockHeight: 4, Index: 4},
|
||||
{Eth1BlockHeight: 6, Index: 6},
|
||||
{Eth1BlockHeight: 8, Index: 8},
|
||||
{Eth1BlockHeight: 10, Index: 10},
|
||||
{Eth1BlockHeight: 12, Index: 12},
|
||||
}
|
||||
|
||||
dc.PrunePendingDeposits(context.Background(), 0)
|
||||
expected := []*ethpb.DepositContainer{
|
||||
{Eth1BlockHeight: 2, Index: 2},
|
||||
{Eth1BlockHeight: 4, Index: 4},
|
||||
{Eth1BlockHeight: 6, Index: 6},
|
||||
{Eth1BlockHeight: 8, Index: 8},
|
||||
{Eth1BlockHeight: 10, Index: 10},
|
||||
{Eth1BlockHeight: 12, Index: 12},
|
||||
}
|
||||
assert.DeepEqual(t, expected, dc.pendingDeposits)
|
||||
}
|
||||
|
||||
func TestPrunePendingDeposits_OK(t *testing.T) {
|
||||
dc := Cache{}
|
||||
|
||||
dc.pendingDeposits = []*ethpb.DepositContainer{
|
||||
{Eth1BlockHeight: 2, Index: 2},
|
||||
{Eth1BlockHeight: 4, Index: 4},
|
||||
{Eth1BlockHeight: 6, Index: 6},
|
||||
{Eth1BlockHeight: 8, Index: 8},
|
||||
{Eth1BlockHeight: 10, Index: 10},
|
||||
{Eth1BlockHeight: 12, Index: 12},
|
||||
}
|
||||
|
||||
dc.PrunePendingDeposits(context.Background(), 6)
|
||||
expected := []*ethpb.DepositContainer{
|
||||
{Eth1BlockHeight: 6, Index: 6},
|
||||
{Eth1BlockHeight: 8, Index: 8},
|
||||
{Eth1BlockHeight: 10, Index: 10},
|
||||
{Eth1BlockHeight: 12, Index: 12},
|
||||
}
|
||||
|
||||
assert.DeepEqual(t, expected, dc.pendingDeposits)
|
||||
|
||||
dc.pendingDeposits = []*ethpb.DepositContainer{
|
||||
{Eth1BlockHeight: 2, Index: 2},
|
||||
{Eth1BlockHeight: 4, Index: 4},
|
||||
{Eth1BlockHeight: 6, Index: 6},
|
||||
{Eth1BlockHeight: 8, Index: 8},
|
||||
{Eth1BlockHeight: 10, Index: 10},
|
||||
{Eth1BlockHeight: 12, Index: 12},
|
||||
}
|
||||
|
||||
dc.PrunePendingDeposits(context.Background(), 10)
|
||||
expected = []*ethpb.DepositContainer{
|
||||
{Eth1BlockHeight: 10, Index: 10},
|
||||
{Eth1BlockHeight: 12, Index: 12},
|
||||
}
|
||||
|
||||
assert.DeepEqual(t, expected, dc.pendingDeposits)
|
||||
}
|
||||
|
||||
func TestPruneAllPendingDeposits(t *testing.T) {
|
||||
dc := Cache{}
|
||||
|
||||
dc.pendingDeposits = []*ethpb.DepositContainer{
|
||||
{Eth1BlockHeight: 2, Index: 2},
|
||||
{Eth1BlockHeight: 4, Index: 4},
|
||||
{Eth1BlockHeight: 6, Index: 6},
|
||||
{Eth1BlockHeight: 8, Index: 8},
|
||||
{Eth1BlockHeight: 10, Index: 10},
|
||||
{Eth1BlockHeight: 12, Index: 12},
|
||||
}
|
||||
|
||||
dc.PruneAllPendingDeposits(context.Background())
|
||||
expected := []*ethpb.DepositContainer{}
|
||||
|
||||
assert.DeepEqual(t, expected, dc.pendingDeposits)
|
||||
}
|
||||
|
||||
func TestPruneProofs_Ok(t *testing.T) {
|
||||
dc, err := New()
|
||||
require.NoError(t, err)
|
||||
|
||||
deposits := []struct {
|
||||
blkNum uint64
|
||||
deposit *ethpb.Deposit
|
||||
index int64
|
||||
}{
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk0"), 48)}},
|
||||
index: 0,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk1"), 48)}},
|
||||
index: 1,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk2"), 48)}},
|
||||
index: 2,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk3"), 48)}},
|
||||
index: 3,
|
||||
},
|
||||
}
|
||||
|
||||
for _, ins := range deposits {
|
||||
assert.NoError(t, dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{}))
|
||||
}
|
||||
|
||||
require.NoError(t, dc.PruneProofs(context.Background(), 1))
|
||||
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[0].Deposit.Proof)
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[1].Deposit.Proof)
|
||||
assert.NotNil(t, dc.deposits[2].Deposit.Proof)
|
||||
assert.NotNil(t, dc.deposits[3].Deposit.Proof)
|
||||
}
|
||||
|
||||
func TestPruneProofs_SomeAlreadyPruned(t *testing.T) {
|
||||
dc, err := New()
|
||||
require.NoError(t, err)
|
||||
|
||||
deposits := []struct {
|
||||
blkNum uint64
|
||||
deposit *ethpb.Deposit
|
||||
index int64
|
||||
}{
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: nil, Data: ðpb.Deposit_Data{
|
||||
PublicKey: bytesutil.PadTo([]byte("pk0"), 48)}},
|
||||
index: 0,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: nil, Data: ðpb.Deposit_Data{
|
||||
PublicKey: bytesutil.PadTo([]byte("pk1"), 48)}}, index: 1,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(), Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk2"), 48)}},
|
||||
index: 2,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk3"), 48)}},
|
||||
index: 3,
|
||||
},
|
||||
}
|
||||
|
||||
for _, ins := range deposits {
|
||||
assert.NoError(t, dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{}))
|
||||
}
|
||||
|
||||
require.NoError(t, dc.PruneProofs(context.Background(), 2))
|
||||
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[2].Deposit.Proof)
|
||||
}
|
||||
|
||||
func TestPruneProofs_PruneAllWhenDepositIndexTooBig(t *testing.T) {
|
||||
dc, err := New()
|
||||
require.NoError(t, err)
|
||||
|
||||
deposits := []struct {
|
||||
blkNum uint64
|
||||
deposit *ethpb.Deposit
|
||||
index int64
|
||||
}{
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk0"), 48)}},
|
||||
index: 0,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk1"), 48)}},
|
||||
index: 1,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk2"), 48)}},
|
||||
index: 2,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk3"), 48)}},
|
||||
index: 3,
|
||||
},
|
||||
}
|
||||
|
||||
for _, ins := range deposits {
|
||||
assert.NoError(t, dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{}))
|
||||
}
|
||||
|
||||
require.NoError(t, dc.PruneProofs(context.Background(), 99))
|
||||
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[0].Deposit.Proof)
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[1].Deposit.Proof)
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[2].Deposit.Proof)
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[3].Deposit.Proof)
|
||||
}
|
||||
|
||||
func TestPruneProofs_CorrectlyHandleLastIndex(t *testing.T) {
|
||||
dc, err := New()
|
||||
require.NoError(t, err)
|
||||
|
||||
deposits := []struct {
|
||||
blkNum uint64
|
||||
deposit *ethpb.Deposit
|
||||
index int64
|
||||
}{
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk0"), 48)}},
|
||||
index: 0,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk1"), 48)}},
|
||||
index: 1,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk2"), 48)}},
|
||||
index: 2,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk3"), 48)}},
|
||||
index: 3,
|
||||
},
|
||||
}
|
||||
|
||||
for _, ins := range deposits {
|
||||
assert.NoError(t, dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{}))
|
||||
}
|
||||
|
||||
require.NoError(t, dc.PruneProofs(context.Background(), 4))
|
||||
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[0].Deposit.Proof)
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[1].Deposit.Proof)
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[2].Deposit.Proof)
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[3].Deposit.Proof)
|
||||
}
|
||||
|
||||
func TestPruneAllProofs(t *testing.T) {
|
||||
dc, err := New()
|
||||
require.NoError(t, err)
|
||||
|
||||
deposits := []struct {
|
||||
blkNum uint64
|
||||
deposit *ethpb.Deposit
|
||||
index int64
|
||||
}{
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk0"), 48)}},
|
||||
index: 0,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk1"), 48)}},
|
||||
index: 1,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk2"), 48)}},
|
||||
index: 2,
|
||||
},
|
||||
{
|
||||
blkNum: 0,
|
||||
deposit: ðpb.Deposit{Proof: makeDepositProof(),
|
||||
Data: ðpb.Deposit_Data{PublicKey: bytesutil.PadTo([]byte("pk3"), 48)}},
|
||||
index: 3,
|
||||
},
|
||||
}
|
||||
|
||||
for _, ins := range deposits {
|
||||
assert.NoError(t, dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{}))
|
||||
}
|
||||
|
||||
dc.PruneAllProofs(context.Background())
|
||||
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[0].Deposit.Proof)
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[1].Deposit.Proof)
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[2].Deposit.Proof)
|
||||
assert.DeepEqual(t, [][]byte(nil), dc.deposits[3].Deposit.Proof)
|
||||
}
|
||||
11
beacon-chain/cache/interfaces.go
vendored
11
beacon-chain/cache/interfaces.go
vendored
@@ -12,6 +12,7 @@ import (
|
||||
type DepositCache interface {
|
||||
DepositFetcher
|
||||
DepositInserter
|
||||
DepositPruner
|
||||
}
|
||||
|
||||
// DepositFetcher defines a struct which can retrieve deposit information from a store.
|
||||
@@ -23,8 +24,6 @@ type DepositFetcher interface {
|
||||
InsertPendingDeposit(ctx context.Context, d *ethpb.Deposit, blockNum uint64, index int64, depositRoot [32]byte)
|
||||
PendingDeposits(ctx context.Context, untilBlk *big.Int) []*ethpb.Deposit
|
||||
PendingContainers(ctx context.Context, untilBlk *big.Int) []*ethpb.DepositContainer
|
||||
PrunePendingDeposits(ctx context.Context, merkleTreeIndex int64)
|
||||
PruneProofs(ctx context.Context, untilDepositIndex int64) error
|
||||
FinalizedFetcher
|
||||
}
|
||||
|
||||
@@ -42,6 +41,14 @@ type FinalizedFetcher interface {
|
||||
NonFinalizedDeposits(ctx context.Context, lastFinalizedIndex int64, untilBlk *big.Int) []*ethpb.Deposit
|
||||
}
|
||||
|
||||
// DepositPruner is an interface for pruning deposits and proofs.
|
||||
type DepositPruner interface {
|
||||
PrunePendingDeposits(ctx context.Context, merkleTreeIndex int64)
|
||||
PruneAllPendingDeposits(ctx context.Context)
|
||||
PruneProofs(ctx context.Context, untilDepositIndex int64) error
|
||||
PruneAllProofs(ctx context.Context)
|
||||
}
|
||||
|
||||
// FinalizedDeposits defines a method to access a merkle tree containing deposits and their indexes.
|
||||
type FinalizedDeposits interface {
|
||||
Deposits() MerkleTree
|
||||
|
||||
@@ -7,12 +7,14 @@ import (
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/blocks"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/signing"
|
||||
v "github.com/prysmaticlabs/prysm/v5/beacon-chain/core/validators"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/state"
|
||||
state_native "github.com/prysmaticlabs/prysm/v5/beacon-chain/state/state-native"
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
"github.com/prysmaticlabs/prysm/v5/crypto/bls"
|
||||
"github.com/prysmaticlabs/prysm/v5/encoding/bytesutil"
|
||||
ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"
|
||||
"github.com/prysmaticlabs/prysm/v5/runtime/version"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/assert"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/require"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/util"
|
||||
@@ -105,293 +107,162 @@ func TestProcessAttesterSlashings_IndexedAttestationFailedToVerify(t *testing.T)
|
||||
}
|
||||
|
||||
func TestProcessAttesterSlashings_AppliesCorrectStatus(t *testing.T) {
|
||||
beaconState, privKeys := util.DeterministicGenesisState(t, 100)
|
||||
for _, vv := range beaconState.Validators() {
|
||||
vv.WithdrawableEpoch = primitives.Epoch(params.BeaconConfig().SlotsPerEpoch)
|
||||
}
|
||||
statePhase0, keysPhase0 := util.DeterministicGenesisState(t, 100)
|
||||
stateAltair, keysAltair := util.DeterministicGenesisStateAltair(t, 100)
|
||||
stateBellatrix, keysBellatrix := util.DeterministicGenesisStateBellatrix(t, 100)
|
||||
stateCapella, keysCapella := util.DeterministicGenesisStateCapella(t, 100)
|
||||
stateDeneb, keysDeneb := util.DeterministicGenesisStateDeneb(t, 100)
|
||||
stateElectra, keysElectra := util.DeterministicGenesisStateElectra(t, 100)
|
||||
|
||||
att1 := util.HydrateIndexedAttestation(ðpb.IndexedAttestation{
|
||||
att1Phase0 := util.HydrateIndexedAttestation(ðpb.IndexedAttestation{
|
||||
Data: ðpb.AttestationData{
|
||||
Source: ðpb.Checkpoint{Epoch: 1},
|
||||
},
|
||||
AttestingIndices: []uint64{0, 1},
|
||||
})
|
||||
domain, err := signing.Domain(beaconState.Fork(), 0, params.BeaconConfig().DomainBeaconAttester, beaconState.GenesisValidatorsRoot())
|
||||
require.NoError(t, err)
|
||||
signingRoot, err := signing.ComputeSigningRoot(att1.Data, domain)
|
||||
assert.NoError(t, err, "Could not get signing root of beacon block header")
|
||||
sig0 := privKeys[0].Sign(signingRoot[:])
|
||||
sig1 := privKeys[1].Sign(signingRoot[:])
|
||||
aggregateSig := bls.AggregateSignatures([]bls.Signature{sig0, sig1})
|
||||
att1.Signature = aggregateSig.Marshal()
|
||||
|
||||
att2 := util.HydrateIndexedAttestation(ðpb.IndexedAttestation{
|
||||
att2Phase0 := util.HydrateIndexedAttestation(ðpb.IndexedAttestation{
|
||||
AttestingIndices: []uint64{0, 1},
|
||||
})
|
||||
signingRoot, err = signing.ComputeSigningRoot(att2.Data, domain)
|
||||
assert.NoError(t, err, "Could not get signing root of beacon block header")
|
||||
sig0 = privKeys[0].Sign(signingRoot[:])
|
||||
sig1 = privKeys[1].Sign(signingRoot[:])
|
||||
aggregateSig = bls.AggregateSignatures([]bls.Signature{sig0, sig1})
|
||||
att2.Signature = aggregateSig.Marshal()
|
||||
|
||||
slashings := []*ethpb.AttesterSlashing{
|
||||
{
|
||||
Attestation_1: att1,
|
||||
Attestation_2: att2,
|
||||
},
|
||||
}
|
||||
|
||||
currentSlot := 2 * params.BeaconConfig().SlotsPerEpoch
|
||||
require.NoError(t, beaconState.SetSlot(currentSlot))
|
||||
|
||||
b := util.NewBeaconBlock()
|
||||
b.Block = ðpb.BeaconBlock{
|
||||
Body: ðpb.BeaconBlockBody{
|
||||
AttesterSlashings: slashings,
|
||||
},
|
||||
}
|
||||
|
||||
ss := make([]ethpb.AttSlashing, len(b.Block.Body.AttesterSlashings))
|
||||
for i, s := range b.Block.Body.AttesterSlashings {
|
||||
ss[i] = s
|
||||
}
|
||||
newState, err := blocks.ProcessAttesterSlashings(context.Background(), beaconState, ss, v.SlashValidator)
|
||||
require.NoError(t, err)
|
||||
newRegistry := newState.Validators()
|
||||
|
||||
// Given the intersection of slashable indices is [1], only validator
|
||||
// at index 1 should be slashed and exited. We confirm this below.
|
||||
if newRegistry[1].ExitEpoch != beaconState.Validators()[1].ExitEpoch {
|
||||
t.Errorf(
|
||||
`
|
||||
Expected validator at index 1's exit epoch to match
|
||||
%d, received %d instead
|
||||
`,
|
||||
beaconState.Validators()[1].ExitEpoch,
|
||||
newRegistry[1].ExitEpoch,
|
||||
)
|
||||
}
|
||||
|
||||
require.Equal(t, uint64(31750000000), newState.Balances()[1])
|
||||
require.Equal(t, uint64(32000000000), newState.Balances()[2])
|
||||
}
|
||||
|
||||
func TestProcessAttesterSlashings_AppliesCorrectStatusAltair(t *testing.T) {
|
||||
beaconState, privKeys := util.DeterministicGenesisStateAltair(t, 100)
|
||||
for _, vv := range beaconState.Validators() {
|
||||
vv.WithdrawableEpoch = primitives.Epoch(params.BeaconConfig().SlotsPerEpoch)
|
||||
}
|
||||
|
||||
att1 := util.HydrateIndexedAttestation(ðpb.IndexedAttestation{
|
||||
att1Electra := util.HydrateIndexedAttestationElectra(ðpb.IndexedAttestationElectra{
|
||||
Data: ðpb.AttestationData{
|
||||
Source: ðpb.Checkpoint{Epoch: 1},
|
||||
},
|
||||
AttestingIndices: []uint64{0, 1},
|
||||
})
|
||||
domain, err := signing.Domain(beaconState.Fork(), 0, params.BeaconConfig().DomainBeaconAttester, beaconState.GenesisValidatorsRoot())
|
||||
require.NoError(t, err)
|
||||
signingRoot, err := signing.ComputeSigningRoot(att1.Data, domain)
|
||||
assert.NoError(t, err, "Could not get signing root of beacon block header")
|
||||
sig0 := privKeys[0].Sign(signingRoot[:])
|
||||
sig1 := privKeys[1].Sign(signingRoot[:])
|
||||
aggregateSig := bls.AggregateSignatures([]bls.Signature{sig0, sig1})
|
||||
att1.Signature = aggregateSig.Marshal()
|
||||
|
||||
att2 := util.HydrateIndexedAttestation(ðpb.IndexedAttestation{
|
||||
att2Electra := util.HydrateIndexedAttestationElectra(ðpb.IndexedAttestationElectra{
|
||||
AttestingIndices: []uint64{0, 1},
|
||||
})
|
||||
signingRoot, err = signing.ComputeSigningRoot(att2.Data, domain)
|
||||
assert.NoError(t, err, "Could not get signing root of beacon block header")
|
||||
sig0 = privKeys[0].Sign(signingRoot[:])
|
||||
sig1 = privKeys[1].Sign(signingRoot[:])
|
||||
aggregateSig = bls.AggregateSignatures([]bls.Signature{sig0, sig1})
|
||||
att2.Signature = aggregateSig.Marshal()
|
||||
|
||||
slashings := []*ethpb.AttesterSlashing{
|
||||
slashingPhase0 := ðpb.AttesterSlashing{
|
||||
Attestation_1: att1Phase0,
|
||||
Attestation_2: att2Phase0,
|
||||
}
|
||||
slashingElectra := ðpb.AttesterSlashingElectra{
|
||||
Attestation_1: att1Electra,
|
||||
Attestation_2: att2Electra,
|
||||
}
|
||||
|
||||
type testCase struct {
|
||||
name string
|
||||
st state.BeaconState
|
||||
keys []bls.SecretKey
|
||||
att1 ethpb.IndexedAtt
|
||||
att2 ethpb.IndexedAtt
|
||||
slashing ethpb.AttSlashing
|
||||
slashedBalance uint64
|
||||
}
|
||||
|
||||
testCases := []testCase{
|
||||
{
|
||||
Attestation_1: att1,
|
||||
Attestation_2: att2,
|
||||
name: "phase0",
|
||||
st: statePhase0,
|
||||
keys: keysPhase0,
|
||||
att1: att1Phase0,
|
||||
att2: att2Phase0,
|
||||
slashing: slashingPhase0,
|
||||
slashedBalance: 31750000000,
|
||||
},
|
||||
{
|
||||
name: "altair",
|
||||
st: stateAltair,
|
||||
keys: keysAltair,
|
||||
att1: att1Phase0,
|
||||
att2: att2Phase0,
|
||||
slashing: slashingPhase0,
|
||||
slashedBalance: 31500000000,
|
||||
},
|
||||
{
|
||||
name: "bellatrix",
|
||||
st: stateBellatrix,
|
||||
keys: keysBellatrix,
|
||||
att1: att1Phase0,
|
||||
att2: att2Phase0,
|
||||
slashing: slashingPhase0,
|
||||
slashedBalance: 31000000000,
|
||||
},
|
||||
{
|
||||
name: "capella",
|
||||
st: stateCapella,
|
||||
keys: keysCapella,
|
||||
att1: att1Phase0,
|
||||
att2: att2Phase0,
|
||||
slashing: slashingPhase0,
|
||||
slashedBalance: 31000000000,
|
||||
},
|
||||
{
|
||||
name: "deneb",
|
||||
st: stateDeneb,
|
||||
keys: keysDeneb,
|
||||
att1: att1Phase0,
|
||||
att2: att2Phase0,
|
||||
slashing: slashingPhase0,
|
||||
slashedBalance: 31000000000,
|
||||
},
|
||||
{
|
||||
name: "electra",
|
||||
st: stateElectra,
|
||||
keys: keysElectra,
|
||||
att1: att1Electra,
|
||||
att2: att2Electra,
|
||||
slashing: slashingElectra,
|
||||
slashedBalance: 31992187500,
|
||||
},
|
||||
}
|
||||
|
||||
currentSlot := 2 * params.BeaconConfig().SlotsPerEpoch
|
||||
require.NoError(t, beaconState.SetSlot(currentSlot))
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
for _, vv := range tc.st.Validators() {
|
||||
vv.WithdrawableEpoch = primitives.Epoch(params.BeaconConfig().SlotsPerEpoch)
|
||||
}
|
||||
|
||||
b := util.NewBeaconBlock()
|
||||
b.Block = ðpb.BeaconBlock{
|
||||
Body: ðpb.BeaconBlockBody{
|
||||
AttesterSlashings: slashings,
|
||||
},
|
||||
}
|
||||
domain, err := signing.Domain(tc.st.Fork(), 0, params.BeaconConfig().DomainBeaconAttester, tc.st.GenesisValidatorsRoot())
|
||||
require.NoError(t, err)
|
||||
signingRoot, err := signing.ComputeSigningRoot(tc.att1.GetData(), domain)
|
||||
assert.NoError(t, err, "Could not get signing root of beacon block header")
|
||||
sig0 := tc.keys[0].Sign(signingRoot[:])
|
||||
sig1 := tc.keys[1].Sign(signingRoot[:])
|
||||
aggregateSig := bls.AggregateSignatures([]bls.Signature{sig0, sig1})
|
||||
|
||||
ss := make([]ethpb.AttSlashing, len(b.Block.Body.AttesterSlashings))
|
||||
for i, s := range b.Block.Body.AttesterSlashings {
|
||||
ss[i] = s
|
||||
}
|
||||
newState, err := blocks.ProcessAttesterSlashings(context.Background(), beaconState, ss, v.SlashValidator)
|
||||
require.NoError(t, err)
|
||||
newRegistry := newState.Validators()
|
||||
if tc.att1.Version() >= version.Electra {
|
||||
tc.att1.(*ethpb.IndexedAttestationElectra).Signature = aggregateSig.Marshal()
|
||||
} else {
|
||||
tc.att1.(*ethpb.IndexedAttestation).Signature = aggregateSig.Marshal()
|
||||
}
|
||||
|
||||
// Given the intersection of slashable indices is [1], only validator
|
||||
// at index 1 should be slashed and exited. We confirm this below.
|
||||
if newRegistry[1].ExitEpoch != beaconState.Validators()[1].ExitEpoch {
|
||||
t.Errorf(
|
||||
`
|
||||
signingRoot, err = signing.ComputeSigningRoot(tc.att2.GetData(), domain)
|
||||
assert.NoError(t, err, "Could not get signing root of beacon block header")
|
||||
sig0 = tc.keys[0].Sign(signingRoot[:])
|
||||
sig1 = tc.keys[1].Sign(signingRoot[:])
|
||||
aggregateSig = bls.AggregateSignatures([]bls.Signature{sig0, sig1})
|
||||
|
||||
if tc.att2.Version() >= version.Electra {
|
||||
tc.att2.(*ethpb.IndexedAttestationElectra).Signature = aggregateSig.Marshal()
|
||||
} else {
|
||||
tc.att2.(*ethpb.IndexedAttestation).Signature = aggregateSig.Marshal()
|
||||
}
|
||||
|
||||
currentSlot := 2 * params.BeaconConfig().SlotsPerEpoch
|
||||
require.NoError(t, tc.st.SetSlot(currentSlot))
|
||||
|
||||
newState, err := blocks.ProcessAttesterSlashings(context.Background(), tc.st, []ethpb.AttSlashing{tc.slashing}, v.SlashValidator)
|
||||
require.NoError(t, err)
|
||||
newRegistry := newState.Validators()
|
||||
|
||||
// Given the intersection of slashable indices is [1], only validator
|
||||
// at index 1 should be slashed and exited. We confirm this below.
|
||||
if newRegistry[1].ExitEpoch != tc.st.Validators()[1].ExitEpoch {
|
||||
t.Errorf(
|
||||
`
|
||||
Expected validator at index 1's exit epoch to match
|
||||
%d, received %d instead
|
||||
`,
|
||||
beaconState.Validators()[1].ExitEpoch,
|
||||
newRegistry[1].ExitEpoch,
|
||||
)
|
||||
}
|
||||
tc.st.Validators()[1].ExitEpoch,
|
||||
newRegistry[1].ExitEpoch,
|
||||
)
|
||||
}
|
||||
|
||||
require.Equal(t, uint64(31500000000), newState.Balances()[1])
|
||||
require.Equal(t, uint64(32000000000), newState.Balances()[2])
|
||||
}
|
||||
|
||||
func TestProcessAttesterSlashings_AppliesCorrectStatusBellatrix(t *testing.T) {
|
||||
beaconState, privKeys := util.DeterministicGenesisStateBellatrix(t, 100)
|
||||
for _, vv := range beaconState.Validators() {
|
||||
vv.WithdrawableEpoch = primitives.Epoch(params.BeaconConfig().SlotsPerEpoch)
|
||||
}
|
||||
|
||||
att1 := util.HydrateIndexedAttestation(ðpb.IndexedAttestation{
|
||||
Data: ðpb.AttestationData{
|
||||
Source: ðpb.Checkpoint{Epoch: 1},
|
||||
},
|
||||
AttestingIndices: []uint64{0, 1},
|
||||
})
|
||||
domain, err := signing.Domain(beaconState.Fork(), 0, params.BeaconConfig().DomainBeaconAttester, beaconState.GenesisValidatorsRoot())
|
||||
require.NoError(t, err)
|
||||
signingRoot, err := signing.ComputeSigningRoot(att1.Data, domain)
|
||||
assert.NoError(t, err, "Could not get signing root of beacon block header")
|
||||
sig0 := privKeys[0].Sign(signingRoot[:])
|
||||
sig1 := privKeys[1].Sign(signingRoot[:])
|
||||
aggregateSig := bls.AggregateSignatures([]bls.Signature{sig0, sig1})
|
||||
att1.Signature = aggregateSig.Marshal()
|
||||
|
||||
att2 := util.HydrateIndexedAttestation(ðpb.IndexedAttestation{
|
||||
AttestingIndices: []uint64{0, 1},
|
||||
})
|
||||
signingRoot, err = signing.ComputeSigningRoot(att2.Data, domain)
|
||||
assert.NoError(t, err, "Could not get signing root of beacon block header")
|
||||
sig0 = privKeys[0].Sign(signingRoot[:])
|
||||
sig1 = privKeys[1].Sign(signingRoot[:])
|
||||
aggregateSig = bls.AggregateSignatures([]bls.Signature{sig0, sig1})
|
||||
att2.Signature = aggregateSig.Marshal()
|
||||
|
||||
slashings := []*ethpb.AttesterSlashing{
|
||||
{
|
||||
Attestation_1: att1,
|
||||
Attestation_2: att2,
|
||||
},
|
||||
}
|
||||
|
||||
currentSlot := 2 * params.BeaconConfig().SlotsPerEpoch
|
||||
require.NoError(t, beaconState.SetSlot(currentSlot))
|
||||
|
||||
b := util.NewBeaconBlock()
|
||||
b.Block = ðpb.BeaconBlock{
|
||||
Body: ðpb.BeaconBlockBody{
|
||||
AttesterSlashings: slashings,
|
||||
},
|
||||
}
|
||||
|
||||
ss := make([]ethpb.AttSlashing, len(b.Block.Body.AttesterSlashings))
|
||||
for i, s := range b.Block.Body.AttesterSlashings {
|
||||
ss[i] = s
|
||||
}
|
||||
newState, err := blocks.ProcessAttesterSlashings(context.Background(), beaconState, ss, v.SlashValidator)
|
||||
require.NoError(t, err)
|
||||
newRegistry := newState.Validators()
|
||||
|
||||
// Given the intersection of slashable indices is [1], only validator
|
||||
// at index 1 should be slashed and exited. We confirm this below.
|
||||
if newRegistry[1].ExitEpoch != beaconState.Validators()[1].ExitEpoch {
|
||||
t.Errorf(
|
||||
`
|
||||
Expected validator at index 1's exit epoch to match
|
||||
%d, received %d instead
|
||||
`,
|
||||
beaconState.Validators()[1].ExitEpoch,
|
||||
newRegistry[1].ExitEpoch,
|
||||
)
|
||||
}
|
||||
|
||||
require.Equal(t, uint64(31000000000), newState.Balances()[1])
|
||||
require.Equal(t, uint64(32000000000), newState.Balances()[2])
|
||||
}
|
||||
|
||||
func TestProcessAttesterSlashings_AppliesCorrectStatusCapella(t *testing.T) {
|
||||
beaconState, privKeys := util.DeterministicGenesisStateCapella(t, 100)
|
||||
for _, vv := range beaconState.Validators() {
|
||||
vv.WithdrawableEpoch = primitives.Epoch(params.BeaconConfig().SlotsPerEpoch)
|
||||
}
|
||||
|
||||
att1 := util.HydrateIndexedAttestation(ðpb.IndexedAttestation{
|
||||
Data: ðpb.AttestationData{
|
||||
Source: ðpb.Checkpoint{Epoch: 1},
|
||||
},
|
||||
AttestingIndices: []uint64{0, 1},
|
||||
})
|
||||
domain, err := signing.Domain(beaconState.Fork(), 0, params.BeaconConfig().DomainBeaconAttester, beaconState.GenesisValidatorsRoot())
|
||||
require.NoError(t, err)
|
||||
signingRoot, err := signing.ComputeSigningRoot(att1.Data, domain)
|
||||
assert.NoError(t, err, "Could not get signing root of beacon block header")
|
||||
sig0 := privKeys[0].Sign(signingRoot[:])
|
||||
sig1 := privKeys[1].Sign(signingRoot[:])
|
||||
aggregateSig := bls.AggregateSignatures([]bls.Signature{sig0, sig1})
|
||||
att1.Signature = aggregateSig.Marshal()
|
||||
|
||||
att2 := util.HydrateIndexedAttestation(ðpb.IndexedAttestation{
|
||||
AttestingIndices: []uint64{0, 1},
|
||||
})
|
||||
signingRoot, err = signing.ComputeSigningRoot(att2.Data, domain)
|
||||
assert.NoError(t, err, "Could not get signing root of beacon block header")
|
||||
sig0 = privKeys[0].Sign(signingRoot[:])
|
||||
sig1 = privKeys[1].Sign(signingRoot[:])
|
||||
aggregateSig = bls.AggregateSignatures([]bls.Signature{sig0, sig1})
|
||||
att2.Signature = aggregateSig.Marshal()
|
||||
|
||||
slashings := []*ethpb.AttesterSlashing{
|
||||
{
|
||||
Attestation_1: att1,
|
||||
Attestation_2: att2,
|
||||
},
|
||||
}
|
||||
|
||||
currentSlot := 2 * params.BeaconConfig().SlotsPerEpoch
|
||||
require.NoError(t, beaconState.SetSlot(currentSlot))
|
||||
|
||||
b := util.NewBeaconBlock()
|
||||
b.Block = ðpb.BeaconBlock{
|
||||
Body: ðpb.BeaconBlockBody{
|
||||
AttesterSlashings: slashings,
|
||||
},
|
||||
}
|
||||
|
||||
ss := make([]ethpb.AttSlashing, len(b.Block.Body.AttesterSlashings))
|
||||
for i, s := range b.Block.Body.AttesterSlashings {
|
||||
ss[i] = s
|
||||
}
|
||||
newState, err := blocks.ProcessAttesterSlashings(context.Background(), beaconState, ss, v.SlashValidator)
|
||||
require.NoError(t, err)
|
||||
newRegistry := newState.Validators()
|
||||
|
||||
// Given the intersection of slashable indices is [1], only validator
|
||||
// at index 1 should be slashed and exited. We confirm this below.
|
||||
if newRegistry[1].ExitEpoch != beaconState.Validators()[1].ExitEpoch {
|
||||
t.Errorf(
|
||||
`
|
||||
Expected validator at index 1's exit epoch to match
|
||||
%d, received %d instead
|
||||
`,
|
||||
beaconState.Validators()[1].ExitEpoch,
|
||||
newRegistry[1].ExitEpoch,
|
||||
)
|
||||
}
|
||||
|
||||
require.Equal(t, uint64(31000000000), newState.Balances()[1])
|
||||
require.Equal(t, uint64(32000000000), newState.Balances()[2])
|
||||
require.Equal(t, tc.slashedBalance, newState.Balances()[1])
|
||||
require.Equal(t, uint64(32000000000), newState.Balances()[2])
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ go_library(
|
||||
"consolidations.go",
|
||||
"deposits.go",
|
||||
"effective_balance_updates.go",
|
||||
"error.go",
|
||||
"registry_updates.go",
|
||||
"transition.go",
|
||||
"transition_no_verify_sig.go",
|
||||
@@ -55,13 +56,16 @@ go_test(
|
||||
"deposit_fuzz_test.go",
|
||||
"deposits_test.go",
|
||||
"effective_balance_updates_test.go",
|
||||
"error_test.go",
|
||||
"export_test.go",
|
||||
"registry_updates_test.go",
|
||||
"transition_no_verify_sig_test.go",
|
||||
"transition_test.go",
|
||||
"upgrade_test.go",
|
||||
"validator_test.go",
|
||||
"withdrawals_test.go",
|
||||
],
|
||||
data = glob(["testdata/**"]),
|
||||
embed = [":go_default_library"],
|
||||
deps = [
|
||||
"//beacon-chain/core/helpers:go_default_library",
|
||||
@@ -86,6 +90,7 @@ go_test(
|
||||
"@com_github_ethereum_go_ethereum//common:go_default_library",
|
||||
"@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
|
||||
"@com_github_google_gofuzz//:go_default_library",
|
||||
"@com_github_pkg_errors//:go_default_library",
|
||||
"@com_github_sirupsen_logrus//:go_default_library",
|
||||
"@com_github_sirupsen_logrus//hooks/test:go_default_library",
|
||||
],
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/electra"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/helpers"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/state"
|
||||
state_native "github.com/prysmaticlabs/prysm/v5/beacon-chain/state/state-native"
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
@@ -129,6 +130,57 @@ func TestComputeConsolidationEpochAndUpdateChurn(t *testing.T) {
|
||||
expectedEpoch: 16, // Flows into another epoch.
|
||||
expectedConsolidationBalanceToConsume: 200000000000, // 200 ETH
|
||||
},
|
||||
{
|
||||
name: "balance to consume is zero, consolidation balance at limit",
|
||||
state: func(t *testing.T) state.BeaconState {
|
||||
activeBal := 32000000000000000 // 32M ETH
|
||||
s, err := state_native.InitializeFromProtoUnsafeElectra(ð.BeaconStateElectra{
|
||||
Slot: slots.UnsafeEpochStart(10),
|
||||
EarliestConsolidationEpoch: 16,
|
||||
ConsolidationBalanceToConsume: 0,
|
||||
Validators: createValidatorsWithTotalActiveBalance(primitives.Gwei(activeBal)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return s
|
||||
}(t),
|
||||
consolidationBalance: helpers.ConsolidationChurnLimit(32000000000000000),
|
||||
expectedEpoch: 17, // Flows into another epoch.
|
||||
expectedConsolidationBalanceToConsume: 0,
|
||||
},
|
||||
{
|
||||
name: "consolidation balance equals consolidation balance to consume",
|
||||
state: func(t *testing.T) state.BeaconState {
|
||||
activeBal := 32000000000000000 // 32M ETH
|
||||
s, err := state_native.InitializeFromProtoUnsafeElectra(ð.BeaconStateElectra{
|
||||
Slot: slots.UnsafeEpochStart(10),
|
||||
EarliestConsolidationEpoch: 16,
|
||||
ConsolidationBalanceToConsume: helpers.ConsolidationChurnLimit(32000000000000000),
|
||||
Validators: createValidatorsWithTotalActiveBalance(primitives.Gwei(activeBal)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return s
|
||||
}(t),
|
||||
consolidationBalance: helpers.ConsolidationChurnLimit(32000000000000000),
|
||||
expectedEpoch: 16,
|
||||
expectedConsolidationBalanceToConsume: 0,
|
||||
},
|
||||
{
|
||||
name: "consolidation balance exceeds limit by one",
|
||||
state: func(t *testing.T) state.BeaconState {
|
||||
activeBal := 32000000000000000 // 32M ETH
|
||||
s, err := state_native.InitializeFromProtoUnsafeElectra(ð.BeaconStateElectra{
|
||||
Slot: slots.UnsafeEpochStart(10),
|
||||
EarliestConsolidationEpoch: 16,
|
||||
ConsolidationBalanceToConsume: 0,
|
||||
Validators: createValidatorsWithTotalActiveBalance(primitives.Gwei(activeBal)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return s
|
||||
}(t),
|
||||
consolidationBalance: helpers.ConsolidationChurnLimit(32000000000000000) + 1,
|
||||
expectedEpoch: 18, // Flows into another epoch.
|
||||
expectedConsolidationBalanceToConsume: helpers.ConsolidationChurnLimit(32000000000000000) - 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/helpers"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/state"
|
||||
state_native "github.com/prysmaticlabs/prysm/v5/beacon-chain/state/state-native"
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
"github.com/prysmaticlabs/prysm/v5/encoding/bytesutil"
|
||||
@@ -184,6 +185,9 @@ func ProcessConsolidationRequests(ctx context.Context, st state.BeaconState, req
|
||||
pcLimit := params.BeaconConfig().PendingConsolidationsLimit
|
||||
|
||||
for _, cr := range reqs {
|
||||
if cr == nil {
|
||||
return errors.New("nil consolidation request")
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return fmt.Errorf("cannot process consolidation requests: %w", ctx.Err())
|
||||
}
|
||||
@@ -233,13 +237,18 @@ func ProcessConsolidationRequests(ctx context.Context, st state.BeaconState, req
|
||||
return fmt.Errorf("failed to fetch source validator: %w", err) // This should never happen.
|
||||
}
|
||||
|
||||
roSrcV, err := state_native.NewValidator(srcV)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tgtV, err := st.ValidatorAtIndexReadOnly(tgtIdx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to fetch target validator: %w", err) // This should never happen.
|
||||
}
|
||||
|
||||
// Verify source withdrawal credentials
|
||||
if !helpers.HasExecutionWithdrawalCredentials(srcV) {
|
||||
if !roSrcV.HasExecutionWithdrawalCredentials() {
|
||||
continue
|
||||
}
|
||||
// Confirm source_validator.withdrawal_credentials[12:] == consolidation_request.source_address
|
||||
@@ -248,7 +257,7 @@ func ProcessConsolidationRequests(ctx context.Context, st state.BeaconState, req
|
||||
}
|
||||
|
||||
// Target validator must have their withdrawal credentials set appropriately.
|
||||
if !helpers.HasCompoundingWithdrawalCredential(tgtV) {
|
||||
if !tgtV.HasCompoundingWithdrawalCredentials() {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -256,7 +265,7 @@ func ProcessConsolidationRequests(ctx context.Context, st state.BeaconState, req
|
||||
if !helpers.IsActiveValidator(srcV, curEpoch) || !helpers.IsActiveValidatorUsingTrie(tgtV, curEpoch) {
|
||||
continue
|
||||
}
|
||||
// Neither validator are exiting.
|
||||
// Neither validator is exiting.
|
||||
if srcV.ExitEpoch != ffe || tgtV.ExitEpoch() != ffe {
|
||||
continue
|
||||
}
|
||||
@@ -364,7 +373,7 @@ func IsValidSwitchToCompoundingRequest(st state.BeaconState, req *enginev1.Conso
|
||||
return false
|
||||
}
|
||||
|
||||
if !helpers.HasETH1WithdrawalCredential(srcV) {
|
||||
if !srcV.HasETH1WithdrawalCredentials() {
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
@@ -209,7 +209,22 @@ func TestProcessConsolidationRequests(t *testing.T) {
|
||||
state state.BeaconState
|
||||
reqs []*enginev1.ConsolidationRequest
|
||||
validate func(*testing.T, state.BeaconState)
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "nil request",
|
||||
state: func() state.BeaconState {
|
||||
st := ð.BeaconStateElectra{}
|
||||
s, err := state_native.InitializeFromProtoElectra(st)
|
||||
require.NoError(t, err)
|
||||
return s
|
||||
}(),
|
||||
reqs: []*enginev1.ConsolidationRequest{nil},
|
||||
validate: func(t *testing.T, st state.BeaconState) {
|
||||
require.DeepEqual(t, st, st)
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "one valid request",
|
||||
state: func() state.BeaconState {
|
||||
@@ -405,7 +420,13 @@ func TestProcessConsolidationRequests(t *testing.T) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := electra.ProcessConsolidationRequests(context.TODO(), tt.state, tt.reqs)
|
||||
require.NoError(t, err)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("ProcessWithdrawalRequests() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if !tt.wantErr {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
if tt.validate != nil {
|
||||
tt.validate(t, tt.state)
|
||||
}
|
||||
|
||||
@@ -385,14 +385,8 @@ func batchProcessNewPendingDeposits(ctx context.Context, state state.BeaconState
|
||||
return errors.Wrap(err, "batch signature verification failed")
|
||||
}
|
||||
|
||||
pubKeyMap := make(map[[48]byte]struct{}, len(pendingDeposits))
|
||||
|
||||
// Process each deposit individually
|
||||
for _, pendingDeposit := range pendingDeposits {
|
||||
_, found := pubKeyMap[bytesutil.ToBytes48(pendingDeposit.PublicKey)]
|
||||
if !found {
|
||||
pubKeyMap[bytesutil.ToBytes48(pendingDeposit.PublicKey)] = struct{}{}
|
||||
}
|
||||
validSignature := allSignaturesVerified
|
||||
|
||||
// If batch verification failed, check the individual deposit signature
|
||||
@@ -410,7 +404,8 @@ func batchProcessNewPendingDeposits(ctx context.Context, state state.BeaconState
|
||||
|
||||
// Add validator to the registry if the signature is valid
|
||||
if validSignature {
|
||||
if found {
|
||||
_, has := state.ValidatorIndexByPubkey(bytesutil.ToBytes48(pendingDeposit.PublicKey))
|
||||
if has {
|
||||
index, _ := state.ValidatorIndexByPubkey(bytesutil.ToBytes48(pendingDeposit.PublicKey))
|
||||
if err := helpers.IncreaseBalance(state, index, pendingDeposit.Amount); err != nil {
|
||||
return errors.Wrap(err, "could not increase balance")
|
||||
@@ -592,10 +587,10 @@ func processDepositRequest(beaconState state.BeaconState, request *enginev1.Depo
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not get deposit requests start index")
|
||||
}
|
||||
if request == nil {
|
||||
return nil, errors.New("nil deposit request")
|
||||
}
|
||||
if requestsStartIndex == params.BeaconConfig().UnsetDepositRequestsStartIndex {
|
||||
if request == nil {
|
||||
return nil, errors.New("nil deposit request")
|
||||
}
|
||||
if err := beaconState.SetDepositRequestsStartIndex(request.Index); err != nil {
|
||||
return nil, errors.Wrap(err, "could not set deposit requests start index")
|
||||
}
|
||||
|
||||
@@ -333,6 +333,7 @@ func TestProcessDepositRequests(t *testing.T) {
|
||||
st, _ := util.DeterministicGenesisStateElectra(t, 1)
|
||||
sk, err := bls.RandKey()
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, st.SetDepositRequestsStartIndex(1))
|
||||
|
||||
t.Run("empty requests continues", func(t *testing.T) {
|
||||
newSt, err := electra.ProcessDepositRequests(context.Background(), st, []*enginev1.DepositRequest{})
|
||||
|
||||
@@ -3,7 +3,6 @@ package electra
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/helpers"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/state"
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"
|
||||
@@ -40,7 +39,7 @@ func ProcessEffectiveBalanceUpdates(st state.BeaconState) error {
|
||||
|
||||
// Update effective balances with hysteresis.
|
||||
validatorFunc := func(idx int, val state.ReadOnlyValidator) (newVal *ethpb.Validator, err error) {
|
||||
if val == nil {
|
||||
if val.IsNil() {
|
||||
return nil, fmt.Errorf("validator %d is nil in state", idx)
|
||||
}
|
||||
if idx >= len(bals) {
|
||||
@@ -49,7 +48,7 @@ func ProcessEffectiveBalanceUpdates(st state.BeaconState) error {
|
||||
balance := bals[idx]
|
||||
|
||||
effectiveBalanceLimit := params.BeaconConfig().MinActivationBalance
|
||||
if helpers.HasCompoundingWithdrawalCredential(val) {
|
||||
if val.HasCompoundingWithdrawalCredentials() {
|
||||
effectiveBalanceLimit = params.BeaconConfig().MaxEffectiveBalanceElectra
|
||||
}
|
||||
|
||||
|
||||
@@ -77,7 +77,7 @@ func TestProcessEffectiveBalnceUpdates(t *testing.T) {
|
||||
Validators: []*eth.Validator{
|
||||
{
|
||||
EffectiveBalance: params.BeaconConfig().MinActivationBalance / 2,
|
||||
WithdrawalCredentials: nil,
|
||||
WithdrawalCredentials: make([]byte, 32),
|
||||
},
|
||||
},
|
||||
Balances: []uint64{
|
||||
|
||||
16
beacon-chain/core/electra/error.go
Normal file
16
beacon-chain/core/electra/error.go
Normal file
@@ -0,0 +1,16 @@
|
||||
package electra
|
||||
|
||||
import "github.com/pkg/errors"
|
||||
|
||||
type execReqErr struct {
|
||||
error
|
||||
}
|
||||
|
||||
// IsExecutionRequestError returns true if the error has `execReqErr`.
|
||||
func IsExecutionRequestError(e error) bool {
|
||||
if e == nil {
|
||||
return false
|
||||
}
|
||||
var d execReqErr
|
||||
return errors.As(e, &d)
|
||||
}
|
||||
45
beacon-chain/core/electra/error_test.go
Normal file
45
beacon-chain/core/electra/error_test.go
Normal file
@@ -0,0 +1,45 @@
|
||||
package electra
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
func TestIsExecutionRequestError(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "nil error",
|
||||
err: nil,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "random error",
|
||||
err: errors.New("some error"),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "execution request error",
|
||||
err: execReqErr{errors.New("execution request failed")},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "wrapped execution request error",
|
||||
err: errors.Wrap(execReqErr{errors.New("execution request failed")}, "wrapped"),
|
||||
want: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := IsExecutionRequestError(tt.err)
|
||||
if got != tt.want {
|
||||
t.Errorf("IsExecutionRequestError(%v) = %v, want %v", tt.err, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package electra
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/blocks"
|
||||
@@ -82,16 +81,31 @@ func ProcessOperations(
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not get execution requests")
|
||||
}
|
||||
for _, d := range requests.Deposits {
|
||||
if d == nil {
|
||||
return nil, errors.New("nil deposit request")
|
||||
}
|
||||
}
|
||||
st, err = ProcessDepositRequests(ctx, st, requests.Deposits)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not process deposit requests")
|
||||
return nil, execReqErr{errors.Wrap(err, "could not process deposit requests")}
|
||||
}
|
||||
for _, w := range requests.Withdrawals {
|
||||
if w == nil {
|
||||
return nil, errors.New("nil withdrawal request")
|
||||
}
|
||||
}
|
||||
st, err = ProcessWithdrawalRequests(ctx, st, requests.Withdrawals)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not process withdrawal requests")
|
||||
return nil, execReqErr{errors.Wrap(err, "could not process withdrawal requests")}
|
||||
}
|
||||
for _, c := range requests.Consolidations {
|
||||
if c == nil {
|
||||
return nil, errors.New("nil consolidation request")
|
||||
}
|
||||
}
|
||||
if err := ProcessConsolidationRequests(ctx, st, requests.Consolidations); err != nil {
|
||||
return nil, fmt.Errorf("could not process consolidation requests: %w", err)
|
||||
return nil, execReqErr{errors.Wrap(err, "could not process consolidation requests")}
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
|
||||
61
beacon-chain/core/electra/transition_no_verify_sig_test.go
Normal file
61
beacon-chain/core/electra/transition_no_verify_sig_test.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package electra_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/electra"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/blocks"
|
||||
enginev1 "github.com/prysmaticlabs/prysm/v5/proto/engine/v1"
|
||||
ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/require"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/util"
|
||||
)
|
||||
|
||||
func TestProcessOperationsWithNilRequests(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
modifyBlk func(blockElectra *ethpb.SignedBeaconBlockElectra)
|
||||
errMsg string
|
||||
}{
|
||||
{
|
||||
name: "Nil deposit request",
|
||||
modifyBlk: func(blk *ethpb.SignedBeaconBlockElectra) {
|
||||
blk.Block.Body.ExecutionRequests.Deposits = []*enginev1.DepositRequest{nil}
|
||||
},
|
||||
errMsg: "nil deposit request",
|
||||
},
|
||||
{
|
||||
name: "Nil withdrawal request",
|
||||
modifyBlk: func(blk *ethpb.SignedBeaconBlockElectra) {
|
||||
blk.Block.Body.ExecutionRequests.Withdrawals = []*enginev1.WithdrawalRequest{nil}
|
||||
},
|
||||
errMsg: "nil withdrawal request",
|
||||
},
|
||||
{
|
||||
name: "Nil consolidation request",
|
||||
modifyBlk: func(blk *ethpb.SignedBeaconBlockElectra) {
|
||||
blk.Block.Body.ExecutionRequests.Consolidations = []*enginev1.ConsolidationRequest{nil}
|
||||
},
|
||||
errMsg: "nil consolidation request",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
st, ks := util.DeterministicGenesisStateElectra(t, 128)
|
||||
blk, err := util.GenerateFullBlockElectra(st, ks, util.DefaultBlockGenConfig(), 1)
|
||||
require.NoError(t, err)
|
||||
|
||||
tc.modifyBlk(blk)
|
||||
|
||||
b, err := blocks.NewSignedBeaconBlock(blk)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, st.SetSlot(1))
|
||||
|
||||
_, err = electra.ProcessOperations(context.Background(), st, b.Block())
|
||||
require.ErrorContains(t, tc.errMsg, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -194,7 +194,7 @@ func UpgradeToElectra(beaconState state.BeaconState) (state.BeaconState, error)
|
||||
if val.ActivationEpoch() == params.BeaconConfig().FarFutureEpoch {
|
||||
preActivationIndices = append(preActivationIndices, primitives.ValidatorIndex(index))
|
||||
}
|
||||
if helpers.HasCompoundingWithdrawalCredential(val) {
|
||||
if val.HasCompoundingWithdrawalCredentials() {
|
||||
compoundWithdrawalIndices = append(compoundWithdrawalIndices, primitives.ValidatorIndex(index))
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -116,7 +116,7 @@ func ProcessWithdrawalRequests(ctx context.Context, st state.BeaconState, wrs []
|
||||
return nil, err
|
||||
}
|
||||
// Verify withdrawal credentials
|
||||
hasCorrectCredential := helpers.HasExecutionWithdrawalCredentials(validator)
|
||||
hasCorrectCredential := validator.HasExecutionWithdrawalCredentials()
|
||||
wc := validator.GetWithdrawalCredentials()
|
||||
isCorrectSourceAddress := bytes.Equal(wc[12:], wr.SourceAddress)
|
||||
if !hasCorrectCredential || !isCorrectSourceAddress {
|
||||
@@ -165,7 +165,7 @@ func ProcessWithdrawalRequests(ctx context.Context, st state.BeaconState, wrs []
|
||||
hasExcessBalance := vBal > params.BeaconConfig().MinActivationBalance+pendingBalanceToWithdraw
|
||||
|
||||
// Only allow partial withdrawals with compounding withdrawal credentials
|
||||
if helpers.HasCompoundingWithdrawalCredential(validator) && hasSufficientEffectiveBalance && hasExcessBalance {
|
||||
if validator.HasCompoundingWithdrawalCredentials() && hasSufficientEffectiveBalance && hasExcessBalance {
|
||||
// Spec definition:
|
||||
// to_withdraw = min(
|
||||
// state.balances[index] - MIN_ACTIVATION_BALANCE - pending_balance_to_withdraw,
|
||||
|
||||
@@ -38,6 +38,17 @@ func TestProcessWithdrawRequests(t *testing.T) {
|
||||
wantFn func(t *testing.T, got state.BeaconState)
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "nil request",
|
||||
args: args{
|
||||
st: func() state.BeaconState { return st }(),
|
||||
wrs: []*enginev1.WithdrawalRequest{nil},
|
||||
},
|
||||
wantErr: true,
|
||||
wantFn: func(t *testing.T, got state.BeaconState) {
|
||||
require.DeepEqual(t, got, nil)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "happy path exit and withdrawal only",
|
||||
args: args{
|
||||
|
||||
@@ -32,6 +32,9 @@ const (
|
||||
|
||||
// AttesterSlashingReceived is sent after an attester slashing is received from gossip or rpc
|
||||
AttesterSlashingReceived = 8
|
||||
|
||||
// SingleAttReceived is sent after a single attestation object is received from gossip or rpc
|
||||
SingleAttReceived = 9
|
||||
)
|
||||
|
||||
// UnAggregatedAttReceivedData is the data sent with UnaggregatedAttReceived events.
|
||||
@@ -43,7 +46,7 @@ type UnAggregatedAttReceivedData struct {
|
||||
// AggregatedAttReceivedData is the data sent with AggregatedAttReceived events.
|
||||
type AggregatedAttReceivedData struct {
|
||||
// Attestation is the aggregated attestation object.
|
||||
Attestation *ethpb.AggregateAttestationAndProof
|
||||
Attestation ethpb.AggregateAttAndProof
|
||||
}
|
||||
|
||||
// ExitReceivedData is the data sent with ExitReceived events.
|
||||
@@ -77,3 +80,8 @@ type ProposerSlashingReceivedData struct {
|
||||
type AttesterSlashingReceivedData struct {
|
||||
AttesterSlashing ethpb.AttSlashing
|
||||
}
|
||||
|
||||
// SingleAttReceivedData is the data sent with SingleAttReceived events.
|
||||
type SingleAttReceivedData struct {
|
||||
Attestation ethpb.Att
|
||||
}
|
||||
|
||||
@@ -6,15 +6,12 @@ go_library(
|
||||
importpath = "github.com/prysmaticlabs/prysm/v5/beacon-chain/core/fulu",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//beacon-chain/core/helpers:go_default_library",
|
||||
"//beacon-chain/core/time:go_default_library",
|
||||
"//beacon-chain/state:go_default_library",
|
||||
"//beacon-chain/state/state-native:go_default_library",
|
||||
"//config/params:go_default_library",
|
||||
"//consensus-types/primitives:go_default_library",
|
||||
"//proto/engine/v1:go_default_library",
|
||||
"//proto/prysm/v1alpha1:go_default_library",
|
||||
"//time/slots:go_default_library",
|
||||
"@com_github_pkg_errors//:go_default_library",
|
||||
],
|
||||
)
|
||||
@@ -24,7 +21,6 @@ go_test(
|
||||
srcs = ["upgrade_test.go"],
|
||||
deps = [
|
||||
":go_default_library",
|
||||
"//beacon-chain/core/helpers:go_default_library",
|
||||
"//beacon-chain/core/time:go_default_library",
|
||||
"//config/params:go_default_library",
|
||||
"//consensus-types/primitives:go_default_library",
|
||||
@@ -32,6 +28,5 @@ go_test(
|
||||
"//proto/prysm/v1alpha1:go_default_library",
|
||||
"//testing/require:go_default_library",
|
||||
"//testing/util:go_default_library",
|
||||
"//time/slots:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -1,18 +1,13 @@
|
||||
package fulu
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/helpers"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/time"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/state"
|
||||
state_native "github.com/prysmaticlabs/prysm/v5/beacon-chain/state/state-native"
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
enginev1 "github.com/prysmaticlabs/prysm/v5/proto/engine/v1"
|
||||
ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"
|
||||
"github.com/prysmaticlabs/prysm/v5/time/slots"
|
||||
)
|
||||
|
||||
// UpgradeToFulu updates inputs a generic state to return the version Fulu state.
|
||||
@@ -74,32 +69,37 @@ func UpgradeToFulu(beaconState state.BeaconState) (state.BeaconState, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
earliestExitEpoch := helpers.ActivationExitEpoch(time.CurrentEpoch(beaconState))
|
||||
preActivationIndices := make([]primitives.ValidatorIndex, 0)
|
||||
compoundWithdrawalIndices := make([]primitives.ValidatorIndex, 0)
|
||||
if err = beaconState.ReadFromEveryValidator(func(index int, val state.ReadOnlyValidator) error {
|
||||
if val.ExitEpoch() != params.BeaconConfig().FarFutureEpoch && val.ExitEpoch() > earliestExitEpoch {
|
||||
earliestExitEpoch = val.ExitEpoch()
|
||||
}
|
||||
if val.ActivationEpoch() == params.BeaconConfig().FarFutureEpoch {
|
||||
preActivationIndices = append(preActivationIndices, primitives.ValidatorIndex(index))
|
||||
}
|
||||
if helpers.HasCompoundingWithdrawalCredential(val) {
|
||||
compoundWithdrawalIndices = append(compoundWithdrawalIndices, primitives.ValidatorIndex(index))
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
depositBalanceToConsume, err := beaconState.DepositBalanceToConsume()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
earliestExitEpoch++ // Increment to find the earliest possible exit epoch
|
||||
|
||||
// note: should be the same in prestate and post beaconState.
|
||||
// we are deviating from the specs a bit as it calls for using the post beaconState
|
||||
tab, err := helpers.TotalActiveBalance(beaconState)
|
||||
exitBalanceToConsume, err := beaconState.ExitBalanceToConsume()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to get total active balance")
|
||||
return nil, err
|
||||
}
|
||||
earliestExitEpoch, err := beaconState.EarliestExitEpoch()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
consolidationBalanceToConsume, err := beaconState.ConsolidationBalanceToConsume()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
earliestConsolidationEpoch, err := beaconState.EarliestConsolidationEpoch()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pendingDeposits, err := beaconState.PendingDeposits()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pendingPartialWithdrawals, err := beaconState.PendingPartialWithdrawals()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pendingConsolidations, err := beaconState.PendingConsolidations()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s := ðpb.BeaconStateFulu{
|
||||
@@ -155,25 +155,16 @@ func UpgradeToFulu(beaconState state.BeaconState) (state.BeaconState, error) {
|
||||
HistoricalSummaries: summaries,
|
||||
|
||||
DepositRequestsStartIndex: params.BeaconConfig().UnsetDepositRequestsStartIndex,
|
||||
DepositBalanceToConsume: 0,
|
||||
ExitBalanceToConsume: helpers.ActivationExitChurnLimit(primitives.Gwei(tab)),
|
||||
DepositBalanceToConsume: depositBalanceToConsume,
|
||||
ExitBalanceToConsume: exitBalanceToConsume,
|
||||
EarliestExitEpoch: earliestExitEpoch,
|
||||
ConsolidationBalanceToConsume: helpers.ConsolidationChurnLimit(primitives.Gwei(tab)),
|
||||
EarliestConsolidationEpoch: helpers.ActivationExitEpoch(slots.ToEpoch(beaconState.Slot())),
|
||||
PendingDeposits: make([]*ethpb.PendingDeposit, 0),
|
||||
PendingPartialWithdrawals: make([]*ethpb.PendingPartialWithdrawal, 0),
|
||||
PendingConsolidations: make([]*ethpb.PendingConsolidation, 0),
|
||||
ConsolidationBalanceToConsume: consolidationBalanceToConsume,
|
||||
EarliestConsolidationEpoch: earliestConsolidationEpoch,
|
||||
PendingDeposits: pendingDeposits,
|
||||
PendingPartialWithdrawals: pendingPartialWithdrawals,
|
||||
PendingConsolidations: pendingConsolidations,
|
||||
}
|
||||
|
||||
// Sorting preActivationIndices based on a custom criteria
|
||||
sort.Slice(preActivationIndices, func(i, j int) bool {
|
||||
// Comparing based on ActivationEligibilityEpoch and then by index if the epochs are the same
|
||||
if s.Validators[preActivationIndices[i]].ActivationEligibilityEpoch == s.Validators[preActivationIndices[j]].ActivationEligibilityEpoch {
|
||||
return preActivationIndices[i] < preActivationIndices[j]
|
||||
}
|
||||
return s.Validators[preActivationIndices[i]].ActivationEligibilityEpoch < s.Validators[preActivationIndices[j]].ActivationEligibilityEpoch
|
||||
})
|
||||
|
||||
// Need to cast the beaconState to use in helper functions
|
||||
post, err := state_native.InitializeFromProtoUnsafeFulu(s)
|
||||
if err != nil {
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/fulu"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/helpers"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/time"
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
@@ -12,7 +11,6 @@ import (
|
||||
ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/require"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/util"
|
||||
"github.com/prysmaticlabs/prysm/v5/time/slots"
|
||||
)
|
||||
|
||||
func TestUpgradeToFulu(t *testing.T) {
|
||||
@@ -33,57 +31,6 @@ func TestUpgradeToFulu(t *testing.T) {
|
||||
require.Equal(t, preForkState.GenesisTime(), mSt.GenesisTime())
|
||||
require.DeepSSZEqual(t, preForkState.GenesisValidatorsRoot(), mSt.GenesisValidatorsRoot())
|
||||
require.Equal(t, preForkState.Slot(), mSt.Slot())
|
||||
require.DeepSSZEqual(t, preForkState.LatestBlockHeader(), mSt.LatestBlockHeader())
|
||||
require.DeepSSZEqual(t, preForkState.BlockRoots(), mSt.BlockRoots())
|
||||
require.DeepSSZEqual(t, preForkState.StateRoots(), mSt.StateRoots())
|
||||
require.DeepSSZEqual(t, preForkState.Validators()[2:], mSt.Validators()[2:])
|
||||
require.DeepSSZEqual(t, preForkState.Balances()[2:], mSt.Balances()[2:])
|
||||
require.DeepSSZEqual(t, preForkState.Eth1Data(), mSt.Eth1Data())
|
||||
require.DeepSSZEqual(t, preForkState.Eth1DataVotes(), mSt.Eth1DataVotes())
|
||||
require.DeepSSZEqual(t, preForkState.Eth1DepositIndex(), mSt.Eth1DepositIndex())
|
||||
require.DeepSSZEqual(t, preForkState.RandaoMixes(), mSt.RandaoMixes())
|
||||
require.DeepSSZEqual(t, preForkState.Slashings(), mSt.Slashings())
|
||||
require.DeepSSZEqual(t, preForkState.JustificationBits(), mSt.JustificationBits())
|
||||
require.DeepSSZEqual(t, preForkState.PreviousJustifiedCheckpoint(), mSt.PreviousJustifiedCheckpoint())
|
||||
require.DeepSSZEqual(t, preForkState.CurrentJustifiedCheckpoint(), mSt.CurrentJustifiedCheckpoint())
|
||||
require.DeepSSZEqual(t, preForkState.FinalizedCheckpoint(), mSt.FinalizedCheckpoint())
|
||||
|
||||
require.Equal(t, len(preForkState.Validators()), len(mSt.Validators()))
|
||||
|
||||
preVal, err := preForkState.ValidatorAtIndex(0)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, params.BeaconConfig().MaxEffectiveBalance, preVal.EffectiveBalance)
|
||||
|
||||
preVal2, err := preForkState.ValidatorAtIndex(1)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, params.BeaconConfig().MaxEffectiveBalance, preVal2.EffectiveBalance)
|
||||
|
||||
// TODO: Fix this test
|
||||
// mVal, err := mSt.ValidatorAtIndex(0)
|
||||
_, err = mSt.ValidatorAtIndex(0)
|
||||
require.NoError(t, err)
|
||||
// require.Equal(t, uint64(0), mVal.EffectiveBalance)
|
||||
|
||||
mVal2, err := mSt.ValidatorAtIndex(1)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, params.BeaconConfig().MinActivationBalance, mVal2.EffectiveBalance)
|
||||
|
||||
numValidators := mSt.NumValidators()
|
||||
p, err := mSt.PreviousEpochParticipation()
|
||||
require.NoError(t, err)
|
||||
require.DeepSSZEqual(t, make([]byte, numValidators), p)
|
||||
p, err = mSt.CurrentEpochParticipation()
|
||||
require.NoError(t, err)
|
||||
require.DeepSSZEqual(t, make([]byte, numValidators), p)
|
||||
s, err := mSt.InactivityScores()
|
||||
require.NoError(t, err)
|
||||
require.DeepSSZEqual(t, make([]uint64, numValidators), s)
|
||||
|
||||
hr1, err := preForkState.HistoricalRoots()
|
||||
require.NoError(t, err)
|
||||
hr2, err := mSt.HistoricalRoots()
|
||||
require.NoError(t, err)
|
||||
require.DeepEqual(t, hr1, hr2)
|
||||
|
||||
f := mSt.Fork()
|
||||
require.DeepSSZEqual(t, ðpb.Fork{
|
||||
@@ -91,11 +38,50 @@ func TestUpgradeToFulu(t *testing.T) {
|
||||
CurrentVersion: params.BeaconConfig().FuluForkVersion,
|
||||
Epoch: time.CurrentEpoch(st),
|
||||
}, f)
|
||||
|
||||
require.DeepSSZEqual(t, preForkState.LatestBlockHeader(), mSt.LatestBlockHeader())
|
||||
require.DeepSSZEqual(t, preForkState.BlockRoots(), mSt.BlockRoots())
|
||||
require.DeepSSZEqual(t, preForkState.StateRoots(), mSt.StateRoots())
|
||||
|
||||
hr1, err := preForkState.HistoricalRoots()
|
||||
require.NoError(t, err)
|
||||
hr2, err := mSt.HistoricalRoots()
|
||||
require.NoError(t, err)
|
||||
require.DeepEqual(t, hr1, hr2)
|
||||
|
||||
require.DeepSSZEqual(t, preForkState.Eth1Data(), mSt.Eth1Data())
|
||||
require.DeepSSZEqual(t, preForkState.Eth1DataVotes(), mSt.Eth1DataVotes())
|
||||
require.DeepSSZEqual(t, preForkState.Eth1DepositIndex(), mSt.Eth1DepositIndex())
|
||||
require.DeepSSZEqual(t, preForkState.Validators(), mSt.Validators())
|
||||
require.DeepSSZEqual(t, preForkState.Balances(), mSt.Balances())
|
||||
require.DeepSSZEqual(t, preForkState.RandaoMixes(), mSt.RandaoMixes())
|
||||
require.DeepSSZEqual(t, preForkState.Slashings(), mSt.Slashings())
|
||||
|
||||
numValidators := mSt.NumValidators()
|
||||
|
||||
p, err := mSt.PreviousEpochParticipation()
|
||||
require.NoError(t, err)
|
||||
require.DeepSSZEqual(t, make([]byte, numValidators), p)
|
||||
|
||||
p, err = mSt.CurrentEpochParticipation()
|
||||
require.NoError(t, err)
|
||||
require.DeepSSZEqual(t, make([]byte, numValidators), p)
|
||||
|
||||
require.DeepSSZEqual(t, preForkState.JustificationBits(), mSt.JustificationBits())
|
||||
require.DeepSSZEqual(t, preForkState.PreviousJustifiedCheckpoint(), mSt.PreviousJustifiedCheckpoint())
|
||||
require.DeepSSZEqual(t, preForkState.CurrentJustifiedCheckpoint(), mSt.CurrentJustifiedCheckpoint())
|
||||
require.DeepSSZEqual(t, preForkState.FinalizedCheckpoint(), mSt.FinalizedCheckpoint())
|
||||
|
||||
s, err := mSt.InactivityScores()
|
||||
require.NoError(t, err)
|
||||
require.DeepSSZEqual(t, make([]uint64, numValidators), s)
|
||||
|
||||
csc, err := mSt.CurrentSyncCommittee()
|
||||
require.NoError(t, err)
|
||||
psc, err := preForkState.CurrentSyncCommittee()
|
||||
require.NoError(t, err)
|
||||
require.DeepSSZEqual(t, psc, csc)
|
||||
|
||||
nsc, err := mSt.NextSyncCommittee()
|
||||
require.NoError(t, err)
|
||||
psc, err = preForkState.NextSyncCommittee()
|
||||
@@ -110,7 +96,6 @@ func TestUpgradeToFulu(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
txRoot, err := prevHeader.TransactionsRoot()
|
||||
require.NoError(t, err)
|
||||
|
||||
wdRoot, err := prevHeader.WithdrawalsRoot()
|
||||
require.NoError(t, err)
|
||||
wanted := &enginev1.ExecutionPayloadHeaderDeneb{
|
||||
@@ -144,45 +129,57 @@ func TestUpgradeToFulu(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, len(summaries))
|
||||
|
||||
startIndex, err := mSt.DepositRequestsStartIndex()
|
||||
preDepositRequestsStartIndex, err := preForkState.DepositRequestsStartIndex()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, params.BeaconConfig().UnsetDepositRequestsStartIndex, startIndex)
|
||||
postDepositRequestsStartIndex, err := mSt.DepositRequestsStartIndex()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, preDepositRequestsStartIndex, postDepositRequestsStartIndex)
|
||||
|
||||
balance, err := mSt.DepositBalanceToConsume()
|
||||
preDepositBalanceToConsume, err := preForkState.DepositBalanceToConsume()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, primitives.Gwei(0), balance)
|
||||
postDepositBalanceToConsume, err := mSt.DepositBalanceToConsume()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, preDepositBalanceToConsume, postDepositBalanceToConsume)
|
||||
|
||||
tab, err := helpers.TotalActiveBalance(mSt)
|
||||
preExitBalanceToConsume, err := preForkState.ExitBalanceToConsume()
|
||||
require.NoError(t, err)
|
||||
postExitBalanceToConsume, err := mSt.ExitBalanceToConsume()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, preExitBalanceToConsume, postExitBalanceToConsume)
|
||||
|
||||
ebtc, err := mSt.ExitBalanceToConsume()
|
||||
preEarliestExitEpoch, err := preForkState.EarliestExitEpoch()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, helpers.ActivationExitChurnLimit(primitives.Gwei(tab)), ebtc)
|
||||
postEarliestExitEpoch, err := mSt.EarliestExitEpoch()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, preEarliestExitEpoch, postEarliestExitEpoch)
|
||||
|
||||
eee, err := mSt.EarliestExitEpoch()
|
||||
preConsolidationBalanceToConsume, err := preForkState.ConsolidationBalanceToConsume()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, helpers.ActivationExitEpoch(primitives.Epoch(1)), eee)
|
||||
postConsolidationBalanceToConsume, err := mSt.ConsolidationBalanceToConsume()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, preConsolidationBalanceToConsume, postConsolidationBalanceToConsume)
|
||||
|
||||
cbtc, err := mSt.ConsolidationBalanceToConsume()
|
||||
preEarliesConsolidationEoch, err := preForkState.EarliestConsolidationEpoch()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, helpers.ConsolidationChurnLimit(primitives.Gwei(tab)), cbtc)
|
||||
postEarliestConsolidationEpoch, err := mSt.EarliestConsolidationEpoch()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, preEarliesConsolidationEoch, postEarliestConsolidationEpoch)
|
||||
|
||||
earliestConsolidationEpoch, err := mSt.EarliestConsolidationEpoch()
|
||||
prePendingDeposits, err := preForkState.PendingDeposits()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, helpers.ActivationExitEpoch(slots.ToEpoch(preForkState.Slot())), earliestConsolidationEpoch)
|
||||
postPendingDeposits, err := mSt.PendingDeposits()
|
||||
require.NoError(t, err)
|
||||
require.DeepSSZEqual(t, prePendingDeposits, postPendingDeposits)
|
||||
|
||||
// TODO: Fix this test
|
||||
// pendingDeposits, err := mSt.PendingDeposits()
|
||||
_, err = mSt.PendingDeposits()
|
||||
prePendingPartialWithdrawals, err := preForkState.PendingPartialWithdrawals()
|
||||
require.NoError(t, err)
|
||||
// require.Equal(t, 2, len(pendingDeposits))
|
||||
// require.Equal(t, uint64(1000), pendingDeposits[1].Amount)
|
||||
postPendingPartialWithdrawals, err := mSt.PendingPartialWithdrawals()
|
||||
require.NoError(t, err)
|
||||
require.DeepSSZEqual(t, prePendingPartialWithdrawals, postPendingPartialWithdrawals)
|
||||
|
||||
numPendingPartialWithdrawals, err := mSt.NumPendingPartialWithdrawals()
|
||||
prePendingConsolidations, err := preForkState.PendingConsolidations()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(0), numPendingPartialWithdrawals)
|
||||
|
||||
consolidations, err := mSt.PendingConsolidations()
|
||||
postPendingConsolidations, err := mSt.PendingConsolidations()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, len(consolidations))
|
||||
require.DeepSSZEqual(t, prePendingConsolidations, postPendingConsolidations)
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ go_library(
|
||||
"beacon_committee.go",
|
||||
"block.go",
|
||||
"genesis.go",
|
||||
"legacy.go",
|
||||
"metrics.go",
|
||||
"randao.go",
|
||||
"rewards_penalties.go",
|
||||
@@ -25,7 +26,6 @@ go_library(
|
||||
"//beacon-chain/state:go_default_library",
|
||||
"//config/fieldparams:go_default_library",
|
||||
"//config/params:go_default_library",
|
||||
"//consensus-types/interfaces:go_default_library",
|
||||
"//consensus-types/primitives:go_default_library",
|
||||
"//container/slice:go_default_library",
|
||||
"//container/trie:go_default_library",
|
||||
@@ -53,6 +53,7 @@ go_test(
|
||||
"attestation_test.go",
|
||||
"beacon_committee_test.go",
|
||||
"block_test.go",
|
||||
"legacy_test.go",
|
||||
"private_access_fuzz_noop_test.go", # keep
|
||||
"private_access_test.go",
|
||||
"randao_test.go",
|
||||
@@ -87,5 +88,6 @@ go_test(
|
||||
"//time:go_default_library",
|
||||
"//time/slots:go_default_library",
|
||||
"@com_github_prysmaticlabs_go_bitfield//:go_default_library",
|
||||
"@com_github_stretchr_testify//require:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
20
beacon-chain/core/helpers/legacy.go
Normal file
20
beacon-chain/core/helpers/legacy.go
Normal file
@@ -0,0 +1,20 @@
|
||||
package helpers
|
||||
|
||||
import (
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/state"
|
||||
"github.com/prysmaticlabs/prysm/v5/runtime/version"
|
||||
)
|
||||
|
||||
// DepositRequestsStarted determines if the deposit requests have started.
|
||||
func DepositRequestsStarted(beaconState state.BeaconState) bool {
|
||||
if beaconState.Version() < version.Electra {
|
||||
return false
|
||||
}
|
||||
|
||||
requestsStartIndex, err := beaconState.DepositRequestsStartIndex()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return beaconState.Eth1DepositIndex() == requestsStartIndex
|
||||
}
|
||||
33
beacon-chain/core/helpers/legacy_test.go
Normal file
33
beacon-chain/core/helpers/legacy_test.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package helpers_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/helpers"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/util"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestDepositRequestHaveStarted contains several test cases for depositRequestHaveStarted.
|
||||
func TestDepositRequestHaveStarted(t *testing.T) {
|
||||
t.Run("Version below Electra returns false", func(t *testing.T) {
|
||||
st, _ := util.DeterministicGenesisStateBellatrix(t, 1)
|
||||
result := helpers.DepositRequestsStarted(st)
|
||||
require.False(t, result)
|
||||
})
|
||||
|
||||
t.Run("Version is Electra or higher, no error, but Eth1DepositIndex != requestsStartIndex returns false", func(t *testing.T) {
|
||||
st, _ := util.DeterministicGenesisStateElectra(t, 1)
|
||||
require.NoError(t, st.SetEth1DepositIndex(1))
|
||||
result := helpers.DepositRequestsStarted(st)
|
||||
require.False(t, result)
|
||||
})
|
||||
|
||||
t.Run("Version is Electra or higher, no error, and Eth1DepositIndex == requestsStartIndex returns true", func(t *testing.T) {
|
||||
st, _ := util.DeterministicGenesisStateElectra(t, 1)
|
||||
require.NoError(t, st.SetEth1DepositIndex(33))
|
||||
require.NoError(t, st.SetDepositRequestsStartIndex(33))
|
||||
result := helpers.DepositRequestsStarted(st)
|
||||
require.True(t, result)
|
||||
})
|
||||
}
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
func BalanceChurnLimit(activeBalance primitives.Gwei) primitives.Gwei {
|
||||
churn := max(
|
||||
params.BeaconConfig().MinPerEpochChurnLimitElectra,
|
||||
(uint64(activeBalance) / params.BeaconConfig().ChurnLimitQuotient),
|
||||
uint64(activeBalance)/params.BeaconConfig().ChurnLimitQuotient,
|
||||
)
|
||||
return primitives.Gwei(churn - churn%params.BeaconConfig().EffectiveBalanceIncrement)
|
||||
}
|
||||
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/state"
|
||||
fieldparams "github.com/prysmaticlabs/prysm/v5/config/fieldparams"
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/interfaces"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
"github.com/prysmaticlabs/prysm/v5/crypto/hash"
|
||||
"github.com/prysmaticlabs/prysm/v5/encoding/bytesutil"
|
||||
@@ -394,9 +393,9 @@ func ComputeProposerIndex(bState state.ReadOnlyBeaconState, activeIndices []prim
|
||||
effectiveBal := v.EffectiveBalance()
|
||||
if bState.Version() >= version.Electra {
|
||||
binary.LittleEndian.PutUint64(seedBuffer[len(seed):], i/16)
|
||||
randomByte := hashFunc(seedBuffer)
|
||||
randomBytes := hashFunc(seedBuffer)
|
||||
offset := (i % 16) * 2
|
||||
randomValue := uint64(randomByte[offset]) | uint64(randomByte[offset+1])<<8
|
||||
randomValue := uint64(randomBytes[offset]) | uint64(randomBytes[offset+1])<<8
|
||||
|
||||
if effectiveBal*fieldparams.MaxRandomValueElectra >= beaconConfig.MaxEffectiveBalanceElectra*randomValue {
|
||||
return candidateIndex, nil
|
||||
@@ -515,63 +514,6 @@ func LastActivatedValidatorIndex(ctx context.Context, st state.ReadOnlyBeaconSta
|
||||
return lastActivatedvalidatorIndex, nil
|
||||
}
|
||||
|
||||
// hasETH1WithdrawalCredential returns whether the validator has an ETH1
|
||||
// Withdrawal prefix. It assumes that the caller has a lock on the state
|
||||
func HasETH1WithdrawalCredential(val interfaces.WithWithdrawalCredentials) bool {
|
||||
if val == nil {
|
||||
return false
|
||||
}
|
||||
return isETH1WithdrawalCredential(val.GetWithdrawalCredentials())
|
||||
}
|
||||
|
||||
func isETH1WithdrawalCredential(creds []byte) bool {
|
||||
return bytes.HasPrefix(creds, []byte{params.BeaconConfig().ETH1AddressWithdrawalPrefixByte})
|
||||
}
|
||||
|
||||
// HasCompoundingWithdrawalCredential checks if the validator has a compounding withdrawal credential.
|
||||
// New in Electra EIP-7251: https://eips.ethereum.org/EIPS/eip-7251
|
||||
//
|
||||
// Spec definition:
|
||||
//
|
||||
// def has_compounding_withdrawal_credential(validator: Validator) -> bool:
|
||||
// """
|
||||
// Check if ``validator`` has an 0x02 prefixed "compounding" withdrawal credential.
|
||||
// """
|
||||
// return is_compounding_withdrawal_credential(validator.withdrawal_credentials)
|
||||
func HasCompoundingWithdrawalCredential(v interfaces.WithWithdrawalCredentials) bool {
|
||||
if v == nil {
|
||||
return false
|
||||
}
|
||||
return IsCompoundingWithdrawalCredential(v.GetWithdrawalCredentials())
|
||||
}
|
||||
|
||||
// IsCompoundingWithdrawalCredential checks if the credentials are a compounding withdrawal credential.
|
||||
//
|
||||
// Spec definition:
|
||||
//
|
||||
// def is_compounding_withdrawal_credential(withdrawal_credentials: Bytes32) -> bool:
|
||||
// return withdrawal_credentials[:1] == COMPOUNDING_WITHDRAWAL_PREFIX
|
||||
func IsCompoundingWithdrawalCredential(creds []byte) bool {
|
||||
return bytes.HasPrefix(creds, []byte{params.BeaconConfig().CompoundingWithdrawalPrefixByte})
|
||||
}
|
||||
|
||||
// HasExecutionWithdrawalCredentials checks if the validator has an execution withdrawal credential or compounding credential.
|
||||
// New in Electra EIP-7251: https://eips.ethereum.org/EIPS/eip-7251
|
||||
//
|
||||
// Spec definition:
|
||||
//
|
||||
// def has_execution_withdrawal_credential(validator: Validator) -> bool:
|
||||
// """
|
||||
// Check if ``validator`` has a 0x01 or 0x02 prefixed withdrawal credential.
|
||||
// """
|
||||
// return has_compounding_withdrawal_credential(validator) or has_eth1_withdrawal_credential(validator)
|
||||
func HasExecutionWithdrawalCredentials(v interfaces.WithWithdrawalCredentials) bool {
|
||||
if v == nil {
|
||||
return false
|
||||
}
|
||||
return HasCompoundingWithdrawalCredential(v) || HasETH1WithdrawalCredential(v)
|
||||
}
|
||||
|
||||
// IsSameWithdrawalCredentials returns true if both validators have the same withdrawal credentials.
|
||||
//
|
||||
// return a.withdrawal_credentials[12:] == b.withdrawal_credentials[12:]
|
||||
@@ -606,10 +548,10 @@ func IsFullyWithdrawableValidator(val state.ReadOnlyValidator, balance uint64, e
|
||||
|
||||
// Electra / EIP-7251 logic
|
||||
if fork >= version.Electra {
|
||||
return HasExecutionWithdrawalCredentials(val) && val.WithdrawableEpoch() <= epoch
|
||||
return val.HasExecutionWithdrawalCredentials() && val.WithdrawableEpoch() <= epoch
|
||||
}
|
||||
|
||||
return HasETH1WithdrawalCredential(val) && val.WithdrawableEpoch() <= epoch
|
||||
return val.HasETH1WithdrawalCredentials() && val.WithdrawableEpoch() <= epoch
|
||||
}
|
||||
|
||||
// IsPartiallyWithdrawableValidator returns whether the validator is able to perform a
|
||||
@@ -637,7 +579,7 @@ func IsPartiallyWithdrawableValidator(val state.ReadOnlyValidator, balance uint6
|
||||
// """
|
||||
// Check if ``validator`` is partially withdrawable.
|
||||
// """
|
||||
// max_effective_balance = get_validator_max_effective_balance(validator)
|
||||
// max_effective_balance = get_max_effective_balance(validator)
|
||||
// has_max_effective_balance = validator.effective_balance == max_effective_balance # [Modified in Electra:EIP7251]
|
||||
// has_excess_balance = balance > max_effective_balance # [Modified in Electra:EIP7251]
|
||||
// return (
|
||||
@@ -650,7 +592,7 @@ func isPartiallyWithdrawableValidatorElectra(val state.ReadOnlyValidator, balanc
|
||||
hasMaxBalance := val.EffectiveBalance() == maxEB
|
||||
hasExcessBalance := balance > maxEB
|
||||
|
||||
return HasExecutionWithdrawalCredentials(val) &&
|
||||
return val.HasExecutionWithdrawalCredentials() &&
|
||||
hasMaxBalance &&
|
||||
hasExcessBalance
|
||||
}
|
||||
@@ -670,14 +612,14 @@ func isPartiallyWithdrawableValidatorElectra(val state.ReadOnlyValidator, balanc
|
||||
func isPartiallyWithdrawableValidatorCapella(val state.ReadOnlyValidator, balance uint64, epoch primitives.Epoch) bool {
|
||||
hasMaxBalance := val.EffectiveBalance() == params.BeaconConfig().MaxEffectiveBalance
|
||||
hasExcessBalance := balance > params.BeaconConfig().MaxEffectiveBalance
|
||||
return HasETH1WithdrawalCredential(val) && hasExcessBalance && hasMaxBalance
|
||||
return val.HasETH1WithdrawalCredentials() && hasExcessBalance && hasMaxBalance
|
||||
}
|
||||
|
||||
// ValidatorMaxEffectiveBalance returns the maximum effective balance for a validator.
|
||||
//
|
||||
// Spec definition:
|
||||
//
|
||||
// def get_validator_max_effective_balance(validator: Validator) -> Gwei:
|
||||
// def get_max_effective_balance(validator: Validator) -> Gwei:
|
||||
// """
|
||||
// Get max effective balance for ``validator``.
|
||||
// """
|
||||
@@ -686,7 +628,7 @@ func isPartiallyWithdrawableValidatorCapella(val state.ReadOnlyValidator, balanc
|
||||
// else:
|
||||
// return MIN_ACTIVATION_BALANCE
|
||||
func ValidatorMaxEffectiveBalance(val state.ReadOnlyValidator) uint64 {
|
||||
if HasCompoundingWithdrawalCredential(val) {
|
||||
if val.HasCompoundingWithdrawalCredentials() {
|
||||
return params.BeaconConfig().MaxEffectiveBalanceElectra
|
||||
}
|
||||
return params.BeaconConfig().MinActivationBalance
|
||||
|
||||
@@ -910,13 +910,15 @@ func TestProposerIndexFromCheckpoint(t *testing.T) {
|
||||
func TestHasETH1WithdrawalCredentials(t *testing.T) {
|
||||
creds := []byte{0xFA, 0xCC}
|
||||
v := ðpb.Validator{WithdrawalCredentials: creds}
|
||||
require.Equal(t, false, helpers.HasETH1WithdrawalCredential(v))
|
||||
roV, err := state_native.NewValidator(v)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, false, roV.HasETH1WithdrawalCredentials())
|
||||
creds = []byte{params.BeaconConfig().ETH1AddressWithdrawalPrefixByte, 0xCC}
|
||||
v = ðpb.Validator{WithdrawalCredentials: creds}
|
||||
require.Equal(t, true, helpers.HasETH1WithdrawalCredential(v))
|
||||
roV, err = state_native.NewValidator(v)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, true, roV.HasETH1WithdrawalCredentials())
|
||||
// No Withdrawal cred
|
||||
v = ðpb.Validator{}
|
||||
require.Equal(t, false, helpers.HasETH1WithdrawalCredential(v))
|
||||
}
|
||||
|
||||
func TestHasCompoundingWithdrawalCredential(t *testing.T) {
|
||||
@@ -931,11 +933,12 @@ func TestHasCompoundingWithdrawalCredential(t *testing.T) {
|
||||
{"Does not have compounding withdrawal credential",
|
||||
ðpb.Validator{WithdrawalCredentials: bytesutil.PadTo([]byte{0x00}, 32)},
|
||||
false},
|
||||
{"Handles nil case", nil, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, helpers.HasCompoundingWithdrawalCredential(tt.validator))
|
||||
roV, err := state_native.NewValidator(tt.validator)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.want, roV.HasCompoundingWithdrawalCredentials())
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -955,11 +958,12 @@ func TestHasExecutionWithdrawalCredentials(t *testing.T) {
|
||||
{"Does not have compounding withdrawal credential or eth1 withdrawal credential",
|
||||
ðpb.Validator{WithdrawalCredentials: bytesutil.PadTo([]byte{0x00}, 32)},
|
||||
false},
|
||||
{"Handles nil case", nil, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, helpers.HasExecutionWithdrawalCredentials(tt.validator))
|
||||
roV, err := state_native.NewValidator(tt.validator)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.want, roV.HasExecutionWithdrawalCredentials())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,15 @@ import (
|
||||
)
|
||||
|
||||
func TestLightClient_NewLightClientOptimisticUpdateFromBeaconState(t *testing.T) {
|
||||
params.SetupTestConfigCleanup(t)
|
||||
cfg := params.BeaconConfig()
|
||||
cfg.AltairForkEpoch = 1
|
||||
cfg.BellatrixForkEpoch = 2
|
||||
cfg.CapellaForkEpoch = 3
|
||||
cfg.DenebForkEpoch = 4
|
||||
cfg.ElectraForkEpoch = 5
|
||||
params.OverrideBeaconConfig(cfg)
|
||||
|
||||
t.Run("Altair", func(t *testing.T) {
|
||||
l := util.NewTestLightClient(t).SetupTestAltair()
|
||||
|
||||
@@ -59,9 +68,31 @@ func TestLightClient_NewLightClientOptimisticUpdateFromBeaconState(t *testing.T)
|
||||
l.CheckSyncAggregate(update.SyncAggregate())
|
||||
l.CheckAttestedHeader(update.AttestedHeader())
|
||||
})
|
||||
|
||||
t.Run("Electra", func(t *testing.T) {
|
||||
l := util.NewTestLightClient(t).SetupTestElectra(false)
|
||||
|
||||
update, err := lightClient.NewLightClientOptimisticUpdateFromBeaconState(l.Ctx, l.State.Slot(), l.State, l.Block, l.AttestedState, l.AttestedBlock)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, update, "update is nil")
|
||||
|
||||
require.Equal(t, l.Block.Block().Slot(), update.SignatureSlot(), "Signature slot is not equal")
|
||||
|
||||
l.CheckSyncAggregate(update.SyncAggregate())
|
||||
l.CheckAttestedHeader(update.AttestedHeader())
|
||||
})
|
||||
}
|
||||
|
||||
func TestLightClient_NewLightClientFinalityUpdateFromBeaconState(t *testing.T) {
|
||||
params.SetupTestConfigCleanup(t)
|
||||
cfg := params.BeaconConfig()
|
||||
cfg.AltairForkEpoch = 1
|
||||
cfg.BellatrixForkEpoch = 2
|
||||
cfg.CapellaForkEpoch = 3
|
||||
cfg.DenebForkEpoch = 4
|
||||
cfg.ElectraForkEpoch = 5
|
||||
params.OverrideBeaconConfig(cfg)
|
||||
|
||||
t.Run("Altair", func(t *testing.T) {
|
||||
l := util.NewTestLightClient(t).SetupTestAltair()
|
||||
|
||||
@@ -356,6 +387,157 @@ func TestLightClient_NewLightClientFinalityUpdateFromBeaconState(t *testing.T) {
|
||||
require.DeepSSZEqual(t, execution, updateExecution.Proto(), "Finalized Block Execution is not equal")
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("Electra", func(t *testing.T) {
|
||||
t.Run("FinalizedBlock Not Nil", func(t *testing.T) {
|
||||
l := util.NewTestLightClient(t).SetupTestElectra(false)
|
||||
|
||||
update, err := lightClient.NewLightClientFinalityUpdateFromBeaconState(l.Ctx, l.State.Slot(), l.State, l.Block, l.AttestedState, l.AttestedBlock, l.FinalizedBlock)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, update, "update is nil")
|
||||
|
||||
require.Equal(t, l.Block.Block().Slot(), update.SignatureSlot(), "Signature slot is not equal")
|
||||
|
||||
l.CheckSyncAggregate(update.SyncAggregate())
|
||||
l.CheckAttestedHeader(update.AttestedHeader())
|
||||
|
||||
//zeroHash := params.BeaconConfig().ZeroHash[:]
|
||||
finalizedBlockHeader, err := l.FinalizedBlock.Header()
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, update.FinalizedHeader(), "Finalized header is nil")
|
||||
updateFinalizedHeaderBeacon := update.FinalizedHeader().Beacon()
|
||||
require.Equal(t, finalizedBlockHeader.Header.Slot, updateFinalizedHeaderBeacon.Slot, "Finalized header slot is not equal")
|
||||
require.Equal(t, finalizedBlockHeader.Header.ProposerIndex, updateFinalizedHeaderBeacon.ProposerIndex, "Finalized header proposer index is not equal")
|
||||
require.DeepSSZEqual(t, finalizedBlockHeader.Header.ParentRoot, updateFinalizedHeaderBeacon.ParentRoot, "Finalized header parent root is not equal")
|
||||
require.DeepSSZEqual(t, finalizedBlockHeader.Header.StateRoot, updateFinalizedHeaderBeacon.StateRoot, "Finalized header state root is not equal")
|
||||
require.DeepSSZEqual(t, finalizedBlockHeader.Header.BodyRoot, updateFinalizedHeaderBeacon.BodyRoot, "Finalized header body root is not equal")
|
||||
fb, err := update.FinalityBranchElectra()
|
||||
require.NoError(t, err)
|
||||
proof, err := l.AttestedState.FinalizedRootProof(l.Ctx)
|
||||
require.NoError(t, err)
|
||||
for i, leaf := range fb {
|
||||
require.DeepSSZEqual(t, proof[i], leaf[:], "Leaf is not equal")
|
||||
}
|
||||
|
||||
// Check Execution BlockHash
|
||||
payloadInterface, err := l.FinalizedBlock.Block().Body().Execution()
|
||||
require.NoError(t, err)
|
||||
transactionsRoot, err := payloadInterface.TransactionsRoot()
|
||||
if errors.Is(err, consensustypes.ErrUnsupportedField) {
|
||||
transactions, err := payloadInterface.Transactions()
|
||||
require.NoError(t, err)
|
||||
transactionsRootArray, err := ssz.TransactionsRoot(transactions)
|
||||
require.NoError(t, err)
|
||||
transactionsRoot = transactionsRootArray[:]
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
withdrawalsRoot, err := payloadInterface.WithdrawalsRoot()
|
||||
if errors.Is(err, consensustypes.ErrUnsupportedField) {
|
||||
withdrawals, err := payloadInterface.Withdrawals()
|
||||
require.NoError(t, err)
|
||||
withdrawalsRootArray, err := ssz.WithdrawalSliceRoot(withdrawals, fieldparams.MaxWithdrawalsPerPayload)
|
||||
require.NoError(t, err)
|
||||
withdrawalsRoot = withdrawalsRootArray[:]
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
execution := &v11.ExecutionPayloadHeaderDeneb{
|
||||
ParentHash: payloadInterface.ParentHash(),
|
||||
FeeRecipient: payloadInterface.FeeRecipient(),
|
||||
StateRoot: payloadInterface.StateRoot(),
|
||||
ReceiptsRoot: payloadInterface.ReceiptsRoot(),
|
||||
LogsBloom: payloadInterface.LogsBloom(),
|
||||
PrevRandao: payloadInterface.PrevRandao(),
|
||||
BlockNumber: payloadInterface.BlockNumber(),
|
||||
GasLimit: payloadInterface.GasLimit(),
|
||||
GasUsed: payloadInterface.GasUsed(),
|
||||
Timestamp: payloadInterface.Timestamp(),
|
||||
ExtraData: payloadInterface.ExtraData(),
|
||||
BaseFeePerGas: payloadInterface.BaseFeePerGas(),
|
||||
BlockHash: payloadInterface.BlockHash(),
|
||||
TransactionsRoot: transactionsRoot,
|
||||
WithdrawalsRoot: withdrawalsRoot,
|
||||
}
|
||||
updateExecution, err := update.FinalizedHeader().Execution()
|
||||
require.NoError(t, err)
|
||||
require.DeepSSZEqual(t, execution, updateExecution.Proto(), "Finalized Block Execution is not equal")
|
||||
})
|
||||
|
||||
t.Run("FinalizedBlock In Previous Fork", func(t *testing.T) {
|
||||
l := util.NewTestLightClient(t).SetupTestElectraFinalizedBlockDeneb(false)
|
||||
|
||||
update, err := lightClient.NewLightClientFinalityUpdateFromBeaconState(l.Ctx, l.State.Slot(), l.State, l.Block, l.AttestedState, l.AttestedBlock, l.FinalizedBlock)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, update, "update is nil")
|
||||
|
||||
require.Equal(t, l.Block.Block().Slot(), update.SignatureSlot(), "Signature slot is not equal")
|
||||
|
||||
l.CheckSyncAggregate(update.SyncAggregate())
|
||||
l.CheckAttestedHeader(update.AttestedHeader())
|
||||
|
||||
finalizedBlockHeader, err := l.FinalizedBlock.Header()
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, update.FinalizedHeader(), "Finalized header is nil")
|
||||
updateFinalizedHeaderBeacon := update.FinalizedHeader().Beacon()
|
||||
require.Equal(t, finalizedBlockHeader.Header.Slot, updateFinalizedHeaderBeacon.Slot, "Finalized header slot is not equal")
|
||||
require.Equal(t, finalizedBlockHeader.Header.ProposerIndex, updateFinalizedHeaderBeacon.ProposerIndex, "Finalized header proposer index is not equal")
|
||||
require.DeepSSZEqual(t, finalizedBlockHeader.Header.ParentRoot, updateFinalizedHeaderBeacon.ParentRoot, "Finalized header parent root is not equal")
|
||||
require.DeepSSZEqual(t, finalizedBlockHeader.Header.StateRoot, updateFinalizedHeaderBeacon.StateRoot, "Finalized header state root is not equal")
|
||||
require.DeepSSZEqual(t, finalizedBlockHeader.Header.BodyRoot, updateFinalizedHeaderBeacon.BodyRoot, "Finalized header body root is not equal")
|
||||
fb, err := update.FinalityBranchElectra()
|
||||
require.NoError(t, err)
|
||||
proof, err := l.AttestedState.FinalizedRootProof(l.Ctx)
|
||||
require.NoError(t, err)
|
||||
for i, leaf := range fb {
|
||||
require.DeepSSZEqual(t, proof[i], leaf[:], "Leaf is not equal")
|
||||
}
|
||||
|
||||
// Check Execution BlockHash
|
||||
payloadInterface, err := l.FinalizedBlock.Block().Body().Execution()
|
||||
require.NoError(t, err)
|
||||
transactionsRoot, err := payloadInterface.TransactionsRoot()
|
||||
if errors.Is(err, consensustypes.ErrUnsupportedField) {
|
||||
transactions, err := payloadInterface.Transactions()
|
||||
require.NoError(t, err)
|
||||
transactionsRootArray, err := ssz.TransactionsRoot(transactions)
|
||||
require.NoError(t, err)
|
||||
transactionsRoot = transactionsRootArray[:]
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
withdrawalsRoot, err := payloadInterface.WithdrawalsRoot()
|
||||
if errors.Is(err, consensustypes.ErrUnsupportedField) {
|
||||
withdrawals, err := payloadInterface.Withdrawals()
|
||||
require.NoError(t, err)
|
||||
withdrawalsRootArray, err := ssz.WithdrawalSliceRoot(withdrawals, fieldparams.MaxWithdrawalsPerPayload)
|
||||
require.NoError(t, err)
|
||||
withdrawalsRoot = withdrawalsRootArray[:]
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
execution := &v11.ExecutionPayloadHeaderDeneb{
|
||||
ParentHash: payloadInterface.ParentHash(),
|
||||
FeeRecipient: payloadInterface.FeeRecipient(),
|
||||
StateRoot: payloadInterface.StateRoot(),
|
||||
ReceiptsRoot: payloadInterface.ReceiptsRoot(),
|
||||
LogsBloom: payloadInterface.LogsBloom(),
|
||||
PrevRandao: payloadInterface.PrevRandao(),
|
||||
BlockNumber: payloadInterface.BlockNumber(),
|
||||
GasLimit: payloadInterface.GasLimit(),
|
||||
GasUsed: payloadInterface.GasUsed(),
|
||||
Timestamp: payloadInterface.Timestamp(),
|
||||
ExtraData: payloadInterface.ExtraData(),
|
||||
BaseFeePerGas: payloadInterface.BaseFeePerGas(),
|
||||
BlockHash: payloadInterface.BlockHash(),
|
||||
TransactionsRoot: transactionsRoot,
|
||||
WithdrawalsRoot: withdrawalsRoot,
|
||||
}
|
||||
updateExecution, err := update.FinalizedHeader().Execution()
|
||||
require.NoError(t, err)
|
||||
require.DeepSSZEqual(t, execution, updateExecution.Proto(), "Finalized Block Execution is not equal")
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestLightClient_BlockToLightClientHeader(t *testing.T) {
|
||||
|
||||
@@ -403,11 +403,15 @@ func VerifyOperationLengths(_ context.Context, state state.BeaconState, b interf
|
||||
)
|
||||
}
|
||||
|
||||
if uint64(len(body.AttesterSlashings())) > params.BeaconConfig().MaxAttesterSlashings {
|
||||
maxSlashings := params.BeaconConfig().MaxAttesterSlashings
|
||||
if body.Version() >= version.Electra {
|
||||
maxSlashings = params.BeaconConfig().MaxAttesterSlashingsElectra
|
||||
}
|
||||
if uint64(len(body.AttesterSlashings())) > maxSlashings {
|
||||
return nil, fmt.Errorf(
|
||||
"number of attester slashings (%d) in block body exceeds allowed threshold of %d",
|
||||
len(body.AttesterSlashings()),
|
||||
params.BeaconConfig().MaxAttesterSlashings,
|
||||
maxSlashings,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -437,6 +437,25 @@ func TestProcessBlock_OverMaxAttesterSlashings(t *testing.T) {
|
||||
assert.ErrorContains(t, want, err)
|
||||
}
|
||||
|
||||
func TestProcessBlock_OverMaxAttesterSlashingsElectra(t *testing.T) {
|
||||
maxSlashings := params.BeaconConfig().MaxAttesterSlashingsElectra
|
||||
b := ðpb.SignedBeaconBlockElectra{
|
||||
Block: ðpb.BeaconBlockElectra{
|
||||
Body: ðpb.BeaconBlockBodyElectra{
|
||||
AttesterSlashings: make([]*ethpb.AttesterSlashingElectra, maxSlashings+1),
|
||||
},
|
||||
},
|
||||
}
|
||||
want := fmt.Sprintf("number of attester slashings (%d) in block body exceeds allowed threshold of %d",
|
||||
len(b.Block.Body.AttesterSlashings), params.BeaconConfig().MaxAttesterSlashingsElectra)
|
||||
s, err := state_native.InitializeFromProtoUnsafeElectra(ðpb.BeaconStateElectra{})
|
||||
require.NoError(t, err)
|
||||
wsb, err := consensusblocks.NewSignedBeaconBlock(b)
|
||||
require.NoError(t, err)
|
||||
_, err = transition.VerifyOperationLengths(context.Background(), s, wsb.Block())
|
||||
assert.ErrorContains(t, want, err)
|
||||
}
|
||||
|
||||
func TestProcessBlock_OverMaxAttestations(t *testing.T) {
|
||||
b := ðpb.SignedBeaconBlock{
|
||||
Block: ðpb.BeaconBlock{
|
||||
|
||||
@@ -75,7 +75,7 @@ func InitiateValidatorExit(ctx context.Context, s state.BeaconState, idx primiti
|
||||
|
||||
// Compute exit queue epoch.
|
||||
if s.Version() < version.Electra {
|
||||
// Relevant spec code from deneb:
|
||||
// Relevant spec code from phase0:
|
||||
//
|
||||
// exit_epochs = [v.exit_epoch for v in state.validators if v.exit_epoch != FAR_FUTURE_EPOCH]
|
||||
// exit_queue_epoch = max(exit_epochs + [compute_activation_exit_epoch(get_current_epoch(state))])
|
||||
|
||||
@@ -94,14 +94,7 @@ func (s *LazilyPersistentStore) IsDataAvailable(ctx context.Context, current pri
|
||||
entry := s.cache.ensure(key)
|
||||
defer s.cache.delete(key)
|
||||
root := b.Root()
|
||||
sumz, err := s.store.WaitForSummarizer(ctx)
|
||||
if err != nil {
|
||||
log.WithField("root", fmt.Sprintf("%#x", b.Root())).
|
||||
WithError(err).
|
||||
Debug("Failed to receive BlobStorageSummarizer within IsDataAvailable")
|
||||
} else {
|
||||
entry.setDiskSummary(sumz.Summary(root))
|
||||
}
|
||||
entry.setDiskSummary(s.store.Summary(root))
|
||||
|
||||
// Verify we have all the expected sidecars, and fail fast if any are missing or inconsistent.
|
||||
// We don't try to salvage problematic batches because this indicates a misbehaving peer and we'd rather
|
||||
|
||||
@@ -5,6 +5,10 @@ go_library(
|
||||
srcs = [
|
||||
"blob.go",
|
||||
"cache.go",
|
||||
"iteration.go",
|
||||
"layout.go",
|
||||
"layout_by_epoch.go",
|
||||
"layout_flat.go",
|
||||
"log.go",
|
||||
"metrics.go",
|
||||
"mock.go",
|
||||
@@ -13,6 +17,7 @@ go_library(
|
||||
importpath = "github.com/prysmaticlabs/prysm/v5/beacon-chain/db/filesystem",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//beacon-chain/db:go_default_library",
|
||||
"//beacon-chain/verification:go_default_library",
|
||||
"//config/fieldparams:go_default_library",
|
||||
"//config/params:go_default_library",
|
||||
@@ -20,7 +25,6 @@ go_library(
|
||||
"//consensus-types/primitives:go_default_library",
|
||||
"//encoding/bytesutil:go_default_library",
|
||||
"//io/file:go_default_library",
|
||||
"//proto/prysm/v1alpha1:go_default_library",
|
||||
"//runtime/logging:go_default_library",
|
||||
"//time/slots:go_default_library",
|
||||
"@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
|
||||
@@ -37,10 +41,14 @@ go_test(
|
||||
srcs = [
|
||||
"blob_test.go",
|
||||
"cache_test.go",
|
||||
"iteration_test.go",
|
||||
"layout_test.go",
|
||||
"migration_test.go",
|
||||
"pruner_test.go",
|
||||
],
|
||||
embed = [":go_default_library"],
|
||||
deps = [
|
||||
"//beacon-chain/db:go_default_library",
|
||||
"//beacon-chain/verification:go_default_library",
|
||||
"//config/params:go_default_library",
|
||||
"//consensus-types/primitives:go_default_library",
|
||||
@@ -48,6 +56,7 @@ go_test(
|
||||
"//proto/prysm/v1alpha1:go_default_library",
|
||||
"//testing/require:go_default_library",
|
||||
"//testing/util:go_default_library",
|
||||
"//time/slots:go_default_library",
|
||||
"@com_github_prysmaticlabs_fastssz//:go_default_library",
|
||||
"@com_github_spf13_afero//:go_default_library",
|
||||
],
|
||||
|
||||
@@ -1,42 +1,31 @@
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common/hexutil"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/verification"
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/blocks"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
"github.com/prysmaticlabs/prysm/v5/encoding/bytesutil"
|
||||
"github.com/prysmaticlabs/prysm/v5/io/file"
|
||||
ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"
|
||||
"github.com/prysmaticlabs/prysm/v5/runtime/logging"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
func directoryPermissions() os.FileMode {
|
||||
return params.BeaconIoConfig().ReadWriteExecutePermissions
|
||||
}
|
||||
|
||||
var (
|
||||
errIndexOutOfBounds = errors.New("blob index in file name >= DeprecatedMaxBlobsPerBlock")
|
||||
errEmptyBlobWritten = errors.New("zero bytes written to disk when saving blob sidecar")
|
||||
errIndexOutOfBounds = errors.New("blob index in file name >= MAX_BLOBS_PER_BLOCK")
|
||||
errSidecarEmptySSZData = errors.New("sidecar marshalled to an empty ssz byte slice")
|
||||
errNoBasePath = errors.New("BlobStorage base path not specified in init")
|
||||
errInvalidRootString = errors.New("Could not parse hex string as a [32]byte")
|
||||
)
|
||||
|
||||
const (
|
||||
sszExt = "ssz"
|
||||
partExt = "part"
|
||||
|
||||
directoryPermissions = 0700
|
||||
)
|
||||
|
||||
// BlobStorageOption is a functional option for configuring a BlobStorage.
|
||||
@@ -66,6 +55,23 @@ func WithSaveFsync(fsync bool) BlobStorageOption {
|
||||
}
|
||||
}
|
||||
|
||||
// WithFs allows the afero.Fs implementation to be customized. Used by tests
|
||||
// to substitute an in-memory filesystem.
|
||||
func WithFs(fs afero.Fs) BlobStorageOption {
|
||||
return func(b *BlobStorage) error {
|
||||
b.fs = fs
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithLayout enables the user to specify which layout scheme to use, dictating how blob files are stored on disk.
|
||||
func WithLayout(name string) BlobStorageOption {
|
||||
return func(b *BlobStorage) error {
|
||||
b.layoutName = name
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// NewBlobStorage creates a new instance of the BlobStorage object. Note that the implementation of BlobStorage may
|
||||
// attempt to hold a file lock to guarantee exclusive control of the blob storage directory, so this should only be
|
||||
// initialized once per beacon node.
|
||||
@@ -76,19 +82,27 @@ func NewBlobStorage(opts ...BlobStorageOption) (*BlobStorage, error) {
|
||||
return nil, errors.Wrap(err, "failed to create blob storage")
|
||||
}
|
||||
}
|
||||
if b.base == "" {
|
||||
return nil, errNoBasePath
|
||||
// Allow tests to set up a different fs using WithFs.
|
||||
if b.fs == nil {
|
||||
if b.base == "" {
|
||||
return nil, errNoBasePath
|
||||
}
|
||||
b.base = path.Clean(b.base)
|
||||
if err := file.MkdirAll(b.base); err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to create blob storage at %s", b.base)
|
||||
}
|
||||
b.fs = afero.NewBasePathFs(afero.NewOsFs(), b.base)
|
||||
}
|
||||
b.base = path.Clean(b.base)
|
||||
if err := file.MkdirAll(b.base); err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to create blob storage at %s", b.base)
|
||||
b.cache = newBlobStorageCache()
|
||||
pruner := newBlobPruner(b.retentionEpochs)
|
||||
if b.layoutName == "" {
|
||||
b.layoutName = LayoutNameFlat
|
||||
}
|
||||
b.fs = afero.NewBasePathFs(afero.NewOsFs(), b.base)
|
||||
pruner, err := newBlobPruner(b.fs, b.retentionEpochs)
|
||||
layout, err := newLayout(b.layoutName, b.fs, b.cache, pruner)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b.pruner = pruner
|
||||
b.layout = layout
|
||||
return b, nil
|
||||
}
|
||||
|
||||
@@ -96,47 +110,103 @@ func NewBlobStorage(opts ...BlobStorageOption) (*BlobStorage, error) {
|
||||
type BlobStorage struct {
|
||||
base string
|
||||
retentionEpochs primitives.Epoch
|
||||
layoutName string
|
||||
fsync bool
|
||||
fs afero.Fs
|
||||
pruner *blobPruner
|
||||
layout fsLayout
|
||||
cache *blobStorageSummaryCache
|
||||
}
|
||||
|
||||
// WarmCache runs the prune routine with an expiration of slot of 0, so nothing will be pruned, but the pruner's cache
|
||||
// will be populated at node startup, avoiding a costly cold prune (~4s in syscalls) during syncing.
|
||||
func (bs *BlobStorage) WarmCache() {
|
||||
if bs.pruner == nil {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
start := time.Now()
|
||||
start := time.Now()
|
||||
if bs.layoutName == LayoutNameFlat {
|
||||
log.Info("Blob filesystem cache warm-up started. This may take a few minutes.")
|
||||
if err := bs.pruner.warmCache(); err != nil {
|
||||
log.WithError(err).Error("Error encountered while warming up blob pruner cache")
|
||||
}
|
||||
log.WithField("elapsed", time.Since(start)).Info("Blob filesystem cache warm-up complete")
|
||||
}()
|
||||
} else {
|
||||
log.Info("Blob filesystem cache warm-up started.")
|
||||
}
|
||||
|
||||
if err := warmCache(bs.layout, bs.cache); err != nil {
|
||||
log.WithError(err).Error("Error encountered while warming up blob filesystem cache.")
|
||||
}
|
||||
if err := bs.migrateLayouts(); err != nil {
|
||||
log.WithError(err).Error("Error encountered while migrating blob storage.")
|
||||
}
|
||||
log.WithField("elapsed", time.Since(start)).Info("Blob filesystem cache warm-up complete.")
|
||||
}
|
||||
|
||||
// ErrBlobStorageSummarizerUnavailable is a sentinel error returned when there is no pruner/cache available.
|
||||
// This should be used by code that optionally uses the summarizer to optimize rpc requests. Being able to
|
||||
// fallback when there is no summarizer allows client code to avoid test complexity where the summarizer doesn't matter.
|
||||
var ErrBlobStorageSummarizerUnavailable = errors.New("BlobStorage not initialized with a pruner or cache")
|
||||
|
||||
// WaitForSummarizer blocks until the BlobStorageSummarizer is ready to use.
|
||||
// BlobStorageSummarizer is not ready immediately on node startup because it needs to sample the blob filesystem to
|
||||
// determine which blobs are available.
|
||||
func (bs *BlobStorage) WaitForSummarizer(ctx context.Context) (BlobStorageSummarizer, error) {
|
||||
if bs == nil || bs.pruner == nil {
|
||||
return nil, ErrBlobStorageSummarizerUnavailable
|
||||
// If any blob storage directories are found for layouts besides the configured layout, migrate them.
|
||||
func (bs *BlobStorage) migrateLayouts() error {
|
||||
for _, name := range LayoutNames {
|
||||
if name == bs.layoutName {
|
||||
continue
|
||||
}
|
||||
from, err := newLayout(name, bs.fs, bs.cache, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := migrateLayout(bs.fs, from, bs.layout, bs.cache); err != nil {
|
||||
if errors.Is(err, errLayoutNotDetected) {
|
||||
continue
|
||||
}
|
||||
return errors.Wrapf(err, "failed to migrate layout from %s to %s", name, bs.layoutName)
|
||||
}
|
||||
}
|
||||
return bs.pruner.waitForCache(ctx)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (bs *BlobStorage) writePart(sidecar blocks.VerifiedROBlob) (ppath string, err error) {
|
||||
ident := identForSidecar(sidecar)
|
||||
sidecarData, err := sidecar.MarshalSSZ()
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to serialize sidecar data")
|
||||
}
|
||||
if len(sidecarData) == 0 {
|
||||
return "", errSidecarEmptySSZData
|
||||
}
|
||||
|
||||
if err := bs.fs.MkdirAll(bs.layout.dir(ident), directoryPermissions()); err != nil {
|
||||
return "", err
|
||||
}
|
||||
ppath = bs.layout.partPath(ident, fmt.Sprintf("%p", sidecarData))
|
||||
|
||||
// Create a partial file and write the serialized data to it.
|
||||
partialFile, err := bs.fs.Create(ppath)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to create partial file")
|
||||
}
|
||||
defer func() {
|
||||
cerr := partialFile.Close()
|
||||
// The close error is probably less important than any existing error, so only overwrite nil err.
|
||||
if cerr != nil && err == nil {
|
||||
err = cerr
|
||||
}
|
||||
}()
|
||||
|
||||
n, err := partialFile.Write(sidecarData)
|
||||
if err != nil {
|
||||
return ppath, errors.Wrap(err, "failed to write to partial file")
|
||||
}
|
||||
if bs.fsync {
|
||||
if err := partialFile.Sync(); err != nil {
|
||||
return ppath, err
|
||||
}
|
||||
}
|
||||
|
||||
if n != len(sidecarData) {
|
||||
return ppath, fmt.Errorf("failed to write the full bytes of sidecarData, wrote only %d of %d bytes", n, len(sidecarData))
|
||||
}
|
||||
|
||||
return ppath, nil
|
||||
}
|
||||
|
||||
// Save saves blobs given a list of sidecars.
|
||||
func (bs *BlobStorage) Save(sidecar blocks.VerifiedROBlob) error {
|
||||
startTime := time.Now()
|
||||
fname := namerForSidecar(sidecar)
|
||||
sszPath := fname.path()
|
||||
|
||||
ident := identForSidecar(sidecar)
|
||||
sszPath := bs.layout.sszPath(ident)
|
||||
exists, err := afero.Exists(bs.fs, sszPath)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -145,78 +215,36 @@ func (bs *BlobStorage) Save(sidecar blocks.VerifiedROBlob) error {
|
||||
log.WithFields(logging.BlobFields(sidecar.ROBlob)).Debug("Ignoring a duplicate blob sidecar save attempt")
|
||||
return nil
|
||||
}
|
||||
if bs.pruner != nil {
|
||||
if err := bs.pruner.notify(sidecar.BlockRoot(), sidecar.Slot(), sidecar.Index); err != nil {
|
||||
return errors.Wrapf(err, "problem maintaining pruning cache/metrics for sidecar with root=%#x", sidecar.BlockRoot())
|
||||
}
|
||||
}
|
||||
|
||||
// Serialize the ethpb.BlobSidecar to binary data using SSZ.
|
||||
sidecarData, err := sidecar.MarshalSSZ()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to serialize sidecar data")
|
||||
} else if len(sidecarData) == 0 {
|
||||
return errSidecarEmptySSZData
|
||||
}
|
||||
|
||||
if err := bs.fs.MkdirAll(fname.dir(), directoryPermissions); err != nil {
|
||||
return err
|
||||
}
|
||||
partPath := fname.partPath(fmt.Sprintf("%p", sidecarData))
|
||||
|
||||
partialMoved := false
|
||||
partPath, err := bs.writePart(sidecar)
|
||||
// Ensure the partial file is deleted.
|
||||
defer func() {
|
||||
if partialMoved {
|
||||
if partialMoved || partPath == "" {
|
||||
return
|
||||
}
|
||||
// It's expected to error if the save is successful.
|
||||
err = bs.fs.Remove(partPath)
|
||||
err := bs.fs.Remove(partPath)
|
||||
if err == nil {
|
||||
log.WithFields(logrus.Fields{
|
||||
"partPath": partPath,
|
||||
}).Debugf("Removed partial file")
|
||||
}
|
||||
}()
|
||||
|
||||
// Create a partial file and write the serialized data to it.
|
||||
partialFile, err := bs.fs.Create(partPath)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create partial file")
|
||||
}
|
||||
|
||||
n, err := partialFile.Write(sidecarData)
|
||||
if err != nil {
|
||||
closeErr := partialFile.Close()
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
return errors.Wrap(err, "failed to write to partial file")
|
||||
}
|
||||
if bs.fsync {
|
||||
if err := partialFile.Sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := partialFile.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if n != len(sidecarData) {
|
||||
return fmt.Errorf("failed to write the full bytes of sidecarData, wrote only %d of %d bytes", n, len(sidecarData))
|
||||
}
|
||||
|
||||
if n == 0 {
|
||||
return errEmptyBlobWritten
|
||||
}
|
||||
|
||||
// Atomically rename the partial file to its final name.
|
||||
err = bs.fs.Rename(partPath, sszPath)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to rename partial file to final name")
|
||||
}
|
||||
partialMoved = true
|
||||
|
||||
if err := bs.layout.notify(ident); err != nil {
|
||||
return errors.Wrapf(err, "problem maintaining pruning cache/metrics for sidecar with root=%#x", sidecar.BlockRoot())
|
||||
}
|
||||
blobsWrittenCounter.Inc()
|
||||
blobSaveLatency.Observe(float64(time.Since(startTime).Milliseconds()))
|
||||
|
||||
@@ -228,70 +256,30 @@ func (bs *BlobStorage) Save(sidecar blocks.VerifiedROBlob) error {
|
||||
// value is always a VerifiedROBlob.
|
||||
func (bs *BlobStorage) Get(root [32]byte, idx uint64) (blocks.VerifiedROBlob, error) {
|
||||
startTime := time.Now()
|
||||
expected := blobNamer{root: root, index: idx}
|
||||
encoded, err := afero.ReadFile(bs.fs, expected.path())
|
||||
var v blocks.VerifiedROBlob
|
||||
ident, err := bs.layout.ident(root, idx)
|
||||
if err != nil {
|
||||
return v, err
|
||||
}
|
||||
s := ðpb.BlobSidecar{}
|
||||
if err := s.UnmarshalSSZ(encoded); err != nil {
|
||||
return v, err
|
||||
}
|
||||
ro, err := blocks.NewROBlobWithRoot(s, root)
|
||||
if err != nil {
|
||||
return blocks.VerifiedROBlob{}, err
|
||||
return verification.VerifiedROBlobError(err)
|
||||
}
|
||||
defer func() {
|
||||
blobFetchLatency.Observe(float64(time.Since(startTime).Milliseconds()))
|
||||
}()
|
||||
return verification.BlobSidecarNoop(ro)
|
||||
return verification.VerifiedROBlobFromDisk(bs.fs, root, bs.layout.sszPath(ident))
|
||||
}
|
||||
|
||||
// Remove removes all blobs for a given root.
|
||||
func (bs *BlobStorage) Remove(root [32]byte) error {
|
||||
rootDir := blobNamer{root: root}.dir()
|
||||
return bs.fs.RemoveAll(rootDir)
|
||||
dirIdent, err := bs.layout.dirIdent(root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = bs.layout.remove(dirIdent)
|
||||
return err
|
||||
}
|
||||
|
||||
// Indices generates a bitmap representing which BlobSidecar.Index values are present on disk for a given root.
|
||||
// This value can be compared to the commitments observed in a block to determine which indices need to be found
|
||||
// on the network to confirm data availability.
|
||||
func (bs *BlobStorage) Indices(root [32]byte, s primitives.Slot) ([]bool, error) {
|
||||
maxBlobsPerBlock := params.BeaconConfig().MaxBlobsPerBlock(s)
|
||||
mask := make([]bool, maxBlobsPerBlock)
|
||||
|
||||
rootDir := blobNamer{root: root}.dir()
|
||||
entries, err := afero.ReadDir(bs.fs, rootDir)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return mask, nil
|
||||
}
|
||||
return mask, err
|
||||
}
|
||||
|
||||
for i := range entries {
|
||||
if entries[i].IsDir() {
|
||||
continue
|
||||
}
|
||||
name := entries[i].Name()
|
||||
if !strings.HasSuffix(name, sszExt) {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(name, ".")
|
||||
if len(parts) != 2 {
|
||||
continue
|
||||
}
|
||||
u, err := strconv.ParseUint(parts[0], 10, 64)
|
||||
if err != nil {
|
||||
return mask, errors.Wrapf(err, "unexpected directory entry breaks listing, %s", parts[0])
|
||||
}
|
||||
if u >= uint64(maxBlobsPerBlock) {
|
||||
return mask, errIndexOutOfBounds
|
||||
}
|
||||
mask[u] = true
|
||||
}
|
||||
return mask, nil
|
||||
// Summary returns the BlobStorageSummary from the layout.
|
||||
// Internally, this is a cached representation of the directory listing for the given root.
|
||||
func (bs *BlobStorage) Summary(root [32]byte) BlobStorageSummary {
|
||||
return bs.layout.summary(root)
|
||||
}
|
||||
|
||||
// Clear deletes all files on the filesystem.
|
||||
@@ -316,36 +304,3 @@ func (bs *BlobStorage) WithinRetentionPeriod(requested, current primitives.Epoch
|
||||
}
|
||||
return requested+bs.retentionEpochs >= current
|
||||
}
|
||||
|
||||
type blobNamer struct {
|
||||
root [32]byte
|
||||
index uint64
|
||||
}
|
||||
|
||||
func namerForSidecar(sc blocks.VerifiedROBlob) blobNamer {
|
||||
return blobNamer{root: sc.BlockRoot(), index: sc.Index}
|
||||
}
|
||||
|
||||
func (p blobNamer) dir() string {
|
||||
return rootString(p.root)
|
||||
}
|
||||
|
||||
func (p blobNamer) partPath(entropy string) string {
|
||||
return path.Join(p.dir(), fmt.Sprintf("%s-%d.%s", entropy, p.index, partExt))
|
||||
}
|
||||
|
||||
func (p blobNamer) path() string {
|
||||
return path.Join(p.dir(), fmt.Sprintf("%d.%s", p.index, sszExt))
|
||||
}
|
||||
|
||||
func rootString(root [32]byte) string {
|
||||
return fmt.Sprintf("%#x", root)
|
||||
}
|
||||
|
||||
func stringToRoot(str string) ([32]byte, error) {
|
||||
slice, err := hexutil.Decode(str)
|
||||
if err != nil {
|
||||
return [32]byte{}, errors.Wrapf(errInvalidRootString, "input=%s", str)
|
||||
}
|
||||
return bytesutil.ToBytes32(slice), nil
|
||||
}
|
||||
|
||||
@@ -9,26 +9,26 @@ import (
|
||||
"testing"
|
||||
|
||||
ssz "github.com/prysmaticlabs/fastssz"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/db"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/verification"
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
"github.com/prysmaticlabs/prysm/v5/encoding/bytesutil"
|
||||
ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/require"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/util"
|
||||
"github.com/prysmaticlabs/prysm/v5/time/slots"
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
func TestBlobStorage_SaveBlobData(t *testing.T) {
|
||||
_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, 1, params.BeaconConfig().MaxBlobsPerBlock(1))
|
||||
testSidecars, err := verification.BlobSidecarSliceNoop(sidecars)
|
||||
require.NoError(t, err)
|
||||
testSidecars := verification.FakeVerifySliceForTest(t, sidecars)
|
||||
|
||||
t.Run("no error for duplicate", func(t *testing.T) {
|
||||
fs, bs := NewEphemeralBlobStorageWithFs(t)
|
||||
fs, bs := NewEphemeralBlobStorageAndFs(t)
|
||||
existingSidecar := testSidecars[0]
|
||||
|
||||
blobPath := namerForSidecar(existingSidecar).path()
|
||||
blobPath := bs.layout.sszPath(identForSidecar(existingSidecar))
|
||||
// Serialize the existing BlobSidecar to binary data.
|
||||
existingSidecarData, err := ssz.MarshalSSZ(existingSidecar)
|
||||
require.NoError(t, err)
|
||||
@@ -56,8 +56,8 @@ func TestBlobStorage_SaveBlobData(t *testing.T) {
|
||||
require.NoError(t, bs.Save(sc))
|
||||
actualSc, err := bs.Get(sc.BlockRoot(), sc.Index)
|
||||
require.NoError(t, err)
|
||||
expectedIdx := []bool{false, false, true, false, false, false}
|
||||
actualIdx, err := bs.Indices(actualSc.BlockRoot(), 100)
|
||||
expectedIdx := blobIndexMask{false, false, true, false, false, false}
|
||||
actualIdx := bs.Summary(actualSc.BlockRoot()).mask
|
||||
require.NoError(t, err)
|
||||
require.DeepEqual(t, expectedIdx, actualIdx)
|
||||
})
|
||||
@@ -85,7 +85,7 @@ func TestBlobStorage_SaveBlobData(t *testing.T) {
|
||||
|
||||
require.NoError(t, bs.Remove(expected.BlockRoot()))
|
||||
_, err = bs.Get(expected.BlockRoot(), expected.Index)
|
||||
require.ErrorContains(t, "file does not exist", err)
|
||||
require.Equal(t, true, db.IsNotFound(err))
|
||||
})
|
||||
|
||||
t.Run("clear", func(t *testing.T) {
|
||||
@@ -126,16 +126,14 @@ func TestBlobStorage_SaveBlobData(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// pollUntil polls a condition function until it returns true or a timeout is reached.
|
||||
|
||||
func TestBlobIndicesBounds(t *testing.T) {
|
||||
fs, bs := NewEphemeralBlobStorageWithFs(t)
|
||||
fs := afero.NewMemMapFs()
|
||||
root := [32]byte{}
|
||||
|
||||
okIdx := uint64(params.BeaconConfig().MaxBlobsPerBlock(0)) - 1
|
||||
writeFakeSSZ(t, fs, root, okIdx)
|
||||
indices, err := bs.Indices(root, 100)
|
||||
require.NoError(t, err)
|
||||
writeFakeSSZ(t, fs, root, 0, okIdx)
|
||||
bs := NewWarmedEphemeralBlobStorageUsingFs(t, fs, WithLayout(LayoutNameByEpoch))
|
||||
indices := bs.Summary(root).mask
|
||||
expected := make([]bool, params.BeaconConfig().MaxBlobsPerBlock(0))
|
||||
expected[okIdx] = true
|
||||
for i := range expected {
|
||||
@@ -143,102 +141,23 @@ func TestBlobIndicesBounds(t *testing.T) {
|
||||
}
|
||||
|
||||
oobIdx := uint64(params.BeaconConfig().MaxBlobsPerBlock(0))
|
||||
writeFakeSSZ(t, fs, root, oobIdx)
|
||||
_, err = bs.Indices(root, 100)
|
||||
require.ErrorIs(t, err, errIndexOutOfBounds)
|
||||
writeFakeSSZ(t, fs, root, 0, oobIdx)
|
||||
// This now fails at cache warmup time.
|
||||
require.ErrorIs(t, warmCache(bs.layout, bs.cache), errIndexOutOfBounds)
|
||||
}
|
||||
|
||||
func writeFakeSSZ(t *testing.T, fs afero.Fs, root [32]byte, idx uint64) {
|
||||
namer := blobNamer{root: root, index: idx}
|
||||
require.NoError(t, fs.MkdirAll(namer.dir(), 0700))
|
||||
fh, err := fs.Create(namer.path())
|
||||
func writeFakeSSZ(t *testing.T, fs afero.Fs, root [32]byte, slot primitives.Slot, idx uint64) {
|
||||
epoch := slots.ToEpoch(slot)
|
||||
namer := newBlobIdent(root, epoch, idx)
|
||||
layout := periodicEpochLayout{}
|
||||
require.NoError(t, fs.MkdirAll(layout.dir(namer), 0700))
|
||||
fh, err := fs.Create(layout.sszPath(namer))
|
||||
require.NoError(t, err)
|
||||
_, err = fh.Write([]byte("derp"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, fh.Close())
|
||||
}
|
||||
|
||||
func TestBlobStoragePrune(t *testing.T) {
|
||||
currentSlot := primitives.Slot(200000)
|
||||
fs, bs := NewEphemeralBlobStorageWithFs(t)
|
||||
|
||||
t.Run("PruneOne", func(t *testing.T) {
|
||||
_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, 300, params.BeaconConfig().MaxBlobsPerBlock(0))
|
||||
testSidecars, err := verification.BlobSidecarSliceNoop(sidecars)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, sidecar := range testSidecars {
|
||||
require.NoError(t, bs.Save(sidecar))
|
||||
}
|
||||
|
||||
require.NoError(t, bs.pruner.prune(currentSlot-bs.pruner.windowSize))
|
||||
|
||||
remainingFolders, err := afero.ReadDir(fs, ".")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, len(remainingFolders))
|
||||
})
|
||||
t.Run("Prune dangling blob", func(t *testing.T) {
|
||||
_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, 299, params.BeaconConfig().MaxBlobsPerBlock(0))
|
||||
testSidecars, err := verification.BlobSidecarSliceNoop(sidecars)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, sidecar := range testSidecars[4:] {
|
||||
require.NoError(t, bs.Save(sidecar))
|
||||
}
|
||||
|
||||
require.NoError(t, bs.pruner.prune(currentSlot-bs.pruner.windowSize))
|
||||
|
||||
remainingFolders, err := afero.ReadDir(fs, ".")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, len(remainingFolders))
|
||||
})
|
||||
t.Run("PruneMany", func(t *testing.T) {
|
||||
blockQty := 10
|
||||
slot := primitives.Slot(1)
|
||||
|
||||
for j := 0; j <= blockQty; j++ {
|
||||
root := bytesutil.ToBytes32(bytesutil.ToBytes(uint64(slot), 32))
|
||||
_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, root, slot, params.BeaconConfig().MaxBlobsPerBlock(0))
|
||||
testSidecars, err := verification.BlobSidecarSliceNoop(sidecars)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, bs.Save(testSidecars[0]))
|
||||
|
||||
slot += 10000
|
||||
}
|
||||
|
||||
require.NoError(t, bs.pruner.prune(currentSlot-bs.pruner.windowSize))
|
||||
|
||||
remainingFolders, err := afero.ReadDir(fs, ".")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 4, len(remainingFolders))
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkPruning(b *testing.B) {
|
||||
var t *testing.T
|
||||
_, bs := NewEphemeralBlobStorageWithFs(t)
|
||||
|
||||
blockQty := 10000
|
||||
currentSlot := primitives.Slot(150000)
|
||||
slot := primitives.Slot(0)
|
||||
|
||||
for j := 0; j <= blockQty; j++ {
|
||||
root := bytesutil.ToBytes32(bytesutil.ToBytes(uint64(slot), 32))
|
||||
_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, root, slot, params.BeaconConfig().MaxBlobsPerBlock(0))
|
||||
testSidecars, err := verification.BlobSidecarSliceNoop(sidecars)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, bs.Save(testSidecars[0]))
|
||||
|
||||
slot += 100
|
||||
}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := bs.pruner.prune(currentSlot)
|
||||
require.NoError(b, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewBlobStorage(t *testing.T) {
|
||||
_, err := NewBlobStorage()
|
||||
require.ErrorIs(t, err, errNoBasePath)
|
||||
@@ -292,3 +211,13 @@ func TestConfig_WithinRetentionPeriod(t *testing.T) {
|
||||
require.Equal(t, true, storage.WithinRetentionPeriod(1, 1))
|
||||
})
|
||||
}
|
||||
|
||||
func TestLayoutNames(t *testing.T) {
|
||||
badLayoutName := "bad"
|
||||
for _, name := range LayoutNames {
|
||||
_, err := newLayout(name, nil, nil, nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
_, err := newLayout(badLayoutName, nil, nil, nil)
|
||||
require.ErrorIs(t, err, errInvalidLayoutName)
|
||||
}
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/db"
|
||||
fieldparams "github.com/prysmaticlabs/prysm/v5/config/fieldparams"
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
@@ -13,17 +15,12 @@ type blobIndexMask []bool
|
||||
|
||||
// BlobStorageSummary represents cached information about the BlobSidecars on disk for each root the cache knows about.
|
||||
type BlobStorageSummary struct {
|
||||
slot primitives.Slot
|
||||
mask blobIndexMask
|
||||
epoch primitives.Epoch
|
||||
mask blobIndexMask
|
||||
}
|
||||
|
||||
// HasIndex returns true if the BlobSidecar at the given index is available in the filesystem.
|
||||
func (s BlobStorageSummary) HasIndex(idx uint64) bool {
|
||||
// Protect from panic, but assume callers are sophisticated enough to not need an error telling them they have an invalid idx.
|
||||
maxBlobsPerBlock := params.BeaconConfig().MaxBlobsPerBlock(s.slot)
|
||||
if idx >= uint64(maxBlobsPerBlock) {
|
||||
return false
|
||||
}
|
||||
if idx >= uint64(len(s.mask)) {
|
||||
return false
|
||||
}
|
||||
@@ -32,10 +29,6 @@ func (s BlobStorageSummary) HasIndex(idx uint64) bool {
|
||||
|
||||
// AllAvailable returns true if we have all blobs for all indices from 0 to count-1.
|
||||
func (s BlobStorageSummary) AllAvailable(count int) bool {
|
||||
maxBlobsPerBlock := params.BeaconConfig().MaxBlobsPerBlock(s.slot)
|
||||
if count > maxBlobsPerBlock {
|
||||
return false
|
||||
}
|
||||
if count > len(s.mask) {
|
||||
return false
|
||||
}
|
||||
@@ -47,83 +40,121 @@ func (s BlobStorageSummary) AllAvailable(count int) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (s BlobStorageSummary) MaxBlobsForEpoch() uint64 {
|
||||
return uint64(params.BeaconConfig().MaxBlobsPerBlockAtEpoch(s.epoch))
|
||||
}
|
||||
|
||||
// NewBlobStorageSummary creates a new BlobStorageSummary for a given epoch and mask.
|
||||
func NewBlobStorageSummary(epoch primitives.Epoch, mask []bool) (BlobStorageSummary, error) {
|
||||
c := params.BeaconConfig().MaxBlobsPerBlockAtEpoch(epoch)
|
||||
if len(mask) != c {
|
||||
return BlobStorageSummary{}, fmt.Errorf("mask length %d does not match expected %d for epoch %d", len(mask), c, epoch)
|
||||
}
|
||||
return BlobStorageSummary{
|
||||
epoch: epoch,
|
||||
mask: mask,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// BlobStorageSummarizer can be used to receive a summary of metadata about blobs on disk for a given root.
|
||||
// The BlobStorageSummary can be used to check which indices (if any) are available for a given block by root.
|
||||
type BlobStorageSummarizer interface {
|
||||
Summary(root [32]byte) BlobStorageSummary
|
||||
}
|
||||
|
||||
type blobStorageCache struct {
|
||||
type blobStorageSummaryCache struct {
|
||||
mu sync.RWMutex
|
||||
nBlobs float64
|
||||
cache map[[32]byte]BlobStorageSummary
|
||||
}
|
||||
|
||||
var _ BlobStorageSummarizer = &blobStorageCache{}
|
||||
var _ BlobStorageSummarizer = &blobStorageSummaryCache{}
|
||||
|
||||
func newBlobStorageCache() *blobStorageCache {
|
||||
return &blobStorageCache{
|
||||
cache: make(map[[32]byte]BlobStorageSummary, params.BeaconConfig().MinEpochsForBlobsSidecarsRequest*fieldparams.SlotsPerEpoch),
|
||||
func newBlobStorageCache() *blobStorageSummaryCache {
|
||||
return &blobStorageSummaryCache{
|
||||
cache: make(map[[32]byte]BlobStorageSummary),
|
||||
}
|
||||
}
|
||||
|
||||
// Summary returns the BlobStorageSummary for `root`. The BlobStorageSummary can be used to check for the presence of
|
||||
// BlobSidecars based on Index.
|
||||
func (s *blobStorageCache) Summary(root [32]byte) BlobStorageSummary {
|
||||
func (s *blobStorageSummaryCache) Summary(root [32]byte) BlobStorageSummary {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return s.cache[root]
|
||||
}
|
||||
|
||||
func (s *blobStorageCache) ensure(key [32]byte, slot primitives.Slot, idx uint64) error {
|
||||
maxBlobsPerBlock := params.BeaconConfig().MaxBlobsPerBlock(slot)
|
||||
if idx >= uint64(maxBlobsPerBlock) {
|
||||
func (s *blobStorageSummaryCache) ensure(ident blobIdent) error {
|
||||
maxBlobsPerBlock := params.BeaconConfig().MaxBlobsPerBlockAtEpoch(ident.epoch)
|
||||
if ident.index >= uint64(maxBlobsPerBlock) {
|
||||
return errIndexOutOfBounds
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
v := s.cache[key]
|
||||
v.slot = slot
|
||||
v := s.cache[ident.root]
|
||||
v.epoch = ident.epoch
|
||||
if v.mask == nil {
|
||||
v.mask = make(blobIndexMask, maxBlobsPerBlock)
|
||||
}
|
||||
if !v.mask[idx] {
|
||||
if !v.mask[ident.index] {
|
||||
s.updateMetrics(1)
|
||||
}
|
||||
v.mask[idx] = true
|
||||
s.cache[key] = v
|
||||
v.mask[ident.index] = true
|
||||
s.cache[ident.root] = v
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *blobStorageCache) slot(key [32]byte) (primitives.Slot, bool) {
|
||||
func (s *blobStorageSummaryCache) get(key [32]byte) (BlobStorageSummary, bool) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
v, ok := s.cache[key]
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
return v.slot, ok
|
||||
return v, ok
|
||||
}
|
||||
|
||||
func (s *blobStorageCache) evict(key [32]byte) {
|
||||
var deleted float64
|
||||
func (s *blobStorageSummaryCache) identForIdx(key [32]byte, idx uint64) (blobIdent, error) {
|
||||
v, ok := s.get(key)
|
||||
if !ok || !v.HasIndex(idx) {
|
||||
return blobIdent{}, db.ErrNotFound
|
||||
}
|
||||
return blobIdent{
|
||||
root: key,
|
||||
index: idx,
|
||||
epoch: v.epoch,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *blobStorageSummaryCache) identForRoot(key [32]byte) (blobIdent, error) {
|
||||
v, ok := s.get(key)
|
||||
if !ok {
|
||||
return blobIdent{}, db.ErrNotFound
|
||||
}
|
||||
return blobIdent{
|
||||
root: key,
|
||||
epoch: v.epoch,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *blobStorageSummaryCache) evict(key [32]byte) int {
|
||||
deleted := 0
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
v, ok := s.cache[key]
|
||||
if ok {
|
||||
for i := range v.mask {
|
||||
if v.mask[i] {
|
||||
deleted += 1
|
||||
}
|
||||
if !ok {
|
||||
return 0
|
||||
}
|
||||
for i := range v.mask {
|
||||
if v.mask[i] {
|
||||
deleted += 1
|
||||
}
|
||||
}
|
||||
delete(s.cache, key)
|
||||
s.mu.Unlock()
|
||||
if deleted > 0 {
|
||||
s.updateMetrics(-deleted)
|
||||
s.updateMetrics(-float64(deleted))
|
||||
}
|
||||
return deleted
|
||||
}
|
||||
|
||||
func (s *blobStorageCache) updateMetrics(delta float64) {
|
||||
func (s *blobStorageSummaryCache) updateMetrics(delta float64) {
|
||||
s.nBlobs += delta
|
||||
blobDiskCount.Set(s.nBlobs)
|
||||
blobDiskSize.Set(s.nBlobs * fieldparams.BlobSidecarSize)
|
||||
|
||||
@@ -53,7 +53,7 @@ func TestSlotByRoot_Summary(t *testing.T) {
|
||||
for _, c := range cases {
|
||||
if c.expected != nil {
|
||||
key := bytesutil.ToBytes32([]byte(c.name))
|
||||
sc.cache[key] = BlobStorageSummary{slot: 0, mask: c.expected}
|
||||
sc.cache[key] = BlobStorageSummary{epoch: 0, mask: c.expected}
|
||||
}
|
||||
}
|
||||
for _, c := range cases {
|
||||
|
||||
238
beacon-chain/db/filesystem/iteration.go
Normal file
238
beacon-chain/db/filesystem/iteration.go
Normal file
@@ -0,0 +1,238 @@
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common/hexutil"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prysmaticlabs/prysm/v5/encoding/bytesutil"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
var errIdentFailure = errors.New("failed to determine blob metadata, ignoring all sub-paths.")
|
||||
|
||||
type identificationError struct {
|
||||
err error
|
||||
path string
|
||||
ident blobIdent
|
||||
}
|
||||
|
||||
func (ide *identificationError) Error() string {
|
||||
return fmt.Sprintf("%s path=%s, err=%s", errIdentFailure.Error(), ide.path, ide.err.Error())
|
||||
}
|
||||
|
||||
func (ide *identificationError) Unwrap() error {
|
||||
return ide.err
|
||||
}
|
||||
|
||||
func (*identificationError) Is(err error) bool {
|
||||
return err == errIdentFailure
|
||||
}
|
||||
|
||||
func (ide *identificationError) LogFields() logrus.Fields {
|
||||
fields := ide.ident.logFields()
|
||||
fields["path"] = ide.path
|
||||
return fields
|
||||
}
|
||||
|
||||
func newIdentificationError(path string, ident blobIdent, err error) *identificationError {
|
||||
return &identificationError{path: path, ident: ident, err: err}
|
||||
}
|
||||
|
||||
func listDir(fs afero.Fs, dir string) ([]string, error) {
|
||||
top, err := fs.Open(dir)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to open directory descriptor")
|
||||
}
|
||||
defer func() {
|
||||
if err := top.Close(); err != nil {
|
||||
log.WithError(err).Errorf("Could not close file %s", dir)
|
||||
}
|
||||
}()
|
||||
// re the -1 param: "If n <= 0, Readdirnames returns all the names from the directory in a single slice"
|
||||
dirs, err := top.Readdirnames(-1)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to read directory listing")
|
||||
}
|
||||
return dirs, nil
|
||||
}
|
||||
|
||||
// identPopulator is a function that sets values in the blobIdent for a given layer of the filesystem layout.
|
||||
type identPopulator func(blobIdent, string) (blobIdent, error)
|
||||
|
||||
// layoutLayer represents a layer of the nested directory scheme. Each layer is defined by a filter function that
|
||||
// ensures any entries at that layer of the scheme are named in a valid way, and a populateIdent function that
|
||||
// parses the directory name into a blobIdent object, used for iterating across the layout in a layout-independent way.
|
||||
type layoutLayer struct {
|
||||
populateIdent identPopulator
|
||||
filter func(string) bool
|
||||
}
|
||||
|
||||
// identIterator moves through the filesystem in order to yield blobIdents.
|
||||
// layoutLayers (in the 'layers' field) allows a filesystem layout to control how the
|
||||
// layout is traversed. A layoutLayer can filter out entries from the directory listing
|
||||
// via the filter function, and populate fields in the blobIdent via the populateIdent function.
|
||||
// The blobIdent is populated from an empty value at the root, accumulating values for its fields at each layer.
|
||||
// The fully populated blobIdent is returned when the iterator reaches the leaf layer.
|
||||
type identIterator struct {
|
||||
fs afero.Fs
|
||||
path string
|
||||
child *identIterator
|
||||
ident blobIdent
|
||||
// layoutLayers are the heart of how the layout defines the nesting of the components of the path.
|
||||
// Each layer of the layout represents a different layer of the directory layout hierarchy,
|
||||
// from the relative root at the zero index to the blob files at the end.
|
||||
layers []layoutLayer
|
||||
entries []string
|
||||
offset int
|
||||
eof bool
|
||||
}
|
||||
|
||||
// atEOF can be used to peek at the iterator to see if it's already finished. This is useful for the migration code to check
|
||||
// if there are any entries in the directory indicated by the migration.
|
||||
func (iter *identIterator) atEOF() bool {
|
||||
return iter.eof
|
||||
}
|
||||
|
||||
// next is the only method that a user of the identIterator needs to call.
|
||||
// identIterator will yield blobIdents in a breadth-first fashion,
|
||||
// returning an empty blobIdent and io.EOF once all branches have been traversed.
|
||||
func (iter *identIterator) next() (blobIdent, error) {
|
||||
if iter.eof {
|
||||
return blobIdent{}, io.EOF
|
||||
}
|
||||
if iter.child != nil {
|
||||
next, err := iter.child.next()
|
||||
if err == nil {
|
||||
return next, nil
|
||||
}
|
||||
if !errors.Is(err, io.EOF) {
|
||||
return blobIdent{}, err
|
||||
}
|
||||
}
|
||||
return iter.advanceChild()
|
||||
}
|
||||
|
||||
// advanceChild is used to move to the next directory at each layer of the tree, either when
|
||||
// the nodes are first being initialized at a layer, or when a sub-branch has been exhausted.
|
||||
func (iter *identIterator) advanceChild() (blobIdent, error) {
|
||||
defer func() {
|
||||
iter.offset += 1
|
||||
}()
|
||||
for i := iter.offset; i < len(iter.entries); i++ {
|
||||
iter.offset = i
|
||||
nextPath := filepath.Join(iter.path, iter.entries[iter.offset])
|
||||
nextLayer := iter.layers[0]
|
||||
if !nextLayer.filter(nextPath) {
|
||||
continue
|
||||
}
|
||||
ident, err := nextLayer.populateIdent(iter.ident, nextPath)
|
||||
if err != nil {
|
||||
return ident, newIdentificationError(nextPath, ident, err)
|
||||
}
|
||||
// if we're at the leaf layer , we can return the updated ident.
|
||||
if len(iter.layers) == 1 {
|
||||
return ident, nil
|
||||
}
|
||||
|
||||
entries, err := listDir(iter.fs, nextPath)
|
||||
if err != nil {
|
||||
return blobIdent{}, err
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
continue
|
||||
}
|
||||
iter.child = &identIterator{
|
||||
fs: iter.fs,
|
||||
path: nextPath,
|
||||
ident: ident,
|
||||
layers: iter.layers[1:],
|
||||
entries: entries,
|
||||
}
|
||||
return iter.child.next()
|
||||
}
|
||||
|
||||
return blobIdent{}, io.EOF
|
||||
}
|
||||
|
||||
func populateNoop(namer blobIdent, _ string) (blobIdent, error) {
|
||||
return namer, nil
|
||||
}
|
||||
|
||||
func populateRoot(namer blobIdent, dir string) (blobIdent, error) {
|
||||
root, err := rootFromPath(dir)
|
||||
if err != nil {
|
||||
return namer, err
|
||||
}
|
||||
namer.root = root
|
||||
return namer, nil
|
||||
}
|
||||
|
||||
func populateIndex(namer blobIdent, fname string) (blobIdent, error) {
|
||||
idx, err := idxFromPath(fname)
|
||||
if err != nil {
|
||||
return namer, err
|
||||
}
|
||||
namer.index = idx
|
||||
return namer, nil
|
||||
}
|
||||
|
||||
func rootFromPath(p string) ([32]byte, error) {
|
||||
subdir := filepath.Base(p)
|
||||
root, err := stringToRoot(subdir)
|
||||
if err != nil {
|
||||
return root, errors.Wrapf(err, "invalid directory, could not parse subdir as root %s", p)
|
||||
}
|
||||
return root, nil
|
||||
}
|
||||
|
||||
func idxFromPath(p string) (uint64, error) {
|
||||
p = path.Base(p)
|
||||
|
||||
if !isSszFile(p) {
|
||||
return 0, errors.Wrap(errNotBlobSSZ, "does not have .ssz extension")
|
||||
}
|
||||
parts := strings.Split(p, ".")
|
||||
if len(parts) != 2 {
|
||||
return 0, errors.Wrap(errNotBlobSSZ, "unexpected filename structure (want <index>.ssz)")
|
||||
}
|
||||
idx, err := strconv.ParseUint(parts[0], 10, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return idx, nil
|
||||
}
|
||||
|
||||
func filterNoop(_ string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func isRootDir(p string) bool {
|
||||
dir := filepath.Base(p)
|
||||
return len(dir) == rootStringLen && strings.HasPrefix(dir, "0x")
|
||||
}
|
||||
|
||||
func isSszFile(s string) bool {
|
||||
return filepath.Ext(s) == "."+sszExt
|
||||
}
|
||||
|
||||
func rootToString(root [32]byte) string {
|
||||
return fmt.Sprintf("%#x", root)
|
||||
}
|
||||
|
||||
func stringToRoot(str string) ([32]byte, error) {
|
||||
if len(str) != rootStringLen {
|
||||
return [32]byte{}, errors.Wrapf(errInvalidRootString, "incorrect len for input=%s", str)
|
||||
}
|
||||
slice, err := hexutil.Decode(str)
|
||||
if err != nil {
|
||||
return [32]byte{}, errors.Wrapf(errInvalidRootString, "input=%s", str)
|
||||
}
|
||||
return bytesutil.ToBytes32(slice), nil
|
||||
}
|
||||
304
beacon-chain/db/filesystem/iteration_test.go
Normal file
304
beacon-chain/db/filesystem/iteration_test.go
Normal file
@@ -0,0 +1,304 @@
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/verification"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/require"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/util"
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
func TestRootFromDir(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
dir string
|
||||
err error
|
||||
root [32]byte
|
||||
}{
|
||||
{
|
||||
name: "happy path",
|
||||
dir: "0xffff875e1d985c5ccb214894983f2428edb271f0f87b68ba7010e4a99df3b5cb",
|
||||
root: [32]byte{255, 255, 135, 94, 29, 152, 92, 92, 203, 33, 72, 148, 152, 63, 36, 40,
|
||||
237, 178, 113, 240, 248, 123, 104, 186, 112, 16, 228, 169, 157, 243, 181, 203},
|
||||
},
|
||||
{
|
||||
name: "too short",
|
||||
dir: "0xffff875e1d985c5ccb214894983f2428edb271f0f87b68ba7010e4a99df3b5c",
|
||||
err: errInvalidRootString,
|
||||
},
|
||||
{
|
||||
name: "too log",
|
||||
dir: "0xffff875e1d985c5ccb214894983f2428edb271f0f87b68ba7010e4a99df3b5cbb",
|
||||
err: errInvalidRootString,
|
||||
},
|
||||
{
|
||||
name: "missing prefix",
|
||||
dir: "ffff875e1d985c5ccb214894983f2428edb271f0f87b68ba7010e4a99df3b5cb",
|
||||
err: errInvalidRootString,
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
root, err := stringToRoot(c.dir)
|
||||
if c.err != nil {
|
||||
require.ErrorIs(t, err, c.err)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, c.root, root)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlotFromFile(t *testing.T) {
|
||||
cases := []struct {
|
||||
slot primitives.Slot
|
||||
}{
|
||||
{slot: 0},
|
||||
{slot: 2},
|
||||
{slot: 1123581321},
|
||||
{slot: math.MaxUint64},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(fmt.Sprintf("slot %d", c.slot), func(t *testing.T) {
|
||||
fs, bs := NewEphemeralBlobStorageAndFs(t)
|
||||
_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, c.slot, 1)
|
||||
sc := verification.FakeVerifyForTest(t, sidecars[0])
|
||||
require.NoError(t, bs.Save(sc))
|
||||
namer := identForSidecar(sc)
|
||||
sszPath := bs.layout.sszPath(namer)
|
||||
slot, err := slotFromFile(sszPath, fs)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, c.slot, slot)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type dirFiles struct {
|
||||
name string
|
||||
isDir bool
|
||||
children []dirFiles
|
||||
}
|
||||
|
||||
func (df dirFiles) reify(t *testing.T, fs afero.Fs, base string) {
|
||||
fullPath := path.Join(base, df.name)
|
||||
if df.isDir {
|
||||
if df.name != "" {
|
||||
require.NoError(t, fs.Mkdir(fullPath, directoryPermissions()))
|
||||
}
|
||||
for _, c := range df.children {
|
||||
c.reify(t, fs, fullPath)
|
||||
}
|
||||
} else {
|
||||
fp, err := fs.Create(fullPath)
|
||||
require.NoError(t, err)
|
||||
_, err = fp.WriteString("derp")
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (df dirFiles) childNames() []string {
|
||||
cn := make([]string, len(df.children))
|
||||
for i := range df.children {
|
||||
cn[i] = df.children[i].name
|
||||
}
|
||||
return cn
|
||||
}
|
||||
|
||||
func TestListDir(t *testing.T) {
|
||||
fs := afero.NewMemMapFs()
|
||||
rootStrs := []string{
|
||||
"0x0023dc5d063c7c1b37016bb54963c6ff4bfe5dfdf6dac29e7ceeb2b8fa81ed7a",
|
||||
"0xff30526cd634a5af3a09cc9bff67f33a621fc5b975750bb4432f74df077554b4",
|
||||
"0x23f5f795aaeb78c01fadaf3d06da2e99bd4b3622ae4dfea61b05b7d9adb119c2",
|
||||
}
|
||||
|
||||
// parent directory
|
||||
tree := dirFiles{isDir: true}
|
||||
// break out each subdir for easier assertions
|
||||
notABlob := dirFiles{name: "notABlob", isDir: true}
|
||||
childlessBlob := dirFiles{name: rootStrs[0], isDir: true}
|
||||
blobWithSsz := dirFiles{name: rootStrs[1], isDir: true,
|
||||
children: []dirFiles{{name: "1.ssz"}, {name: "2.ssz"}},
|
||||
}
|
||||
blobWithSszAndTmp := dirFiles{name: rootStrs[2], isDir: true,
|
||||
children: []dirFiles{{name: "5.ssz"}, {name: "0.part"}}}
|
||||
tree.children = append(tree.children,
|
||||
notABlob, childlessBlob, blobWithSsz, blobWithSszAndTmp)
|
||||
|
||||
topChildren := make([]string, len(tree.children))
|
||||
for i := range tree.children {
|
||||
topChildren[i] = tree.children[i].name
|
||||
}
|
||||
|
||||
var filter = func(entries []string, filt func(string) bool) []string {
|
||||
filtered := make([]string, 0, len(entries))
|
||||
for i := range entries {
|
||||
if filt(entries[i]) {
|
||||
filtered = append(filtered, entries[i])
|
||||
}
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
tree.reify(t, fs, "")
|
||||
cases := []struct {
|
||||
name string
|
||||
dirPath string
|
||||
expected []string
|
||||
filter func(string) bool
|
||||
err error
|
||||
}{
|
||||
{
|
||||
name: "non-existent",
|
||||
dirPath: "derp",
|
||||
expected: []string{},
|
||||
err: os.ErrNotExist,
|
||||
},
|
||||
{
|
||||
name: "empty",
|
||||
dirPath: childlessBlob.name,
|
||||
expected: []string{},
|
||||
},
|
||||
{
|
||||
name: "top",
|
||||
dirPath: ".",
|
||||
expected: topChildren,
|
||||
},
|
||||
{
|
||||
name: "custom filter: only notABlob",
|
||||
dirPath: ".",
|
||||
expected: []string{notABlob.name},
|
||||
filter: func(s string) bool {
|
||||
return s == notABlob.name
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "root filter",
|
||||
dirPath: ".",
|
||||
expected: []string{childlessBlob.name, blobWithSsz.name, blobWithSszAndTmp.name},
|
||||
filter: isRootDir,
|
||||
},
|
||||
{
|
||||
name: "ssz filter",
|
||||
dirPath: blobWithSsz.name,
|
||||
expected: blobWithSsz.childNames(),
|
||||
filter: isSszFile,
|
||||
},
|
||||
{
|
||||
name: "ssz mixed filter",
|
||||
dirPath: blobWithSszAndTmp.name,
|
||||
expected: []string{"5.ssz"},
|
||||
filter: isSszFile,
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
result, err := listDir(fs, c.dirPath)
|
||||
if c.filter != nil {
|
||||
result = filter(result, c.filter)
|
||||
}
|
||||
if c.err != nil {
|
||||
require.ErrorIs(t, err, c.err)
|
||||
require.Equal(t, 0, len(result))
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
sort.Strings(c.expected)
|
||||
sort.Strings(result)
|
||||
require.DeepEqual(t, c.expected, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlotFromBlob(t *testing.T) {
|
||||
cases := []struct {
|
||||
slot primitives.Slot
|
||||
}{
|
||||
{slot: 0},
|
||||
{slot: 2},
|
||||
{slot: 1123581321},
|
||||
{slot: math.MaxUint64},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(fmt.Sprintf("slot %d", c.slot), func(t *testing.T) {
|
||||
_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, c.slot, 1)
|
||||
sc := sidecars[0]
|
||||
enc, err := sc.MarshalSSZ()
|
||||
require.NoError(t, err)
|
||||
slot, err := slotFromBlob(bytes.NewReader(enc))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, c.slot, slot)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIterationComplete(t *testing.T) {
|
||||
targets := []migrationTestTarget{
|
||||
{
|
||||
ident: ezIdent(t, "0x0125e54c64c925018c9296965a5b622d9f5ab626c10917860dcfb6aa09a0a00b", 1234, 0),
|
||||
path: "by-epoch/0/1234/0x0125e54c64c925018c9296965a5b622d9f5ab626c10917860dcfb6aa09a0a00b/0.ssz",
|
||||
},
|
||||
{
|
||||
ident: ezIdent(t, "0x0127dba6fd30fdbb47e73e861d5c6e602b38ac3ddc945bb6a2fc4e10761e9a86", 5330, 0),
|
||||
slotOffset: 31,
|
||||
path: "by-epoch/1/5330/0x0127dba6fd30fdbb47e73e861d5c6e602b38ac3ddc945bb6a2fc4e10761e9a86/0.ssz",
|
||||
},
|
||||
{
|
||||
ident: ezIdent(t, "0x0127dba6fd30fdbb47e73e861d5c6e602b38ac3ddc945bb6a2fc4e10761e9a86", 5330, 1),
|
||||
slotOffset: 31,
|
||||
path: "by-epoch/1/5330/0x0127dba6fd30fdbb47e73e861d5c6e602b38ac3ddc945bb6a2fc4e10761e9a86/1.ssz",
|
||||
},
|
||||
{
|
||||
ident: ezIdent(t, "0x0232521756a0b965eab2c2245d7ad85feaeaf5f427cd14d1a7531f9d555b415c", 16777216, 0),
|
||||
slotOffset: 16,
|
||||
path: "by-epoch/4096/16777216/0x0232521756a0b965eab2c2245d7ad85feaeaf5f427cd14d1a7531f9d555b415c/0.ssz",
|
||||
},
|
||||
{
|
||||
ident: ezIdent(t, "0x0232521756a0b965eab2c2245d7ad85feaeaf5f427cd14d1a7531f9d555b415c", 16777216, 1),
|
||||
slotOffset: 16,
|
||||
path: "by-epoch/4096/16777216/0x0232521756a0b965eab2c2245d7ad85feaeaf5f427cd14d1a7531f9d555b415c/1.ssz",
|
||||
},
|
||||
{
|
||||
ident: ezIdent(t, "0x42eabe3d2c125410cd226de6f2825fb7575ab896c3f52e43de1fa29e4c809aba", 16777217, 0),
|
||||
slotOffset: 16,
|
||||
path: "by-epoch/4096/16777217/0x42eabe3d2c125410cd226de6f2825fb7575ab896c3f52e43de1fa29e4c809aba/0.ssz",
|
||||
},
|
||||
{
|
||||
ident: ezIdent(t, "0x666cea5034e22bd3b849cb33914cad59afd88ee08e4d5bc0e997411c945fbc1d", 11235, 1),
|
||||
path: "by-epoch/2/11235/0x666cea5034e22bd3b849cb33914cad59afd88ee08e4d5bc0e997411c945fbc1d/1.ssz",
|
||||
},
|
||||
}
|
||||
fs := afero.NewMemMapFs()
|
||||
cache := newBlobStorageCache()
|
||||
byEpoch, err := newLayout(LayoutNameByEpoch, fs, cache, nil)
|
||||
require.NoError(t, err)
|
||||
for _, tar := range targets {
|
||||
setupTestBlobFile(t, tar.ident, tar.slotOffset, fs, byEpoch)
|
||||
}
|
||||
iter, err := byEpoch.iterateIdents(0)
|
||||
require.NoError(t, err)
|
||||
nIdents := 0
|
||||
for ident, err := iter.next(); err != io.EOF; ident, err = iter.next() {
|
||||
require.NoError(t, err)
|
||||
nIdents++
|
||||
require.NoError(t, cache.ensure(ident))
|
||||
}
|
||||
require.Equal(t, len(targets), nIdents)
|
||||
for _, tar := range targets {
|
||||
entry, ok := cache.get(tar.ident.root)
|
||||
require.Equal(t, true, ok)
|
||||
require.Equal(t, tar.ident.epoch, entry.epoch)
|
||||
require.Equal(t, true, entry.HasIndex(tar.ident.index))
|
||||
require.Equal(t, tar.path, byEpoch.sszPath(tar.ident))
|
||||
}
|
||||
}
|
||||
291
beacon-chain/db/filesystem/layout.go
Normal file
291
beacon-chain/db/filesystem/layout.go
Normal file
@@ -0,0 +1,291 @@
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/blocks"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
"github.com/prysmaticlabs/prysm/v5/time/slots"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
const (
|
||||
// Full root in directory will be 66 chars, eg:
|
||||
// >>> len('0x0002fb4db510b8618b04dc82d023793739c26346a8b02eb73482e24b0fec0555') == 66
|
||||
rootStringLen = 66
|
||||
sszExt = "ssz"
|
||||
partExt = "part"
|
||||
periodicEpochBaseDir = "by-epoch"
|
||||
)
|
||||
|
||||
const (
|
||||
LayoutNameFlat = "flat"
|
||||
LayoutNameByEpoch = "by-epoch"
|
||||
)
|
||||
|
||||
var LayoutNames = []string{LayoutNameFlat, LayoutNameByEpoch}
|
||||
|
||||
var (
|
||||
errMigrationFailure = errors.New("unable to migrate blob directory between old and new layout")
|
||||
errCacheWarmFailed = errors.New("failed to warm blob filesystem cache")
|
||||
errPruneFailed = errors.New("failed to prune root")
|
||||
errInvalidRootString = errors.New("Could not parse hex string as a [32]byte")
|
||||
errInvalidDirectoryLayout = errors.New("Could not parse blob directory path")
|
||||
errInvalidLayoutName = errors.New("unknown layout name")
|
||||
errLayoutNotDetected = errors.New("given layout not observed in the blob filesystem tree")
|
||||
)
|
||||
|
||||
type blobIdent struct {
|
||||
root [32]byte
|
||||
epoch primitives.Epoch
|
||||
index uint64
|
||||
}
|
||||
|
||||
func newBlobIdent(root [32]byte, epoch primitives.Epoch, index uint64) blobIdent {
|
||||
return blobIdent{root: root, epoch: epoch, index: index}
|
||||
}
|
||||
|
||||
func identForSidecar(sc blocks.VerifiedROBlob) blobIdent {
|
||||
return newBlobIdent(sc.BlockRoot(), slots.ToEpoch(sc.Slot()), sc.Index)
|
||||
}
|
||||
|
||||
func (n blobIdent) sszFname() string {
|
||||
return fmt.Sprintf("%d.%s", n.index, sszExt)
|
||||
}
|
||||
|
||||
func (n blobIdent) partFname(entropy string) string {
|
||||
return fmt.Sprintf("%s-%d.%s", entropy, n.index, partExt)
|
||||
}
|
||||
|
||||
func (n blobIdent) logFields() logrus.Fields {
|
||||
return logrus.Fields{
|
||||
"root": fmt.Sprintf("%#x", n.root),
|
||||
"epoch": n.epoch,
|
||||
"index": n.index,
|
||||
}
|
||||
}
|
||||
|
||||
type fsLayout interface {
|
||||
name() string
|
||||
dir(n blobIdent) string
|
||||
sszPath(n blobIdent) string
|
||||
partPath(n blobIdent, entropy string) string
|
||||
iterateIdents(before primitives.Epoch) (*identIterator, error)
|
||||
ident(root [32]byte, idx uint64) (blobIdent, error)
|
||||
dirIdent(root [32]byte) (blobIdent, error)
|
||||
summary(root [32]byte) BlobStorageSummary
|
||||
notify(ident blobIdent) error
|
||||
pruneBefore(before primitives.Epoch) (*pruneSummary, error)
|
||||
remove(ident blobIdent) (int, error)
|
||||
blockParentDirs(ident blobIdent) []string
|
||||
}
|
||||
|
||||
func newLayout(name string, fs afero.Fs, cache *blobStorageSummaryCache, pruner *blobPruner) (fsLayout, error) {
|
||||
switch name {
|
||||
case LayoutNameFlat:
|
||||
return newFlatLayout(fs, cache, pruner), nil
|
||||
case LayoutNameByEpoch:
|
||||
return newPeriodicEpochLayout(fs, cache, pruner), nil
|
||||
default:
|
||||
return nil, errors.Wrapf(errInvalidLayoutName, "name=%s", name)
|
||||
}
|
||||
}
|
||||
|
||||
func warmCache(l fsLayout, cache *blobStorageSummaryCache) error {
|
||||
iter, err := l.iterateIdents(0)
|
||||
if err != nil {
|
||||
return errors.Wrap(errCacheWarmFailed, err.Error())
|
||||
}
|
||||
for ident, err := iter.next(); !errors.Is(err, io.EOF); ident, err = iter.next() {
|
||||
if errors.Is(err, errIdentFailure) {
|
||||
idf := &identificationError{}
|
||||
if errors.As(err, &idf) {
|
||||
log.WithFields(idf.LogFields()).WithError(err).Error("Failed to cache blob data for path")
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: failed to populate blob data cache: %w", errCacheWarmFailed, err)
|
||||
}
|
||||
if err := cache.ensure(ident); err != nil {
|
||||
return fmt.Errorf("%w: failed to write cache entry for %s: %w", errCacheWarmFailed, l.sszPath(ident), err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateLayout(fs afero.Fs, from, to fsLayout, cache *blobStorageSummaryCache) error {
|
||||
start := time.Now()
|
||||
iter, err := from.iterateIdents(0)
|
||||
if err != nil {
|
||||
return errors.Wrapf(errMigrationFailure, "failed to iterate legacy structure while migrating blobs, err=%s", err.Error())
|
||||
}
|
||||
if iter.atEOF() {
|
||||
return errLayoutNotDetected
|
||||
}
|
||||
log.WithField("fromLayout", from.name()).WithField("toLayout", to.name()).Info("Migrating blob filesystem layout. This one-time operation can take extra time (up to a few minutes for systems with extended blob storage and a cold disk cache).")
|
||||
lastMoved := ""
|
||||
parentDirs := make(map[string]bool) // this map should have < 65k keys by design
|
||||
moved := 0
|
||||
dc := newDirCleaner()
|
||||
for ident, err := iter.next(); !errors.Is(err, io.EOF); ident, err = iter.next() {
|
||||
if err != nil {
|
||||
if errors.Is(err, errIdentFailure) {
|
||||
idf := &identificationError{}
|
||||
if errors.As(err, &idf) {
|
||||
log.WithFields(idf.LogFields()).WithError(err).Error("Failed to migrate blob path")
|
||||
}
|
||||
continue
|
||||
}
|
||||
return errors.Wrapf(errMigrationFailure, "failed to iterate previous layout structure while migrating blobs, err=%s", err.Error())
|
||||
}
|
||||
src := from.dir(ident)
|
||||
target := to.dir(ident)
|
||||
if src != lastMoved {
|
||||
targetParent := filepath.Dir(target)
|
||||
if targetParent != "" && targetParent != "." && !parentDirs[targetParent] {
|
||||
if err := fs.MkdirAll(targetParent, directoryPermissions()); err != nil {
|
||||
return errors.Wrapf(errMigrationFailure, "failed to make enclosing path before moving %s to %s", src, target)
|
||||
}
|
||||
parentDirs[targetParent] = true
|
||||
}
|
||||
if err := fs.Rename(src, target); err != nil {
|
||||
return errors.Wrapf(errMigrationFailure, "could not rename %s to %s", src, target)
|
||||
}
|
||||
moved += 1
|
||||
lastMoved = src
|
||||
for _, dir := range from.blockParentDirs(ident) {
|
||||
dc.add(dir)
|
||||
}
|
||||
}
|
||||
if err := cache.ensure(ident); err != nil {
|
||||
return errors.Wrapf(errMigrationFailure, "could not cache path %s, err=%s", to.sszPath(ident), err.Error())
|
||||
}
|
||||
}
|
||||
dc.clean(fs)
|
||||
if moved > 0 {
|
||||
log.WithField("dirsMoved", moved).WithField("elapsed", time.Since(start)).
|
||||
Info("Blob filesystem migration complete.")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type dirCleaner struct {
|
||||
maxDepth int
|
||||
layers map[int]map[string]struct{}
|
||||
}
|
||||
|
||||
func newDirCleaner() *dirCleaner {
|
||||
return &dirCleaner{layers: make(map[int]map[string]struct{})}
|
||||
}
|
||||
|
||||
func (d *dirCleaner) add(dir string) {
|
||||
nLayers := len(strings.Split(dir, string(filepath.Separator)))
|
||||
_, ok := d.layers[nLayers]
|
||||
if !ok {
|
||||
d.layers[nLayers] = make(map[string]struct{})
|
||||
}
|
||||
d.layers[nLayers][dir] = struct{}{}
|
||||
if nLayers > d.maxDepth {
|
||||
d.maxDepth = nLayers
|
||||
}
|
||||
}
|
||||
|
||||
func (d *dirCleaner) clean(fs afero.Fs) {
|
||||
for i := d.maxDepth; i >= 0; i-- {
|
||||
d.cleanLayer(fs, i)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *dirCleaner) cleanLayer(fs afero.Fs, layer int) {
|
||||
dirs, ok := d.layers[layer]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
for dir := range dirs {
|
||||
// Use Remove rather than RemoveAll to make sure we're only removing empty directories
|
||||
if err := fs.Remove(dir); err != nil {
|
||||
log.WithField("dir", dir).WithError(err).Error("Failed to remove blob directory, please remove it manually if desired.")
|
||||
contents, err := listDir(fs, dir)
|
||||
if err != nil {
|
||||
log.WithField("dir", dir).WithError(err).Error("Could not list blob directory contents to find reason for removal failure.")
|
||||
continue
|
||||
}
|
||||
for _, c := range contents {
|
||||
log.WithField("file", c).WithField("dir", dir).Debug("Unexpected file blocking migrated blob directory cleanup.")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type pruneSummary struct {
|
||||
blobsPruned int
|
||||
failedRemovals []string
|
||||
}
|
||||
|
||||
func (s pruneSummary) LogFields() logrus.Fields {
|
||||
return logrus.Fields{
|
||||
"blobsPruned": s.blobsPruned,
|
||||
"failedRemovals": len(s.failedRemovals),
|
||||
}
|
||||
}
|
||||
|
||||
func pruneBefore(before primitives.Epoch, l fsLayout) (map[primitives.Epoch]*pruneSummary, error) {
|
||||
sums := make(map[primitives.Epoch]*pruneSummary)
|
||||
iter, err := l.iterateIdents(before)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to iterate blob paths for pruning")
|
||||
}
|
||||
|
||||
// We will get an ident for each index, but want to prune all indexes for the given root together.
|
||||
var lastIdent blobIdent
|
||||
for ident, err := iter.next(); !errors.Is(err, io.EOF); ident, err = iter.next() {
|
||||
if err != nil {
|
||||
if errors.Is(err, errIdentFailure) {
|
||||
idf := &identificationError{}
|
||||
if errors.As(err, &idf) {
|
||||
log.WithFields(idf.LogFields()).WithError(err).Error("Failed to prune blob path due to identification errors")
|
||||
}
|
||||
continue
|
||||
}
|
||||
log.WithError(err).Error("encountered unhandled error during pruning")
|
||||
return nil, errors.Wrap(errPruneFailed, err.Error())
|
||||
}
|
||||
if ident.epoch >= before {
|
||||
continue
|
||||
}
|
||||
if lastIdent.root != ident.root {
|
||||
pruneOne(lastIdent, l, sums)
|
||||
lastIdent = ident
|
||||
}
|
||||
}
|
||||
// handle the final ident
|
||||
pruneOne(lastIdent, l, sums)
|
||||
|
||||
return sums, nil
|
||||
}
|
||||
|
||||
func pruneOne(ident blobIdent, l fsLayout, sums map[primitives.Epoch]*pruneSummary) {
|
||||
// Skip pruning the n-1 ident if we're on the first real ident (lastIdent will be zero value).
|
||||
if ident.root == params.BeaconConfig().ZeroHash {
|
||||
return
|
||||
}
|
||||
_, ok := sums[ident.epoch]
|
||||
if !ok {
|
||||
sums[ident.epoch] = &pruneSummary{}
|
||||
}
|
||||
s := sums[ident.epoch]
|
||||
removed, err := l.remove(ident)
|
||||
if err != nil {
|
||||
s.failedRemovals = append(s.failedRemovals, l.dir(ident))
|
||||
log.WithField("root", fmt.Sprintf("%#x", ident.root)).Error("Failed to delete blob directory for root")
|
||||
}
|
||||
s.blobsPruned += removed
|
||||
}
|
||||
212
beacon-chain/db/filesystem/layout_by_epoch.go
Normal file
212
beacon-chain/db/filesystem/layout_by_epoch.go
Normal file
@@ -0,0 +1,212 @@
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
const epochsPerDirectory = 4096
|
||||
|
||||
type periodicEpochLayout struct {
|
||||
fs afero.Fs
|
||||
cache *blobStorageSummaryCache
|
||||
pruner *blobPruner
|
||||
}
|
||||
|
||||
var _ fsLayout = &periodicEpochLayout{}
|
||||
|
||||
func newPeriodicEpochLayout(fs afero.Fs, cache *blobStorageSummaryCache, pruner *blobPruner) fsLayout {
|
||||
l := &periodicEpochLayout{fs: fs, cache: cache, pruner: pruner}
|
||||
return l
|
||||
}
|
||||
|
||||
func (l *periodicEpochLayout) name() string {
|
||||
return LayoutNameByEpoch
|
||||
}
|
||||
|
||||
func (l *periodicEpochLayout) blockParentDirs(ident blobIdent) []string {
|
||||
return []string{
|
||||
periodicEpochBaseDir,
|
||||
l.periodDir(ident.epoch),
|
||||
l.epochDir(ident.epoch),
|
||||
}
|
||||
}
|
||||
|
||||
func (l *periodicEpochLayout) notify(ident blobIdent) error {
|
||||
if err := l.cache.ensure(ident); err != nil {
|
||||
return err
|
||||
}
|
||||
l.pruner.notify(ident.epoch, l)
|
||||
return nil
|
||||
}
|
||||
|
||||
// If before == 0, it won't be used as a filter and all idents will be returned.
|
||||
func (l *periodicEpochLayout) iterateIdents(before primitives.Epoch) (*identIterator, error) {
|
||||
_, err := l.fs.Stat(periodicEpochBaseDir)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return &identIterator{eof: true}, nil // The directory is non-existent, which is fine; stop iteration.
|
||||
}
|
||||
return nil, errors.Wrapf(err, "error reading path %s", periodicEpochBaseDir)
|
||||
}
|
||||
// iterate root, which should have directories named by "period"
|
||||
entries, err := listDir(l.fs, periodicEpochBaseDir)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to list %s", periodicEpochBaseDir)
|
||||
}
|
||||
|
||||
return &identIterator{
|
||||
fs: l.fs,
|
||||
path: periodicEpochBaseDir,
|
||||
// Please see comments on the `layers` field in `identIterator`` if the role of the layers is unclear.
|
||||
layers: []layoutLayer{
|
||||
{populateIdent: populateNoop, filter: isBeforePeriod(before)},
|
||||
{populateIdent: populateEpoch, filter: isBeforeEpoch(before)},
|
||||
{populateIdent: populateRoot, filter: isRootDir}, // extract root from path
|
||||
{populateIdent: populateIndex, filter: isSszFile}, // extract index from filename
|
||||
},
|
||||
entries: entries,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (l *periodicEpochLayout) ident(root [32]byte, idx uint64) (blobIdent, error) {
|
||||
return l.cache.identForIdx(root, idx)
|
||||
}
|
||||
|
||||
func (l *periodicEpochLayout) dirIdent(root [32]byte) (blobIdent, error) {
|
||||
return l.cache.identForRoot(root)
|
||||
}
|
||||
|
||||
func (l *periodicEpochLayout) summary(root [32]byte) BlobStorageSummary {
|
||||
return l.cache.Summary(root)
|
||||
}
|
||||
|
||||
func (l *periodicEpochLayout) dir(n blobIdent) string {
|
||||
return filepath.Join(l.epochDir(n.epoch), rootToString(n.root))
|
||||
}
|
||||
|
||||
func (l *periodicEpochLayout) epochDir(epoch primitives.Epoch) string {
|
||||
return filepath.Join(l.periodDir(epoch), fmt.Sprintf("%d", epoch))
|
||||
}
|
||||
|
||||
func (l *periodicEpochLayout) periodDir(epoch primitives.Epoch) string {
|
||||
return filepath.Join(periodicEpochBaseDir, fmt.Sprintf("%d", periodForEpoch(epoch)))
|
||||
}
|
||||
|
||||
func (l *periodicEpochLayout) sszPath(n blobIdent) string {
|
||||
return filepath.Join(l.dir(n), n.sszFname())
|
||||
}
|
||||
|
||||
func (l *periodicEpochLayout) partPath(n blobIdent, entropy string) string {
|
||||
return path.Join(l.dir(n), n.partFname(entropy))
|
||||
}
|
||||
|
||||
func (l *periodicEpochLayout) pruneBefore(before primitives.Epoch) (*pruneSummary, error) {
|
||||
sums, err := pruneBefore(before, l)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Roll up summaries and clean up per-epoch directories.
|
||||
rollup := &pruneSummary{}
|
||||
for epoch, sum := range sums {
|
||||
rollup.blobsPruned += sum.blobsPruned
|
||||
rollup.failedRemovals = append(rollup.failedRemovals, sum.failedRemovals...)
|
||||
rmdir := l.epochDir(epoch)
|
||||
if len(sum.failedRemovals) == 0 {
|
||||
if err := l.fs.Remove(rmdir); err != nil {
|
||||
log.WithField("dir", rmdir).WithError(err).Error("Failed to remove epoch directory while pruning")
|
||||
}
|
||||
} else {
|
||||
log.WithField("dir", rmdir).WithField("numFailed", len(sum.failedRemovals)).WithError(err).Error("Unable to remove epoch directory due to pruning failures")
|
||||
}
|
||||
}
|
||||
|
||||
return rollup, nil
|
||||
}
|
||||
|
||||
func (l *periodicEpochLayout) remove(ident blobIdent) (int, error) {
|
||||
removed := l.cache.evict(ident.root)
|
||||
// Skip the syscall if there are no blobs to remove.
|
||||
if removed == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if err := l.fs.RemoveAll(l.dir(ident)); err != nil {
|
||||
return removed, err
|
||||
}
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
func periodForEpoch(epoch primitives.Epoch) primitives.Epoch {
|
||||
return epoch / params.BeaconConfig().MinEpochsForBlobsSidecarsRequest
|
||||
}
|
||||
|
||||
// Funcs below this line are iteration support methods that are specific to the epoch layout.
|
||||
|
||||
func isBeforePeriod(before primitives.Epoch) func(string) bool {
|
||||
if before == 0 {
|
||||
return filterNoop
|
||||
}
|
||||
beforePeriod := periodForEpoch(before)
|
||||
if before%epochsPerDirectory != 0 {
|
||||
// Add one because we need to include the period the epoch is in, unless it is the first epoch in the period,
|
||||
// in which case we can just look at any previous period.
|
||||
beforePeriod += 1
|
||||
}
|
||||
return func(p string) bool {
|
||||
period, err := periodFromPath(p)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return primitives.Epoch(period) < beforePeriod
|
||||
}
|
||||
}
|
||||
|
||||
func isBeforeEpoch(before primitives.Epoch) func(string) bool {
|
||||
if before == 0 {
|
||||
return filterNoop
|
||||
}
|
||||
return func(p string) bool {
|
||||
epoch, err := epochFromPath(p)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return epoch < before
|
||||
}
|
||||
}
|
||||
|
||||
func epochFromPath(p string) (primitives.Epoch, error) {
|
||||
subdir := filepath.Base(p)
|
||||
epoch, err := strconv.ParseUint(subdir, 10, 64)
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(errInvalidDirectoryLayout,
|
||||
"failed to decode epoch as uint, err=%s, dir=%s", err.Error(), p)
|
||||
}
|
||||
return primitives.Epoch(epoch), nil
|
||||
}
|
||||
|
||||
func periodFromPath(p string) (uint64, error) {
|
||||
subdir := filepath.Base(p)
|
||||
period, err := strconv.ParseUint(subdir, 10, 64)
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(errInvalidDirectoryLayout,
|
||||
"failed to decode period from path as uint, err=%s, dir=%s", err.Error(), p)
|
||||
}
|
||||
return period, nil
|
||||
}
|
||||
|
||||
func populateEpoch(namer blobIdent, dir string) (blobIdent, error) {
|
||||
epoch, err := epochFromPath(dir)
|
||||
if err != nil {
|
||||
return namer, err
|
||||
}
|
||||
namer.epoch = epoch
|
||||
return namer, nil
|
||||
}
|
||||
219
beacon-chain/db/filesystem/layout_flat.go
Normal file
219
beacon-chain/db/filesystem/layout_flat.go
Normal file
@@ -0,0 +1,219 @@
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
"github.com/prysmaticlabs/prysm/v5/time/slots"
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
type flatLayout struct {
|
||||
fs afero.Fs
|
||||
cache *blobStorageSummaryCache
|
||||
pruner *blobPruner
|
||||
}
|
||||
|
||||
var _ fsLayout = &flatLayout{}
|
||||
|
||||
func newFlatLayout(fs afero.Fs, cache *blobStorageSummaryCache, pruner *blobPruner) fsLayout {
|
||||
l := &flatLayout{fs: fs, cache: cache, pruner: pruner}
|
||||
return l
|
||||
}
|
||||
|
||||
func (l *flatLayout) iterateIdents(before primitives.Epoch) (*identIterator, error) {
|
||||
if _, err := l.fs.Stat("."); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return &identIterator{eof: true}, nil // The directory is non-existent, which is fine; stop iteration.
|
||||
}
|
||||
return nil, errors.Wrapf(err, "error reading path %s", periodicEpochBaseDir)
|
||||
}
|
||||
entries, err := listDir(l.fs, ".")
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "could not list root directory")
|
||||
}
|
||||
slotAndIndex := &flatSlotReader{fs: l.fs, cache: l.cache, before: before}
|
||||
return &identIterator{
|
||||
fs: l.fs,
|
||||
// Please see comments on the `layers` field in `identIterator`` if the role of the layers is unclear.
|
||||
layers: []layoutLayer{
|
||||
{populateIdent: populateRoot, filter: isFlatCachedAndBefore(l.cache, before)},
|
||||
{populateIdent: slotAndIndex.populateEpoch, filter: slotAndIndex.isSSZAndBefore}},
|
||||
entries: entries,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (*flatLayout) name() string {
|
||||
return LayoutNameFlat
|
||||
}
|
||||
|
||||
func (l *flatLayout) blockParentDirs(ident blobIdent) []string {
|
||||
return []string{}
|
||||
}
|
||||
|
||||
func (*flatLayout) dir(n blobIdent) string {
|
||||
return rootToString(n.root)
|
||||
}
|
||||
|
||||
func (l *flatLayout) sszPath(n blobIdent) string {
|
||||
return path.Join(l.dir(n), n.sszFname())
|
||||
}
|
||||
|
||||
func (l *flatLayout) partPath(n blobIdent, entropy string) string {
|
||||
return path.Join(l.dir(n), n.partFname(entropy))
|
||||
}
|
||||
|
||||
func (l *flatLayout) ident(root [32]byte, idx uint64) (blobIdent, error) {
|
||||
return l.cache.identForIdx(root, idx)
|
||||
}
|
||||
|
||||
func (l *flatLayout) dirIdent(root [32]byte) (blobIdent, error) {
|
||||
return l.cache.identForRoot(root)
|
||||
}
|
||||
|
||||
func (l *flatLayout) summary(root [32]byte) BlobStorageSummary {
|
||||
return l.cache.Summary(root)
|
||||
}
|
||||
|
||||
func (l *flatLayout) remove(ident blobIdent) (int, error) {
|
||||
removed := l.cache.evict(ident.root)
|
||||
if err := l.fs.RemoveAll(l.dir(ident)); err != nil {
|
||||
return removed, err
|
||||
}
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
func (l *flatLayout) notify(ident blobIdent) error {
|
||||
if err := l.cache.ensure(ident); err != nil {
|
||||
return err
|
||||
}
|
||||
l.pruner.notify(ident.epoch, l)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *flatLayout) pruneBefore(before primitives.Epoch) (*pruneSummary, error) {
|
||||
sums, err := pruneBefore(before, l)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Roll up summaries and clean up per-epoch directories.
|
||||
rollup := &pruneSummary{}
|
||||
for _, sum := range sums {
|
||||
rollup.blobsPruned += sum.blobsPruned
|
||||
rollup.failedRemovals = append(rollup.failedRemovals, sum.failedRemovals...)
|
||||
}
|
||||
|
||||
return rollup, nil
|
||||
}
|
||||
|
||||
// Below this line are iteration support funcs and types that are specific to the flat layout.
|
||||
|
||||
// Read slot from marshaled BlobSidecar data in the given file. See slotFromBlob for details.
|
||||
func slotFromFile(name string, fs afero.Fs) (primitives.Slot, error) {
|
||||
f, err := fs.Open(name)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() {
|
||||
if err := f.Close(); err != nil {
|
||||
log.WithError(err).Errorf("Could not close blob file")
|
||||
}
|
||||
}()
|
||||
return slotFromBlob(f)
|
||||
}
|
||||
|
||||
// slotFromBlob reads the ssz data of a file at the specified offset (8 + 131072 + 48 + 48 = 131176 bytes),
|
||||
// which is calculated based on the size of the BlobSidecar struct and is based on the size of the fields
|
||||
// preceding the slot information within SignedBeaconBlockHeader.
|
||||
func slotFromBlob(at io.ReaderAt) (primitives.Slot, error) {
|
||||
b := make([]byte, 8)
|
||||
_, err := at.ReadAt(b, 131176)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rawSlot := binary.LittleEndian.Uint64(b)
|
||||
return primitives.Slot(rawSlot), nil
|
||||
}
|
||||
|
||||
type flatSlotReader struct {
|
||||
before primitives.Epoch
|
||||
fs afero.Fs
|
||||
cache *blobStorageSummaryCache
|
||||
}
|
||||
|
||||
func (l *flatSlotReader) populateEpoch(ident blobIdent, fname string) (blobIdent, error) {
|
||||
ident, err := populateIndex(ident, fname)
|
||||
if err != nil {
|
||||
return ident, err
|
||||
}
|
||||
sum, ok := l.cache.get(ident.root)
|
||||
if ok {
|
||||
ident.epoch = sum.epoch
|
||||
// Return early if the index is already known to the cache.
|
||||
if sum.HasIndex(ident.index) {
|
||||
return ident, nil
|
||||
}
|
||||
} else {
|
||||
// If the root is not in the cache, we need to read the slot from the file.
|
||||
slot, err := slotFromFile(fname, l.fs)
|
||||
if err != nil {
|
||||
return ident, err
|
||||
}
|
||||
ident.epoch = slots.ToEpoch(slot)
|
||||
}
|
||||
return ident, l.cache.ensure(ident)
|
||||
}
|
||||
|
||||
func (l *flatSlotReader) isSSZAndBefore(fname string) bool {
|
||||
if !isSszFile(fname) {
|
||||
return false
|
||||
}
|
||||
// If 'before' != 0, assuming isSSZAndBefore is used as a filter on the same layer with populateEpoch, this will typically
|
||||
// call populateEpoch before the iteration code calls it. So we can guarantee that the cache gets populated
|
||||
// in either case, because if it is filtered out here, we either have a malformed path (root can't be determined) in which case
|
||||
// the filter code won't call it anyway, or we have a valid path and the cache will be populated before the epoch can be compared.
|
||||
if l.before == 0 {
|
||||
return true
|
||||
}
|
||||
ident, err := populateRoot(blobIdent{}, path.Dir(fname))
|
||||
// Filter out the path if we can't determine its root.
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
ident, err = l.populateEpoch(ident, fname)
|
||||
// Filter out the path if we can't determine its epoch or properly cache it.
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return ident.epoch < l.before
|
||||
}
|
||||
|
||||
// isFlatCachedAndBefore returns a filter callback function to exclude roots that are known to be after the given epoch
|
||||
// based on the cache. It's an opportunistic filter; if the cache is not populated, it will not attempt to populate it.
|
||||
// isSSZAndBefore on the other hand, is a strict filter that will only return true if the file is an SSZ file and
|
||||
// the epoch can be determined.
|
||||
func isFlatCachedAndBefore(cache *blobStorageSummaryCache, before primitives.Epoch) func(string) bool {
|
||||
if before == 0 {
|
||||
return isRootDir
|
||||
}
|
||||
return func(p string) bool {
|
||||
if !isRootDir(p) {
|
||||
return false
|
||||
}
|
||||
root, err := rootFromPath(p)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
sum, ok := cache.get(root)
|
||||
// If we don't know the epoch by looking at the root, don't try to filter it.
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
return sum.epoch < before
|
||||
}
|
||||
}
|
||||
75
beacon-chain/db/filesystem/layout_test.go
Normal file
75
beacon-chain/db/filesystem/layout_test.go
Normal file
@@ -0,0 +1,75 @@
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/require"
|
||||
)
|
||||
|
||||
type mockLayout struct {
|
||||
pruneBeforeFunc func(primitives.Epoch) (*pruneSummary, error)
|
||||
}
|
||||
|
||||
var _ fsLayout = &mockLayout{}
|
||||
|
||||
func (m *mockLayout) name() string {
|
||||
return "mock"
|
||||
}
|
||||
|
||||
func (*mockLayout) dir(_ blobIdent) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (*mockLayout) blockParentDirs(id blobIdent) []string {
|
||||
return []string{}
|
||||
}
|
||||
|
||||
func (*mockLayout) sszPath(_ blobIdent) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (*mockLayout) partPath(_ blobIdent, _ string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (*mockLayout) iterateIdents(_ primitives.Epoch) (*identIterator, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (*mockLayout) ident(_ [32]byte, _ uint64) (blobIdent, error) {
|
||||
return blobIdent{}, nil
|
||||
}
|
||||
|
||||
func (*mockLayout) dirIdent(_ [32]byte) (blobIdent, error) {
|
||||
return blobIdent{}, nil
|
||||
}
|
||||
|
||||
func (*mockLayout) summary(_ [32]byte) BlobStorageSummary {
|
||||
return BlobStorageSummary{}
|
||||
}
|
||||
|
||||
func (*mockLayout) notify(blobIdent) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockLayout) pruneBefore(before primitives.Epoch) (*pruneSummary, error) {
|
||||
return m.pruneBeforeFunc(before)
|
||||
}
|
||||
|
||||
func (*mockLayout) remove(ident blobIdent) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var _ fsLayout = &mockLayout{}
|
||||
|
||||
func TestCleaner(t *testing.T) {
|
||||
l := &periodicEpochLayout{}
|
||||
p := l.periodDir(11235813)
|
||||
e := l.epochDir(11235813)
|
||||
dc := newDirCleaner()
|
||||
dc.add(p)
|
||||
require.Equal(t, 2, dc.maxDepth)
|
||||
dc.add(e)
|
||||
require.Equal(t, 3, dc.maxDepth)
|
||||
}
|
||||
180
beacon-chain/db/filesystem/migration_test.go
Normal file
180
beacon-chain/db/filesystem/migration_test.go
Normal file
@@ -0,0 +1,180 @@
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/require"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/util"
|
||||
"github.com/prysmaticlabs/prysm/v5/time/slots"
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
func ezIdent(t *testing.T, rootStr string, epoch primitives.Epoch, index uint64) blobIdent {
|
||||
r, err := stringToRoot(rootStr)
|
||||
require.NoError(t, err)
|
||||
return blobIdent{root: r, epoch: epoch, index: index}
|
||||
}
|
||||
|
||||
func setupTestBlobFile(t *testing.T, ident blobIdent, offset primitives.Slot, fs afero.Fs, l fsLayout) {
|
||||
slot, err := slots.EpochStart(ident.epoch)
|
||||
require.NoError(t, err)
|
||||
slot += offset
|
||||
_, sc := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, slot, 1)
|
||||
scb, err := sc[0].MarshalSSZ()
|
||||
require.NoError(t, err)
|
||||
dir := l.dir(ident)
|
||||
require.NoError(t, fs.MkdirAll(dir, directoryPermissions()))
|
||||
p := l.sszPath(ident)
|
||||
require.NoError(t, afero.WriteFile(fs, p, scb, 0666))
|
||||
_, err = fs.Stat(p)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
type migrationTestTarget struct {
|
||||
ident blobIdent
|
||||
slotOffset primitives.Slot
|
||||
migrated bool
|
||||
path string
|
||||
}
|
||||
|
||||
func testAssertFsMigrated(t *testing.T, fs afero.Fs, ident blobIdent, before, after fsLayout) {
|
||||
// Assert the pre-migration path is gone.
|
||||
_, err := fs.Stat(before.sszPath(ident))
|
||||
require.ErrorIs(t, err, os.ErrNotExist)
|
||||
dir := before.dir(ident)
|
||||
_, err = listDir(fs, dir)
|
||||
require.ErrorIs(t, err, os.ErrNotExist)
|
||||
|
||||
// Assert the post-migration path present.
|
||||
_, err = fs.Stat(after.sszPath(ident))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestMigrations(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
forwardLayout string
|
||||
backwardLayout string
|
||||
targets []migrationTestTarget
|
||||
}{
|
||||
{
|
||||
name: "all need migration",
|
||||
backwardLayout: LayoutNameFlat,
|
||||
forwardLayout: LayoutNameByEpoch,
|
||||
targets: []migrationTestTarget{
|
||||
{
|
||||
ident: ezIdent(t, "0x0125e54c64c925018c9296965a5b622d9f5ab626c10917860dcfb6aa09a0a00b", 1234, 0),
|
||||
},
|
||||
{
|
||||
ident: ezIdent(t, "0x0127dba6fd30fdbb47e73e861d5c6e602b38ac3ddc945bb6a2fc4e10761e9a86", 5330, 0),
|
||||
slotOffset: 31,
|
||||
},
|
||||
{
|
||||
ident: ezIdent(t, "0x0127dba6fd30fdbb47e73e861d5c6e602b38ac3ddc945bb6a2fc4e10761e9a86", 5330, 1),
|
||||
slotOffset: 31,
|
||||
},
|
||||
{
|
||||
ident: ezIdent(t, "0x0232521756a0b965eab2c2245d7ad85feaeaf5f427cd14d1a7531f9d555b415c", 16777216, 0),
|
||||
slotOffset: 16,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "mix old and new",
|
||||
backwardLayout: LayoutNameFlat,
|
||||
forwardLayout: LayoutNameByEpoch,
|
||||
targets: []migrationTestTarget{
|
||||
{
|
||||
ident: ezIdent(t, "0x0125e54c64c925018c9296965a5b622d9f5ab626c10917860dcfb6aa09a0a00b", 1234, 0),
|
||||
},
|
||||
{
|
||||
ident: ezIdent(t, "0x0127dba6fd30fdbb47e73e861d5c6e602b38ac3ddc945bb6a2fc4e10761e9a86", 5330, 0),
|
||||
slotOffset: 31,
|
||||
},
|
||||
{
|
||||
ident: ezIdent(t, "0x0127dba6fd30fdbb47e73e861d5c6e602b38ac3ddc945bb6a2fc4e10761e9a86", 5330, 1),
|
||||
slotOffset: 31,
|
||||
},
|
||||
{
|
||||
ident: ezIdent(t, "0x0232521756a0b965eab2c2245d7ad85feaeaf5f427cd14d1a7531f9d555b415c", 16777216, 0),
|
||||
slotOffset: 16,
|
||||
migrated: true,
|
||||
},
|
||||
{
|
||||
ident: ezIdent(t, "0x0232521756a0b965eab2c2245d7ad85feaeaf5f427cd14d1a7531f9d555b415c", 16777216, 1),
|
||||
slotOffset: 16,
|
||||
migrated: true,
|
||||
},
|
||||
{
|
||||
ident: ezIdent(t, "0x42eabe3d2c125410cd226de6f2825fb7575ab896c3f52e43de1fa29e4c809aba", 16777217, 0),
|
||||
slotOffset: 16,
|
||||
migrated: true,
|
||||
},
|
||||
{
|
||||
ident: ezIdent(t, "0x666cea5034e22bd3b849cb33914cad59afd88ee08e4d5bc0e997411c945fbc1d", 11235, 1),
|
||||
migrated: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
t.Run("forward", func(t *testing.T) {
|
||||
testMigration(t, c.forwardLayout, c.backwardLayout, c.targets)
|
||||
})
|
||||
// run the same test in reverse - to cover both directions while making the test table smaller.
|
||||
t.Run("backward", func(t *testing.T) {
|
||||
testMigration(t, c.forwardLayout, c.backwardLayout, c.targets)
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func testMigration(t *testing.T, forwardName, backwardName string, targets []migrationTestTarget) {
|
||||
fs := afero.NewMemMapFs()
|
||||
cache := newBlobStorageCache()
|
||||
forward, err := newLayout(forwardName, fs, cache, nil)
|
||||
require.NoError(t, err)
|
||||
backward, err := newLayout(backwardName, fs, cache, nil)
|
||||
require.NoError(t, err)
|
||||
for _, tar := range targets {
|
||||
if tar.migrated {
|
||||
setupTestBlobFile(t, tar.ident, tar.slotOffset, fs, forward)
|
||||
} else {
|
||||
setupTestBlobFile(t, tar.ident, tar.slotOffset, fs, backward)
|
||||
}
|
||||
}
|
||||
require.NoError(t, migrateLayout(fs, backward, forward, cache))
|
||||
for _, tar := range targets {
|
||||
// Make sure the file wound up in the right spot, according to the forward layout
|
||||
// and that the old file is gone, according to the backward layout.
|
||||
testAssertFsMigrated(t, fs, tar.ident, backward, forward)
|
||||
entry, ok := cache.get(tar.ident.root)
|
||||
// we only expect cache to be populated here by files that needed to be moved.
|
||||
if !tar.migrated {
|
||||
require.Equal(t, true, ok)
|
||||
require.Equal(t, true, entry.HasIndex(tar.ident.index))
|
||||
require.Equal(t, tar.ident.epoch, entry.epoch)
|
||||
}
|
||||
}
|
||||
|
||||
// Run migration in reverse - testing "undo"
|
||||
cache = newBlobStorageCache()
|
||||
forward, err = newLayout(forwardName, fs, cache, nil)
|
||||
require.NoError(t, err)
|
||||
backward, err = newLayout(backwardName, fs, cache, nil)
|
||||
require.NoError(t, err)
|
||||
// forward and backward are flipped compared to the above
|
||||
require.NoError(t, migrateLayout(fs, forward, backward, cache))
|
||||
for _, tar := range targets {
|
||||
// just like the above, but forward and backward are flipped
|
||||
testAssertFsMigrated(t, fs, tar.ident, forward, backward)
|
||||
entry, ok := cache.get(tar.ident.root)
|
||||
require.Equal(t, true, ok)
|
||||
require.Equal(t, true, entry.HasIndex(tar.ident.index))
|
||||
require.Equal(t, tar.ident.epoch, entry.epoch)
|
||||
}
|
||||
}
|
||||
@@ -4,30 +4,41 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
"github.com/prysmaticlabs/prysm/v5/time/slots"
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
// NewEphemeralBlobStorage should only be used for tests.
|
||||
// The instance of BlobStorage returned is backed by an in-memory virtual filesystem,
|
||||
// improving test performance and simplifying cleanup.
|
||||
func NewEphemeralBlobStorage(t testing.TB) *BlobStorage {
|
||||
fs := afero.NewMemMapFs()
|
||||
pruner, err := newBlobPruner(fs, params.BeaconConfig().MinEpochsForBlobsSidecarsRequest, withWarmedCache())
|
||||
if err != nil {
|
||||
t.Fatal("test setup issue", err)
|
||||
}
|
||||
return &BlobStorage{fs: fs, pruner: pruner}
|
||||
func NewEphemeralBlobStorage(t testing.TB, opts ...BlobStorageOption) *BlobStorage {
|
||||
return NewWarmedEphemeralBlobStorageUsingFs(t, afero.NewMemMapFs(), opts...)
|
||||
}
|
||||
|
||||
// NewEphemeralBlobStorageWithFs can be used by tests that want access to the virtual filesystem
|
||||
// NewEphemeralBlobStorageAndFs can be used by tests that want access to the virtual filesystem
|
||||
// in order to interact with it outside the parameters of the BlobStorage api.
|
||||
func NewEphemeralBlobStorageWithFs(t testing.TB) (afero.Fs, *BlobStorage) {
|
||||
func NewEphemeralBlobStorageAndFs(t testing.TB, opts ...BlobStorageOption) (afero.Fs, *BlobStorage) {
|
||||
fs := afero.NewMemMapFs()
|
||||
pruner, err := newBlobPruner(fs, params.BeaconConfig().MinEpochsForBlobsSidecarsRequest, withWarmedCache())
|
||||
bs := NewWarmedEphemeralBlobStorageUsingFs(t, fs, opts...)
|
||||
return fs, bs
|
||||
}
|
||||
|
||||
func NewEphemeralBlobStorageUsingFs(t testing.TB, fs afero.Fs, opts ...BlobStorageOption) *BlobStorage {
|
||||
opts = append(opts,
|
||||
WithBlobRetentionEpochs(params.BeaconConfig().MinEpochsForBlobsSidecarsRequest),
|
||||
WithFs(fs))
|
||||
bs, err := NewBlobStorage(opts...)
|
||||
if err != nil {
|
||||
t.Fatal("test setup issue", err)
|
||||
t.Fatalf("error initializing test BlobStorage, err=%s", err.Error())
|
||||
}
|
||||
return fs, &BlobStorage{fs: fs, pruner: pruner}
|
||||
return bs
|
||||
}
|
||||
|
||||
func NewWarmedEphemeralBlobStorageUsingFs(t testing.TB, fs afero.Fs, opts ...BlobStorageOption) *BlobStorage {
|
||||
bs := NewEphemeralBlobStorageUsingFs(t, fs, opts...)
|
||||
bs.WarmCache()
|
||||
return bs
|
||||
}
|
||||
|
||||
type BlobMocker struct {
|
||||
@@ -37,17 +48,9 @@ type BlobMocker struct {
|
||||
|
||||
// CreateFakeIndices creates empty blob sidecar files at the expected path for the given
|
||||
// root and indices to influence the result of Indices().
|
||||
func (bm *BlobMocker) CreateFakeIndices(root [32]byte, indices ...uint64) error {
|
||||
func (bm *BlobMocker) CreateFakeIndices(root [32]byte, slot primitives.Slot, indices ...uint64) error {
|
||||
for i := range indices {
|
||||
n := blobNamer{root: root, index: indices[i]}
|
||||
if err := bm.fs.MkdirAll(n.dir(), directoryPermissions); err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := bm.fs.Create(n.path())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
if err := bm.bs.layout.notify(newBlobIdent(root, slots.ToEpoch(slot), indices[i])); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -56,9 +59,8 @@ func (bm *BlobMocker) CreateFakeIndices(root [32]byte, indices ...uint64) error
|
||||
|
||||
// NewEphemeralBlobStorageWithMocker returns a *BlobMocker value in addition to the BlobStorage value.
|
||||
// BlockMocker encapsulates things blob path construction to avoid leaking implementation details.
|
||||
func NewEphemeralBlobStorageWithMocker(_ testing.TB) (*BlobMocker, *BlobStorage) {
|
||||
fs := afero.NewMemMapFs()
|
||||
bs := &BlobStorage{fs: fs}
|
||||
func NewEphemeralBlobStorageWithMocker(t testing.TB) (*BlobMocker, *BlobStorage) {
|
||||
fs, bs := NewEphemeralBlobStorageAndFs(t)
|
||||
return &BlobMocker{fs: fs, bs: bs}, bs
|
||||
}
|
||||
|
||||
@@ -66,7 +68,7 @@ func NewMockBlobStorageSummarizer(t *testing.T, set map[[32]byte][]int) BlobStor
|
||||
c := newBlobStorageCache()
|
||||
for k, v := range set {
|
||||
for i := range v {
|
||||
if err := c.ensure(k, 0, uint64(v[i])); err != nil {
|
||||
if err := c.ensure(blobIdent{root: k, epoch: 0, index: uint64(v[i])}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,319 +1,67 @@
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
"github.com/prysmaticlabs/prysm/v5/time/slots"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
const retentionBuffer primitives.Epoch = 2
|
||||
|
||||
var (
|
||||
errPruningFailures = errors.New("blobs could not be pruned for some roots")
|
||||
errNotBlobSSZ = errors.New("not a blob ssz file")
|
||||
)
|
||||
var errNotBlobSSZ = errors.New("not a blob ssz file")
|
||||
|
||||
// blobPruner keeps track of the tail end of the retention period, based only the blobs it has seen via the notify method.
|
||||
// If the retention period advances in response to notify being called,
|
||||
// the pruner will invoke the pruneBefore method of the given layout in a new goroutine.
|
||||
// The details of pruning are left entirely to the layout, with the pruner's only responsibility being to
|
||||
// schedule just one pruning operation at a time, for each forward movement of the minimum retention epoch.
|
||||
type blobPruner struct {
|
||||
sync.Mutex
|
||||
prunedBefore atomic.Uint64
|
||||
windowSize primitives.Slot
|
||||
cache *blobStorageCache
|
||||
cacheReady chan struct{}
|
||||
warmed bool
|
||||
fs afero.Fs
|
||||
mu sync.Mutex
|
||||
prunedBefore atomic.Uint64
|
||||
retentionPeriod primitives.Epoch
|
||||
}
|
||||
|
||||
type prunerOpt func(*blobPruner) error
|
||||
|
||||
func withWarmedCache() prunerOpt {
|
||||
return func(p *blobPruner) error {
|
||||
return p.warmCache()
|
||||
}
|
||||
func newBlobPruner(retain primitives.Epoch) *blobPruner {
|
||||
p := &blobPruner{retentionPeriod: retain + retentionBuffer}
|
||||
return p
|
||||
}
|
||||
|
||||
func newBlobPruner(fs afero.Fs, retain primitives.Epoch, opts ...prunerOpt) (*blobPruner, error) {
|
||||
r, err := slots.EpochStart(retain + retentionBuffer)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "could not set retentionSlots")
|
||||
}
|
||||
cw := make(chan struct{})
|
||||
p := &blobPruner{fs: fs, windowSize: r, cache: newBlobStorageCache(), cacheReady: cw}
|
||||
for _, o := range opts {
|
||||
if err := o(p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// notify updates the pruner's view of root->blob mappings. This allows the pruner to build a cache
|
||||
// of root->slot mappings and decide when to evict old blobs based on the age of present blobs.
|
||||
func (p *blobPruner) notify(root [32]byte, latest primitives.Slot, idx uint64) error {
|
||||
if err := p.cache.ensure(root, latest, idx); err != nil {
|
||||
return err
|
||||
}
|
||||
pruned := uint64(windowMin(latest, p.windowSize))
|
||||
if p.prunedBefore.Swap(pruned) == pruned {
|
||||
return nil
|
||||
// notify returns a channel that is closed when the pruning operation is complete.
|
||||
// This is useful for tests, but at runtime fsLayouts or BlobStorage should not wait for completion.
|
||||
func (p *blobPruner) notify(latest primitives.Epoch, layout fsLayout) chan struct{} {
|
||||
done := make(chan struct{})
|
||||
floor := periodFloor(latest, p.retentionPeriod)
|
||||
if primitives.Epoch(p.prunedBefore.Swap(uint64(floor))) >= floor {
|
||||
// Only trigger pruning if the atomic swap changed the previous value of prunedBefore.
|
||||
close(done)
|
||||
return done
|
||||
}
|
||||
go func() {
|
||||
p.Lock()
|
||||
defer p.Unlock()
|
||||
if err := p.prune(primitives.Slot(pruned)); err != nil {
|
||||
log.WithError(err).Errorf("Failed to prune blobs from slot %d", latest)
|
||||
p.mu.Lock()
|
||||
start := time.Now()
|
||||
defer p.mu.Unlock()
|
||||
sum, err := layout.pruneBefore(floor)
|
||||
if err != nil {
|
||||
log.WithError(err).WithFields(sum.LogFields()).Warn("Encountered errors during blob pruning.")
|
||||
}
|
||||
log.WithFields(logrus.Fields{
|
||||
"upToEpoch": floor,
|
||||
"duration": time.Since(start).String(),
|
||||
"filesRemoved": sum.blobsPruned,
|
||||
}).Debug("Pruned old blobs")
|
||||
blobsPrunedCounter.Add(float64(sum.blobsPruned))
|
||||
close(done)
|
||||
}()
|
||||
return nil
|
||||
return done
|
||||
}
|
||||
|
||||
func windowMin(latest, offset primitives.Slot) primitives.Slot {
|
||||
// Safely compute the first slot in the epoch for the latest slot
|
||||
latest = latest - latest%params.BeaconConfig().SlotsPerEpoch
|
||||
if latest < offset {
|
||||
func periodFloor(latest, period primitives.Epoch) primitives.Epoch {
|
||||
if latest < period {
|
||||
return 0
|
||||
}
|
||||
return latest - offset
|
||||
}
|
||||
|
||||
func (p *blobPruner) warmCache() error {
|
||||
p.Lock()
|
||||
defer func() {
|
||||
if !p.warmed {
|
||||
p.warmed = true
|
||||
close(p.cacheReady)
|
||||
}
|
||||
p.Unlock()
|
||||
}()
|
||||
if err := p.prune(0); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *blobPruner) waitForCache(ctx context.Context) (*blobStorageCache, error) {
|
||||
select {
|
||||
case <-p.cacheReady:
|
||||
return p.cache, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// Prune prunes blobs in the base directory based on the retention epoch.
|
||||
// It deletes blobs older than currentEpoch - (retentionEpochs+bufferEpochs).
|
||||
// This is so that we keep a slight buffer and blobs are deleted after n+2 epochs.
|
||||
func (p *blobPruner) prune(pruneBefore primitives.Slot) error {
|
||||
start := time.Now()
|
||||
totalPruned, totalErr := 0, 0
|
||||
// Customize logging/metrics behavior for the initial cache warmup when slot=0.
|
||||
// We'll never see a prune request for slot 0, unless this is the initial call to warm up the cache.
|
||||
if pruneBefore == 0 {
|
||||
defer func() {
|
||||
log.WithField("duration", time.Since(start).String()).Debug("Warmed up pruner cache")
|
||||
}()
|
||||
} else {
|
||||
defer func() {
|
||||
log.WithFields(logrus.Fields{
|
||||
"upToEpoch": slots.ToEpoch(pruneBefore),
|
||||
"duration": time.Since(start).String(),
|
||||
"filesRemoved": totalPruned,
|
||||
}).Debug("Pruned old blobs")
|
||||
blobsPrunedCounter.Add(float64(totalPruned))
|
||||
}()
|
||||
}
|
||||
|
||||
entries, err := listDir(p.fs, ".")
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "unable to list root blobs directory")
|
||||
}
|
||||
dirs := filter(entries, filterRoot)
|
||||
for _, dir := range dirs {
|
||||
pruned, err := p.tryPruneDir(dir, pruneBefore)
|
||||
if err != nil {
|
||||
totalErr += 1
|
||||
log.WithError(err).WithField("directory", dir).Error("Unable to prune directory")
|
||||
}
|
||||
totalPruned += pruned
|
||||
}
|
||||
|
||||
if totalErr > 0 {
|
||||
return errors.Wrapf(errPruningFailures, "pruning failed for %d root directories", totalErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func shouldRetain(slot, pruneBefore primitives.Slot) bool {
|
||||
return slot >= pruneBefore
|
||||
}
|
||||
|
||||
func (p *blobPruner) tryPruneDir(dir string, pruneBefore primitives.Slot) (int, error) {
|
||||
root, err := rootFromDir(dir)
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "invalid directory, could not parse subdir as root %s", dir)
|
||||
}
|
||||
slot, slotCached := p.cache.slot(root)
|
||||
// Return early if the slot is cached and doesn't need pruning.
|
||||
if slotCached && shouldRetain(slot, pruneBefore) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// entries will include things that aren't ssz files, like dangling .part files. We need these to
|
||||
// completely clean up the directory.
|
||||
entries, err := listDir(p.fs, dir)
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "failed to list blobs in directory %s", dir)
|
||||
}
|
||||
// scFiles filters the dir listing down to the ssz encoded BlobSidecar files. This allows us to peek
|
||||
// at the first one in the list to figure out the slot.
|
||||
scFiles := filter(entries, filterSsz)
|
||||
if len(scFiles) == 0 {
|
||||
log.WithField("dir", dir).Warn("Pruner ignoring directory with no blob files")
|
||||
return 0, nil
|
||||
}
|
||||
if !slotCached {
|
||||
slot, err = slotFromFile(path.Join(dir, scFiles[0]), p.fs)
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "slot could not be read from blob file %s", scFiles[0])
|
||||
}
|
||||
for i := range scFiles {
|
||||
idx, err := idxFromPath(scFiles[i])
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "index could not be determined for blob file %s", scFiles[i])
|
||||
}
|
||||
if err := p.cache.ensure(root, slot, idx); err != nil {
|
||||
return 0, errors.Wrapf(err, "could not update prune cache for blob file %s", scFiles[i])
|
||||
}
|
||||
}
|
||||
if shouldRetain(slot, pruneBefore) {
|
||||
return 0, nil
|
||||
}
|
||||
}
|
||||
|
||||
removed := 0
|
||||
for _, fname := range entries {
|
||||
fullName := path.Join(dir, fname)
|
||||
if err := p.fs.Remove(fullName); err != nil {
|
||||
return removed, errors.Wrapf(err, "unable to remove %s", fullName)
|
||||
}
|
||||
// Don't count other files that happen to be in the dir, like dangling .part files.
|
||||
if filterSsz(fname) {
|
||||
removed += 1
|
||||
}
|
||||
// Log a warning whenever we clean up a .part file
|
||||
if filterPart(fullName) {
|
||||
log.WithField("file", fullName).Warn("Deleting abandoned blob .part file")
|
||||
}
|
||||
}
|
||||
if err := p.fs.Remove(dir); err != nil {
|
||||
return removed, errors.Wrapf(err, "unable to remove blob directory %s", dir)
|
||||
}
|
||||
|
||||
p.cache.evict(root)
|
||||
return len(scFiles), nil
|
||||
}
|
||||
|
||||
func idxFromPath(fname string) (uint64, error) {
|
||||
fname = path.Base(fname)
|
||||
|
||||
if filepath.Ext(fname) != dotSszExt {
|
||||
return 0, errors.Wrap(errNotBlobSSZ, "does not have .ssz extension")
|
||||
}
|
||||
parts := strings.Split(fname, ".")
|
||||
if len(parts) != 2 {
|
||||
return 0, errors.Wrap(errNotBlobSSZ, "unexpected filename structure (want <index>.ssz)")
|
||||
}
|
||||
return strconv.ParseUint(parts[0], 10, 64)
|
||||
}
|
||||
|
||||
func rootFromDir(dir string) ([32]byte, error) {
|
||||
subdir := filepath.Base(dir) // end of the path should be the blob directory, named by hex encoding of root
|
||||
root, err := stringToRoot(subdir)
|
||||
if err != nil {
|
||||
return root, errors.Wrapf(err, "invalid directory, could not parse subdir as root %s", dir)
|
||||
}
|
||||
return root, nil
|
||||
}
|
||||
|
||||
// Read slot from marshaled BlobSidecar data in the given file. See slotFromBlob for details.
|
||||
func slotFromFile(file string, fs afero.Fs) (primitives.Slot, error) {
|
||||
f, err := fs.Open(file)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() {
|
||||
if err := f.Close(); err != nil {
|
||||
log.WithError(err).Errorf("Could not close blob file")
|
||||
}
|
||||
}()
|
||||
return slotFromBlob(f)
|
||||
}
|
||||
|
||||
// slotFromBlob reads the ssz data of a file at the specified offset (8 + 131072 + 48 + 48 = 131176 bytes),
|
||||
// which is calculated based on the size of the BlobSidecar struct and is based on the size of the fields
|
||||
// preceding the slot information within SignedBeaconBlockHeader.
|
||||
func slotFromBlob(at io.ReaderAt) (primitives.Slot, error) {
|
||||
b := make([]byte, 8)
|
||||
_, err := at.ReadAt(b, 131176)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rawSlot := binary.LittleEndian.Uint64(b)
|
||||
return primitives.Slot(rawSlot), nil
|
||||
}
|
||||
|
||||
func listDir(fs afero.Fs, dir string) ([]string, error) {
|
||||
top, err := fs.Open(dir)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to open directory descriptor")
|
||||
}
|
||||
defer func() {
|
||||
if err := top.Close(); err != nil {
|
||||
log.WithError(err).Errorf("Could not close file %s", dir)
|
||||
}
|
||||
}()
|
||||
// re the -1 param: "If n <= 0, Readdirnames returns all the names from the directory in a single slice"
|
||||
dirs, err := top.Readdirnames(-1)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to read directory listing")
|
||||
}
|
||||
return dirs, nil
|
||||
}
|
||||
|
||||
func filter(entries []string, filt func(string) bool) []string {
|
||||
filtered := make([]string, 0, len(entries))
|
||||
for i := range entries {
|
||||
if filt(entries[i]) {
|
||||
filtered = append(filtered, entries[i])
|
||||
}
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
func filterRoot(s string) bool {
|
||||
return strings.HasPrefix(s, "0x")
|
||||
}
|
||||
|
||||
var dotSszExt = "." + sszExt
|
||||
var dotPartExt = "." + partExt
|
||||
|
||||
func filterSsz(s string) bool {
|
||||
return filepath.Ext(s) == dotSszExt
|
||||
}
|
||||
|
||||
func filterPart(s string) bool {
|
||||
return filepath.Ext(s) == dotPartExt
|
||||
return latest - period
|
||||
}
|
||||
|
||||
@@ -1,394 +1,197 @@
|
||||
package filesystem
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"encoding/binary"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/verification"
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
"github.com/prysmaticlabs/prysm/v5/encoding/bytesutil"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/require"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/util"
|
||||
"github.com/prysmaticlabs/prysm/v5/time/slots"
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
func TestTryPruneDir_CachedNotExpired(t *testing.T) {
|
||||
fs := afero.NewMemMapFs()
|
||||
pr, err := newBlobPruner(fs, 0)
|
||||
require.NoError(t, err)
|
||||
slot := pr.windowSize
|
||||
_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, slot, params.BeaconConfig().MaxBlobsPerBlock(slot))
|
||||
sc, err := verification.BlobSidecarNoop(sidecars[0])
|
||||
require.NoError(t, err)
|
||||
rootStr := rootString(sc.BlockRoot())
|
||||
// This slot is right on the edge of what would need to be pruned, so by adding it to the cache and
|
||||
// skipping any other test setup, we can be certain the hot cache path never touches the filesystem.
|
||||
require.NoError(t, pr.cache.ensure(sc.BlockRoot(), sc.Slot(), 0))
|
||||
pruned, err := pr.tryPruneDir(rootStr, pr.windowSize)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, pruned)
|
||||
type prunerScenario struct {
|
||||
name string
|
||||
prunedBefore primitives.Epoch
|
||||
retentionPeriod primitives.Epoch
|
||||
latest primitives.Epoch
|
||||
expected pruneExpectation
|
||||
}
|
||||
|
||||
func TestCacheWarmFail(t *testing.T) {
|
||||
fs := afero.NewMemMapFs()
|
||||
n := blobNamer{root: bytesutil.ToBytes32([]byte("derp")), index: 0}
|
||||
bp := n.path()
|
||||
mkdir := path.Dir(bp)
|
||||
require.NoError(t, fs.MkdirAll(mkdir, directoryPermissions))
|
||||
|
||||
// Create an empty blob index in the fs by touching the file at a seemingly valid path.
|
||||
fi, err := fs.Create(bp)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, fi.Close())
|
||||
|
||||
// Cache warm should fail due to the unexpected EOF.
|
||||
pr, err := newBlobPruner(fs, 0)
|
||||
require.NoError(t, err)
|
||||
require.ErrorIs(t, pr.warmCache(), errPruningFailures)
|
||||
|
||||
// The cache warm has finished, so calling waitForCache with a super short deadline
|
||||
// should not block or hit the context deadline.
|
||||
ctx := context.Background()
|
||||
ctx, cancel := context.WithDeadline(ctx, time.Now().Add(1*time.Millisecond))
|
||||
defer cancel()
|
||||
c, err := pr.waitForCache(ctx)
|
||||
// We will get an error and a nil value for the cache if we hit the deadline.
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, c)
|
||||
type pruneExpectation struct {
|
||||
called bool
|
||||
arg primitives.Epoch
|
||||
summary *pruneSummary
|
||||
err error
|
||||
}
|
||||
|
||||
func TestTryPruneDir_CachedExpired(t *testing.T) {
|
||||
t.Run("empty directory", func(t *testing.T) {
|
||||
fs := afero.NewMemMapFs()
|
||||
pr, err := newBlobPruner(fs, 0)
|
||||
require.NoError(t, err)
|
||||
var slot primitives.Slot = 0
|
||||
_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, slot, 1)
|
||||
sc, err := verification.BlobSidecarNoop(sidecars[0])
|
||||
require.NoError(t, err)
|
||||
rootStr := rootString(sc.BlockRoot())
|
||||
require.NoError(t, fs.Mkdir(rootStr, directoryPermissions)) // make empty directory
|
||||
require.NoError(t, pr.cache.ensure(sc.BlockRoot(), sc.Slot(), 0))
|
||||
pruned, err := pr.tryPruneDir(rootStr, slot+1)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, pruned)
|
||||
})
|
||||
t.Run("blobs to delete", func(t *testing.T) {
|
||||
fs, bs := NewEphemeralBlobStorageWithFs(t)
|
||||
var slot primitives.Slot = 0
|
||||
_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, slot, 2)
|
||||
scs, err := verification.BlobSidecarSliceNoop(sidecars)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, bs.Save(scs[0]))
|
||||
require.NoError(t, bs.Save(scs[1]))
|
||||
|
||||
// check that the root->slot is cached
|
||||
root := scs[0].BlockRoot()
|
||||
rootStr := rootString(root)
|
||||
cs, cok := bs.pruner.cache.slot(scs[0].BlockRoot())
|
||||
require.Equal(t, true, cok)
|
||||
require.Equal(t, slot, cs)
|
||||
|
||||
// ensure that we see the saved files in the filesystem
|
||||
files, err := listDir(fs, rootStr)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, len(files))
|
||||
|
||||
pruned, err := bs.pruner.tryPruneDir(rootStr, slot+1)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, pruned)
|
||||
files, err = listDir(fs, rootStr)
|
||||
require.ErrorIs(t, err, os.ErrNotExist)
|
||||
require.Equal(t, 0, len(files))
|
||||
})
|
||||
func (e *pruneExpectation) record(before primitives.Epoch) (*pruneSummary, error) {
|
||||
e.called = true
|
||||
e.arg = before
|
||||
if e.summary == nil {
|
||||
e.summary = &pruneSummary{}
|
||||
}
|
||||
return e.summary, e.err
|
||||
}
|
||||
|
||||
func TestTryPruneDir_SlotFromFile(t *testing.T) {
|
||||
t.Run("expired blobs deleted", func(t *testing.T) {
|
||||
fs, bs := NewEphemeralBlobStorageWithFs(t)
|
||||
var slot primitives.Slot = 0
|
||||
_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, slot, 2)
|
||||
scs, err := verification.BlobSidecarSliceNoop(sidecars)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, bs.Save(scs[0]))
|
||||
require.NoError(t, bs.Save(scs[1]))
|
||||
|
||||
// check that the root->slot is cached
|
||||
root := scs[0].BlockRoot()
|
||||
rootStr := rootString(root)
|
||||
cs, ok := bs.pruner.cache.slot(root)
|
||||
require.Equal(t, true, ok)
|
||||
require.Equal(t, slot, cs)
|
||||
// evict it from the cache so that we trigger the file read path
|
||||
bs.pruner.cache.evict(root)
|
||||
_, ok = bs.pruner.cache.slot(root)
|
||||
require.Equal(t, false, ok)
|
||||
|
||||
// ensure that we see the saved files in the filesystem
|
||||
files, err := listDir(fs, rootStr)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, len(files))
|
||||
|
||||
pruned, err := bs.pruner.tryPruneDir(rootStr, slot+1)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, pruned)
|
||||
files, err = listDir(fs, rootStr)
|
||||
require.ErrorIs(t, err, os.ErrNotExist)
|
||||
require.Equal(t, 0, len(files))
|
||||
})
|
||||
t.Run("not expired, intact", func(t *testing.T) {
|
||||
fs, bs := NewEphemeralBlobStorageWithFs(t)
|
||||
// Set slot equal to the window size, so it should be retained.
|
||||
slot := bs.pruner.windowSize
|
||||
_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, slot, 2)
|
||||
scs, err := verification.BlobSidecarSliceNoop(sidecars)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, bs.Save(scs[0]))
|
||||
require.NoError(t, bs.Save(scs[1]))
|
||||
|
||||
// Evict slot mapping from the cache so that we trigger the file read path.
|
||||
root := scs[0].BlockRoot()
|
||||
rootStr := rootString(root)
|
||||
bs.pruner.cache.evict(root)
|
||||
_, ok := bs.pruner.cache.slot(root)
|
||||
require.Equal(t, false, ok)
|
||||
|
||||
// Ensure that we see the saved files in the filesystem.
|
||||
files, err := listDir(fs, rootStr)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, len(files))
|
||||
|
||||
// This should use the slotFromFile code (simulating restart).
|
||||
// Setting pruneBefore == slot, so that the slot will be outside the window (at the boundary).
|
||||
pruned, err := bs.pruner.tryPruneDir(rootStr, slot)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, pruned)
|
||||
|
||||
// Ensure files are still present.
|
||||
files, err = listDir(fs, rootStr)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, len(files))
|
||||
})
|
||||
}
|
||||
|
||||
func TestSlotFromBlob(t *testing.T) {
|
||||
cases := []struct {
|
||||
slot primitives.Slot
|
||||
}{
|
||||
{slot: 0},
|
||||
{slot: 2},
|
||||
{slot: 1123581321},
|
||||
{slot: math.MaxUint64},
|
||||
func TestPrunerNotify(t *testing.T) {
|
||||
defaultRetention := params.BeaconConfig().MinEpochsForBlobsSidecarsRequest
|
||||
cases := []prunerScenario{
|
||||
{
|
||||
name: "last epoch of period",
|
||||
retentionPeriod: defaultRetention,
|
||||
prunedBefore: 11235,
|
||||
latest: defaultRetention + 11235,
|
||||
expected: pruneExpectation{called: false},
|
||||
},
|
||||
{
|
||||
name: "within period",
|
||||
retentionPeriod: defaultRetention,
|
||||
prunedBefore: 11235,
|
||||
latest: 11235 + defaultRetention - 1,
|
||||
expected: pruneExpectation{called: false},
|
||||
},
|
||||
{
|
||||
name: "triggers",
|
||||
retentionPeriod: defaultRetention,
|
||||
prunedBefore: 11235,
|
||||
latest: 11235 + 1 + defaultRetention,
|
||||
expected: pruneExpectation{called: true, arg: 11235 + 1},
|
||||
},
|
||||
{
|
||||
name: "from zero - before first period",
|
||||
retentionPeriod: defaultRetention,
|
||||
prunedBefore: 0,
|
||||
latest: defaultRetention - 1,
|
||||
expected: pruneExpectation{called: false},
|
||||
},
|
||||
{
|
||||
name: "from zero - at boundary",
|
||||
retentionPeriod: defaultRetention,
|
||||
prunedBefore: 0,
|
||||
latest: defaultRetention,
|
||||
expected: pruneExpectation{called: false},
|
||||
},
|
||||
{
|
||||
name: "from zero - triggers",
|
||||
retentionPeriod: defaultRetention,
|
||||
prunedBefore: 0,
|
||||
latest: defaultRetention + 1,
|
||||
expected: pruneExpectation{called: true, arg: 1},
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(fmt.Sprintf("slot %d", c.slot), func(t *testing.T) {
|
||||
_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, c.slot, 1)
|
||||
sc := sidecars[0]
|
||||
enc, err := sc.MarshalSSZ()
|
||||
require.NoError(t, err)
|
||||
slot, err := slotFromBlob(bytes.NewReader(enc))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, c.slot, slot)
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
actual := &pruneExpectation{}
|
||||
l := &mockLayout{pruneBeforeFunc: actual.record}
|
||||
pruner := &blobPruner{retentionPeriod: c.retentionPeriod}
|
||||
pruner.prunedBefore.Store(uint64(c.prunedBefore))
|
||||
done := pruner.notify(c.latest, l)
|
||||
<-done
|
||||
require.Equal(t, c.expected.called, actual.called)
|
||||
require.Equal(t, c.expected.arg, actual.arg)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlotFromFile(t *testing.T) {
|
||||
cases := []struct {
|
||||
slot primitives.Slot
|
||||
}{
|
||||
{slot: 0},
|
||||
{slot: 2},
|
||||
{slot: 1123581321},
|
||||
{slot: math.MaxUint64},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(fmt.Sprintf("slot %d", c.slot), func(t *testing.T) {
|
||||
fs, bs := NewEphemeralBlobStorageWithFs(t)
|
||||
_, sidecars := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, c.slot, 1)
|
||||
sc, err := verification.BlobSidecarNoop(sidecars[0])
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, bs.Save(sc))
|
||||
fname := namerForSidecar(sc)
|
||||
sszPath := fname.path()
|
||||
slot, err := slotFromFile(sszPath, fs)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, c.slot, slot)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type dirFiles struct {
|
||||
name string
|
||||
isDir bool
|
||||
children []dirFiles
|
||||
}
|
||||
|
||||
func (df dirFiles) reify(t *testing.T, fs afero.Fs, base string) {
|
||||
fullPath := path.Join(base, df.name)
|
||||
if df.isDir {
|
||||
if df.name != "" {
|
||||
require.NoError(t, fs.Mkdir(fullPath, directoryPermissions))
|
||||
}
|
||||
for _, c := range df.children {
|
||||
c.reify(t, fs, fullPath)
|
||||
}
|
||||
} else {
|
||||
fp, err := fs.Create(fullPath)
|
||||
func testSetupBlobIdentPaths(t *testing.T, fs afero.Fs, bs *BlobStorage, idents []testIdent) []blobIdent {
|
||||
created := make([]blobIdent, len(idents))
|
||||
for i, id := range idents {
|
||||
slot, err := slots.EpochStart(id.epoch)
|
||||
require.NoError(t, err)
|
||||
_, err = fp.WriteString("derp")
|
||||
slot += id.offset
|
||||
_, scs := util.GenerateTestDenebBlockWithSidecar(t, [32]byte{}, slot, 1)
|
||||
sc := verification.FakeVerifyForTest(t, scs[0])
|
||||
require.NoError(t, bs.Save(sc))
|
||||
ident := identForSidecar(sc)
|
||||
_, err = fs.Stat(bs.layout.sszPath(ident))
|
||||
require.NoError(t, err)
|
||||
created[i] = ident
|
||||
}
|
||||
return created
|
||||
}
|
||||
|
||||
func testAssertBlobsPruned(t *testing.T, fs afero.Fs, bs *BlobStorage, pruned, remain []blobIdent) {
|
||||
for _, id := range pruned {
|
||||
_, err := fs.Stat(bs.layout.sszPath(id))
|
||||
require.NotNil(t, err)
|
||||
require.Equal(t, true, os.IsNotExist(err))
|
||||
}
|
||||
for _, id := range remain {
|
||||
_, err := fs.Stat(bs.layout.sszPath(id))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (df dirFiles) childNames() []string {
|
||||
cn := make([]string, len(df.children))
|
||||
for i := range df.children {
|
||||
cn[i] = df.children[i].name
|
||||
}
|
||||
return cn
|
||||
type testIdent struct {
|
||||
blobIdent
|
||||
offset primitives.Slot
|
||||
}
|
||||
|
||||
func TestListDir(t *testing.T) {
|
||||
fs := afero.NewMemMapFs()
|
||||
|
||||
// parent directory
|
||||
fsLayout := dirFiles{isDir: true}
|
||||
// break out each subdir for easier assertions
|
||||
notABlob := dirFiles{name: "notABlob", isDir: true}
|
||||
childlessBlob := dirFiles{name: "0x0987654321", isDir: true}
|
||||
blobWithSsz := dirFiles{name: "0x1123581321", isDir: true,
|
||||
children: []dirFiles{{name: "1.ssz"}, {name: "2.ssz"}},
|
||||
func testRoots(n int) [][32]byte {
|
||||
roots := make([][32]byte, n)
|
||||
for i := range roots {
|
||||
binary.LittleEndian.PutUint32(roots[i][:], uint32(1+i))
|
||||
}
|
||||
blobWithSszAndTmp := dirFiles{name: "0x1234567890", isDir: true,
|
||||
children: []dirFiles{{name: "5.ssz"}, {name: "0.part"}}}
|
||||
fsLayout.children = append(fsLayout.children,
|
||||
notABlob, childlessBlob, blobWithSsz, blobWithSszAndTmp)
|
||||
return roots
|
||||
}
|
||||
|
||||
topChildren := make([]string, len(fsLayout.children))
|
||||
for i := range fsLayout.children {
|
||||
topChildren[i] = fsLayout.children[i].name
|
||||
}
|
||||
|
||||
fsLayout.reify(t, fs, "")
|
||||
func TestLayoutPruneBefore(t *testing.T) {
|
||||
roots := testRoots(10)
|
||||
cases := []struct {
|
||||
name string
|
||||
dirPath string
|
||||
expected []string
|
||||
filter func(string) bool
|
||||
err error
|
||||
name string
|
||||
pruned []testIdent
|
||||
remain []testIdent
|
||||
pruneBefore primitives.Epoch
|
||||
err error
|
||||
sum pruneSummary
|
||||
}{
|
||||
{
|
||||
name: "non-existent",
|
||||
dirPath: "derp",
|
||||
expected: []string{},
|
||||
err: os.ErrNotExist,
|
||||
},
|
||||
{
|
||||
name: "empty",
|
||||
dirPath: childlessBlob.name,
|
||||
expected: []string{},
|
||||
},
|
||||
{
|
||||
name: "top",
|
||||
dirPath: ".",
|
||||
expected: topChildren,
|
||||
},
|
||||
{
|
||||
name: "custom filter: only notABlob",
|
||||
dirPath: ".",
|
||||
expected: []string{notABlob.name},
|
||||
filter: func(s string) bool {
|
||||
return s == notABlob.name
|
||||
name: "none pruned",
|
||||
pruneBefore: 1,
|
||||
pruned: []testIdent{},
|
||||
remain: []testIdent{
|
||||
{offset: 1, blobIdent: blobIdent{root: roots[0], epoch: 1, index: 0}},
|
||||
{offset: 1, blobIdent: blobIdent{root: roots[1], epoch: 1, index: 0}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "root filter",
|
||||
dirPath: ".",
|
||||
expected: []string{childlessBlob.name, blobWithSsz.name, blobWithSszAndTmp.name},
|
||||
filter: filterRoot,
|
||||
},
|
||||
{
|
||||
name: "ssz filter",
|
||||
dirPath: blobWithSsz.name,
|
||||
expected: blobWithSsz.childNames(),
|
||||
filter: filterSsz,
|
||||
},
|
||||
{
|
||||
name: "ssz mixed filter",
|
||||
dirPath: blobWithSszAndTmp.name,
|
||||
expected: []string{"5.ssz"},
|
||||
filter: filterSsz,
|
||||
name: "expected pruned before epoch",
|
||||
pruneBefore: 3,
|
||||
pruned: []testIdent{
|
||||
{offset: 0, blobIdent: blobIdent{root: roots[0], epoch: 1, index: 0}},
|
||||
{offset: 31, blobIdent: blobIdent{root: roots[1], epoch: 1, index: 5}},
|
||||
{offset: 0, blobIdent: blobIdent{root: roots[2], epoch: 2, index: 0}},
|
||||
{offset: 31, blobIdent: blobIdent{root: roots[3], epoch: 2, index: 3}},
|
||||
},
|
||||
remain: []testIdent{
|
||||
{offset: 0, blobIdent: blobIdent{root: roots[4], epoch: 3, index: 2}}, // boundary
|
||||
{offset: 31, blobIdent: blobIdent{root: roots[5], epoch: 3, index: 0}}, // boundary
|
||||
{offset: 0, blobIdent: blobIdent{root: roots[6], epoch: 4, index: 1}},
|
||||
{offset: 31, blobIdent: blobIdent{root: roots[7], epoch: 4, index: 5}},
|
||||
},
|
||||
sum: pruneSummary{blobsPruned: 4},
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
result, err := listDir(fs, c.dirPath)
|
||||
if c.filter != nil {
|
||||
result = filter(result, c.filter)
|
||||
}
|
||||
if c.err != nil {
|
||||
require.ErrorIs(t, err, c.err)
|
||||
require.Equal(t, 0, len(result))
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
sort.Strings(c.expected)
|
||||
sort.Strings(result)
|
||||
require.DeepEqual(t, c.expected, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRootFromDir(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
dir string
|
||||
err error
|
||||
root [32]byte
|
||||
}{
|
||||
{
|
||||
name: "happy path",
|
||||
dir: "0xffff875e1d985c5ccb214894983f2428edb271f0f87b68ba7010e4a99df3b5cb",
|
||||
root: [32]byte{255, 255, 135, 94, 29, 152, 92, 92, 203, 33, 72, 148, 152, 63, 36, 40,
|
||||
237, 178, 113, 240, 248, 123, 104, 186, 112, 16, 228, 169, 157, 243, 181, 203},
|
||||
},
|
||||
{
|
||||
name: "too short",
|
||||
dir: "0xffff875e1d985c5ccb214894983f2428edb271f0f87b68ba7010e4a99df3b5c",
|
||||
err: errInvalidRootString,
|
||||
},
|
||||
{
|
||||
name: "too log",
|
||||
dir: "0xffff875e1d985c5ccb214894983f2428edb271f0f87b68ba7010e4a99df3b5cbb",
|
||||
err: errInvalidRootString,
|
||||
},
|
||||
{
|
||||
name: "missing prefix",
|
||||
dir: "ffff875e1d985c5ccb214894983f2428edb271f0f87b68ba7010e4a99df3b5cb",
|
||||
err: errInvalidRootString,
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
root, err := stringToRoot(c.dir)
|
||||
fs, bs := NewEphemeralBlobStorageAndFs(t, WithLayout(LayoutNameByEpoch))
|
||||
pruned := testSetupBlobIdentPaths(t, fs, bs, c.pruned)
|
||||
remain := testSetupBlobIdentPaths(t, fs, bs, c.remain)
|
||||
sum, err := bs.layout.pruneBefore(c.pruneBefore)
|
||||
if c.err != nil {
|
||||
require.ErrorIs(t, err, c.err)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, c.root, root)
|
||||
testAssertBlobsPruned(t, fs, bs, pruned, remain)
|
||||
require.Equal(t, c.sum.blobsPruned, sum.blobsPruned)
|
||||
require.Equal(t, len(c.pruned), sum.blobsPruned)
|
||||
require.Equal(t, len(c.sum.failedRemovals), len(sum.failedRemovals))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,6 +101,7 @@ type NoHeadAccessDatabase interface {
|
||||
SaveLightClientBootstrap(ctx context.Context, blockRoot []byte, bootstrap interfaces.LightClientBootstrap) error
|
||||
|
||||
CleanUpDirtyStates(ctx context.Context, slotsPerArchivedPoint primitives.Slot) error
|
||||
DeleteHistoricalDataBeforeSlot(ctx context.Context, slot primitives.Slot) error
|
||||
}
|
||||
|
||||
// HeadAccessDatabase defines a struct with access to reading chain head data.
|
||||
|
||||
@@ -227,10 +227,7 @@ func (s *Store) DeleteBlock(ctx context.Context, root [32]byte) error {
|
||||
return ErrDeleteJustifiedAndFinalized
|
||||
}
|
||||
|
||||
if err := tx.Bucket(blocksBucket).Delete(root[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Bucket(blockParentRootIndicesBucket).Delete(root[:]); err != nil {
|
||||
if err := s.deleteBlock(tx, root[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
s.blockCache.Del(string(root[:]))
|
||||
@@ -238,6 +235,89 @@ func (s *Store) DeleteBlock(ctx context.Context, root [32]byte) error {
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteHistoricalDataBeforeSlot deletes all blocks and states before the given slot.
|
||||
// This function deletes data from the following buckets:
|
||||
// - blocksBucket
|
||||
// - blockParentRootIndicesBucket
|
||||
// - finalizedBlockRootsIndexBucket
|
||||
// - stateBucket
|
||||
// - stateSummaryBucket
|
||||
// - blockRootValidatorHashesBucket
|
||||
// - blockSlotIndicesBucket
|
||||
// - stateSlotIndicesBucket
|
||||
func (s *Store) DeleteHistoricalDataBeforeSlot(ctx context.Context, cutoffSlot primitives.Slot) error {
|
||||
ctx, span := trace.StartSpan(ctx, "BeaconDB.DeleteHistoricalDataBeforeSlot")
|
||||
defer span.End()
|
||||
|
||||
// Collect slot/root pairs to perform deletions in a separate read only transaction.
|
||||
var (
|
||||
roots [][]byte
|
||||
slts []primitives.Slot
|
||||
)
|
||||
err := s.db.View(func(tx *bolt.Tx) error {
|
||||
var err error
|
||||
roots, slts, err = blockRootsBySlotRange(ctx, tx.Bucket(blockSlotIndicesBucket), primitives.Slot(0), cutoffSlot, nil, nil, nil)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not retrieve block roots")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not retrieve block roots and slots")
|
||||
}
|
||||
|
||||
// Perform all deletions in a single transaction for atomicity
|
||||
return s.db.Update(func(tx *bolt.Tx) error {
|
||||
for _, root := range roots {
|
||||
// Delete block
|
||||
if err = s.deleteBlock(tx, root); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete finalized block roots index
|
||||
if err = tx.Bucket(finalizedBlockRootsIndexBucket).Delete(root); err != nil {
|
||||
return errors.Wrap(err, "could not delete finalized block root index")
|
||||
}
|
||||
|
||||
// Delete state
|
||||
if err = tx.Bucket(stateBucket).Delete(root); err != nil {
|
||||
return errors.Wrap(err, "could not delete state")
|
||||
}
|
||||
|
||||
// Delete state summary
|
||||
if err = tx.Bucket(stateSummaryBucket).Delete(root); err != nil {
|
||||
return errors.Wrap(err, "could not delete state summary")
|
||||
}
|
||||
|
||||
// Delete validator entries
|
||||
if err = s.deleteValidatorHashes(tx, root); err != nil {
|
||||
return errors.Wrap(err, "could not delete validators")
|
||||
}
|
||||
}
|
||||
|
||||
for _, slot := range slts {
|
||||
// Delete slot indices
|
||||
if err = tx.Bucket(blockSlotIndicesBucket).Delete(bytesutil.SlotToBytesBigEndian(slot)); err != nil {
|
||||
return errors.Wrap(err, "could not delete block slot index")
|
||||
}
|
||||
if err = tx.Bucket(stateSlotIndicesBucket).Delete(bytesutil.SlotToBytesBigEndian(slot)); err != nil {
|
||||
return errors.Wrap(err, "could not delete state slot index")
|
||||
}
|
||||
}
|
||||
|
||||
// Delete all caches after we have deleted everything from buckets.
|
||||
// This is done after the buckets are deleted to avoid any issues in case of transaction rollback.
|
||||
for _, root := range roots {
|
||||
// Delete block from cache
|
||||
s.blockCache.Del(string(root))
|
||||
// Delete state summary from cache
|
||||
s.stateSummaryCache.delete([32]byte(root))
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// SaveBlock to the db.
|
||||
func (s *Store) SaveBlock(ctx context.Context, signed interfaces.ReadOnlySignedBeaconBlock) error {
|
||||
ctx, span := trace.StartSpan(ctx, "BeaconDB.SaveBlock")
|
||||
@@ -609,7 +689,7 @@ func blockRootsByFilter(ctx context.Context, tx *bolt.Tx, f *filters.QueryFilter
|
||||
|
||||
// We retrieve block roots that match a filter criteria of slot ranges, if specified.
|
||||
filtersMap := f.Filters()
|
||||
rootsBySlotRange, err := blockRootsBySlotRange(
|
||||
rootsBySlotRange, _, err := blockRootsBySlotRange(
|
||||
ctx,
|
||||
tx.Bucket(blockSlotIndicesBucket),
|
||||
filtersMap[filters.StartSlot],
|
||||
@@ -627,6 +707,7 @@ func blockRootsByFilter(ctx context.Context, tx *bolt.Tx, f *filters.QueryFilter
|
||||
// that list of roots to lookup the block. These block will
|
||||
// meet the filter criteria.
|
||||
indices := lookupValuesForIndices(ctx, indicesByBucket, tx)
|
||||
|
||||
keys := rootsBySlotRange
|
||||
if len(indices) > 0 {
|
||||
// If we have found indices that meet the filter criteria, and there are also
|
||||
@@ -653,13 +734,13 @@ func blockRootsBySlotRange(
|
||||
ctx context.Context,
|
||||
bkt *bolt.Bucket,
|
||||
startSlotEncoded, endSlotEncoded, startEpochEncoded, endEpochEncoded, slotStepEncoded interface{},
|
||||
) ([][]byte, error) {
|
||||
) ([][]byte, []primitives.Slot, error) {
|
||||
_, span := trace.StartSpan(ctx, "BeaconDB.blockRootsBySlotRange")
|
||||
defer span.End()
|
||||
|
||||
// Return nothing when all slot parameters are missing
|
||||
if startSlotEncoded == nil && endSlotEncoded == nil && startEpochEncoded == nil && endEpochEncoded == nil {
|
||||
return [][]byte{}, nil
|
||||
return [][]byte{}, nil, nil
|
||||
}
|
||||
|
||||
var startSlot, endSlot primitives.Slot
|
||||
@@ -680,11 +761,11 @@ func blockRootsBySlotRange(
|
||||
if startEpochOk && endEpochOk {
|
||||
startSlot, err = slots.EpochStart(startEpoch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
endSlot, err = slots.EpochStart(endEpoch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
endSlot = endSlot + params.BeaconConfig().SlotsPerEpoch - 1
|
||||
}
|
||||
@@ -695,14 +776,15 @@ func blockRootsBySlotRange(
|
||||
return key != nil && bytes.Compare(key, max) <= 0
|
||||
}
|
||||
if endSlot < startSlot {
|
||||
return nil, errInvalidSlotRange
|
||||
return nil, nil, errInvalidSlotRange
|
||||
}
|
||||
rootsRange := endSlot.SubSlot(startSlot).Div(step)
|
||||
roots := make([][]byte, 0, rootsRange)
|
||||
var slts []primitives.Slot
|
||||
c := bkt.Cursor()
|
||||
for k, v := c.Seek(min); conditional(k, max); k, v = c.Next() {
|
||||
slot := bytesutil.BytesToSlotBigEndian(k)
|
||||
if step > 1 {
|
||||
slot := bytesutil.BytesToSlotBigEndian(k)
|
||||
if slot.SubSlot(startSlot).Mod(step) != 0 {
|
||||
continue
|
||||
}
|
||||
@@ -713,8 +795,9 @@ func blockRootsBySlotRange(
|
||||
splitRoots = append(splitRoots, v[i:i+32])
|
||||
}
|
||||
roots = append(roots, splitRoots...)
|
||||
slts = append(slts, slot)
|
||||
}
|
||||
return roots, nil
|
||||
return roots, slts, nil
|
||||
}
|
||||
|
||||
// blockRootsBySlot retrieves the block roots by slot
|
||||
@@ -813,9 +896,9 @@ func unmarshalBlock(_ context.Context, enc []byte) (interfaces.ReadOnlySignedBea
|
||||
if err := rawBlock.UnmarshalSSZ(enc[len(denebBlindKey):]); err != nil {
|
||||
return nil, errors.Wrap(err, "could not unmarshal blinded Deneb block")
|
||||
}
|
||||
case hasElectraKey(enc):
|
||||
case HasElectraKey(enc):
|
||||
rawBlock = ðpb.SignedBeaconBlockElectra{}
|
||||
if err := rawBlock.UnmarshalSSZ(enc[len(electraKey):]); err != nil {
|
||||
if err := rawBlock.UnmarshalSSZ(enc[len(ElectraKey):]); err != nil {
|
||||
return nil, errors.Wrap(err, "could not unmarshal Electra block")
|
||||
}
|
||||
case hasElectraBlindKey(enc):
|
||||
@@ -874,7 +957,7 @@ func keyForBlock(blk interfaces.ReadOnlySignedBeaconBlock) ([]byte, error) {
|
||||
if blk.IsBlinded() {
|
||||
return electraBlindKey, nil
|
||||
}
|
||||
return electraKey, nil
|
||||
return ElectraKey, nil
|
||||
}
|
||||
|
||||
if v >= version.Deneb {
|
||||
@@ -908,3 +991,32 @@ func keyForBlock(blk interfaces.ReadOnlySignedBeaconBlock) ([]byte, error) {
|
||||
|
||||
return nil, fmt.Errorf("unsupported block version: %v", blk.Version())
|
||||
}
|
||||
|
||||
func (s *Store) deleteBlock(tx *bolt.Tx, root []byte) error {
|
||||
if err := tx.Bucket(blocksBucket).Delete(root); err != nil {
|
||||
return errors.Wrap(err, "could not delete block")
|
||||
}
|
||||
|
||||
if err := tx.Bucket(blockParentRootIndicesBucket).Delete(root); err != nil {
|
||||
return errors.Wrap(err, "could not delete block parent indices")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) deleteValidatorHashes(tx *bolt.Tx, root []byte) error {
|
||||
ok, err := s.isStateValidatorMigrationOver()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete the validator hash index
|
||||
if err = tx.Bucket(blockRootValidatorHashesBucket).Delete(root); err != nil {
|
||||
return errors.Wrap(err, "could not delete validator index")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -2,12 +2,14 @@ package kv
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/db/filters"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/state"
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/blocks"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/interfaces"
|
||||
@@ -18,6 +20,7 @@ import (
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/assert"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/require"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/util"
|
||||
bolt "go.etcd.io/bbolt"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
@@ -353,6 +356,189 @@ func TestStore_DeleteFinalizedBlock(t *testing.T) {
|
||||
require.NoError(t, db.SaveFinalizedCheckpoint(ctx, cp))
|
||||
require.ErrorIs(t, db.DeleteBlock(ctx, root), ErrDeleteJustifiedAndFinalized)
|
||||
}
|
||||
|
||||
func TestStore_HistoricalDataBeforeSlot(t *testing.T) {
|
||||
slotsPerEpoch := uint64(params.BeaconConfig().SlotsPerEpoch)
|
||||
db := setupDB(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// Save genesis block root
|
||||
require.NoError(t, db.SaveGenesisBlockRoot(ctx, genesisBlockRoot))
|
||||
|
||||
// Create and save blocks for 4 epochs
|
||||
blks := makeBlocks(t, 0, slotsPerEpoch*4, genesisBlockRoot)
|
||||
require.NoError(t, db.SaveBlocks(ctx, blks))
|
||||
|
||||
// Mark state validator migration as complete
|
||||
err := db.db.Update(func(tx *bolt.Tx) error {
|
||||
return tx.Bucket(migrationsBucket).Put(migrationStateValidatorsKey, migrationCompleted)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
migrated, err := db.isStateValidatorMigrationOver()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, true, migrated)
|
||||
|
||||
// Create state summaries and states for each block
|
||||
ss := make([]*ethpb.StateSummary, len(blks))
|
||||
states := make([]state.BeaconState, len(blks))
|
||||
|
||||
for i, blk := range blks {
|
||||
slot := blk.Block().Slot()
|
||||
r, err := blk.Block().HashTreeRoot()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create and save state summary
|
||||
ss[i] = ðpb.StateSummary{
|
||||
Slot: slot,
|
||||
Root: r[:],
|
||||
}
|
||||
|
||||
// Create and save state with validator entries
|
||||
vals := make([]*ethpb.Validator, 2)
|
||||
for j := range vals {
|
||||
vals[j] = ðpb.Validator{
|
||||
PublicKey: bytesutil.PadTo([]byte{byte(i*j + 1)}, 48),
|
||||
WithdrawalCredentials: bytesutil.PadTo([]byte{byte(i*j + 2)}, 32),
|
||||
}
|
||||
}
|
||||
|
||||
st, err := util.NewBeaconState(func(state *ethpb.BeaconState) error {
|
||||
state.Validators = vals
|
||||
state.Slot = slot
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.SaveState(ctx, st, r))
|
||||
states[i] = st
|
||||
|
||||
// Verify validator entries are saved to db
|
||||
valsActual, err := db.validatorEntries(ctx, r)
|
||||
require.NoError(t, err)
|
||||
for j, val := range valsActual {
|
||||
require.DeepEqual(t, vals[j], val)
|
||||
}
|
||||
}
|
||||
require.NoError(t, db.SaveStateSummaries(ctx, ss))
|
||||
|
||||
// Verify slot indices exist before deletion
|
||||
err = db.db.View(func(tx *bolt.Tx) error {
|
||||
blockSlotBkt := tx.Bucket(blockSlotIndicesBucket)
|
||||
stateSlotBkt := tx.Bucket(stateSlotIndicesBucket)
|
||||
|
||||
for i := uint64(0); i < slotsPerEpoch; i++ {
|
||||
slot := bytesutil.SlotToBytesBigEndian(primitives.Slot(i + 1))
|
||||
assert.NotNil(t, blockSlotBkt.Get(slot), "Expected block slot index to exist")
|
||||
assert.NotNil(t, stateSlotBkt.Get(slot), "Expected state slot index to exist", i)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Delete data before slot at epoch 1
|
||||
require.NoError(t, db.DeleteHistoricalDataBeforeSlot(ctx, primitives.Slot(slotsPerEpoch)))
|
||||
|
||||
// Verify blocks from epoch 0 are deleted
|
||||
for i := uint64(0); i < slotsPerEpoch; i++ {
|
||||
root, err := blks[i].Block().HashTreeRoot()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Check block is deleted
|
||||
retrievedBlocks, err := db.BlocksBySlot(ctx, primitives.Slot(i))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, len(retrievedBlocks))
|
||||
|
||||
// Verify block does not exist
|
||||
assert.Equal(t, false, db.HasBlock(ctx, root))
|
||||
|
||||
// Verify block parent root does not exist
|
||||
err = db.db.View(func(tx *bolt.Tx) error {
|
||||
require.Equal(t, 0, len(tx.Bucket(blockParentRootIndicesBucket).Get(root[:])))
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify state is deleted
|
||||
hasState := db.HasState(ctx, root)
|
||||
assert.Equal(t, false, hasState)
|
||||
|
||||
// Verify state summary is deleted
|
||||
hasSummary := db.HasStateSummary(ctx, root)
|
||||
assert.Equal(t, false, hasSummary)
|
||||
|
||||
// Verify validator hashes for block roots are deleted
|
||||
err = db.db.View(func(tx *bolt.Tx) error {
|
||||
assert.Equal(t, 0, len(tx.Bucket(blockRootValidatorHashesBucket).Get(root[:])))
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Verify slot indices are deleted
|
||||
err = db.db.View(func(tx *bolt.Tx) error {
|
||||
blockSlotBkt := tx.Bucket(blockSlotIndicesBucket)
|
||||
stateSlotBkt := tx.Bucket(stateSlotIndicesBucket)
|
||||
|
||||
for i := uint64(0); i < slotsPerEpoch; i++ {
|
||||
slot := bytesutil.SlotToBytesBigEndian(primitives.Slot(i + 1))
|
||||
assert.Equal(t, 0, len(blockSlotBkt.Get(slot)), fmt.Sprintf("Expected block slot index to be deleted, slot: %d", slot))
|
||||
assert.Equal(t, 0, len(stateSlotBkt.Get(slot)), fmt.Sprintf("Expected state slot index to be deleted, slot: %d", slot))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify blocks from epochs 1-3 still exist
|
||||
for i := slotsPerEpoch; i < slotsPerEpoch*4; i++ {
|
||||
root, err := blks[i].Block().HashTreeRoot()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify block exists
|
||||
assert.Equal(t, true, db.HasBlock(ctx, root))
|
||||
|
||||
// Verify remaining block parent root exists, except last slot since we store parent roots of each block.
|
||||
if i < slotsPerEpoch*4-1 {
|
||||
err = db.db.View(func(tx *bolt.Tx) error {
|
||||
require.NotNil(t, tx.Bucket(blockParentRootIndicesBucket).Get(root[:]), fmt.Sprintf("Expected block parent index to be deleted, slot: %d", i))
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Verify state exists
|
||||
hasState := db.HasState(ctx, root)
|
||||
assert.Equal(t, true, hasState)
|
||||
|
||||
// Verify state summary exists
|
||||
hasSummary := db.HasStateSummary(ctx, root)
|
||||
assert.Equal(t, true, hasSummary)
|
||||
|
||||
// Verify slot indices still exist
|
||||
err = db.db.View(func(tx *bolt.Tx) error {
|
||||
blockSlotBkt := tx.Bucket(blockSlotIndicesBucket)
|
||||
stateSlotBkt := tx.Bucket(stateSlotIndicesBucket)
|
||||
|
||||
slot := bytesutil.SlotToBytesBigEndian(primitives.Slot(i + 1))
|
||||
assert.NotNil(t, blockSlotBkt.Get(slot), "Expected block slot index to exist")
|
||||
assert.NotNil(t, stateSlotBkt.Get(slot), "Expected state slot index to exist")
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify validator entries still exist
|
||||
valsActual, err := db.validatorEntries(ctx, root)
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, valsActual)
|
||||
|
||||
// Verify remaining validator hashes for block roots exists
|
||||
err = db.db.View(func(tx *bolt.Tx) error {
|
||||
assert.NotNil(t, tx.Bucket(blockRootValidatorHashesBucket).Get(root[:]))
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStore_GenesisBlock(t *testing.T) {
|
||||
db := setupDB(t)
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -52,11 +52,12 @@ func hasDenebBlindKey(enc []byte) bool {
|
||||
return bytes.Equal(enc[:len(denebBlindKey)], denebBlindKey)
|
||||
}
|
||||
|
||||
func hasElectraKey(enc []byte) bool {
|
||||
if len(electraKey) >= len(enc) {
|
||||
// HasElectraKey verifies if the encoding is Electra compatible.
|
||||
func HasElectraKey(enc []byte) bool {
|
||||
if len(ElectraKey) >= len(enc) {
|
||||
return false
|
||||
}
|
||||
return bytes.Equal(enc[:len(electraKey)], electraKey)
|
||||
return bytes.Equal(enc[:len(ElectraKey)], ElectraKey)
|
||||
}
|
||||
|
||||
func hasElectraBlindKey(enc []byte) bool {
|
||||
|
||||
@@ -167,13 +167,13 @@ func decodeLightClientBootstrap(enc []byte) (interfaces.LightClientBootstrap, []
|
||||
}
|
||||
m = bootstrap
|
||||
syncCommitteeHash = enc[len(denebKey) : len(denebKey)+32]
|
||||
case hasElectraKey(enc):
|
||||
case HasElectraKey(enc):
|
||||
bootstrap := ðpb.LightClientBootstrapElectra{}
|
||||
if err := bootstrap.UnmarshalSSZ(enc[len(electraKey)+32:]); err != nil {
|
||||
if err := bootstrap.UnmarshalSSZ(enc[len(ElectraKey)+32:]); err != nil {
|
||||
return nil, nil, errors.Wrap(err, "could not unmarshal Electra light client bootstrap")
|
||||
}
|
||||
m = bootstrap
|
||||
syncCommitteeHash = enc[len(electraKey) : len(electraKey)+32]
|
||||
syncCommitteeHash = enc[len(ElectraKey) : len(ElectraKey)+32]
|
||||
default:
|
||||
return nil, nil, errors.New("decoding of saved light client bootstrap is unsupported")
|
||||
}
|
||||
@@ -277,9 +277,9 @@ func decodeLightClientUpdate(enc []byte) (interfaces.LightClientUpdate, error) {
|
||||
return nil, errors.Wrap(err, "could not unmarshal Deneb light client update")
|
||||
}
|
||||
m = update
|
||||
case hasElectraKey(enc):
|
||||
case HasElectraKey(enc):
|
||||
update := ðpb.LightClientUpdateElectra{}
|
||||
if err := update.UnmarshalSSZ(enc[len(electraKey):]); err != nil {
|
||||
if err := update.UnmarshalSSZ(enc[len(ElectraKey):]); err != nil {
|
||||
return nil, errors.Wrap(err, "could not unmarshal Electra light client update")
|
||||
}
|
||||
m = update
|
||||
@@ -292,7 +292,7 @@ func decodeLightClientUpdate(enc []byte) (interfaces.LightClientUpdate, error) {
|
||||
func keyForLightClientUpdate(v int) ([]byte, error) {
|
||||
switch v {
|
||||
case version.Electra:
|
||||
return electraKey, nil
|
||||
return ElectraKey, nil
|
||||
case version.Deneb:
|
||||
return denebKey, nil
|
||||
case version.Capella:
|
||||
|
||||
@@ -53,7 +53,7 @@ var (
|
||||
saveBlindedBeaconBlocksKey = []byte("save-blinded-beacon-blocks")
|
||||
denebKey = []byte("deneb")
|
||||
denebBlindKey = []byte("blind-deneb")
|
||||
electraKey = []byte("electra")
|
||||
ElectraKey = []byte("electra")
|
||||
electraBlindKey = []byte("blind-electra")
|
||||
fuluKey = []byte("fulu")
|
||||
fuluBlindKey = []byte("blind-fulu")
|
||||
|
||||
@@ -357,7 +357,7 @@ func (s *Store) processElectra(ctx context.Context, pbState *ethpb.BeaconStateEl
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
encodedState := snappy.Encode(nil, append(electraKey, rawObj...))
|
||||
encodedState := snappy.Encode(nil, append(ElectraKey, rawObj...))
|
||||
if err := bucket.Put(rootHash, encodedState); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -530,9 +530,9 @@ func (s *Store) unmarshalState(_ context.Context, enc []byte, validatorEntries [
|
||||
protoState.Validators = validatorEntries
|
||||
}
|
||||
return statenative.InitializeFromProtoUnsafeFulu(protoState)
|
||||
case hasElectraKey(enc):
|
||||
case HasElectraKey(enc):
|
||||
protoState := ðpb.BeaconStateElectra{}
|
||||
if err := protoState.UnmarshalSSZ(enc[len(electraKey):]); err != nil {
|
||||
if err := protoState.UnmarshalSSZ(enc[len(ElectraKey):]); err != nil {
|
||||
return nil, errors.Wrap(err, "failed to unmarshal encoding for Electra")
|
||||
}
|
||||
ok, err := s.isStateValidatorMigrationOver()
|
||||
@@ -688,7 +688,7 @@ func marshalState(ctx context.Context, st state.ReadOnlyBeaconState) ([]byte, er
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return snappy.Encode(nil, append(electraKey, rawObj...)), nil
|
||||
return snappy.Encode(nil, append(ElectraKey, rawObj...)), nil
|
||||
case version.Fulu:
|
||||
rState, ok := st.ToProtoUnsafe().(*ethpb.BeaconStateFulu)
|
||||
if !ok {
|
||||
@@ -725,7 +725,7 @@ func (s *Store) validatorEntries(ctx context.Context, blockRoot [32]byte) ([]*et
|
||||
idxBkt := tx.Bucket(blockRootValidatorHashesBucket)
|
||||
valKey := idxBkt.Get(blockRoot[:])
|
||||
if len(valKey) == 0 {
|
||||
return errors.Errorf("invalid compressed validator keys length")
|
||||
return errors.Errorf("validator keys not found for given block root: %x", blockRoot)
|
||||
}
|
||||
|
||||
// decompress the keys and check if they are of proper length.
|
||||
|
||||
38
beacon-chain/db/pruner/BUILD.bazel
Normal file
38
beacon-chain/db/pruner/BUILD.bazel
Normal file
@@ -0,0 +1,38 @@
|
||||
load("@prysm//tools/go:def.bzl", "go_library", "go_test")
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = ["pruner.go"],
|
||||
importpath = "github.com/prysmaticlabs/prysm/v5/beacon-chain/db/pruner",
|
||||
visibility = [
|
||||
"//beacon-chain:__subpackages__",
|
||||
],
|
||||
deps = [
|
||||
"//beacon-chain/core/helpers:go_default_library",
|
||||
"//beacon-chain/db:go_default_library",
|
||||
"//beacon-chain/db/iface:go_default_library",
|
||||
"//config/params:go_default_library",
|
||||
"//consensus-types/primitives:go_default_library",
|
||||
"//time/slots:go_default_library",
|
||||
"@com_github_pkg_errors//:go_default_library",
|
||||
"@com_github_sirupsen_logrus//:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = ["pruner_test.go"],
|
||||
embed = [":go_default_library"],
|
||||
deps = [
|
||||
"//beacon-chain/db/testing:go_default_library",
|
||||
"//config/params:go_default_library",
|
||||
"//consensus-types/blocks:go_default_library",
|
||||
"//consensus-types/primitives:go_default_library",
|
||||
"//proto/prysm/v1alpha1:go_default_library",
|
||||
"//testing/require:go_default_library",
|
||||
"//testing/util:go_default_library",
|
||||
"//time/slots/testing:go_default_library",
|
||||
"@com_github_sirupsen_logrus//:go_default_library",
|
||||
"@com_github_sirupsen_logrus//hooks/test:go_default_library",
|
||||
],
|
||||
)
|
||||
174
beacon-chain/db/pruner/pruner.go
Normal file
174
beacon-chain/db/pruner/pruner.go
Normal file
@@ -0,0 +1,174 @@
|
||||
package pruner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/helpers"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/db"
|
||||
"github.com/prysmaticlabs/prysm/v5/beacon-chain/db/iface"
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
"github.com/prysmaticlabs/prysm/v5/time/slots"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var log = logrus.WithField("prefix", "db-pruner")
|
||||
|
||||
type ServiceOption func(*Service)
|
||||
|
||||
// WithRetentionPeriod allows the user to specify a different data retention period than the spec default.
|
||||
// The retention period is specified in epochs, and must be >= MIN_EPOCHS_FOR_BLOCK_REQUESTS.
|
||||
func WithRetentionPeriod(retentionEpochs primitives.Epoch) ServiceOption {
|
||||
return func(s *Service) {
|
||||
defaultRetentionEpochs := helpers.MinEpochsForBlockRequests() + 1
|
||||
if retentionEpochs < defaultRetentionEpochs {
|
||||
log.WithField("userEpochs", retentionEpochs).
|
||||
WithField("minRequired", defaultRetentionEpochs).
|
||||
Warn("Retention period too low, using minimum required value")
|
||||
}
|
||||
|
||||
s.ps = pruneStartSlotFunc(retentionEpochs)
|
||||
}
|
||||
}
|
||||
|
||||
func WithSlotTicker(slotTicker slots.Ticker) ServiceOption {
|
||||
return func(s *Service) {
|
||||
s.slotTicker = slotTicker
|
||||
}
|
||||
}
|
||||
|
||||
// Service defines a service that prunes beacon chain DB based on MIN_EPOCHS_FOR_BLOCK_REQUESTS.
|
||||
type Service struct {
|
||||
ctx context.Context
|
||||
db db.Database
|
||||
ps func(current primitives.Slot) primitives.Slot
|
||||
prunedUpto primitives.Slot
|
||||
done chan struct{}
|
||||
slotTicker slots.Ticker
|
||||
backfillWaiter func() error
|
||||
initSyncWaiter func() error
|
||||
}
|
||||
|
||||
func New(ctx context.Context, db iface.Database, genesisTime uint64, initSyncWaiter, backfillWaiter func() error, opts ...ServiceOption) (*Service, error) {
|
||||
p := &Service{
|
||||
ctx: ctx,
|
||||
db: db,
|
||||
ps: pruneStartSlotFunc(helpers.MinEpochsForBlockRequests() + 1), // Default retention epochs is MIN_EPOCHS_FOR_BLOCK_REQUESTS + 1 from the current slot.
|
||||
done: make(chan struct{}),
|
||||
slotTicker: slots.NewSlotTicker(slots.StartTime(genesisTime, 0), params.BeaconConfig().SecondsPerSlot),
|
||||
initSyncWaiter: initSyncWaiter,
|
||||
backfillWaiter: backfillWaiter,
|
||||
}
|
||||
|
||||
for _, o := range opts {
|
||||
o(p)
|
||||
}
|
||||
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (p *Service) Start() {
|
||||
log.Info("Starting Beacon DB pruner service")
|
||||
p.run()
|
||||
}
|
||||
|
||||
func (p *Service) Stop() error {
|
||||
log.Info("Stopping Beacon DB pruner service")
|
||||
close(p.done)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Service) Status() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Service) run() {
|
||||
if p.initSyncWaiter != nil {
|
||||
log.Info("Waiting for initial sync service to complete before starting pruner")
|
||||
if err := p.initSyncWaiter(); err != nil {
|
||||
log.WithError(err).Error("Failed to start database pruner, error waiting for initial sync completion")
|
||||
return
|
||||
}
|
||||
}
|
||||
if p.backfillWaiter != nil {
|
||||
log.Info("Waiting for backfill service to complete before starting pruner")
|
||||
if err := p.backfillWaiter(); err != nil {
|
||||
log.WithError(err).Error("Failed to start database pruner, error waiting for backfill completion")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
defer p.slotTicker.Done()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-p.ctx.Done():
|
||||
log.Debug("Stopping Beacon DB pruner service", "prunedUpto", p.prunedUpto)
|
||||
return
|
||||
case <-p.done:
|
||||
log.Debug("Stopping Beacon DB pruner service", "prunedUpto", p.prunedUpto)
|
||||
return
|
||||
case slot := <-p.slotTicker.C():
|
||||
// Prune at the middle of every epoch since we do a lot of things around epoch boundaries.
|
||||
if slots.SinceEpochStarts(slot) != (params.BeaconConfig().SlotsPerEpoch / 2) {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := p.prune(slot); err != nil {
|
||||
log.WithError(err).Error("Failed to prune database")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// prune deletes historical chain data beyond the pruneSlot.
|
||||
func (p *Service) prune(slot primitives.Slot) error {
|
||||
// Prune everything up to this slot (inclusive).
|
||||
pruneUpto := p.ps(slot)
|
||||
|
||||
// Can't prune beyond genesis.
|
||||
if pruneUpto == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Skip if already pruned up to this slot.
|
||||
if pruneUpto <= p.prunedUpto {
|
||||
return nil
|
||||
}
|
||||
|
||||
log.WithFields(logrus.Fields{
|
||||
"pruneUpto": pruneUpto,
|
||||
}).Debug("Pruning chain data")
|
||||
|
||||
tt := time.Now()
|
||||
if err := p.db.DeleteHistoricalDataBeforeSlot(p.ctx, pruneUpto); err != nil {
|
||||
return errors.Wrapf(err, "could not delete upto slot %d", pruneUpto)
|
||||
}
|
||||
|
||||
log.WithFields(logrus.Fields{
|
||||
"prunedUpto": pruneUpto,
|
||||
"duration": time.Since(tt),
|
||||
"currentSlot": slot,
|
||||
}).Debug("Successfully pruned chain data")
|
||||
|
||||
// Update pruning checkpoint.
|
||||
p.prunedUpto = pruneUpto
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// pruneStartSlotFunc returns the function to determine the start slot to start pruning.
|
||||
func pruneStartSlotFunc(retentionEpochs primitives.Epoch) func(primitives.Slot) primitives.Slot {
|
||||
return func(current primitives.Slot) primitives.Slot {
|
||||
if retentionEpochs > slots.MaxSafeEpoch() {
|
||||
retentionEpochs = slots.MaxSafeEpoch()
|
||||
}
|
||||
offset := slots.UnsafeEpochStart(retentionEpochs)
|
||||
if offset >= current {
|
||||
return 0
|
||||
}
|
||||
return current - offset
|
||||
}
|
||||
}
|
||||
135
beacon-chain/db/pruner/pruner_test.go
Normal file
135
beacon-chain/db/pruner/pruner_test.go
Normal file
@@ -0,0 +1,135 @@
|
||||
package pruner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prysmaticlabs/prysm/v5/config/params"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/blocks"
|
||||
eth "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1"
|
||||
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/util"
|
||||
slottest "github.com/prysmaticlabs/prysm/v5/time/slots/testing"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
dbtest "github.com/prysmaticlabs/prysm/v5/beacon-chain/db/testing"
|
||||
"github.com/prysmaticlabs/prysm/v5/consensus-types/primitives"
|
||||
"github.com/prysmaticlabs/prysm/v5/testing/require"
|
||||
logTest "github.com/sirupsen/logrus/hooks/test"
|
||||
)
|
||||
|
||||
func TestPruner_PruningConditions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
synced bool
|
||||
backfillCompleted bool
|
||||
expectedLog string
|
||||
}{
|
||||
{
|
||||
name: "Not synced",
|
||||
synced: false,
|
||||
backfillCompleted: true,
|
||||
expectedLog: "Waiting for initial sync service to complete before starting pruner",
|
||||
},
|
||||
{
|
||||
name: "Backfill incomplete",
|
||||
synced: true,
|
||||
backfillCompleted: false,
|
||||
expectedLog: "Waiting for backfill service to complete before starting pruner",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
logrus.SetLevel(logrus.DebugLevel)
|
||||
hook := logTest.NewGlobal()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
beaconDB := dbtest.SetupDB(t)
|
||||
|
||||
slotTicker := &slottest.MockTicker{Channel: make(chan primitives.Slot)}
|
||||
|
||||
waitChan := make(chan struct{})
|
||||
waiter := func() error {
|
||||
close(waitChan)
|
||||
return nil
|
||||
}
|
||||
|
||||
var initSyncWaiter, backfillWaiter func() error
|
||||
if !tt.synced {
|
||||
initSyncWaiter = waiter
|
||||
}
|
||||
if !tt.backfillCompleted {
|
||||
backfillWaiter = waiter
|
||||
}
|
||||
p, err := New(ctx, beaconDB, uint64(time.Now().Unix()), initSyncWaiter, backfillWaiter, WithSlotTicker(slotTicker))
|
||||
require.NoError(t, err)
|
||||
|
||||
go p.Start()
|
||||
<-waitChan
|
||||
cancel()
|
||||
|
||||
if tt.expectedLog != "" {
|
||||
require.LogsContain(t, hook, tt.expectedLog)
|
||||
}
|
||||
|
||||
require.NoError(t, p.Stop())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruner_PruneSuccess(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
beaconDB := dbtest.SetupDB(t)
|
||||
|
||||
// Create and save some blocks at different slots
|
||||
var blks []*eth.SignedBeaconBlock
|
||||
for slot := primitives.Slot(1); slot <= 32; slot++ {
|
||||
blk := util.NewBeaconBlock()
|
||||
blk.Block.Slot = slot
|
||||
wsb, err := blocks.NewSignedBeaconBlock(blk)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, beaconDB.SaveBlock(ctx, wsb))
|
||||
blks = append(blks, blk)
|
||||
}
|
||||
|
||||
// Create pruner with retention of 2 epochs (64 slots)
|
||||
retentionEpochs := primitives.Epoch(2)
|
||||
slotTicker := &slottest.MockTicker{Channel: make(chan primitives.Slot)}
|
||||
|
||||
p, err := New(
|
||||
ctx,
|
||||
beaconDB,
|
||||
uint64(time.Now().Unix()),
|
||||
nil,
|
||||
nil,
|
||||
WithSlotTicker(slotTicker),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
p.ps = func(current primitives.Slot) primitives.Slot {
|
||||
return current - primitives.Slot(retentionEpochs)*params.BeaconConfig().SlotsPerEpoch
|
||||
}
|
||||
|
||||
// Start pruner and trigger at middle of 3rd epoch (slot 80)
|
||||
go p.Start()
|
||||
currentSlot := primitives.Slot(80) // Middle of 3rd epoch
|
||||
slotTicker.Channel <- currentSlot
|
||||
// Send the same slot again to ensure the pruning operation completes
|
||||
slotTicker.Channel <- currentSlot
|
||||
|
||||
for slot := primitives.Slot(1); slot <= 32; slot++ {
|
||||
root, err := blks[slot-1].Block.HashTreeRoot()
|
||||
require.NoError(t, err)
|
||||
present := beaconDB.HasBlock(ctx, root)
|
||||
if slot <= 16 { // These should be pruned
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, false, present, "Expected present at slot %d to be pruned", slot)
|
||||
} else { // These should remain
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, true, present, "Expected present at slot %d to exist", slot)
|
||||
}
|
||||
}
|
||||
|
||||
require.NoError(t, p.Stop())
|
||||
}
|
||||
@@ -15,6 +15,7 @@ go_library(
|
||||
visibility = ["//beacon-chain:__subpackages__"],
|
||||
deps = [
|
||||
"//beacon-chain/db/iface:go_default_library",
|
||||
"//beacon-chain/db/kv:go_default_library",
|
||||
"//beacon-chain/slasher/types:go_default_library",
|
||||
"//config/params:go_default_library",
|
||||
"//consensus-types/primitives:go_default_library",
|
||||
@@ -22,6 +23,7 @@ go_library(
|
||||
"//io/file:go_default_library",
|
||||
"//monitoring/tracing/trace:go_default_library",
|
||||
"//proto/prysm/v1alpha1:go_default_library",
|
||||
"//runtime/version:go_default_library",
|
||||
"//time/slots:go_default_library",
|
||||
"@com_github_golang_snappy//:go_default_library",
|
||||
"@com_github_pkg_errors//:go_default_library",
|
||||
@@ -50,6 +52,7 @@ go_test(
|
||||
"//consensus-types/primitives:go_default_library",
|
||||
"//encoding/bytesutil:go_default_library",
|
||||
"//proto/prysm/v1alpha1:go_default_library",
|
||||
"//runtime/version:go_default_library",
|
||||
"//testing/require:go_default_library",
|
||||
"//time/slots:go_default_library",
|
||||
"@com_github_prysmaticlabs_fastssz//:go_default_library",
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user