Updates resyncIfBehind() functionality (#7039)

* defines BestNonFinalized
* updates blocks_fetcher
* updates blocks_queue
* Merge branch 'master' into fix-resync
* Nishant's suggestion
* Merge branch 'fix-resync' of github.com:prysmaticlabs/prysm into fix-resync
* unit test
* Merge branch 'master' into fix-resync
* fixes test
* Merge refs/heads/master into fix-resync

.bazelrc

@@ -5,6 +5,9 @@ test --test_verbose_timeout_warnings
 test --build_tests_only
 test --test_output=errors
 
+# Clearly indicate that coverage is enabled to disable certain nogo checks.
+coverage --define=coverage_enabled=1
+
 # Fix for rules_docker. See: https://github.com/bazelbuild/rules_docker/issues/842
 build --host_force_python=PY2
 test --host_force_python=PY2
@@ -30,20 +33,60 @@ build --define kafka_enabled=false
 test --define kafka_enabled=false
 run --define kafka_enabled=false
 
+build:kafka_enabled --define kafka_enabled=true
+build:kafka_enabled --define gotags=kafka_enabled
+
 # Release flags
 build:release --workspace_status_command=./scripts/workspace_status.sh
 build:release --stamp
 build:release --compilation_mode=opt
+build:release --config=llvm
 
 # LLVM compiler for building C/C++ dependencies.
 build:llvm --crosstool_top=@llvm_toolchain//:toolchain
 build:llvm --define compiler=llvm
+build:llvm --copt -fno-sanitize=vptr,function
+build:llvm --linkopt -fno-sanitize=vptr,function
+
+build:asan --copt -fsanitize=address,undefined
+build:asan --copt -fno-omit-frame-pointer
+build:asan --linkopt -fsanitize=address,undefined
+build:asan --copt -fno-sanitize=vptr,function
+build:asan --linkopt -fno-sanitize=vptr,function
+build:asan --copt -DADDRESS_SANITIZER=1
+build:asan --copt -D__SANITIZE_ADDRESS__
+build:asan --linkopt -ldl
+
+build:llvm-asan --config=llvm
+build:llvm-asan --config=asan
+build:llvm-asan --linkopt -fuse-ld=ld.lld
+
+build:fuzz --define=gotags=libfuzzer
+build:fuzz --config=llvm-asan
+build:fuzz --copt=-fsanitize=fuzzer-no-link
+build:fuzz --linkopt=-fsanitize=fuzzer
+build:fuzz --copt=-fno-omit-frame-pointer
+build:fuzz --define=FUZZING_ENGINE=libfuzzer
+build:fuzz --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+build:fuzz --linkopt -Wl,--no-as-needed
+build:fuzz --define=gc_goopts=-d=libfuzzer
+build:fuzz --run_under=//tools:fuzz_wrapper
+build:fuzz --compilation_mode=opt
+
+test:fuzz --local_test_jobs="HOST_CPUS*.5"
+
+test:fuzzit --config=fuzz
+test:fuzzit --test_env=FUZZIT_API_KEY
+test:fuzzit --test_env=PRYSM_BUILD_IMAGE=gcr.io/prysmaticlabs/prysm-fuzzit:v0.11.0
+test:fuzzit --test_timeout=1200
+test:fuzzit --run_under=//tools:fuzzit_wrapper
 
 # Build binary with cgo symbolizer for debugging / profiling.
 build:cgo_symbolizer --config=llvm
 build:cgo_symbolizer --copt=-g
 build:cgo_symbolizer --define=USE_CGO_SYMBOLIZER=true
 build:cgo_symbolizer -c dbg
+build:cgo_symbolizer --define=gotags=cgosymbolizer_enabled
 
 # multi-arch cross-compiling toolchain configs:
 -----------------------------------------------

.bazelversion

@@ -1 +1 @@
-3.0.0
+3.2.0

.buildkite-bazelrc

@@ -2,24 +2,14 @@
 # across machines, developers, and workspaces.
 # 
 # This config is loaded from https://github.com/bazelbuild/bazel-toolchains/blob/master/bazelrc/latest.bazelrc
-build:remote-cache --remote_cache=grpcs://remotebuildexecution.googleapis.com
 build:remote-cache --remote_timeout=3600
-build:remote-cache --auth_enabled=true
 build:remote-cache --spawn_strategy=standalone
 build:remote-cache --strategy=Javac=standalone
 build:remote-cache --strategy=Closure=standalone
 build:remote-cache --strategy=Genrule=standalone
 
-# Build results backend.                                                        
-#build:remote-cache --bes_results_url="https://source.cloud.google.com/results/invocations/"
-#build:remote-cache --bes_backend=buildeventservice.googleapis.com               
-#build:remote-cache --bes_timeout=60s                                            
-#build:remote-cache --project_id=prysmaticlabs
-
 # Prysm specific remote-cache properties.
-build:remote-cache --disk_cache=
-build:remote-cache --host_platform_remote_properties_override='properties:{name:\"cache-silo-key\" value:\"prysm\"}' 
-build:remote-cache --remote_instance_name=projects/prysmaticlabs/instances/default_instance
+#build:remote-cache --disk_cache=
 build:remote-cache --remote_download_minimal
 
 # Import workspace options.
@@ -45,7 +35,15 @@ build --flaky_test_attempts=5
 # build --features=race
 
 # Enable kafka for CI tests only.
-test --define kafka_enabled=true
+test --config=kafka_enabled
 
 build --bes_backend=grpcs://builds.prylabs.net:1985
 build --bes_results_url=https://builds.prylabs.net/invocation/
+
+# Disable flaky test detection for fuzzing.
+test:fuzz --flaky_test_attempts=1
+
+# Expose test environment variables in CI
+test:fuzzit --test_env=GITHUB_REF
+test:fuzzit --test_env=GITHUB_SHA
+test:fuzzit --test_env=DOCKER_HOST

.github/CODEOWNERS

@@ -3,3 +3,6 @@
 
 # Starlark code owners
 *.bzl @prestonvanloon
+
+# Anyone on prylabs team can approve dependency updates.
+deps.bzl @prysmaticlabs/core-team

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,59 @@
+---
+name: "\U0001F41EBug report"
+about: Report a bug or problem with running Prysm
+---
+<!--💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎
+
+Hellooo! 😄 
+
+To help us tend to your issue faster, please search our currently open issues before submitting a new one.
+Existing issues often contain information about workarounds, resolution, or progress updates.
+
+💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎-->
+
+# 🐞 Bug Report
+
+### Description
+
+<!-- ✍️--> A clear and concise description of the problem...
+
+### Has this worked before in a previous version?
+
+<!-- Did this behavior use to work in the previous version? -->
+<!-- ✍️--> Yes, the previous version in which this bug was not present was: ....
+
+## 🔬 Minimal Reproduction
+
+<!--
+Please let us know how we can reproduce this issue. Include the exact method you used to run Prysm along with any flags used in your beacon chain and/or validator. Make sure you don't upload any confidential files or private keys.
+-->
+
+## 🔥 Error
+
+<pre><code>
+<!-- If the issue is accompanied by an error, please share the error logs with us below. If you have a lot of logs, place make a paste bin with your logs and share the link with us here: -->
+<!-- ✍️-->
+
+</code></pre>
+
+
+## 🌍  Your Environment
+
+**Operating System:**
+
+<pre>
+  <code>
+
+  </code>
+</pre>
+
+**What version of Prysm are you running? (Which release)**
+
+<pre>
+  <code>
+
+  </code>
+</pre>
+
+**Anything else relevant (validator index / public key)?**
+

.github/ISSUE_TEMPLATE/discussion.md

@@ -0,0 +1,21 @@
+---
+name: "\U0001F48EGeneral issue / discussion"
+about: Any other type of general issue or discussion
+
+---
+<!--💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎
+
+Hellooo! 😄 
+
+To help us tend to your issue faster, please search our currently open issues before submitting a new one.
+Existing issues often contain information about workarounds, resolution, or progress updates.
+
+💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎-->
+
+# 💎 Issue
+
+### Background
+
+<!-- ✍️--> Context and background information on the discussion...
+
+### Description

.github/ISSUE_TEMPLATE/feature_flag.md

@@ -0,0 +1,38 @@
+---
+name: "\U0001F984Feature Flag Tracking"
+about: Track a new feature, in development, in Prysm. This issue template should only be used by developers or contributors!
+labels: Tracking
+---
+<!--💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎
+
+Hellooo! 😄 
+
+Thanks for taking the time to file a tracking issue for your new feature. These issues really help
+us track progress of features as they work their way through development. Be sure to review 
+shared/featureconfig/README.md for the latest documentation around feature flags.
+
+💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎-->
+
+# 🦄 Feature Tracking
+
+**Current status** (opt-in or opt-out): 
+
+**Opt-in feature flag name**:
+
+**Opt-out feature flag name**: 
+
+### Feature Description
+
+<!-- ✍️--> A clear and concise description of the new feature. Provide links to your technical
+design document or other supporting information.
+
+### Tasks
+
+- [ ] Feature flag created as opt-in
+- [ ] Feature implemented / code complete
+- [ ] Feature tested in production for adequate amount of time
+- [ ] Feature flag is inverted to be opt-out and the opt-in flag is deprecated
+- [ ] Feature has made it to a tagged production release as an opt-out flag
+- [ ] Opt-out feature flag is deprecated and old code paths are cleaned up
+
+This issue should be closed after all of the above tasks are complete.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,27 @@
+---
+name: "\U0001F680Feature request"
+about: Suggest a feature for Prysm
+
+---
+<!--💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎
+
+Hellooo! 😄 
+
+To help us tend to your issue faster, please search our currently open issues before submitting a new one.
+Existing issues often contain information about workarounds, resolution, or progress updates.
+
+💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎-->
+
+# 🚀 Feature Request
+
+### Description
+
+<!-- ✍️--> A clear and concise description of the problem or missing capability...
+
+### Describe the solution you'd like
+
+<!-- ✍️--> If you have a solution in mind, please describe it.
+
+### Describe alternatives you've considered
+
+<!-- ✍️--> Have you considered any alternative solutions or workarounds?

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,30 @@
+<!-- Thanks for sending a PR! Before submitting:
+
+1. If this is your first PR, please read CONTRIBUTING.md and sign the CLA
+   first. We cannot review code without a signed CLA.
+2. Please file an issue *first*. All features and most bug fixes should have
+   an associated issue with a design discussed and decided upon. Small bug
+   fixes and documentation improvements don't need issues.
+3. New features and bug fixes must have tests. Documentation may need to
+   be updated. If you're unsure what to update, send the PR, and we'll discuss
+   in review.
+4. Note that PRs updating dependencies and new Go versions are not accepted.
+   Please file an issue instead.
+-->
+
+**What type of PR is this?**
+
+> Uncomment one line below and remove others.
+>
+> Bug fix
+> Feature
+> Documentation
+> Other
+
+**What does this PR do? Why is it needed?**
+
+**Which issues(s) does this PR fix?**
+
+Fixes #
+
+**Other notes for review**

.github/actions/gofmt/Dockerfile

@@ -0,0 +1,5 @@
+FROM cytopia/gofmt
+
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

.github/actions/gofmt/action.yml

@@ -0,0 +1,12 @@
+name: 'Gofmt checker'
+description: 'Checks that all project files have been properly formatted.'
+inputs:
+  path:
+    description: 'Path to check'
+    required: true
+    default: './'
+runs:
+  using: 'docker'
+  image: 'Dockerfile'
+  args:
+    - ${{ inputs.path }}

.github/actions/gofmt/entrypoint.sh

@@ -0,0 +1,15 @@
+#!/bin/sh -l
+set -e
+
+cd $GITHUB_WORKSPACE
+
+# Check if any files are not formatted.
+nonformatted="$(gofmt -l $1 2>&1)"
+
+# Return if `go fmt` passes.
+[ -z "$nonformatted" ] && exit 0
+
+# Notify of issues with formatting.
+echo "Following files need to be properly formatted:"
+echo "$nonformatted"
+exit 1

.github/actions/gomodtidy/Dockerfile

@@ -0,0 +1,5 @@
+FROM golang:alpine
+
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

.github/actions/gomodtidy/action.yml

@@ -0,0 +1,5 @@
+name: 'Go mod tidy checker'
+description: 'Checks that `go mod tidy` has been applied.'
+runs:
+  using: 'docker'
+  image: 'Dockerfile'

.github/actions/gomodtidy/entrypoint.sh

@@ -0,0 +1,34 @@
+#!/bin/sh -l
+set -e
+export PATH=$PATH:/usr/local/go/bin
+
+cd $GITHUB_WORKSPACE
+
+cp go.mod go.mod.orig
+cp go.sum go.sum.orig
+
+go mod tidy
+
+echo "Checking go.mod and go.sum:"
+checks=0
+if [ "$(diff -s go.mod.orig go.mod | grep -c 'Files go.mod.orig and go.mod are identical')" = 1 ]; then
+  echo "- go.mod is up to date."
+  checks=$((checks + 1))
+else
+  echo "- go.mod is NOT up to date."
+fi
+
+if [ "$(diff -s go.sum.orig go.sum | grep -c 'Files go.sum.orig and go.sum are identical')" = 1 ]; then
+  echo "- go.sum is up to date."
+  checks=$((checks + 1))
+else
+  echo "- go.sum is NOT up to date."
+fi
+
+if [ $checks -eq 2 ]; then
+  exit 0
+fi
+
+# Notify of any issues.
+echo "Run 'go mod tidy' to update."
+exit 1

.github/workflows/dappnode-release-trigger.yml

@@ -0,0 +1,41 @@
+name: Update DAppNodePackages
+
+on:
+  push:
+    tags:
+    - '*'
+
+jobs:
+  dappnode-update-beacon-chain:
+    name: Trigger a beacon-chain release
+    runs-on: ubuntu-latest
+    steps:
+    - name: Get latest tag
+      id: get_tag
+      run: echo ::set-output name=TAG::${GITHUB_REF/refs\/tags\//}
+    - name: Send dispatch event to DAppNodePackage-prysm-beacon-chain
+      env: 
+        DISPATCH_REPO: dappnode/DAppNodePackage-prysm-beacon-chain
+      run: |
+        curl -v -X POST -u "${{ secrets.PAT_GITHUB }}" \
+        -H "Accept: application/vnd.github.everest-preview+json" \
+        -H "Content-Type: application/json" \
+        --data '{"event_type":"new_release", "client_payload": { "tag":"${{ steps.get_tag.outputs.TAG }}"}}' \
+        https://api.github.com/repos/$DISPATCH_REPO/dispatches
+        
+  dappnode-update-validator:
+    name: Trigger a validator release
+    runs-on: ubuntu-latest
+    steps:
+    - name: Get latest tag
+      id: get_tag
+      run: echo ::set-output name=TAG::${GITHUB_REF/refs\/tags\//}
+    - name: Send dispatch event to DAppNodePackage validator repository
+      env: 
+        DISPATCH_REPO: dappnode/DAppNodePackage-prysm-validator
+      run: |
+        curl -v -X POST -u "${{ secrets.PAT_GITHUB }}" \
+        -H "Accept: application/vnd.github.everest-preview+json" \
+        -H "Content-Type: application/json" \
+        --data '{"event_type":"new_release", "client_payload": { "tag":"${{ steps.get_tag.outputs.TAG }}"}}' \
+        https://api.github.com/repos/$DISPATCH_REPO/dispatches

.github/workflows/go.yml

@@ -0,0 +1,50 @@
+name: Go
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ '*' ]
+
+jobs:
+
+  check:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+
+      - name: Go mod tidy checker
+        id: gomodtidy
+        uses: ./.github/actions/gomodtidy
+
+      - name: Gofmt checker
+        id: gofmt
+        uses: ./.github/actions/gofmt
+        with:
+          path: ./
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.14
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Get dependencies
+        run: |
+          go get -v -t -d ./...
+
+      - name: Build
+        run: go build -v ./...
+
+# Tests run via Bazel for now...
+#      - name: Test
+#        run: go test -v ./...

.gitignore

@@ -27,9 +27,5 @@ yarn-error.log
 # Ignore password file
 password.txt
 
-# go dependancy
-/go.mod
-/go.sum
-
 # Dist files
 dist

BUILD.bazel

@@ -4,6 +4,7 @@ load("@com_github_atlassian_bazel_tools//goimports:def.bzl", "goimports")
 load("@io_kubernetes_build//defs:run_in_workspace.bzl", "workspace_binary")
 load("@io_bazel_rules_go//go:def.bzl", "nogo")
 load("@graknlabs_bazel_distribution//common:rules.bzl", "assemble_targz", "assemble_versioned")
+load("@bazel_skylib//rules:common_settings.bzl", "string_setting")
 
 prefix = "github.com/prysmaticlabs/prysm"
 
@@ -12,6 +13,8 @@ exports_files([
 ])
 
 # gazelle:prefix github.com/prysmaticlabs/prysm
+# gazelle:map_kind go_library go_library @prysm//tools/go:def.bzl
+# gazelle:map_kind go_repository go_repository @prysm//tools/go:def.bzl
 gazelle(
     name = "gazelle",
     prefix = prefix,
@@ -43,7 +46,7 @@ alias(
 # Protobuf gRPC gateway compiler
 alias(
     name = "grpc_gateway_proto_compiler",
-    actual = "@grpc_ecosystem_grpc_gateway//protoc-gen-grpc-gateway:go_gen_grpc_gateway",
+    actual = "@com_github_grpc_ecosystem_grpc_gateway//protoc-gen-grpc-gateway:go_gen_grpc_gateway",
     visibility = ["//visibility:public"],
 )
 
@@ -85,15 +88,12 @@ nogo(
         "@org_golang_x_tools//go/analysis/passes/pkgfact:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/nilness:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/nilfunc:go_tool_library",
-        # lost cancel ignore doesn't seem to work when running with coverage
-        #"@org_golang_x_tools//go/analysis/passes/lostcancel:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/loopclosure:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/httpresponse:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/findcall:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/deepequalerrors:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/ctrlflow:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/copylock:go_tool_library",
-        "@org_golang_x_tools//go/analysis/passes/composite:go_tool_library",
         #  "@org_golang_x_tools//go/analysis/passes/cgocall:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/buildtag:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/buildssa:go_tool_library",
@@ -105,8 +105,22 @@ nogo(
         "@org_golang_x_tools//go/analysis/passes/asmdecl:go_tool_library",
         "//tools/analyzers/maligned:go_tool_library",
         "//tools/analyzers/roughtime:go_tool_library",
+        "//tools/analyzers/cryptorand:go_tool_library",
         "//tools/analyzers/errcheck:go_tool_library",
-    ],
+        "//tools/analyzers/featureconfig:go_tool_library",
+    ] + select({
+        # nogo checks that fail with coverage enabled.
+        ":coverage_enabled": [],
+        "//conditions:default": [
+            "@org_golang_x_tools//go/analysis/passes/lostcancel:go_tool_library",
+            "@org_golang_x_tools//go/analysis/passes/composite:go_tool_library",
+        ],
+    }),
+)
+
+config_setting(
+    name = "coverage_enabled",
+    values = {"define": "coverage_enabled=1"},
 )
 
 common_files = {
@@ -119,3 +133,12 @@ toolchain(
     toolchain = "@rules_foreign_cc//tools/build_defs/native_tools:built_cmake",
     toolchain_type = "@rules_foreign_cc//tools/build_defs:cmake_toolchain",
 )
+
+string_setting(
+    name = "gotags",
+    build_setting_default = "",
+    values = [
+        "",
+        "libfuzzer",
+    ],
+)

CONTRIBUTING.md

@@ -1,6 +1,8 @@
 # Contribution Guidelines
 
-Excited by our work and want to get involved in building out our sharding releases? Or maybe you haven't learned as much about the Ethereum protocol but are a savvy developer? Our [READINGS.md](https://github.com/prysmaticlabs/prysm/blob/master/docs/READINGS.md) doc includes comprehensive information on Ethereum and sharding for both part-time and core contributors to the project.
+Note: The latest and most up to date documenation can be found on our [docs portal](https://docs.prylabs.network/docs/contribute/contribution-guidelines).
+
+Excited by our work and want to get involved in building out our sharding releases? Or maybe you haven't learned as much about the Ethereum protocol but are a savvy developer? 
 
 You can explore our [Open Issues](https://github.com/prysmaticlabs/prysm/issues) in-the works for our different releases. Feel free to fork our repo and start creating PR’s after assigning yourself to an issue of interest. We are always chatting on [Discord](https://discord.gg/che9auJ) or [Gitter](https://gitter.im/prysmaticlabs/geth-sharding) drop us a line there if you want to get more involved or have any questions on our implementation!
 

DEPENDENCIES.md

@@ -0,0 +1,69 @@
+# Dependency Managagement in Prysm
+
+Prysm is go project with many complicated dependencies, including some c++ based libraries. There
+are two parts to Prysm's dependency management. Go modules and bazel managed dependencies. Be sure 
+to read [Why Bazel?](https://github.com/prysmaticlabs/documentation/issues/138) to fully
+understand the reasoning behind an additional layer of build tooling via Bazel rather than a pure
+"go build" project.
+
+## Go Module support
+
+The Prysm project officially supports go modules with some caveats. 
+
+### Caveat 1: Some c++ libraries are precompiled archives
+
+Given some of Prysm's c++ dependencies have very complicated project structures which make building
+difficult or impossible with "go build" alone. Additionally, building c++ dependencies with certain
+compilers, like clang / LLVM, offer a significant performance improvement. To get around this 
+issue, c++ dependencies have been precompiled as linkable archives. While there isn't necessarily
+anything bad about precompiled archives, these files are a "blackbox" which a 3rd party author
+could have compiled anything for this archive and detecting undesired behavior would be nearly
+impossible. If your risk tolerance is low, always compile everything from source yourself, 
+including complicated c++ dependencies.
+
+*Recommendation: Use go build only for local development and use bazel build for production.*
+
+### Caveat 2: Generated gRPC protobuf libraries
+
+One key advantage of Bazel over vanilla `go build` is that Bazel automatically (re)builds generated
+pb.go files at build time when file changes are present in any protobuf definition file or after
+any updates to the protobuf compiler or other relevant dependencies. Vanilla go users should run
+the following scripts often to ensure their generated files are up to date. Further more, Prysm
+generates SSZ marshal related code based on defined data structures. These generated files must
+also be updated and checked in as frequently.
+
+```bash
+./scripts/update-go-pbs.sh
+./scripts/update-go-ssz.sh
+```
+
+*Recommendation: Use go build only for local development and use bazel build for production.*
+
+### Caveat 3: Compile-time optimizations 
+
+When Prysmatic Labs builds production binaries, they use the "release" configuration of bazel to
+compile with several compiler optimizations and recommended production build configurations.
+Additionally, the release build properly stamps the built binaries to include helpful metadata
+about how and when the binary was built. 
+
+*Recommendation: Use go build only for local development and use bazel build for production.*
+
+```bash
+bazel build //beacon-chain --config=release
+```
+ 
+## Adding / updating dependencies
+
+1. Add your dependency as you would with go modules. I.e. `go get ...`
+1. Run `gazelle update-repos -from_file=go.mod` to update the bazel managed dependencies.
+
+Example:
+
+```bash
+go get github.com/prysmaticlabs/example@v1.2.3
+bazel run //:gazelle -- update-repos -from_file=go.mod -to_macro=deps.bzl%prysm_deps
+```
+
+The deps.bzl file should have been updated with the dependency and any transitive dependencies. 
+
+Do NOT add new `go_repository` to the WORKSPACE file. All dependencies should live in deps.bzl.

PULL_REQUEST_TEMPLATE.md

@@ -1,13 +0,0 @@
-TODO(you): choose "part of" or "resolves" and the associated github issue #.
-
-[Part of|Resolves] #531 
-
----
-
-# Description
-
-**Write why you are making the changes in this pull request**
-
-**Write a summary of the changes you are making**
-
-**Link anything that would be helpful or relevant to the reviewers**

README.md

@@ -1,7 +1,7 @@
 # Prysm: An Ethereum 2.0 Client Written in Go
 
 [![Build status](https://badge.buildkite.com/b555891daf3614bae4284dcf365b2340cefc0089839526f096.svg?branch=master)](https://buildkite.com/prysmatic-labs/prysm)
-[![ETH2.0_Spec_Version 0.11.1](https://img.shields.io/badge/ETH2.0%20Spec%20Version-v0.11.1-blue.svg)](https://github.com/ethereum/eth2.0-specs/tree/v0.11.1)
+[![ETH2.0_Spec_Version 0.12.2](https://img.shields.io/badge/ETH2.0%20Spec%20Version-v0.12.2-blue.svg)](https://github.com/ethereum/eth2.0-specs/tree/v0.12.2)
 [![Discord](https://user-images.githubusercontent.com/7288322/34471967-1df7808a-efbb-11e7-9088-ed0b04151291.png)](https://discord.gg/KSA7rPr)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/prysmaticlabs/geth-sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 

beacon-chain/BUILD.bazel

@@ -1,4 +1,5 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_test")
 load("@io_bazel_rules_docker//go:image.bzl", "go_image")
 load("@io_bazel_rules_docker//container:container.bzl", "container_bundle")
 load("//tools:go_image.bzl", "go_image_alpine", "go_image_debug")
@@ -19,15 +20,15 @@ go_library(
         "//shared/debug:go_default_library",
         "//shared/featureconfig:go_default_library",
         "//shared/logutil:go_default_library",
+        "//shared/maxprocs:go_default_library",
         "//shared/version:go_default_library",
-        "@com_github_ipfs_go_log//:go_default_library",
+        "@com_github_ethereum_go_ethereum//log:go_default_library",
+        "@com_github_ipfs_go_log_v2//:go_default_library",
         "@com_github_joonix_log//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_whyrusleeping_go_logging//:go_default_library",
+        "@com_github_urfave_cli_v2//:go_default_library",
+        "@com_github_urfave_cli_v2//altsrc:go_default_library",
         "@com_github_x_cray_logrus_prefixed_formatter//:go_default_library",
-        "@in_gopkg_urfave_cli_v2//:go_default_library",
-        "@in_gopkg_urfave_cli_v2//altsrc:go_default_library",
-        "@org_uber_go_automaxprocs//:go_default_library",
     ],
 )
 
@@ -57,14 +58,14 @@ go_image(
         "//shared/featureconfig:go_default_library",
         "//shared/logutil:go_default_library",
         "//shared/version:go_default_library",
-        "@com_github_ipfs_go_log//:go_default_library",
+        "@com_github_ethereum_go_ethereum//log:go_default_library",
+        "@com_github_ipfs_go_log_v2//:go_default_library",
         "@com_github_joonix_log//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_whyrusleeping_go_logging//:go_default_library",
+        "@com_github_urfave_cli_v2//:go_default_library",
+        "@com_github_urfave_cli_v2//altsrc:go_default_library",
         "@com_github_x_cray_logrus_prefixed_formatter//:go_default_library",
-        "@in_gopkg_urfave_cli_v2//:go_default_library",
-        "@in_gopkg_urfave_cli_v2//altsrc:go_default_library",
-        "@org_uber_go_automaxprocs//:go_default_library",
+        "//shared/maxprocs:go_default_library",
     ],
 )
 
@@ -73,6 +74,8 @@ container_bundle(
     images = {
         "gcr.io/prysmaticlabs/prysm/beacon-chain:latest": ":image",
         "gcr.io/prysmaticlabs/prysm/beacon-chain:{DOCKER_TAG}": ":image",
+        "index.docker.io/prysmaticlabs/prysm-beacon-chain:latest": ":image",
+        "index.docker.io/prysmaticlabs/prysm-beacon-chain:{DOCKER_TAG}": ":image",
     },
     tags = ["manual"],
 )
@@ -88,6 +91,8 @@ container_bundle(
     images = {
         "gcr.io/prysmaticlabs/prysm/beacon-chain:latest-debug": ":image_debug",
         "gcr.io/prysmaticlabs/prysm/beacon-chain:{DOCKER_TAG}-debug": ":image_debug",
+        "index.docker.io/prysmaticlabs/prysm-beacon-chain:latest-debug": ":image_debug",
+        "index.docker.io/prysmaticlabs/prysm-beacon-chain:{DOCKER_TAG}-debug": ":image_debug",
     },
     tags = ["manual"],
 )
@@ -103,6 +108,8 @@ container_bundle(
     images = {
         "gcr.io/prysmaticlabs/prysm/beacon-chain:latest-alpine": ":image_alpine",
         "gcr.io/prysmaticlabs/prysm/beacon-chain:{DOCKER_TAG}-alpine": ":image_alpine",
+        "index.docker.io/prysmaticlabs/prysm-beacon-chain:latest-alpine": ":image_alpine",
+        "index.docker.io/prysmaticlabs/prysm-beacon-chain:{DOCKER_TAG}-alpine": ":image_alpine",
     },
     tags = ["manual"],
 )
@@ -141,6 +148,6 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//shared/featureconfig:go_default_library",
-        "@in_gopkg_urfave_cli_v2//:go_default_library",
+        "@com_github_urfave_cli_v2//:go_default_library",
     ],
 )

beacon-chain/blockchain/BUILD.bazel

@@ -1,4 +1,5 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -29,7 +30,6 @@ go_library(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/state:go_default_library",
         "//beacon-chain/db:go_default_library",
-        "//beacon-chain/db/filters:go_default_library",
         "//beacon-chain/flags:go_default_library",
         "//beacon-chain/forkchoice:go_default_library",
         "//beacon-chain/forkchoice/protoarray:go_default_library",
@@ -43,6 +43,7 @@ go_library(
         "//beacon-chain/state/stateutil:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/attestationutil:go_default_library",
+        "//shared/bls:go_default_library",
         "//shared/bytesutil:go_default_library",
         "//shared/featureconfig:go_default_library",
         "//shared/params:go_default_library",
@@ -54,7 +55,6 @@ go_library(
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
-        "@com_github_prysmaticlabs_go_ssz//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@io_opencensus_go//trace:go_default_library",
     ],
@@ -74,14 +74,16 @@ go_test(
     srcs = [
         "chain_info_test.go",
         "head_test.go",
-        "init_sync_process_block_test.go",
+        "info_test.go",
         "process_attestation_test.go",
         "process_block_test.go",
         "receive_attestation_test.go",
+        "receive_block_test.go",
         "service_test.go",
     ],
     embed = [":go_default_library"],
     deps = [
+        "//beacon-chain/blockchain/testing:go_default_library",
         "//beacon-chain/cache/depositcache:go_default_library",
         "//beacon-chain/core/blocks:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
@@ -97,6 +99,8 @@ go_test(
         "//shared/event:go_default_library",
         "//shared/params:go_default_library",
         "//shared/testutil:go_default_library",
+        "//shared/testutil/assert:go_default_library",
+        "//shared/testutil/require:go_default_library",
         "@com_github_ethereum_go_ethereum//:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_ethereum_go_ethereum//core/types:go_default_library",
@@ -114,6 +118,7 @@ go_test(
     name = "go_raceon_test",
     srcs = [
         "chain_info_norace_test.go",
+        "receive_block_test.go",
         "service_norace_test.go",
     ],
     embed = [":go_default_library"],
@@ -127,6 +132,7 @@ go_test(
     race = "on",
     tags = ["race_on"],
     deps = [
+        "//beacon-chain/blockchain/testing:go_default_library",
         "//beacon-chain/cache/depositcache:go_default_library",
         "//beacon-chain/core/blocks:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
@@ -140,6 +146,8 @@ go_test(
         "//shared/event:go_default_library",
         "//shared/params:go_default_library",
         "//shared/testutil:go_default_library",
+        "//shared/testutil/assert:go_default_library",
+        "//shared/testutil/require:go_default_library",
         "@com_github_ethereum_go_ethereum//:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_ethereum_go_ethereum//core/types:go_default_library",

beacon-chain/blockchain/chain_info.go

@@ -1,17 +1,18 @@
 package blockchain
 
 import (
-	"bytes"
 	"context"
 	"time"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
+	"go.opencensus.io/trace"
 )
 
 // ChainInfoFetcher defines a common interface for methods in blockchain service which
@@ -19,6 +20,8 @@ import (
 type ChainInfoFetcher interface {
 	HeadFetcher
 	FinalizationFetcher
+	GenesisFetcher
+	CanonicalFetcher
 }
 
 // TimeFetcher retrieves the Eth2 data that's related to time.
@@ -27,6 +30,11 @@ type TimeFetcher interface {
 	CurrentSlot() uint64
 }
 
+// GenesisFetcher retrieves the eth2 data related to its genesis.
+type GenesisFetcher interface {
+	GenesisValidatorRoot() [32]byte
+}
+
 // HeadFetcher defines a common interface for methods in blockchain service which
 // directly retrieves head related data.
 type HeadFetcher interface {
@@ -34,8 +42,11 @@ type HeadFetcher interface {
 	HeadRoot(ctx context.Context) ([]byte, error)
 	HeadBlock(ctx context.Context) (*ethpb.SignedBeaconBlock, error)
 	HeadState(ctx context.Context) (*state.BeaconState, error)
-	HeadValidatorsIndices(epoch uint64) ([]uint64, error)
-	HeadSeed(epoch uint64) ([32]byte, error)
+	HeadValidatorsIndices(ctx context.Context, epoch uint64) ([]uint64, error)
+	HeadSeed(ctx context.Context, epoch uint64) ([32]byte, error)
+	HeadGenesisValidatorRoot() [32]byte
+	HeadETH1Data() *ethpb.Eth1Data
+	ProtoArrayStore() *protoarray.Store
 }
 
 // ForkFetcher retrieves the current fork information of the Ethereum beacon chain.
@@ -43,6 +54,12 @@ type ForkFetcher interface {
 	CurrentFork() *pb.Fork
 }
 
+// CanonicalFetcher retrieves the current chain's canonical information.
+type CanonicalFetcher interface {
+	IsCanonical(ctx context.Context, blockRoot [32]byte) (bool, error)
+	VerifyBlkDescendant(ctx context.Context, blockRoot [32]byte) error
+}
+
 // FinalizationFetcher defines a common interface for methods in blockchain service which
 // directly retrieves finalization and justification related data.
 type FinalizationFetcher interface {
@@ -51,24 +68,12 @@ type FinalizationFetcher interface {
 	PreviousJustifiedCheckpt() *ethpb.Checkpoint
 }
 
-// ParticipationFetcher defines a common interface for methods in blockchain service which
-// directly retrieves validator participation related data.
-type ParticipationFetcher interface {
-	Participation(epoch uint64) *precompute.Balance
-}
-
 // FinalizedCheckpt returns the latest finalized checkpoint from head state.
 func (s *Service) FinalizedCheckpt() *ethpb.Checkpoint {
 	if s.finalizedCheckpt == nil {
 		return &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
 	}
 
-	// If head state exists but there hasn't been a finalized check point,
-	// the check point's root should refer to genesis block root.
-	if bytes.Equal(s.finalizedCheckpt.Root, params.BeaconConfig().ZeroHash[:]) {
-		return &ethpb.Checkpoint{Root: s.genesisRoot[:]}
-	}
-
 	return state.CopyCheckpoint(s.finalizedCheckpt)
 }
 
@@ -78,12 +83,6 @@ func (s *Service) CurrentJustifiedCheckpt() *ethpb.Checkpoint {
 		return &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
 	}
 
-	// If head state exists but there hasn't been a justified check point,
-	// the check point root should refer to genesis block root.
-	if bytes.Equal(s.justifiedCheckpt.Root, params.BeaconConfig().ZeroHash[:]) {
-		return &ethpb.Checkpoint{Root: s.genesisRoot[:]}
-	}
-
 	return state.CopyCheckpoint(s.justifiedCheckpt)
 }
 
@@ -93,12 +92,6 @@ func (s *Service) PreviousJustifiedCheckpt() *ethpb.Checkpoint {
 		return &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
 	}
 
-	// If head state exists but there hasn't been a justified check point,
-	// the check point root should refer to genesis block root.
-	if bytes.Equal(s.prevJustifiedCheckpt.Root, params.BeaconConfig().ZeroHash[:]) {
-		return &ethpb.Checkpoint{Root: s.genesisRoot[:]}
-	}
-
 	return state.CopyCheckpoint(s.prevJustifiedCheckpt)
 }
 
@@ -126,7 +119,7 @@ func (s *Service) HeadRoot(ctx context.Context) ([]byte, error) {
 		return params.BeaconConfig().ZeroHash[:], nil
 	}
 
-	r, err := ssz.HashTreeRoot(b.Block)
+	r, err := stateutil.BlockRoot(b.Block)
 	if err != nil {
 		return nil, err
 	}
@@ -135,7 +128,7 @@ func (s *Service) HeadRoot(ctx context.Context) ([]byte, error) {
 }
 
 // HeadBlock returns the head block of the chain.
-// If the head state is nil from service struct,
+// If the head is nil from service struct,
 // it will attempt to get the head block from DB.
 func (s *Service) HeadBlock(ctx context.Context) (*ethpb.SignedBeaconBlock, error) {
 	if s.hasHeadState() {
@@ -146,31 +139,59 @@ func (s *Service) HeadBlock(ctx context.Context) (*ethpb.SignedBeaconBlock, erro
 }
 
 // HeadState returns the head state of the chain.
-// If the head state is nil from service struct,
+// If the head is nil from service struct,
 // it will attempt to get the head state from DB.
 func (s *Service) HeadState(ctx context.Context) (*state.BeaconState, error) {
-	if s.hasHeadState() {
-		return s.headState(), nil
+	ctx, span := trace.StartSpan(ctx, "blockChain.HeadState")
+	defer span.End()
+
+	ok := s.hasHeadState()
+	span.AddAttributes(trace.BoolAttribute("cache_hit", ok))
+
+	if ok {
+		return s.headState(ctx), nil
 	}
 
 	return s.beaconDB.HeadState(ctx)
 }
 
 // HeadValidatorsIndices returns a list of active validator indices from the head view of a given epoch.
-func (s *Service) HeadValidatorsIndices(epoch uint64) ([]uint64, error) {
+func (s *Service) HeadValidatorsIndices(ctx context.Context, epoch uint64) ([]uint64, error) {
 	if !s.hasHeadState() {
 		return []uint64{}, nil
 	}
-	return helpers.ActiveValidatorIndices(s.headState(), epoch)
+	return helpers.ActiveValidatorIndices(s.headState(ctx), epoch)
 }
 
 // HeadSeed returns the seed from the head view of a given epoch.
-func (s *Service) HeadSeed(epoch uint64) ([32]byte, error) {
+func (s *Service) HeadSeed(ctx context.Context, epoch uint64) ([32]byte, error) {
 	if !s.hasHeadState() {
 		return [32]byte{}, nil
 	}
 
-	return helpers.Seed(s.headState(), epoch, params.BeaconConfig().DomainBeaconAttester)
+	return helpers.Seed(s.headState(ctx), epoch, params.BeaconConfig().DomainBeaconAttester)
+}
+
+// HeadGenesisValidatorRoot returns genesis validator root of the head state.
+func (s *Service) HeadGenesisValidatorRoot() [32]byte {
+	if !s.hasHeadState() {
+		return [32]byte{}
+	}
+
+	return s.headGenesisValidatorRoot()
+}
+
+// HeadETH1Data returns the eth1data of the current head state.
+func (s *Service) HeadETH1Data() *ethpb.Eth1Data {
+	if !s.hasHeadState() {
+		return &ethpb.Eth1Data{}
+	}
+	return s.head.state.Eth1Data()
+}
+
+// ProtoArrayStore returns the proto array store object.
+func (s *Service) ProtoArrayStore() *protoarray.Store {
+	return s.forkChoiceStore.Store()
 }
 
 // GenesisTime returns the genesis time of beacon chain.
@@ -178,6 +199,15 @@ func (s *Service) GenesisTime() time.Time {
 	return s.genesisTime
 }
 
+// GenesisValidatorRoot returns the genesis validator
+// root of the chain.
+func (s *Service) GenesisValidatorRoot() [32]byte {
+	if !s.hasHeadState() {
+		return [32]byte{}
+	}
+	return bytesutil.ToBytes32(s.head.state.GenesisValidatorRoot())
+}
+
 // CurrentFork retrieves the latest fork information of the beacon chain.
 func (s *Service) CurrentFork() *pb.Fork {
 	if !s.hasHeadState() {
@@ -189,10 +219,16 @@ func (s *Service) CurrentFork() *pb.Fork {
 	return s.head.state.Fork()
 }
 
-// Participation returns the participation stats of a given epoch.
-func (s *Service) Participation(epoch uint64) *precompute.Balance {
-	s.epochParticipationLock.RLock()
-	defer s.epochParticipationLock.RUnlock()
+// IsCanonical returns true if the input block root is part of the canonical chain.
+func (s *Service) IsCanonical(ctx context.Context, blockRoot [32]byte) (bool, error) {
+	// If the block has been finalized, the block will always be part of the canonical chain.
+	if s.beaconDB.IsFinalizedBlock(ctx, blockRoot) {
+		return true, nil
+	}
 
-	return s.epochParticipation[epoch]
+	// If the block has not been finalized, the block must be recent. Check recent canonical roots
+	// mapping which uses proto array fork choice.
+	s.recentCanonicalBlocksLock.RLock()
+	defer s.recentCanonicalBlocksLock.RUnlock()
+	return s.recentCanonicalBlocks[blockRoot], nil
 }

beacon-chain/blockchain/chain_info_norace_test.go

@@ -5,74 +5,59 @@ import (
 	"testing"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 )
 
 func TestHeadSlot_DataRace(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 	s := &Service{
 		beaconDB: db,
 	}
 	go func() {
-		if err := s.saveHead(context.Background(), [32]byte{}, ); err != nil {
-			t.Fatal(err)
-		}
+		require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
 	}()
 	s.HeadSlot()
 }
 
 func TestHeadRoot_DataRace(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 	s := &Service{
 		beaconDB: db,
 		head:     &head{root: [32]byte{'A'}},
-		stateGen: stategen.New(db, cache.NewStateSummaryCache()),
+		stateGen: stategen.New(db, sc),
 	}
 	go func() {
-		if err := s.saveHead(context.Background(), [32]byte{}, ); err != nil {
-			t.Fatal(err)
-		}
+		require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
 	}()
-	if _, err := s.HeadRoot(context.Background()); err != nil {
-		t.Fatal(err)
-	}
+	_, err := s.HeadRoot(context.Background())
+	require.NoError(t, err)
 }
 
 func TestHeadBlock_DataRace(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 	s := &Service{
 		beaconDB: db,
 		head:     &head{block: &ethpb.SignedBeaconBlock{}},
-		stateGen: stategen.New(db, cache.NewStateSummaryCache()),
+		stateGen: stategen.New(db, sc),
 	}
 	go func() {
-		if err := s.saveHead(context.Background(), [32]byte{}, ); err != nil {
-			t.Fatal(err)
-		}
+		require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
 	}()
-	if _, err := s.HeadBlock(context.Background()); err != nil {
-		t.Fatal(err)
-	}
+	_, err := s.HeadBlock(context.Background())
+	require.NoError(t, err)
 }
 
 func TestHeadState_DataRace(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 	s := &Service{
 		beaconDB: db,
-		stateGen: stategen.New(db, cache.NewStateSummaryCache()),
+		stateGen: stategen.New(db, sc),
 	}
 	go func() {
-		if err := s.saveHead(context.Background(), [32]byte{}, ); err != nil {
-			t.Fatal(err)
-		}
+		require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
 	}()
-	if _, err := s.HeadState(context.Background()); err != nil {
-		t.Fatal(err)
-	}
+	_, err := s.HeadState(context.Background())
+	require.NoError(t, err)
 }

beacon-chain/blockchain/chain_info_test.go

@@ -3,7 +3,6 @@ package blockchain
 import (
 	"bytes"
 	"context"
-	"reflect"
 	"testing"
 	"time"
 
@@ -13,6 +12,8 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 )
 
 // Ensure Service implements chain info interface.
@@ -21,47 +22,39 @@ var _ = TimeFetcher(&Service{})
 var _ = ForkFetcher(&Service{})
 
 func TestFinalizedCheckpt_Nil(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	c := setupBeaconChain(t, db)
+	db, sc := testDB.SetupDB(t)
+	c := setupBeaconChain(t, db, sc)
 	if !bytes.Equal(c.FinalizedCheckpt().Root, params.BeaconConfig().ZeroHash[:]) {
 		t.Error("Incorrect pre chain start value")
 	}
 }
 
 func TestHeadRoot_Nil(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	c := setupBeaconChain(t, db)
+	db, sc := testDB.SetupDB(t)
+	c := setupBeaconChain(t, db, sc)
 	headRoot, err := c.HeadRoot(context.Background())
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	if !bytes.Equal(headRoot, params.BeaconConfig().ZeroHash[:]) {
 		t.Error("Incorrect pre chain start value")
 	}
 }
 
 func TestFinalizedCheckpt_CanRetrieve(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	cp := &ethpb.Checkpoint{Epoch: 5, Root: []byte("foo")}
-	c := setupBeaconChain(t, db)
+	c := setupBeaconChain(t, db, sc)
 	c.finalizedCheckpt = cp
 
-	if c.FinalizedCheckpt().Epoch != cp.Epoch {
-		t.Errorf("Finalized epoch at genesis should be %d, got: %d", cp.Epoch, c.FinalizedCheckpt().Epoch)
-	}
+	assert.Equal(t, cp.Epoch, c.FinalizedCheckpt().Epoch, "Unexpected finalized epoch")
 }
 
 func TestFinalizedCheckpt_GenesisRootOk(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	genesisRoot := [32]byte{'A'}
 	cp := &ethpb.Checkpoint{Root: genesisRoot[:]}
-	c := setupBeaconChain(t, db)
+	c := setupBeaconChain(t, db, sc)
 	c.finalizedCheckpt = cp
 	c.genesisRoot = genesisRoot
 
@@ -71,25 +64,21 @@ func TestFinalizedCheckpt_GenesisRootOk(t *testing.T) {
 }
 
 func TestCurrentJustifiedCheckpt_CanRetrieve(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	cp := &ethpb.Checkpoint{Epoch: 6, Root: []byte("foo")}
-	c := setupBeaconChain(t, db)
+	c := setupBeaconChain(t, db, sc)
 	c.justifiedCheckpt = cp
 
-	if c.CurrentJustifiedCheckpt().Epoch != cp.Epoch {
-		t.Errorf("Current Justifiied epoch at genesis should be %d, got: %d", cp.Epoch, c.CurrentJustifiedCheckpt().Epoch)
-	}
+	assert.Equal(t, cp.Epoch, c.CurrentJustifiedCheckpt().Epoch, "Unexpected justified epoch")
 }
 
 func TestJustifiedCheckpt_GenesisRootOk(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	genesisRoot := [32]byte{'B'}
 	cp := &ethpb.Checkpoint{Root: genesisRoot[:]}
-	c := setupBeaconChain(t, db)
+	c := setupBeaconChain(t, db, sc)
 	c.justifiedCheckpt = cp
 	c.genesisRoot = genesisRoot
 
@@ -99,25 +88,21 @@ func TestJustifiedCheckpt_GenesisRootOk(t *testing.T) {
 }
 
 func TestPreviousJustifiedCheckpt_CanRetrieve(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	cp := &ethpb.Checkpoint{Epoch: 7, Root: []byte("foo")}
-	c := setupBeaconChain(t, db)
+	c := setupBeaconChain(t, db, sc)
 	c.prevJustifiedCheckpt = cp
 
-	if c.PreviousJustifiedCheckpt().Epoch != cp.Epoch {
-		t.Errorf("Previous Justifiied epoch at genesis should be %d, got: %d", cp.Epoch, c.PreviousJustifiedCheckpt().Epoch)
-	}
+	assert.Equal(t, cp.Epoch, c.PreviousJustifiedCheckpt().Epoch, "Unexpected previous justified epoch")
 }
 
 func TestPrevJustifiedCheckpt_GenesisRootOk(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	genesisRoot := [32]byte{'C'}
 	cp := &ethpb.Checkpoint{Root: genesisRoot[:]}
-	c := setupBeaconChain(t, db)
+	c := setupBeaconChain(t, db, sc)
 	c.prevJustifiedCheckpt = cp
 	c.genesisRoot = genesisRoot
 
@@ -133,49 +118,34 @@ func TestHeadSlot_CanRetrieve(t *testing.T) {
 		t.Fatal(err)
 	}
 	c.head = &head{slot: 100, state: s}
-	if c.HeadSlot() != 100 {
-		t.Errorf("Wanted head slot: %d, got: %d", 100, c.HeadSlot())
-	}
+	assert.Equal(t, uint64(100), c.headSlot())
 }
 
 func TestHeadRoot_CanRetrieve(t *testing.T) {
 	c := &Service{}
 	c.head = &head{root: [32]byte{'A'}}
-	if [32]byte{'A'} != c.headRoot() {
-		t.Errorf("Wanted head root: %v, got: %d", []byte{'A'}, c.headRoot())
-	}
+	assert.Equal(t, [32]byte{'A'}, c.headRoot())
 }
 
 func TestHeadBlock_CanRetrieve(t *testing.T) {
 	b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1}}
 	s, err := state.InitializeFromProto(&pb.BeaconState{})
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	c := &Service{}
 	c.head = &head{block: b, state: s}
 
 	recevied, err := c.HeadBlock(context.Background())
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if !reflect.DeepEqual(b, recevied) {
-		t.Error("incorrect head block received")
-	}
+	require.NoError(t, err)
+	assert.DeepEqual(t, b, recevied, "Incorrect head block received")
 }
 
 func TestHeadState_CanRetrieve(t *testing.T) {
 	s, err := state.InitializeFromProto(&pb.BeaconState{Slot: 2, GenesisValidatorsRoot: params.BeaconConfig().ZeroHash[:]})
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	c := &Service{}
 	c.head = &head{state: s}
 	headState, err := c.HeadState(context.Background())
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	if !proto.Equal(s.InnerStateUnsafe(), headState.InnerStateUnsafe()) {
 		t.Error("incorrect head state received")
 	}
@@ -184,20 +154,44 @@ func TestHeadState_CanRetrieve(t *testing.T) {
 func TestGenesisTime_CanRetrieve(t *testing.T) {
 	c := &Service{genesisTime: time.Unix(999, 0)}
 	wanted := time.Unix(999, 0)
-	if c.GenesisTime() != wanted {
-		t.Error("Did not get wanted genesis time")
-	}
+	assert.Equal(t, wanted, c.GenesisTime(), "Did not get wanted genesis time")
 }
 
 func TestCurrentFork_CanRetrieve(t *testing.T) {
 	f := &pb.Fork{Epoch: 999}
 	s, err := state.InitializeFromProto(&pb.BeaconState{Fork: f})
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	c := &Service{}
 	c.head = &head{state: s}
 	if !proto.Equal(c.CurrentFork(), f) {
 		t.Error("Received incorrect fork version")
 	}
 }
+
+func TestGenesisValidatorRoot_CanRetrieve(t *testing.T) {
+	// Should not panic if head state is nil.
+	c := &Service{}
+	assert.Equal(t, [32]byte{}, c.GenesisValidatorRoot(), "Did not get correct genesis validator root")
+
+	s, err := state.InitializeFromProto(&pb.BeaconState{GenesisValidatorsRoot: []byte{'a'}})
+	require.NoError(t, err)
+	c.head = &head{state: s}
+	assert.Equal(t, [32]byte{'a'}, c.GenesisValidatorRoot(), "Did not get correct genesis validator root")
+}
+
+func TestHeadETH1Data_Nil(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+	c := setupBeaconChain(t, db, sc)
+	assert.DeepEqual(t, &ethpb.Eth1Data{}, c.HeadETH1Data(), "Incorrect pre chain start value")
+}
+
+func TestHeadETH1Data_CanRetrieve(t *testing.T) {
+	d := &ethpb.Eth1Data{DepositCount: 999}
+	s, err := state.InitializeFromProto(&pb.BeaconState{Eth1Data: d})
+	require.NoError(t, err)
+	c := &Service{}
+	c.head = &head{state: s}
+	if !proto.Equal(c.HeadETH1Data(), d) {
+		t.Error("Received incorrect eth1 data")
+	}
+}

beacon-chain/blockchain/head.go

@@ -1,15 +1,21 @@
 package blockchain
 
 import (
+	"bytes"
 	"context"
+	"fmt"
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
+	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
 )
 
@@ -21,10 +27,10 @@ type head struct {
 	state *state.BeaconState       // current head state.
 }
 
-// This gets head from the fork choice service and saves head related items
-// (ie root, block, state) to the local service cache.
+// Determined the head from the fork choice service and saves its new data
+// (head root, head block, and head state) to the local service cache.
 func (s *Service) updateHead(ctx context.Context, balances []uint64) error {
-	ctx, span := trace.StartSpan(ctx, "blockchain.updateHead")
+	ctx, span := trace.StartSpan(ctx, "blockChain.updateHead")
 	defer span.End()
 
 	// To get the proper head update, a node first checks its best justified
@@ -32,16 +38,29 @@ func (s *Service) updateHead(ctx context.Context, balances []uint64) error {
 	// ensure head gets its best justified info.
 	if s.bestJustifiedCheckpt.Epoch > s.justifiedCheckpt.Epoch {
 		s.justifiedCheckpt = s.bestJustifiedCheckpt
+		if err := s.cacheJustifiedStateBalances(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root)); err != nil {
+			return err
+		}
 	}
 
 	// Get head from the fork choice service.
 	f := s.finalizedCheckpt
 	j := s.justifiedCheckpt
-	headRoot, err := s.forkChoiceStore.Head(ctx, j.Epoch, bytesutil.ToBytes32(j.Root), balances, f.Epoch)
+	// To get head before the first justified epoch, the fork choice will start with genesis root
+	// instead of zero hashes.
+	headStartRoot := bytesutil.ToBytes32(j.Root)
+	if headStartRoot == params.BeaconConfig().ZeroHash {
+		headStartRoot = s.genesisRoot
+	}
+	headRoot, err := s.forkChoiceStore.Head(ctx, j.Epoch, headStartRoot, balances, f.Epoch)
 	if err != nil {
 		return err
 	}
 
+	if err := s.updateRecentCanonicalBlocks(ctx, headRoot); err != nil {
+		return err
+	}
+
 	// Save head to the local service cache.
 	return s.saveHead(ctx, headRoot)
 }
@@ -49,7 +68,7 @@ func (s *Service) updateHead(ctx context.Context, balances []uint64) error {
 // This saves head info to the local service cache, it also saves the
 // new head root to the DB.
 func (s *Service) saveHead(ctx context.Context, headRoot [32]byte) error {
-	ctx, span := trace.StartSpan(ctx, "blockchain.saveHead")
+	ctx, span := trace.StartSpan(ctx, "blockChain.saveHead")
 	defer span.End()
 
 	// Do nothing if head hasn't changed.
@@ -59,15 +78,8 @@ func (s *Service) saveHead(ctx context.Context, headRoot [32]byte) error {
 
 	// If the head state is not available, just return nil.
 	// There's nothing to cache
-	if !featureconfig.Get().DisableNewStateMgmt {
-		if !s.stateGen.StateSummaryExists(ctx, headRoot) {
-			return nil
-		}
-	} else {
-		_, cached := s.initSyncState[headRoot]
-		if !cached && !s.beaconDB.HasState(ctx, headRoot) {
-			return nil
-		}
+	if !s.stateGen.StateSummaryExists(ctx, headRoot) {
+		return nil
 	}
 
 	// Get the new head block from DB.
@@ -80,26 +92,31 @@ func (s *Service) saveHead(ctx context.Context, headRoot [32]byte) error {
 	}
 
 	// Get the new head state from cached state or DB.
-	var newHeadState *state.BeaconState
-	if !featureconfig.Get().DisableNewStateMgmt {
-		newHeadState, err = s.stateGen.StateByRoot(ctx, headRoot)
-		if err != nil {
-			return errors.Wrap(err, "could not retrieve head state in DB")
-		}
-	} else {
-		var exists bool
-		newHeadState, exists = s.initSyncState[headRoot]
-		if !exists {
-			newHeadState, err = s.beaconDB.State(ctx, headRoot)
-			if err != nil {
-				return errors.Wrap(err, "could not retrieve head state in DB")
-			}
-		}
+	newHeadState, err := s.stateGen.StateByRoot(ctx, headRoot)
+	if err != nil {
+		return errors.Wrap(err, "could not retrieve head state in DB")
 	}
 	if newHeadState == nil {
 		return errors.New("cannot save nil head state")
 	}
 
+	// A chain re-org occurred, so we fire an event notifying the rest of the services.
+	if bytesutil.ToBytes32(newHeadBlock.Block.ParentRoot) != s.headRoot() {
+		log.WithFields(logrus.Fields{
+			"newSlot": fmt.Sprintf("%d", newHeadBlock.Block.Slot),
+			"oldSlot": fmt.Sprintf("%d", s.headSlot()),
+		}).Debug("Chain reorg occurred")
+		s.stateNotifier.StateFeed().Send(&feed.Event{
+			Type: statefeed.Reorg,
+			Data: &statefeed.ReorgData{
+				NewSlot: newHeadBlock.Block.Slot,
+				OldSlot: s.headSlot(),
+			},
+		})
+
+		reorgCount.Inc()
+	}
+
 	// Cache the new head info.
 	s.setHead(headRoot, newHeadBlock, newHeadState)
 
@@ -112,40 +129,22 @@ func (s *Service) saveHead(ctx context.Context, headRoot [32]byte) error {
 }
 
 // This gets called to update canonical root mapping. It does not save head block
-// root in DB. With the inception of inital-sync-cache-state flag, it uses finalized
+// root in DB. With the inception of initial-sync-cache-state flag, it uses finalized
 // check point as anchors to resume sync therefore head is no longer needed to be saved on per slot basis.
-func (s *Service) saveHeadNoDB(ctx context.Context, b *ethpb.SignedBeaconBlock, r [32]byte) error {
+func (s *Service) saveHeadNoDB(ctx context.Context, b *ethpb.SignedBeaconBlock, r [32]byte, hs *state.BeaconState) error {
+	cachedHeadRoot, err := s.HeadRoot(ctx)
+	if err != nil {
+		return errors.Wrap(err, "could not get head root from cache")
+	}
+	if bytes.Equal(r[:], cachedHeadRoot) {
+		return nil
+	}
+
 	if b == nil || b.Block == nil {
 		return errors.New("cannot save nil head block")
 	}
 
-	var headState *state.BeaconState
-	var err error
-	if !featureconfig.Get().DisableNewStateMgmt {
-		headState, err = s.stateGen.StateByRoot(ctx, r)
-		if err != nil {
-			return errors.Wrap(err, "could not retrieve head state in DB")
-		}
-	} else {
-		headState, err = s.beaconDB.State(ctx, r)
-		if err != nil {
-			return errors.Wrap(err, "could not retrieve head state in DB")
-		}
-		if headState == nil {
-			s.initSyncStateLock.RLock()
-			cachedHeadState, ok := s.initSyncState[r]
-			if ok {
-				headState = cachedHeadState
-			}
-			s.initSyncStateLock.RUnlock()
-		}
-	}
-	if headState == nil {
-		return errors.New("nil head state")
-	}
-
-	s.setHead(r, stateTrie.CopySignedBeaconBlock(b), headState)
-
+	s.setHeadInitialSync(r, stateTrie.CopySignedBeaconBlock(b), hs)
 	return nil
 }
 
@@ -163,6 +162,22 @@ func (s *Service) setHead(root [32]byte, block *ethpb.SignedBeaconBlock, state *
 	}
 }
 
+// This sets head view object which is used to track the head slot, root, block and state. The method
+// assumes that state being passed into the method will not be modified by any other alternate
+// caller which holds the state's reference.
+func (s *Service) setHeadInitialSync(root [32]byte, block *ethpb.SignedBeaconBlock, state *state.BeaconState) {
+	s.headLock.Lock()
+	defer s.headLock.Unlock()
+
+	// This does a full copy of the block only.
+	s.head = &head{
+		slot:  block.Block.Slot,
+		root:  root,
+		block: stateTrie.CopySignedBeaconBlock(block),
+		state: state,
+	}
+}
+
 // This returns the head slot.
 func (s *Service) headSlot() uint64 {
 	s.headLock.RLock()
@@ -195,13 +210,24 @@ func (s *Service) headBlock() *ethpb.SignedBeaconBlock {
 
 // This returns the head state.
 // It does a full copy on head state for immutability.
-func (s *Service) headState() *state.BeaconState {
+func (s *Service) headState(ctx context.Context) *stateTrie.BeaconState {
+	ctx, span := trace.StartSpan(ctx, "blockChain.headState")
+	defer span.End()
+
 	s.headLock.RLock()
 	defer s.headLock.RUnlock()
 
 	return s.head.state.Copy()
 }
 
+// This returns the genesis validator root of the head state.
+func (s *Service) headGenesisValidatorRoot() [32]byte {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	return bytesutil.ToBytes32(s.head.state.GenesisValidatorRoot())
+}
+
 // Returns true if head state exists.
 func (s *Service) hasHeadState() bool {
 	s.headLock.RLock()
@@ -209,3 +235,82 @@ func (s *Service) hasHeadState() bool {
 
 	return s.head != nil && s.head.state != nil
 }
+
+// This updates recent canonical block mapping. It uses input head root and retrieves
+// all the canonical block roots that are ancestor of the input head block root.
+func (s *Service) updateRecentCanonicalBlocks(ctx context.Context, headRoot [32]byte) error {
+	ctx, span := trace.StartSpan(ctx, "blockChain.updateRecentCanonicalBlocks")
+	defer span.End()
+
+	s.recentCanonicalBlocksLock.Lock()
+	defer s.recentCanonicalBlocksLock.Unlock()
+
+	s.recentCanonicalBlocks = make(map[[32]byte]bool)
+	s.recentCanonicalBlocks[headRoot] = true
+	nodes := s.forkChoiceStore.Nodes()
+	node := s.forkChoiceStore.Node(headRoot)
+	if node == nil {
+		return nil
+	}
+
+	for node.Parent() != protoarray.NonExistentNode {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		node = nodes[node.Parent()]
+		s.recentCanonicalBlocks[node.Root()] = true
+	}
+
+	return nil
+}
+
+// This caches justified state balances to be used for fork choice.
+func (s *Service) cacheJustifiedStateBalances(ctx context.Context, justifiedRoot [32]byte) error {
+	if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+		return err
+	}
+
+	s.clearInitSyncBlocks()
+
+	var justifiedState *stateTrie.BeaconState
+	var err error
+	if justifiedRoot == s.genesisRoot {
+		justifiedState, err = s.beaconDB.GenesisState(ctx)
+		if err != nil {
+			return err
+		}
+	} else {
+		justifiedState, err = s.stateGen.StateByRoot(ctx, justifiedRoot)
+		if err != nil {
+			return err
+		}
+	}
+	if justifiedState == nil {
+		return errors.New("justified state can't be nil")
+	}
+
+	epoch := helpers.CurrentEpoch(justifiedState)
+
+	justifiedBalances := make([]uint64, justifiedState.NumValidators())
+	if err := justifiedState.ReadFromEveryValidator(func(idx int, val *stateTrie.ReadOnlyValidator) error {
+		if helpers.IsActiveValidatorUsingTrie(val, epoch) {
+			justifiedBalances[idx] = val.EffectiveBalance()
+		} else {
+			justifiedBalances[idx] = 0
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	s.justifiedBalancesLock.Lock()
+	defer s.justifiedBalancesLock.Unlock()
+	s.justifiedBalances = justifiedBalances
+	return nil
+}
+
+func (s *Service) getJustifiedBalances() []uint64 {
+	s.justifiedBalancesLock.RLock()
+	defer s.justifiedBalancesLock.RUnlock()
+	return s.justifiedBalances
+}

beacon-chain/blockchain/head_test.go

@@ -3,41 +3,34 @@ package blockchain
 import (
 	"bytes"
 	"context"
-	"reflect"
 	"testing"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+	logTest "github.com/sirupsen/logrus/hooks/test"
 )
 
 func TestSaveHead_Same(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	service := setupBeaconChain(t, db)
+	db, sc := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db, sc)
 
 	r := [32]byte{'A'}
 	service.head = &head{slot: 0, root: r}
 
-	if err := service.saveHead(context.Background(), r); err != nil {
-		t.Fatal(err)
-	}
-
-	if service.headSlot() != 0 {
-		t.Error("Head did not stay the same")
-	}
-
-	if service.headRoot() != r {
-		t.Error("Head did not stay the same")
-	}
+	require.NoError(t, service.saveHead(context.Background(), r))
+	assert.Equal(t, uint64(0), service.headSlot(), "Head did not stay the same")
+	assert.Equal(t, r, service.headRoot(), "Head did not stay the same")
 }
 
 func TestSaveHead_Different(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	service := setupBeaconChain(t, db)
+	ctx := context.Background()
+	db, sc := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db, sc)
 
 	oldRoot := [32]byte{'A'}
 	service.head = &head{slot: 0, root: oldRoot}
@@ -45,42 +38,104 @@ func TestSaveHead_Different(t *testing.T) {
 	newHeadBlock := &ethpb.BeaconBlock{Slot: 1}
 	newHeadSignedBlock := &ethpb.SignedBeaconBlock{Block: newHeadBlock}
 
-	if err := service.beaconDB.SaveBlock(context.Background(), newHeadSignedBlock); err != nil {
-		t.Fatal(err)
-	}
-	newRoot, err := ssz.HashTreeRoot(newHeadBlock)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, service.beaconDB.SaveBlock(context.Background(), newHeadSignedBlock))
+	newRoot, err := stateutil.BlockRoot(newHeadBlock)
+	require.NoError(t, err)
 	headState := testutil.NewBeaconState()
-	if err := headState.SetSlot(1); err != nil {
-		t.Fatal(err)
-	}
-	if err := service.beaconDB.SaveStateSummary(context.Background(), &pb.StateSummary{Slot: 1, Root: newRoot[:]}); err != nil {
-		t.Fatal(err)
-	}
-	if err := service.beaconDB.SaveState(context.Background(), headState, newRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := service.saveHead(context.Background(), newRoot); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, headState.SetSlot(1))
+	require.NoError(t, service.beaconDB.SaveStateSummary(context.Background(), &pb.StateSummary{Slot: 1, Root: newRoot[:]}))
+	require.NoError(t, service.beaconDB.SaveState(context.Background(), headState, newRoot))
+	require.NoError(t, service.saveHead(context.Background(), newRoot))
 
-	if service.HeadSlot() != 1 {
-		t.Error("Head did not change")
-	}
+	assert.Equal(t, uint64(1), service.HeadSlot(), "Head did not change")
 
 	cachedRoot, err := service.HeadRoot(context.Background())
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	if !bytes.Equal(cachedRoot, newRoot[:]) {
 		t.Error("Head did not change")
 	}
-	if !reflect.DeepEqual(service.headBlock(), newHeadSignedBlock) {
-		t.Error("Head did not change")
-	}
-	if !reflect.DeepEqual(service.headState().CloneInnerState(), headState.CloneInnerState()) {
-		t.Error("Head did not change")
-	}
+	assert.DeepEqual(t, newHeadSignedBlock, service.headBlock(), "Head did not change")
+	assert.DeepEqual(t, headState.CloneInnerState(), service.headState(ctx).CloneInnerState(), "Head did not change")
+}
+
+func TestSaveHead_Different_Reorg(t *testing.T) {
+	ctx := context.Background()
+	hook := logTest.NewGlobal()
+	db, sc := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db, sc)
+
+	oldRoot := [32]byte{'A'}
+	service.head = &head{slot: 0, root: oldRoot}
+
+	reorgChainParent := [32]byte{'B'}
+	newHeadBlock := &ethpb.BeaconBlock{
+		Slot:       1,
+		ParentRoot: reorgChainParent[:],
+	}
+	newHeadSignedBlock := &ethpb.SignedBeaconBlock{Block: newHeadBlock}
+
+	require.NoError(t, service.beaconDB.SaveBlock(context.Background(), newHeadSignedBlock))
+	newRoot, err := stateutil.BlockRoot(newHeadBlock)
+	require.NoError(t, err)
+	headState := testutil.NewBeaconState()
+	require.NoError(t, headState.SetSlot(1))
+	require.NoError(t, service.beaconDB.SaveStateSummary(context.Background(), &pb.StateSummary{Slot: 1, Root: newRoot[:]}))
+	require.NoError(t, service.beaconDB.SaveState(context.Background(), headState, newRoot))
+	require.NoError(t, service.saveHead(context.Background(), newRoot))
+
+	assert.Equal(t, uint64(1), service.HeadSlot(), "Head did not change")
+
+	cachedRoot, err := service.HeadRoot(context.Background())
+	require.NoError(t, err)
+	if !bytes.Equal(cachedRoot, newRoot[:]) {
+		t.Error("Head did not change")
+	}
+	assert.DeepEqual(t, newHeadSignedBlock, service.headBlock(), "Head did not change")
+	assert.DeepEqual(t, headState.CloneInnerState(), service.headState(ctx).CloneInnerState(), "Head did not change")
+	require.LogsContain(t, hook, "Chain reorg occurred")
+}
+
+func TestUpdateRecentCanonicalBlocks_CanUpdateWithoutParent(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db, sc)
+
+	r := [32]byte{'a'}
+	require.NoError(t, service.updateRecentCanonicalBlocks(context.Background(), r))
+	canonical, err := service.IsCanonical(context.Background(), r)
+	require.NoError(t, err)
+	assert.Equal(t, true, canonical, "Block should be canonical")
+}
+
+func TestUpdateRecentCanonicalBlocks_CanUpdateWithParent(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db, sc)
+	oldHead := [32]byte{'a'}
+	require.NoError(t, service.forkChoiceStore.ProcessBlock(context.Background(), 1, oldHead, [32]byte{'g'}, [32]byte{}, 0, 0))
+	currentHead := [32]byte{'b'}
+	require.NoError(t, service.forkChoiceStore.ProcessBlock(context.Background(), 3, currentHead, oldHead, [32]byte{}, 0, 0))
+	forkedRoot := [32]byte{'c'}
+	require.NoError(t, service.forkChoiceStore.ProcessBlock(context.Background(), 2, forkedRoot, oldHead, [32]byte{}, 0, 0))
+
+	require.NoError(t, service.updateRecentCanonicalBlocks(context.Background(), currentHead))
+	canonical, err := service.IsCanonical(context.Background(), currentHead)
+	require.NoError(t, err)
+	assert.Equal(t, true, canonical, "Block should be canonical")
+	canonical, err = service.IsCanonical(context.Background(), oldHead)
+	require.NoError(t, err)
+	assert.Equal(t, true, canonical, "Block should be canonical")
+	canonical, err = service.IsCanonical(context.Background(), forkedRoot)
+	require.NoError(t, err)
+	assert.Equal(t, false, canonical, "Block should not be canonical")
+}
+
+func TestCacheJustifiedStateBalances_CanCache(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db, sc)
+
+	state, _ := testutil.DeterministicGenesisState(t, 100)
+	r := [32]byte{'a'}
+	require.NoError(t, service.beaconDB.SaveStateSummary(context.Background(), &pb.StateSummary{Root: r[:]}))
+	require.NoError(t, service.beaconDB.SaveState(context.Background(), state, r))
+	require.NoError(t, service.cacheJustifiedStateBalances(context.Background(), r))
+	require.DeepEqual(t, service.getJustifiedBalances(), state.Balances(), "Incorrect justified balances")
 }

beacon-chain/blockchain/info.go

@@ -1,11 +1,12 @@
 package blockchain
 
 import (
+	"encoding/hex"
 	"fmt"
 	"net/http"
-	"strconv"
 
 	"github.com/emicklei/dot"
+	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
 const template = `<html>
@@ -32,8 +33,8 @@ const template = `<html>
 </html>`
 
 // TreeHandler is a handler to serve /tree page in metrics.
-func (s *Service) TreeHandler(w http.ResponseWriter, _ *http.Request) {
-	if s.headState() == nil {
+func (s *Service) TreeHandler(w http.ResponseWriter, r *http.Request) {
+	if s.headState(r.Context()) == nil {
 		if _, err := w.Write([]byte("Unavailable during initial syncing")); err != nil {
 			log.WithError(err).Error("Failed to render p2p info page")
 		}
@@ -46,23 +47,25 @@ func (s *Service) TreeHandler(w http.ResponseWriter, _ *http.Request) {
 	graph.Attr("labeljust", "l")
 
 	dotNodes := make([]*dot.Node, len(nodes))
-	avgBalance := uint64(averageBalance(s.headState().Balances()))
+	avgBalance := uint64(averageBalance(s.headState(r.Context()).Balances()))
 
 	for i := len(nodes) - 1; i >= 0; i-- {
 		// Construct label for each node.
-		slot := strconv.Itoa(int(nodes[i].Slot))
-		weight := strconv.Itoa(int(nodes[i].Weight / 1e9)) // Convert unit Gwei to unit ETH.
-		votes := strconv.Itoa(int(nodes[i].Weight / 1e9 / avgBalance))
-		bestDescendent := strconv.Itoa(int(nodes[i].BestDescendent))
-		index := strconv.Itoa(int(i))
-		label := "slot: " + slot + "\n index: " + index + "\n bestDescendent: " + bestDescendent + "\n votes: " + votes + "\n weight: " + weight
+		slot := fmt.Sprintf("%d", nodes[i].Slot())
+		weight := fmt.Sprintf("%d", nodes[i].Weight()/1e9) // Convert unit Gwei to unit ETH.
+		votes := fmt.Sprintf("%d", nodes[i].Weight()/1e9/avgBalance)
+		index := fmt.Sprintf("%d", i)
+		g := nodes[i].Graffiti()
+		graffiti := hex.EncodeToString(g[:8])
+		label := "slot: " + slot + "\n votes: " + votes + "\n weight: " + weight + "\n graffiti: " + graffiti
 		var dotN dot.Node
-		if nodes[i].Parent != ^uint64(0) {
+		if nodes[i].Parent() != ^uint64(0) {
 			dotN = graph.Node(index).Box().Attr("label", label)
 		}
 
-		if nodes[i].Slot == s.headSlot() &&
-			nodes[i].BestDescendent == ^uint64(0) {
+		if nodes[i].Slot() == s.headSlot() &&
+			nodes[i].BestDescendant() == ^uint64(0) &&
+			nodes[i].Parent() != ^uint64(0) {
 			dotN = dotN.Attr("color", "green")
 		}
 
@@ -70,8 +73,8 @@ func (s *Service) TreeHandler(w http.ResponseWriter, _ *http.Request) {
 	}
 
 	for i := len(nodes) - 1; i >= 0; i-- {
-		if nodes[i].Parent != ^uint64(0) && nodes[i].Parent < uint64(len(dotNodes)) {
-			graph.Edge(*dotNodes[i], *dotNodes[nodes[i].Parent])
+		if nodes[i].Parent() != ^uint64(0) && nodes[i].Parent() < uint64(len(dotNodes)) {
+			graph.Edge(*dotNodes[i], *dotNodes[nodes[i].Parent()])
 		}
 	}
 
@@ -81,3 +84,11 @@ func (s *Service) TreeHandler(w http.ResponseWriter, _ *http.Request) {
 		log.WithError(err).Error("Failed to render p2p info page")
 	}
 }
+
+func averageBalance(balances []uint64) float64 {
+	total := uint64(0)
+	for i := 0; i < len(balances); i++ {
+		total += balances[i]
+	}
+	return float64(total) / float64(len(balances)) / float64(params.BeaconConfig().GweiPerEth)
+}

beacon-chain/blockchain/info_test.go

@@ -0,0 +1,46 @@
+package blockchain
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+)
+
+func TestService_TreeHandler(t *testing.T) {
+	req, err := http.NewRequest("GET", "/tree", nil)
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	db, sCache := testDB.SetupDB(t)
+	headState := testutil.NewBeaconState()
+	require.NoError(t, headState.SetBalances([]uint64{params.BeaconConfig().GweiPerEth}))
+	cfg := &Config{
+		BeaconDB: db,
+		ForkChoiceStore: protoarray.New(
+			0, // justifiedEpoch
+			0, // finalizedEpoch
+			[32]byte{'a'},
+		),
+		StateGen: stategen.New(db, sCache),
+	}
+	s, err := NewService(ctx, cfg)
+	require.NoError(t, s.forkChoiceStore.ProcessBlock(ctx, 0, [32]byte{'a'}, [32]byte{'g'}, [32]byte{'c'}, 0, 0))
+	require.NoError(t, s.forkChoiceStore.ProcessBlock(ctx, 1, [32]byte{'b'}, [32]byte{'a'}, [32]byte{'c'}, 0, 0))
+	s.setHead([32]byte{'a'}, testutil.NewBeaconBlock(), headState)
+
+	rr := httptest.NewRecorder()
+	handler := http.HandlerFunc(s.TreeHandler)
+
+	handler.ServeHTTP(rr, req)
+
+	assert.Equal(t, http.StatusOK, rr.Code)
+}

beacon-chain/blockchain/init_sync_process_block.go

@@ -1,214 +1,9 @@
 package blockchain
 
 import (
-	"context"
-	"sort"
-
-	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
-const maxCacheSize = 70
-const initialSyncCacheSize = 45
-const minimumCacheSize = initialSyncCacheSize / 3
-
-func (s *Service) persistCachedStates(ctx context.Context, numOfStates int) error {
-	oldStates := make([]*stateTrie.BeaconState, 0, numOfStates)
-
-	// Add slots to the map and add epoch boundary states to the slice.
-	for _, rt := range s.boundaryRoots[:numOfStates-minimumCacheSize] {
-		oldStates = append(oldStates, s.initSyncState[rt])
-	}
-
-	err := s.beaconDB.SaveStates(ctx, oldStates, s.boundaryRoots[:numOfStates-minimumCacheSize])
-	if err != nil {
-		return err
-	}
-	for _, rt := range s.boundaryRoots[:numOfStates-minimumCacheSize] {
-		delete(s.initSyncState, rt)
-	}
-	s.boundaryRoots = s.boundaryRoots[numOfStates-minimumCacheSize:]
-	return nil
-}
-
-// filter out boundary candidates from our currently processed batch of states.
-func (s *Service) filterBoundaryCandidates(ctx context.Context, root [32]byte, postState *stateTrie.BeaconState) {
-	// Only trigger on epoch start.
-	if !helpers.IsEpochStart(postState.Slot()) {
-		return
-	}
-
-	stateSlice := make([][32]byte, 0, len(s.initSyncState))
-	// Add epoch boundary roots to slice.
-	for rt := range s.initSyncState {
-		stateSlice = append(stateSlice, rt)
-	}
-
-	sort.Slice(stateSlice, func(i int, j int) bool {
-		return s.initSyncState[stateSlice[i]].Slot() < s.initSyncState[stateSlice[j]].Slot()
-	})
-	epochLength := params.BeaconConfig().SlotsPerEpoch
-
-	if len(s.boundaryRoots) > 0 {
-		// Retrieve previous boundary root.
-		previousBoundaryRoot := s.boundaryRoots[len(s.boundaryRoots)-1]
-		previousState, ok := s.initSyncState[previousBoundaryRoot]
-		if !ok {
-			// Remove the non-existent root and exit filtering.
-			s.boundaryRoots = s.boundaryRoots[:len(s.boundaryRoots)-1]
-			return
-		}
-		previousSlot := previousState.Slot()
-
-		// Round up slot number to account for skipped slots.
-		previousSlot = helpers.RoundUpToNearestEpoch(previousSlot)
-		if postState.Slot()-previousSlot >= epochLength {
-			targetSlot := postState.Slot()
-			tempRoots := s.loopThroughCandidates(stateSlice, previousBoundaryRoot, previousSlot, targetSlot)
-			s.boundaryRoots = append(s.boundaryRoots, tempRoots...)
-		}
-	}
-	s.boundaryRoots = append(s.boundaryRoots, root)
-	s.pruneOldStates()
-	s.pruneNonBoundaryStates()
-}
-
-// loop-through the provided candidate roots to filter out which would be appropriate boundary roots.
-func (s *Service) loopThroughCandidates(stateSlice [][32]byte, previousBoundaryRoot [32]byte,
-	previousSlot uint64, targetSlot uint64) [][32]byte {
-	tempRoots := [][32]byte{}
-	epochLength := params.BeaconConfig().SlotsPerEpoch
-
-	// Loop through current states to filter for valid boundary states.
-	for i := len(stateSlice) - 1; stateSlice[i] != previousBoundaryRoot && i >= 0; i-- {
-		currentSlot := s.initSyncState[stateSlice[i]].Slot()
-		// Skip if the current slot is larger than the previous epoch
-		// boundary.
-		if currentSlot > targetSlot-epochLength {
-			continue
-		}
-		tempRoots = append(tempRoots, stateSlice[i])
-
-		// Switch target slot if the current slot is greater than
-		// 1 epoch boundary from the previously saved boundary slot.
-		if currentSlot > previousSlot+epochLength {
-			currentSlot = helpers.RoundUpToNearestEpoch(currentSlot)
-			targetSlot = currentSlot
-			continue
-		}
-		break
-	}
-	// Reverse to append the roots in ascending order corresponding
-	// to the respective slots.
-	tempRoots = bytesutil.ReverseBytes32Slice(tempRoots)
-	return tempRoots
-}
-
-// prune for states past the current finalized checkpoint.
-func (s *Service) pruneOldStates() {
-	prunedBoundaryRoots := [][32]byte{}
-	for _, rt := range s.boundaryRoots {
-		st, ok := s.initSyncState[rt]
-		// Skip non-existent roots.
-		if !ok {
-			continue
-		}
-		if st.Slot() < helpers.StartSlot(s.FinalizedCheckpt().Epoch) {
-			delete(s.initSyncState, rt)
-			continue
-		}
-		prunedBoundaryRoots = append(prunedBoundaryRoots, rt)
-	}
-	s.boundaryRoots = prunedBoundaryRoots
-}
-
-// prune cache for non-boundary states.
-func (s *Service) pruneNonBoundaryStates() {
-	boundaryMap := make(map[[32]byte]bool)
-	for i := range s.boundaryRoots {
-		boundaryMap[s.boundaryRoots[i]] = true
-	}
-	for rt := range s.initSyncState {
-		if !boundaryMap[rt] {
-			delete(s.initSyncState, rt)
-		}
-	}
-}
-
-func (s *Service) pruneOldNonFinalizedStates() {
-	stateSlice := make([][32]byte, 0, len(s.initSyncState))
-	// Add epoch boundary roots to slice.
-	for rt := range s.initSyncState {
-		stateSlice = append(stateSlice, rt)
-	}
-
-	// Sort by slots.
-	sort.Slice(stateSlice, func(i int, j int) bool {
-		return s.initSyncState[stateSlice[i]].Slot() < s.initSyncState[stateSlice[j]].Slot()
-	})
-
-	boundaryMap := make(map[[32]byte]bool)
-	for i := range s.boundaryRoots {
-		boundaryMap[s.boundaryRoots[i]] = true
-	}
-	for _, rt := range stateSlice[:initialSyncCacheSize] {
-		if boundaryMap[rt] {
-			continue
-		}
-		delete(s.initSyncState, rt)
-	}
-}
-
-func (s *Service) generateState(ctx context.Context, startRoot [32]byte, endRoot [32]byte) (*stateTrie.BeaconState, error) {
-	preState, err := s.beaconDB.State(ctx, startRoot)
-	if err != nil {
-		return nil, err
-	}
-	if preState == nil {
-		preState, err = s.stateGen.StateByRoot(ctx, startRoot)
-		if err != nil {
-			return nil, err
-		}
-		if preState == nil {
-			return nil, errors.New("finalized state does not exist in db")
-		}
-	}
-
-	if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
-		return nil, err
-	}
-	var endBlock *ethpb.SignedBeaconBlock
-	if !featureconfig.Get().NoInitSyncBatchSaveBlocks && s.hasInitSyncBlock(endRoot) {
-		endBlock = s.getInitSyncBlock(endRoot)
-		s.clearInitSyncBlocks()
-	} else {
-		endBlock, err = s.beaconDB.Block(ctx, endRoot)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	if endBlock == nil {
-		return nil, errors.New("provided block root does not have block saved in the db")
-	}
-	log.Warnf("Generating missing state of slot %d and root %#x", endBlock.Block.Slot, endRoot)
-
-	blocks, err := s.stateGen.LoadBlocks(ctx, preState.Slot()+1, endBlock.Block.Slot, endRoot)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not load the required blocks")
-	}
-	postState, err := s.stateGen.ReplayBlocks(ctx, preState, blocks, endBlock.Block.Slot)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not replay the blocks to generate the resultant state")
-	}
-	return postState, nil
-}
-
 // This saves a beacon block to the initial sync blocks cache.
 func (s *Service) saveInitSyncBlock(r [32]byte, b *ethpb.SignedBeaconBlock) {
 	s.initSyncBlocksLock.Lock()

beacon-chain/blockchain/init_sync_process_block_test.go

@@ -1,318 +0,0 @@
-package blockchain
-
-import (
-	"context"
-	"testing"
-
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
-	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/prysmaticlabs/prysm/shared/testutil"
-	messagediff "gopkg.in/d4l3k/messagediff.v1"
-)
-
-func TestFilterBoundaryCandidates_FilterCorrect(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	cfg := &Config{BeaconDB: db}
-	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	st, err := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := uint64(0); i < 500; i++ {
-		if err := st.SetSlot(i); err != nil {
-			t.Fatal(err)
-		}
-		root := [32]byte{}
-		copy(root[:], bytesutil.Bytes32(i))
-		service.initSyncState[root] = st.Copy()
-		if helpers.IsEpochStart(i) {
-			service.boundaryRoots = append(service.boundaryRoots, root)
-		}
-	}
-	lastIndex := len(service.boundaryRoots) - 1
-	for i := uint64(500); i < 2000; i++ {
-		if err := st.SetSlot(i); err != nil {
-			t.Fatal(err)
-		}
-		root := [32]byte{}
-		copy(root[:], bytesutil.Bytes32(i))
-		service.initSyncState[root] = st.Copy()
-	}
-	// Set current state.
-	latestSlot := helpers.RoundUpToNearestEpoch(2000)
-	if err := st.SetSlot(latestSlot); err != nil {
-		t.Fatal(err)
-	}
-	lastRoot := [32]byte{}
-	copy(lastRoot[:], bytesutil.Bytes32(latestSlot))
-
-	service.initSyncState[lastRoot] = st.Copy()
-	service.finalizedCheckpt = &ethpb.Checkpoint{
-		Epoch: 0,
-		Root:  []byte{},
-	}
-	service.filterBoundaryCandidates(context.Background(), lastRoot, st)
-	if len(service.boundaryRoots[lastIndex+1:]) == 0 {
-		t.Fatal("Wanted non zero added boundary roots")
-	}
-	for _, rt := range service.boundaryRoots[lastIndex+1:] {
-		st, ok := service.initSyncState[rt]
-		if !ok {
-			t.Error("Root doen't exist in cache map")
-			continue
-		}
-		if !(helpers.IsEpochStart(st.Slot()) || helpers.IsEpochStart(st.Slot()-1) || helpers.IsEpochStart(st.Slot()+1)) {
-			t.Errorf("boundary roots not validly stored. They have slot %d", st.Slot())
-		}
-	}
-}
-
-func TestFilterBoundaryCandidates_HandleSkippedSlots(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	cfg := &Config{BeaconDB: db}
-	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	st, err := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := uint64(0); i < 500; i++ {
-		if err := st.SetSlot(i); err != nil {
-			t.Fatal(err)
-		}
-		root := [32]byte{}
-		copy(root[:], bytesutil.Bytes32(i))
-		service.initSyncState[root] = st.Copy()
-		if helpers.IsEpochStart(i) {
-			service.boundaryRoots = append(service.boundaryRoots, root)
-		}
-	}
-	lastIndex := len(service.boundaryRoots) - 1
-	for i := uint64(500); i < 2000; i++ {
-		if err := st.SetSlot(i); err != nil {
-			t.Fatal(err)
-		}
-		root := [32]byte{}
-		copy(root[:], bytesutil.Bytes32(i))
-		// save only for offsetted slots
-		if helpers.IsEpochStart(i + 10) {
-			service.initSyncState[root] = st.Copy()
-		}
-	}
-	// Set current state.
-	latestSlot := helpers.RoundUpToNearestEpoch(2000)
-	if err := st.SetSlot(latestSlot); err != nil {
-		t.Fatal(err)
-	}
-	lastRoot := [32]byte{}
-	copy(lastRoot[:], bytesutil.Bytes32(latestSlot))
-
-	service.initSyncState[lastRoot] = st.Copy()
-	service.finalizedCheckpt = &ethpb.Checkpoint{
-		Epoch: 0,
-		Root:  []byte{},
-	}
-	service.filterBoundaryCandidates(context.Background(), lastRoot, st)
-	if len(service.boundaryRoots[lastIndex+1:]) == 0 {
-		t.Fatal("Wanted non zero added boundary roots")
-	}
-	for _, rt := range service.boundaryRoots[lastIndex+1:] {
-		st, ok := service.initSyncState[rt]
-		if !ok {
-			t.Error("Root doen't exist in cache map")
-			continue
-		}
-		if st.Slot() >= 500 {
-			// Ignore head boundary root.
-			if st.Slot() == 2016 {
-				continue
-			}
-			if !helpers.IsEpochStart(st.Slot() + 10) {
-				t.Errorf("boundary roots not validly stored. They have slot %d "+
-					"instead of the offset from epoch start", st.Slot())
-			}
-		}
-	}
-}
-
-func TestPruneOldStates_AlreadyFinalized(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	cfg := &Config{BeaconDB: db}
-	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	st, err := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := uint64(100); i < 200; i++ {
-		if err := st.SetSlot(i); err != nil {
-			t.Fatal(err)
-		}
-		root := [32]byte{}
-		copy(root[:], bytesutil.Bytes32(i))
-		service.initSyncState[root] = st.Copy()
-		service.boundaryRoots = append(service.boundaryRoots, root)
-	}
-	finalizedEpoch := uint64(5)
-	service.finalizedCheckpt = &ethpb.Checkpoint{Epoch: finalizedEpoch}
-	service.pruneOldStates()
-	for _, rt := range service.boundaryRoots {
-		st, ok := service.initSyncState[rt]
-		if !ok {
-			t.Error("Root doen't exist in cache map")
-			continue
-		}
-		if st.Slot() < helpers.StartSlot(finalizedEpoch) {
-			t.Errorf("State with slot %d still exists and not pruned", st.Slot())
-		}
-	}
-}
-
-func TestPruneNonBoundary_CanPrune(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	cfg := &Config{BeaconDB: db}
-	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	st, err := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := uint64(0); i < 2000; i++ {
-		if err := st.SetSlot(i); err != nil {
-			t.Fatal(err)
-		}
-		root := [32]byte{}
-		copy(root[:], bytesutil.Bytes32(i))
-		service.initSyncState[root] = st.Copy()
-		if helpers.IsEpochStart(i) {
-			service.boundaryRoots = append(service.boundaryRoots, root)
-		}
-	}
-	service.pruneNonBoundaryStates()
-	for _, rt := range service.boundaryRoots {
-		st, ok := service.initSyncState[rt]
-		if !ok {
-			t.Error("Root doesn't exist in cache map")
-			continue
-		}
-		if !helpers.IsEpochStart(st.Slot()) {
-			t.Errorf("Non boundary state with slot %d still exists and not pruned", st.Slot())
-		}
-	}
-}
-
-func TestGenerateState_CorrectlyGenerated(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	cfg := &Config{BeaconDB: db, StateGen: stategen.New(db, cache.NewStateSummaryCache())}
-	service, err := NewService(context.Background(), cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	beaconState, privs := testutil.DeterministicGenesisState(t, 32)
-	genesisBlock := blocks.NewGenesisBlock([]byte{})
-	bodyRoot, err := ssz.HashTreeRoot(genesisBlock.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = beaconState.SetLatestBlockHeader(&ethpb.BeaconBlockHeader{
-		Slot:       genesisBlock.Block.Slot,
-		ParentRoot: genesisBlock.Block.ParentRoot,
-		StateRoot:  params.BeaconConfig().ZeroHash[:],
-		BodyRoot:   bodyRoot[:],
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := beaconState.SetSlashings(make([]uint64, params.BeaconConfig().EpochsPerSlashingsVector)); err != nil {
-		t.Fatal(err)
-	}
-	cp := beaconState.CurrentJustifiedCheckpoint()
-	mockRoot := [32]byte{}
-	copy(mockRoot[:], "hello-world")
-	cp.Root = mockRoot[:]
-	if err := beaconState.SetCurrentJustifiedCheckpoint(cp); err != nil {
-		t.Fatal(err)
-	}
-	if err := beaconState.SetCurrentEpochAttestations([]*pb.PendingAttestation{}); err != nil {
-		t.Fatal(err)
-	}
-	err = db.SaveBlock(context.Background(), genesisBlock)
-	if err != nil {
-		t.Fatal(err)
-	}
-	genRoot, err := ssz.HashTreeRoot(genesisBlock)
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = db.SaveState(context.Background(), beaconState, genRoot)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	lastBlock := &ethpb.SignedBeaconBlock{}
-	for i := uint64(1); i < 10; i++ {
-		block, err := testutil.GenerateFullBlock(beaconState, privs, testutil.DefaultBlockGenConfig(), i)
-		if err != nil {
-			t.Fatal(err)
-		}
-		beaconState, err = state.ExecuteStateTransition(context.Background(), beaconState, block)
-		if err != nil {
-			t.Fatal(err)
-		}
-		err = db.SaveBlock(context.Background(), block)
-		if err != nil {
-			t.Fatal(err)
-		}
-		lastBlock = block
-	}
-	root, err := ssz.HashTreeRoot(lastBlock.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	newState, err := service.generateState(context.Background(), genRoot, root)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !ssz.DeepEqual(newState.InnerStateUnsafe(), beaconState.InnerStateUnsafe()) {
-		diff, _ := messagediff.PrettyDiff(newState.InnerStateUnsafe(), beaconState.InnerStateUnsafe())
-		t.Errorf("Generated state is different from what is expected: %s", diff)
-	}
-}

beacon-chain/blockchain/log.go

@@ -1,12 +1,11 @@
 package blockchain
 
 import (
+	"encoding/hex"
 	"fmt"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
-	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/sirupsen/logrus"
 )
 
@@ -15,7 +14,6 @@ var log = logrus.WithField("prefix", "blockchain")
 // logs state transition related data every slot.
 func logStateTransitionData(b *ethpb.BeaconBlock) {
 	log.WithFields(logrus.Fields{
-		"slot":              b.Slot,
 		"attestations":      len(b.Body.Attestations),
 		"deposits":          len(b.Body.Deposits),
 		"attesterSlashings": len(b.Body.AttesterSlashings),
@@ -24,29 +22,12 @@ func logStateTransitionData(b *ethpb.BeaconBlock) {
 	}).Info("Finished applying state transition")
 }
 
-func logEpochData(beaconState *stateTrie.BeaconState) {
+func logBlockSyncStatus(block *ethpb.BeaconBlock, blockRoot [32]byte, finalized *ethpb.Checkpoint) {
 	log.WithFields(logrus.Fields{
-		"epoch":                  helpers.CurrentEpoch(beaconState),
-		"finalizedEpoch":         beaconState.FinalizedCheckpointEpoch(),
-		"justifiedEpoch":         beaconState.CurrentJustifiedCheckpoint().Epoch,
-		"previousJustifiedEpoch": beaconState.PreviousJustifiedCheckpoint().Epoch,
-	}).Info("Starting next epoch")
-	activeVals, err := helpers.ActiveValidatorIndices(beaconState, helpers.CurrentEpoch(beaconState))
-	if err != nil {
-		log.WithError(err).Error("Could not get active validator indices")
-		return
-	}
-	log.WithFields(logrus.Fields{
-		"totalValidators":  len(beaconState.Validators()),
-		"activeValidators": len(activeVals),
-		"averageBalance":   fmt.Sprintf("%.5f ETH", averageBalance(beaconState.Balances())),
-	}).Info("Validator registry information")
-}
-
-func averageBalance(balances []uint64) float64 {
-	total := uint64(0)
-	for i := 0; i < len(balances); i++ {
-		total += balances[i]
-	}
-	return float64(total) / float64(len(balances)) / float64(params.BeaconConfig().GweiPerEth)
+		"slot":           block.Slot,
+		"block":          fmt.Sprintf("0x%s...", hex.EncodeToString(blockRoot[:])[:8]),
+		"epoch":          helpers.SlotToEpoch(block.Slot),
+		"finalizedEpoch": finalized.Epoch,
+		"finalizedRoot":  fmt.Sprintf("0x%s...", hex.EncodeToString(finalized.Root[:])[:8]),
+	}).Info("Synced new block")
 }

beacon-chain/blockchain/metrics.go

@@ -1,13 +1,17 @@
 package blockchain
 
 import (
+	"time"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/roughtime"
 )
 
 var (
@@ -19,10 +23,11 @@ var (
 		Name: "beacon_head_slot",
 		Help: "Slot of the head block of the beacon chain",
 	})
-	competingBlks = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "competing_blocks",
-		Help: "The # of blocks received and processed from a competing chain",
+	clockTimeSlot = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_clock_time_slot",
+		Help: "The current slot based on the genesis time and current clock",
 	})
+
 	headFinalizedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
 		Name: "head_finalized_epoch",
 		Help: "Last finalized epoch of the head state",
@@ -73,17 +78,36 @@ var (
 	})
 	totalEligibleBalances = promauto.NewGauge(prometheus.GaugeOpts{
 		Name: "total_eligible_balances",
-		Help: "The total amount of ether, in gwei, that has been used in voting attestation target of previous epoch",
+		Help: "The total amount of ether, in gwei, that is eligible for voting of previous epoch",
 	})
 	totalVotedTargetBalances = promauto.NewGauge(prometheus.GaugeOpts{
 		Name: "total_voted_target_balances",
-		Help: "The total amount of ether, in gwei, that is eligible for voting of previous epoch",
+		Help: "The total amount of ether, in gwei, that has been used in voting attestation target of previous epoch",
 	})
+	reorgCount = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "beacon_reorg_total",
+		Help: "Count the number of times beacon chain has a reorg",
+	})
+	sentBlockPropagationHistogram = promauto.NewHistogram(
+		prometheus.HistogramOpts{
+			Name:    "block_sent_latency_milliseconds",
+			Help:    "Captures blocks broadcast time. Blocks sent in milliseconds distribution",
+			Buckets: []float64{1000, 2000, 3000, 4000, 5000, 6000},
+		},
+	)
+	attestationInclusionDelay = promauto.NewHistogram(
+		prometheus.HistogramOpts{
+			Name:    "attestation_inclusion_delay_slots",
+			Help:    "The number of slots between att.Slot and block.Slot",
+			Buckets: []float64{1, 2, 3, 4, 6, 32, 64},
+		},
+	)
 )
 
 // reportSlotMetrics reports slot related metrics.
-func reportSlotMetrics(currentSlot uint64, headSlot uint64, finalizedCheckpoint *ethpb.Checkpoint) {
-	beaconSlot.Set(float64(currentSlot))
+func reportSlotMetrics(stateSlot uint64, headSlot uint64, clockSlot uint64, finalizedCheckpoint *ethpb.Checkpoint) {
+	clockTimeSlot.Set(float64(clockSlot))
+	beaconSlot.Set(float64(stateSlot))
 	beaconHeadSlot.Set(float64(headSlot))
 	if finalizedCheckpoint != nil {
 		headFinalizedEpoch.Set(float64(finalizedCheckpoint.Epoch))
@@ -114,6 +138,7 @@ func reportEpochMetrics(state *stateTrie.BeaconState) {
 	for i, validator := range state.Validators() {
 		bal, err := state.BalanceAtIndex(uint64(i))
 		if err != nil {
+			log.Errorf("Could not load validator balance: %v", err)
 			continue
 		}
 		if validator.Slashed {
@@ -174,7 +199,25 @@ func reportEpochMetrics(state *stateTrie.BeaconState) {
 	currentEth1DataDepositCount.Set(float64(state.Eth1Data().DepositCount))
 
 	if precompute.Balances != nil {
-		totalEligibleBalances.Set(float64(precompute.Balances.PrevEpoch))
-		totalVotedTargetBalances.Set(float64(precompute.Balances.PrevEpochTargetAttesters))
+		totalEligibleBalances.Set(float64(precompute.Balances.ActivePrevEpoch))
+		totalVotedTargetBalances.Set(float64(precompute.Balances.PrevEpochTargetAttested))
+	}
+}
+
+// This captures metrics for block sent time by subtracts slot start time.
+func captureSentTimeMetric(genesisTime uint64, currentSlot uint64) error {
+	startTime, err := helpers.SlotToTime(genesisTime, currentSlot)
+	if err != nil {
+		return err
+	}
+	diffMs := roughtime.Now().Sub(startTime) / time.Millisecond
+	sentBlockPropagationHistogram.Observe(float64(diffMs))
+
+	return nil
+}
+
+func reportAttestationInclusion(blk *ethpb.BeaconBlock) {
+	for _, att := range blk.Body.Attestations {
+		attestationInclusionDelay.Observe(float64(blk.Slot - att.Data.Slot))
 	}
 }

beacon-chain/blockchain/process_attestation.go

@@ -9,6 +9,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/roughtime"
 	"go.opencensus.io/trace"
 )
@@ -18,60 +19,43 @@ import (
 var ErrTargetRootNotInDB = errors.New("target root does not exist in db")
 
 // onAttestation is called whenever an attestation is received, verifies the attestation is valid and saves
-/// it to the DB.
+// it to the DB. As a stateless function, this does not hold nor delay attestation based on the spec descriptions.
+// The delay is handled by the caller in `processAttestation`.
 //
 // Spec pseudocode definition:
-//   def on_attestation(store: Service, attestation: Attestation) -> None:
+//   def on_attestation(store: Store, attestation: Attestation) -> None:
 //    """
 //    Run ``on_attestation`` upon receiving a new ``attestation`` from either within a block or directly on the wire.
 //
 //    An ``attestation`` that is asserted as invalid may be valid at a later time,
 //    consider scheduling it for later processing in such case.
 //    """
-//    target = attestation.data.target
+//    validate_on_attestation(store, attestation)
+//    store_target_checkpoint_state(store, attestation.data.target)
 //
-//    # Attestations must be from the current or previous epoch
-//    current_epoch = compute_epoch_at_slot(get_current_slot(store))
-//    # Use GENESIS_EPOCH for previous when genesis to avoid underflow
-//    previous_epoch = current_epoch - 1 if current_epoch > GENESIS_EPOCH else GENESIS_EPOCH
-//    assert target.epoch in [current_epoch, previous_epoch]
-//    assert target.epoch == compute_epoch_at_slot(attestation.data.slot)
-//
-//    # Attestations target be for a known block. If target block is unknown, delay consideration until the block is found
-//    assert target.root in store.blocks
-//    # Attestations cannot be from future epochs. If they are, delay consideration until the epoch arrives
-//    base_state = store.block_states[target.root].copy()
-//    assert store.time >= base_state.genesis_time + compute_start_slot_at_epoch(target.epoch) * SECONDS_PER_SLOT
-//
-//    # Attestations must be for a known block. If block is unknown, delay consideration until the block is found
-//    assert attestation.data.beacon_block_root in store.blocks
-//    # Attestations must not be for blocks in the future. If not, the attestation should not be considered
-//    assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot
-//
-//    # Service target checkpoint state if not yet seen
-//    if target not in store.checkpoint_states:
-//        process_slots(base_state, compute_start_slot_at_epoch(target.epoch))
-//        store.checkpoint_states[target] = base_state
-//    target_state = store.checkpoint_states[target]
-//
-//    # Attestations can only affect the fork choice of subsequent slots.
-//    # Delay consideration in the fork choice until their slot is in the past.
-//    assert store.time >= (attestation.data.slot + 1) * SECONDS_PER_SLOT
-//
-//    # Get state at the `target` to validate attestation and calculate the committees
+//    # Get state at the `target` to fully validate attestation
+//    target_state = store.checkpoint_states[attestation.data.target]
 //    indexed_attestation = get_indexed_attestation(target_state, attestation)
 //    assert is_valid_indexed_attestation(target_state, indexed_attestation)
 //
-//    # Update latest messages
-//    for i in indexed_attestation.attesting_indices:
-//        if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch:
-//            store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=attestation.data.beacon_block_root)
+//    # Update latest messages for attesting indices
+//    update_latest_messages(store, indexed_attestation.attesting_indices, attestation)
+// TODO(#6072): This code path is highly untested. Requires comprehensive tests and simpler refactoring.
 func (s *Service) onAttestation(ctx context.Context, a *ethpb.Attestation) ([]uint64, error) {
-	ctx, span := trace.StartSpan(ctx, "blockchain.onAttestation")
+	ctx, span := trace.StartSpan(ctx, "blockChain.onAttestation")
 	defer span.End()
 
+	if a == nil {
+		return nil, errors.New("nil attestation")
+	}
+	if a.Data == nil {
+		return nil, errors.New("nil attestation.Data field")
+	}
+	if a.Data.Target == nil {
+		return nil, errors.New("nil attestation.Data.Target field")
+	}
+
 	tgt := stateTrie.CopyCheckpoint(a.Data.Target)
-	tgtSlot := helpers.StartSlot(tgt.Epoch)
 
 	if helpers.SlotToEpoch(a.Data.Slot) != a.Data.Target.Epoch {
 		return nil, fmt.Errorf("data slot is not in the same epoch as target %d != %d", helpers.SlotToEpoch(a.Data.Slot), a.Data.Target.Epoch)
@@ -89,29 +73,29 @@ func (s *Service) onAttestation(ctx context.Context, a *ethpb.Attestation) ([]ui
 		return nil, err
 	}
 
-	genesisTime := baseState.GenesisTime()
+	genesisTime := helpers.GenesisTime(baseState)
 
 	// Verify attestation target is from current epoch or previous epoch.
 	if err := s.verifyAttTargetEpoch(ctx, genesisTime, uint64(roughtime.Now().Unix()), tgt); err != nil {
 		return nil, err
 	}
 
-	// Verify Attestations cannot be from future epochs.
-	if err := helpers.VerifySlotTime(genesisTime, tgtSlot, helpers.TimeShiftTolerance); err != nil {
-		return nil, errors.Wrap(err, "could not verify attestation target slot")
-	}
-
 	// Verify attestation beacon block is known and not from the future.
 	if err := s.verifyBeaconBlock(ctx, a.Data); err != nil {
 		return nil, errors.Wrap(err, "could not verify attestation beacon block")
 	}
 
+	// Verify LMG GHOST and FFG votes are consistent with each other.
+	if err := s.verifyLMDFFGConsistent(ctx, a.Data.Target.Epoch, a.Data.Target.Root, a.Data.BeaconBlockRoot); err != nil {
+		return nil, errors.Wrap(err, "could not verify attestation beacon block")
+	}
+
 	// Verify attestations can only affect the fork choice of subsequent slots.
-	if err := helpers.VerifySlotTime(genesisTime, a.Data.Slot, helpers.TimeShiftTolerance); err != nil {
+	if err := helpers.VerifySlotTime(genesisTime, a.Data.Slot+1, params.BeaconNetworkConfig().MaximumGossipClockDisparity); err != nil {
 		return nil, err
 	}
 
-	// Use the target state to to validate attestation and calculate the committees.
+	// Use the target state to validate attestation and calculate the committees.
 	indexedAtt, err := s.verifyAttestation(ctx, baseState, a)
 	if err != nil {
 		return nil, err
@@ -120,12 +104,6 @@ func (s *Service) onAttestation(ctx context.Context, a *ethpb.Attestation) ([]ui
 	if indexedAtt.AttestingIndices == nil {
 		return nil, errors.New("nil attesting indices")
 	}
-	if a.Data == nil {
-		return nil, errors.New("nil att data")
-	}
-	if a.Data.Target == nil {
-		return nil, errors.New("nil att target")
-	}
 
 	// Update forkchoice store with the new attestation for updating weight.
 	s.forkChoiceStore.ProcessAttestation(ctx, indexedAtt.AttestingIndices, bytesutil.ToBytes32(a.Data.BeaconBlockRoot), a.Data.Target.Epoch)

beacon-chain/blockchain/process_attestation_helpers.go

@@ -3,19 +3,16 @@ package blockchain
 import (
 	"bytes"
 	"context"
-	"encoding/hex"
 	"fmt"
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/attestationutil"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
@@ -23,6 +20,7 @@ import (
 func (s *Service) getAttPreState(ctx context.Context, c *ethpb.Checkpoint) (*stateTrie.BeaconState, error) {
 	s.checkpointStateLock.Lock()
 	defer s.checkpointStateLock.Unlock()
+
 	cachedState, err := s.checkpointState.StateByCheckpoint(c)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not get cached checkpoint state")
@@ -31,58 +29,34 @@ func (s *Service) getAttPreState(ctx context.Context, c *ethpb.Checkpoint) (*sta
 		return cachedState, nil
 	}
 
-	var baseState *stateTrie.BeaconState
-	if !featureconfig.Get().DisableNewStateMgmt {
-		baseState, err = s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(c.Root))
-		if err != nil {
-			return nil, errors.Wrapf(err, "could not get pre state for slot %d", helpers.StartSlot(c.Epoch))
-		}
-	} else {
-		if featureconfig.Get().CheckHeadState {
-			headRoot, err := s.HeadRoot(ctx)
-			if err != nil {
-				return nil, errors.Wrapf(err, "could not get head root")
-			}
-			if bytes.Equal(headRoot, c.Root) {
-				st, err := s.HeadState(ctx)
-				if err != nil {
-					return nil, errors.Wrapf(err, "could not get head state")
-				}
-				if err := s.checkpointState.AddCheckpointState(&cache.CheckpointState{
-					Checkpoint: c,
-					State:      st.Copy(),
-				}); err != nil {
-					return nil, errors.Wrap(err, "could not saved checkpoint state to cache")
-				}
-				return st, nil
-			}
-		}
-
-		baseState, err = s.beaconDB.State(ctx, bytesutil.ToBytes32(c.Root))
-		if err != nil {
-			return nil, errors.Wrapf(err, "could not get pre state for slot %d", helpers.StartSlot(c.Epoch))
-		}
-	}
-
-	if baseState == nil {
-		return nil, fmt.Errorf("pre state of target block %d does not exist", helpers.StartSlot(c.Epoch))
+	baseState, err := s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(c.Root))
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not get pre state for slot %d", helpers.StartSlot(c.Epoch))
 	}
 
 	if helpers.StartSlot(c.Epoch) > baseState.Slot() {
+		baseState = baseState.Copy()
 		baseState, err = state.ProcessSlots(ctx, baseState, helpers.StartSlot(c.Epoch))
 		if err != nil {
 			return nil, errors.Wrapf(err, "could not process slots up to %d", helpers.StartSlot(c.Epoch))
 		}
+		if err := s.checkpointState.AddCheckpointState(c, baseState); err != nil {
+			return nil, errors.Wrap(err, "could not saved checkpoint state to cache")
+		}
+		return baseState, nil
 	}
 
-	if err := s.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: c,
-		State:      baseState.Copy(),
-	}); err != nil {
-		return nil, errors.Wrap(err, "could not saved checkpoint state to cache")
+	has, err := s.stateGen.HasState(ctx, bytesutil.ToBytes32(c.Root))
+	if err != nil {
+		return nil, err
+	}
+	if !has {
+		if err := s.checkpointState.AddCheckpointState(c, baseState); err != nil {
+			return nil, errors.Wrap(err, "could not saved checkpoint state to cache")
+		}
 	}
-
 	return baseState, nil
+
 }
 
 // verifyAttTargetEpoch validates attestation is from the current or previous epoch.
@@ -102,10 +76,16 @@ func (s *Service) verifyAttTargetEpoch(ctx context.Context, genesisTime uint64,
 
 // verifyBeaconBlock verifies beacon head block is known and not from the future.
 func (s *Service) verifyBeaconBlock(ctx context.Context, data *ethpb.AttestationData) error {
-	b, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(data.BeaconBlockRoot))
+	r := bytesutil.ToBytes32(data.BeaconBlockRoot)
+	b, err := s.beaconDB.Block(ctx, r)
 	if err != nil {
 		return err
 	}
+	// If the block does not exist in db, check again if block exists in initial sync block cache.
+	// This could happen as the node first syncs to head.
+	if b == nil && s.hasInitSyncBlock(r) {
+		b = s.getInitSyncBlock(r)
+	}
 	if b == nil || b.Block == nil {
 		return fmt.Errorf("beacon block %#x does not exist", bytesutil.Trunc(data.BeaconBlockRoot))
 	}
@@ -115,6 +95,20 @@ func (s *Service) verifyBeaconBlock(ctx context.Context, data *ethpb.Attestation
 	return nil
 }
 
+// verifyLMDFFGConsistent verifies LMD GHOST and FFG votes are consistent with each other.
+func (s *Service) verifyLMDFFGConsistent(ctx context.Context, ffgEpoch uint64, ffgRoot []byte, lmdRoot []byte) error {
+	ffgSlot := helpers.StartSlot(ffgEpoch)
+	r, err := s.ancestor(ctx, lmdRoot, ffgSlot)
+	if err != nil {
+		return err
+	}
+	if !bytes.Equal(ffgRoot, r) {
+		return errors.New("FFG and LMD votes are not consistent")
+	}
+
+	return nil
+}
+
 // verifyAttestation validates input attestation is valid.
 func (s *Service) verifyAttestation(ctx context.Context, baseState *stateTrie.BeaconState, a *ethpb.Attestation) (*ethpb.IndexedAttestation, error) {
 	committee, err := helpers.BeaconCommitteeFromState(baseState, a.Data.Slot, a.Data.CommitteeIndex)
@@ -123,42 +117,7 @@ func (s *Service) verifyAttestation(ctx context.Context, baseState *stateTrie.Be
 	}
 	indexedAtt := attestationutil.ConvertToIndexed(ctx, a, committee)
 	if err := blocks.VerifyIndexedAttestation(ctx, baseState, indexedAtt); err != nil {
-		if err == helpers.ErrSigFailedToVerify {
-			// When sig fails to verify, check if there's a differences in committees due to
-			// different seeds.
-			var aState *stateTrie.BeaconState
-			var err error
-			if !featureconfig.Get().DisableNewStateMgmt {
-				aState, err = s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
-				if err != nil {
-					return nil, err
-				}
-			} else {
-				aState, err = s.beaconDB.State(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
-				if err != nil {
-					return nil, err
-				}
-			}
-			if aState == nil {
-				return nil, fmt.Errorf("nil state for block root %#x", a.Data.BeaconBlockRoot)
-			}
-			epoch := helpers.SlotToEpoch(a.Data.Slot)
-			origSeed, err := helpers.Seed(baseState, epoch, params.BeaconConfig().DomainBeaconAttester)
-			if err != nil {
-				return nil, errors.Wrap(err, "could not get original seed")
-			}
-
-			aSeed, err := helpers.Seed(aState, epoch, params.BeaconConfig().DomainBeaconAttester)
-			if err != nil {
-				return nil, errors.Wrap(err, "could not get attester's seed")
-			}
-			if origSeed != aSeed {
-				return nil, fmt.Errorf("could not verify indexed attestation due to differences in seeds: %v != %v",
-					hex.EncodeToString(bytesutil.Trunc(origSeed[:])), hex.EncodeToString(bytesutil.Trunc(aSeed[:])))
-			}
-		}
 		return nil, errors.Wrap(err, "could not verify indexed attestation")
 	}
-
 	return indexedAtt, nil
 }

beacon-chain/blockchain/process_attestation_test.go

@@ -7,75 +7,55 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 )
 
 func TestStore_OnAttestation(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	cfg := &Config{
 		BeaconDB:        db,
 		ForkChoiceStore: protoarray.New(0, 0, [32]byte{}),
-		StateGen:        stategen.New(db, cache.NewStateSummaryCache()),
+		StateGen:        stategen.New(db, sc),
 	}
 	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	_, err = blockTree1(db, []byte{'g'})
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	BlkWithOutState := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 0}}
-	if err := db.SaveBlock(ctx, BlkWithOutState); err != nil {
-		t.Fatal(err)
-	}
-	BlkWithOutStateRoot, err := ssz.HashTreeRoot(BlkWithOutState.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, db.SaveBlock(ctx, BlkWithOutState))
+	BlkWithOutStateRoot, err := stateutil.BlockRoot(BlkWithOutState.Block)
+	require.NoError(t, err)
 
 	BlkWithStateBadAtt := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1}}
-	if err := db.SaveBlock(ctx, BlkWithStateBadAtt); err != nil {
-		t.Fatal(err)
-	}
-	BlkWithStateBadAttRoot, err := ssz.HashTreeRoot(BlkWithStateBadAtt.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, db.SaveBlock(ctx, BlkWithStateBadAtt))
+	BlkWithStateBadAttRoot, err := stateutil.BlockRoot(BlkWithStateBadAtt.Block)
+	require.NoError(t, err)
 
 	s := testutil.NewBeaconState()
-	if err := s.SetSlot(100 * params.BeaconConfig().SlotsPerEpoch); err != nil {
-		t.Fatal(err)
-	}
-	if err := service.beaconDB.SaveState(ctx, s, BlkWithStateBadAttRoot); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, s.SetSlot(100*params.BeaconConfig().SlotsPerEpoch))
+	require.NoError(t, service.beaconDB.SaveState(ctx, s, BlkWithStateBadAttRoot))
 
 	BlkWithValidState := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 2}}
-	if err := db.SaveBlock(ctx, BlkWithValidState); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, db.SaveBlock(ctx, BlkWithValidState))
 
-	BlkWithValidStateRoot, err := ssz.HashTreeRoot(BlkWithValidState.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
+	BlkWithValidStateRoot, err := stateutil.BlockRoot(BlkWithValidState.Block)
+	require.NoError(t, err)
 	s = testutil.NewBeaconState()
 	if err := s.SetFork(&pb.Fork{
 		Epoch:           0,
@@ -84,9 +64,7 @@ func TestStore_OnAttestation(t *testing.T) {
 	}); err != nil {
 		t.Fatal(err)
 	}
-	if err := service.beaconDB.SaveState(ctx, s, BlkWithValidStateRoot); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, service.beaconDB.SaveState(ctx, s, BlkWithValidStateRoot))
 
 	tests := []struct {
 		name          string
@@ -114,7 +92,7 @@ func TestStore_OnAttestation(t *testing.T) {
 			a:             &ethpb.Attestation{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{Root: BlkWithOutStateRoot[:]}}},
 			s:             &pb.BeaconState{},
 			wantErr:       true,
-			wantErrString: "could not get pre state for slot 0: unknown boundary state",
+			wantErrString: "could not get pre state for slot 0",
 		},
 		{
 			name: "process attestation doesn't match current epoch",
@@ -122,7 +100,28 @@ func TestStore_OnAttestation(t *testing.T) {
 				Root: BlkWithStateBadAttRoot[:]}}},
 			s:             &pb.BeaconState{Slot: 100 * params.BeaconConfig().SlotsPerEpoch},
 			wantErr:       true,
-			wantErrString: "does not match current epoch",
+			wantErrString: "target epoch 100 does not match current epoch",
+		},
+		{
+			name:          "process nil field (a.Target) in attestation",
+			a:             nil,
+			s:             &pb.BeaconState{},
+			wantErr:       true,
+			wantErrString: "nil attestation",
+		},
+		{
+			name:          "process nil field (a.Data) in attestation",
+			a:             &ethpb.Attestation{},
+			s:             &pb.BeaconState{},
+			wantErr:       true,
+			wantErrString: "nil attestation.Data field",
+		},
+		{
+			name:          "process nil field (a.Target) in attestation",
+			a:             &ethpb.Attestation{Data: &ethpb.AttestationData{}},
+			s:             &pb.BeaconState{},
+			wantErr:       true,
+			wantErrString: "nil attestation.Data.Target field",
 		},
 	}
 
@@ -130,7 +129,7 @@ func TestStore_OnAttestation(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			_, err := service.onAttestation(ctx, tt.a)
 			if tt.wantErr {
-				if !strings.Contains(err.Error(), tt.wantErrString) {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErrString) {
 					t.Errorf("Store.onAttestation() error = %v, wantErr = %v", err, tt.wantErrString)
 				}
 			} else {
@@ -142,17 +141,14 @@ func TestStore_OnAttestation(t *testing.T) {
 
 func TestStore_SaveCheckpointState(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	cfg := &Config{
 		BeaconDB: db,
-		StateGen: stategen.New(db, cache.NewStateSummaryCache()),
+		StateGen: stategen.New(db, sc),
 	}
 	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	s, err := stateTrie.InitializeFromProto(&pb.BeaconState{
 		Fork: &pb.Fork{
@@ -170,13 +166,9 @@ func TestStore_SaveCheckpointState(t *testing.T) {
 		Validators:          []*ethpb.Validator{{PublicKey: bytesutil.PadTo([]byte("foo"), 48)}},
 		Balances:            []uint64{0},
 	})
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	r := [32]byte{'g'}
-	if err := service.beaconDB.SaveState(ctx, s, r); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, service.beaconDB.SaveState(ctx, s, r))
 
 	service.justifiedCheckpt = &ethpb.Checkpoint{Root: r[:]}
 	service.bestJustifiedCheckpt = &ethpb.Checkpoint{Root: r[:]}
@@ -185,142 +177,80 @@ func TestStore_SaveCheckpointState(t *testing.T) {
 
 	r = bytesutil.ToBytes32([]byte{'A'})
 	cp1 := &ethpb.Checkpoint{Epoch: 1, Root: bytesutil.PadTo([]byte{'A'}, 32)}
-	if err := service.beaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'A'})); err != nil {
-		t.Fatal(err)
-	}
-	if err := service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Root: bytesutil.PadTo([]byte{'A'}, 32)}); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, service.beaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'A'})))
+	require.NoError(t, service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Root: bytesutil.PadTo([]byte{'A'}, 32)}))
+
 	s1, err := service.getAttPreState(ctx, cp1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if s1.Slot() != 1*params.BeaconConfig().SlotsPerEpoch {
-		t.Errorf("Wanted state slot: %d, got: %d", 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot())
-	}
+	require.NoError(t, err)
+	assert.Equal(t, 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot(), "Unexpected state slot")
 
 	cp2 := &ethpb.Checkpoint{Epoch: 2, Root: bytesutil.PadTo([]byte{'B'}, 32)}
-	if err := service.beaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'B'})); err != nil {
-		t.Fatal(err)
-	}
-	if err := service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Root: bytesutil.PadTo([]byte{'B'}, 32)}); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, service.beaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'B'})))
+	require.NoError(t, service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Root: bytesutil.PadTo([]byte{'B'}, 32)}))
 	s2, err := service.getAttPreState(ctx, cp2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if s2.Slot() != 2*params.BeaconConfig().SlotsPerEpoch {
-		t.Errorf("Wanted state slot: %d, got: %d", 2*params.BeaconConfig().SlotsPerEpoch, s2.Slot())
-	}
+	require.NoError(t, err)
+	assert.Equal(t, 2*params.BeaconConfig().SlotsPerEpoch, s2.Slot(), "Unexpected state slot")
 
 	s1, err = service.getAttPreState(ctx, cp1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if s1.Slot() != 1*params.BeaconConfig().SlotsPerEpoch {
-		t.Errorf("Wanted state slot: %d, got: %d", 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot())
-	}
+	require.NoError(t, err)
+	assert.Equal(t, 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot(), "Unexpected state slot")
 
 	s1, err = service.checkpointState.StateByCheckpoint(cp1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if s1.Slot() != 1*params.BeaconConfig().SlotsPerEpoch {
-		t.Errorf("Wanted state slot: %d, got: %d", 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot())
-	}
+	require.NoError(t, err)
+	assert.Equal(t, 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot(), "Unexpected state slot")
 
 	s2, err = service.checkpointState.StateByCheckpoint(cp2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if s2.Slot() != 2*params.BeaconConfig().SlotsPerEpoch {
-		t.Errorf("Wanted state slot: %d, got: %d", 2*params.BeaconConfig().SlotsPerEpoch, s2.Slot())
-	}
+	require.NoError(t, err)
+	assert.Equal(t, 2*params.BeaconConfig().SlotsPerEpoch, s2.Slot(), "Unexpected state slot")
 
-	if err := s.SetSlot(params.BeaconConfig().SlotsPerEpoch + 1); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, s.SetSlot(params.BeaconConfig().SlotsPerEpoch+1))
 	service.justifiedCheckpt = &ethpb.Checkpoint{Root: r[:]}
 	service.bestJustifiedCheckpt = &ethpb.Checkpoint{Root: r[:]}
 	service.finalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
 	service.prevFinalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
 	cp3 := &ethpb.Checkpoint{Epoch: 1, Root: bytesutil.PadTo([]byte{'C'}, 32)}
-	if err := service.beaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'C'})); err != nil {
-		t.Fatal(err)
-	}
-	if err := service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Root: bytesutil.PadTo([]byte{'C'}, 32)}); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, service.beaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'C'})))
+	require.NoError(t, service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Root: bytesutil.PadTo([]byte{'C'}, 32)}))
 	s3, err := service.getAttPreState(ctx, cp3)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if s3.Slot() != s.Slot() {
-		t.Errorf("Wanted state slot: %d, got: %d", s.Slot(), s3.Slot())
-	}
+	require.NoError(t, err)
+	assert.Equal(t, s.Slot(), s3.Slot(), "Unexpected state slot")
 }
 
 func TestStore_UpdateCheckpointState(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	cfg := &Config{
 		BeaconDB: db,
-		StateGen: stategen.New(db, cache.NewStateSummaryCache()),
+		StateGen: stategen.New(db, sc),
 	}
 	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	epoch := uint64(1)
 	baseState, _ := testutil.DeterministicGenesisState(t, 1)
-	if err := baseState.SetSlot(epoch * params.BeaconConfig().SlotsPerEpoch); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, baseState.SetSlot(epoch*params.BeaconConfig().SlotsPerEpoch))
 	checkpoint := &ethpb.Checkpoint{Epoch: epoch}
-	if err := service.beaconDB.SaveState(ctx, baseState, bytesutil.ToBytes32(checkpoint.Root)); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, service.beaconDB.SaveState(ctx, baseState, bytesutil.ToBytes32(checkpoint.Root)))
 	returned, err := service.getAttPreState(ctx, checkpoint)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if baseState.Slot() != returned.Slot() {
-		t.Error("Incorrectly returned base state")
-	}
+	require.NoError(t, err)
+	assert.Equal(t, returned.Slot(), baseState.Slot(), "Incorrectly returned base state")
 
 	cached, err := service.checkpointState.StateByCheckpoint(checkpoint)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if cached == nil {
-		t.Error("State should have been cached")
-	}
+	require.NoError(t, err)
+	assert.NotNil(t, cached, "State should have been cached")
 
 	epoch = uint64(2)
 	newCheckpoint := &ethpb.Checkpoint{Epoch: epoch}
-	if err := service.beaconDB.SaveState(ctx, baseState, bytesutil.ToBytes32(newCheckpoint.Root)); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, service.beaconDB.SaveState(ctx, baseState, bytesutil.ToBytes32(newCheckpoint.Root)))
 	returned, err = service.getAttPreState(ctx, newCheckpoint)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	baseState, err = state.ProcessSlots(ctx, baseState, helpers.StartSlot(newCheckpoint.Epoch))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if baseState.Slot() != returned.Slot() {
-		t.Error("Incorrectly returned base state")
-	}
+	require.NoError(t, err)
+	assert.Equal(t, returned.Slot(), baseState.Slot(), "Incorrectly returned base state")
 
 	cached, err = service.checkpointState.StateByCheckpoint(newCheckpoint)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	if !proto.Equal(returned.InnerStateUnsafe(), cached.InnerStateUnsafe()) {
 		t.Error("Incorrectly cached base state")
 	}
@@ -328,14 +258,11 @@ func TestStore_UpdateCheckpointState(t *testing.T) {
 
 func TestAttEpoch_MatchPrevEpoch(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 
 	cfg := &Config{BeaconDB: db}
 	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	if err := service.verifyAttTargetEpoch(
 		ctx,
@@ -348,14 +275,11 @@ func TestAttEpoch_MatchPrevEpoch(t *testing.T) {
 
 func TestAttEpoch_MatchCurrentEpoch(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 
 	cfg := &Config{BeaconDB: db}
 	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	if err := service.verifyAttTargetEpoch(
 		ctx,
@@ -368,14 +292,11 @@ func TestAttEpoch_MatchCurrentEpoch(t *testing.T) {
 
 func TestAttEpoch_NotMatch(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 
 	cfg := &Config{BeaconDB: db}
 	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	if err := service.verifyAttTargetEpoch(
 		ctx,
@@ -389,70 +310,88 @@ func TestAttEpoch_NotMatch(t *testing.T) {
 
 func TestVerifyBeaconBlock_NoBlock(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 
 	cfg := &Config{BeaconDB: db}
 	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	d := &ethpb.AttestationData{}
-	if err := service.verifyBeaconBlock(ctx, d); !strings.Contains(err.Error(), "beacon block  does not exist") {
-		t.Error("Did not receive the wanted error")
-	}
+	assert.ErrorContains(t, "beacon block  does not exist", service.verifyBeaconBlock(ctx, d))
 }
 
 func TestVerifyBeaconBlock_futureBlock(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 
 	cfg := &Config{BeaconDB: db}
 	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 2}}
-	if err := service.beaconDB.SaveBlock(ctx, b); err != nil {
-		t.Fatal(err)
-	}
-	r, err := ssz.HashTreeRoot(b.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, b))
+	r, err := stateutil.BlockRoot(b.Block)
+	require.NoError(t, err)
 	d := &ethpb.AttestationData{Slot: 1, BeaconBlockRoot: r[:]}
 
-	err = service.verifyBeaconBlock(ctx, d)
-	if err == nil || !strings.Contains(err.Error(), "could not process attestation for future block") {
-		t.Error("Did not receive the wanted error")
-	}
+	assert.ErrorContains(t, "could not process attestation for future block", service.verifyBeaconBlock(ctx, d))
 }
 
 func TestVerifyBeaconBlock_OK(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 
 	cfg := &Config{BeaconDB: db}
 	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 2}}
-	if err := service.beaconDB.SaveBlock(ctx, b); err != nil {
-		t.Fatal(err)
-	}
-	r, err := ssz.HashTreeRoot(b.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, b))
+	r, err := stateutil.BlockRoot(b.Block)
+	require.NoError(t, err)
 	d := &ethpb.AttestationData{Slot: 2, BeaconBlockRoot: r[:]}
 
-	if err := service.verifyBeaconBlock(ctx, d); err != nil {
-		t.Error("Did not receive the wanted error")
-	}
+	assert.NoError(t, service.verifyBeaconBlock(ctx, d), "Did not receive the wanted error")
+}
+
+func TestVerifyLMDFFGConsistent_NotOK(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db, ForkChoiceStore: protoarray.New(0, 0, [32]byte{})}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	b32 := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 32}}
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, b32))
+	r32, err := stateutil.BlockRoot(b32.Block)
+	require.NoError(t, err)
+	b33 := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 33, ParentRoot: r32[:]}}
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, b33))
+	r33, err := stateutil.BlockRoot(b33.Block)
+	require.NoError(t, err)
+
+	wanted := "FFG and LMD votes are not consistent"
+	assert.ErrorContains(t, wanted, service.verifyLMDFFGConsistent(context.Background(), 1, []byte{'a'}, r33[:]))
+}
+
+func TestVerifyLMDFFGConsistent_OK(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db, ForkChoiceStore: protoarray.New(0, 0, [32]byte{})}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	b32 := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 32}}
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, b32))
+	r32, err := stateutil.BlockRoot(b32.Block)
+	require.NoError(t, err)
+	b33 := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 33, ParentRoot: r32[:]}}
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, b33))
+	r33, err := stateutil.BlockRoot(b33.Block)
+	require.NoError(t, err)
+
+	err = service.verifyLMDFFGConsistent(context.Background(), 1, r32[:], r33[:])
+	assert.NoError(t, err, "Could not verify LMD and FFG votes to be consistent")
 }

beacon-chain/blockchain/process_block.go

@@ -2,7 +2,6 @@ package blockchain
 
 import (
 	"context"
-	"encoding/hex"
 	"fmt"
 
 	"github.com/pkg/errors"
@@ -10,12 +9,11 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	"github.com/prysmaticlabs/prysm/shared/attestationutil"
+	"github.com/prysmaticlabs/prysm/shared/bls"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
 )
 
@@ -23,175 +21,120 @@ import (
 var initialSyncBlockCacheSize = 2 * params.BeaconConfig().SlotsPerEpoch
 
 // onBlock is called when a gossip block is received. It runs regular state transition on the block.
+// The block's signing root should be computed before calling this method to avoid redundant
+// computation in this method and methods it calls into.
 //
 // Spec pseudocode definition:
-//   def on_block(store: Store, block: BeaconBlock) -> None:
-//    # Make a copy of the state to avoid mutability issues
+//   def on_block(store: Store, signed_block: SignedBeaconBlock) -> None:
+//    block = signed_block.message
+//    # Parent block must be known
 //    assert block.parent_root in store.block_states
-//    pre_state = store.block_states[block.parent_root].copy()
+//    # Make a copy of the state to avoid mutability issues
+//    pre_state = copy(store.block_states[block.parent_root])
 //    # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past.
-//    assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT
-//    # Add new block to the store
-//    store.blocks[signing_root(block)] = block
-//    # Check block is a descendant of the finalized block
-//    assert (
-//        get_ancestor(store, signing_root(block), store.blocks[store.finalized_checkpoint.root].slot) ==
-//        store.finalized_checkpoint.root
-//    )
-//    # Check that block is later than the finalized epoch slot
-//    assert block.slot > compute_start_slot_of_epoch(store.finalized_checkpoint.epoch)
+//    assert get_current_slot(store) >= block.slot
+//
+//    # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor)
+//    finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)
+//    assert block.slot > finalized_slot
+//    # Check block is a descendant of the finalized block at the checkpoint finalized slot
+//    assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root
+//
 //    # Check the block is valid and compute the post-state
-//    state = state_transition(pre_state, block)
+//    state = state_transition(pre_state, signed_block, True)
+//    # Add new block to the store
+//    store.blocks[hash_tree_root(block)] = block
 //    # Add new state for this block to the store
-//    store.block_states[signing_root(block)] = state
+//    store.block_states[hash_tree_root(block)] = state
 //
 //    # Update justified checkpoint
 //    if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch:
 //        if state.current_justified_checkpoint.epoch > store.best_justified_checkpoint.epoch:
 //            store.best_justified_checkpoint = state.current_justified_checkpoint
+//        if should_update_justified_checkpoint(store, state.current_justified_checkpoint):
+//            store.justified_checkpoint = state.current_justified_checkpoint
 //
 //    # Update finalized checkpoint
 //    if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch:
 //        store.finalized_checkpoint = state.finalized_checkpoint
-func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock) (*stateTrie.BeaconState, error) {
-	ctx, span := trace.StartSpan(ctx, "blockchain.onBlock")
+//
+//        # Potentially update justified if different from store
+//        if store.justified_checkpoint != state.current_justified_checkpoint:
+//            # Update justified if new justified is later than store justified
+//            if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch:
+//                store.justified_checkpoint = state.current_justified_checkpoint
+//                return
+//
+//            # Update justified if store justified is not in chain with finalized checkpoint
+//            finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)
+//            ancestor_at_finalized_slot = get_ancestor(store, store.justified_checkpoint.root, finalized_slot)
+//            if ancestor_at_finalized_slot != store.finalized_checkpoint.root:
+//                store.justified_checkpoint = state.current_justified_checkpoint
+func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
+	ctx, span := trace.StartSpan(ctx, "blockChain.onBlock")
 	defer span.End()
 
 	if signed == nil || signed.Block == nil {
-		return nil, errors.New("nil block")
+		return errors.New("nil block")
 	}
-
 	b := signed.Block
 
-	// Retrieve incoming block's pre state.
 	preState, err := s.getBlockPreState(ctx, b)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	root, err := stateutil.BlockRoot(b)
-	if err != nil {
-		return nil, errors.Wrapf(err, "could not get signing root of block %d", b.Slot)
-	}
-	log.WithFields(logrus.Fields{
-		"slot": b.Slot,
-		"root": fmt.Sprintf("0x%s...", hex.EncodeToString(root[:])[:8]),
-	}).Info("Executing state transition on block")
-
 	postState, err := state.ExecuteStateTransition(ctx, preState, signed)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not execute state transition")
+		return errors.Wrap(err, "could not execute state transition")
 	}
 
-	if err := s.beaconDB.SaveBlock(ctx, signed); err != nil {
-		return nil, errors.Wrapf(err, "could not save block from slot %d", b.Slot)
-	}
-
-	if err := s.insertBlockToForkChoiceStore(ctx, b, root, postState); err != nil {
-		return nil, errors.Wrapf(err, "could not insert block %d to fork choice store", b.Slot)
-	}
-
-	if !featureconfig.Get().DisableNewStateMgmt {
-		if err := s.stateGen.SaveState(ctx, root, postState); err != nil {
-			return nil, errors.Wrap(err, "could not save state")
-		}
-	} else {
-		if err := s.beaconDB.SaveState(ctx, postState, root); err != nil {
-			return nil, errors.Wrap(err, "could not save state")
-		}
+	if err := s.savePostStateInfo(ctx, blockRoot, signed, postState, false /* reg sync */); err != nil {
+		return err
 	}
 
 	// Update justified check point.
 	if postState.CurrentJustifiedCheckpoint().Epoch > s.justifiedCheckpt.Epoch {
 		if err := s.updateJustified(ctx, postState); err != nil {
-			return nil, err
+			return err
 		}
 	}
 
-	// Update finalized check point. Prune the block cache and helper caches on every new finalized epoch.
+	// Update finalized check point.
 	if postState.FinalizedCheckpointEpoch() > s.finalizedCheckpt.Epoch {
-		if !featureconfig.Get().NoInitSyncBatchSaveBlocks {
-			if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
-				return nil, err
-			}
-			s.clearInitSyncBlocks()
+		if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+			return err
+		}
+		s.clearInitSyncBlocks()
+
+		if err := s.updateFinalized(ctx, postState.FinalizedCheckpoint()); err != nil {
+			return err
 		}
 
-		if err := s.beaconDB.SaveFinalizedCheckpoint(ctx, postState.FinalizedCheckpoint()); err != nil {
-			return nil, errors.Wrap(err, "could not save finalized checkpoint")
-		}
-
-		if featureconfig.Get().DisableNewStateMgmt {
-			startSlot := helpers.StartSlot(s.prevFinalizedCheckpt.Epoch)
-			endSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
-			if endSlot > startSlot {
-				if err := s.rmStatesOlderThanLastFinalized(ctx, startSlot, endSlot); err != nil {
-					return nil, errors.Wrapf(err, "could not delete states prior to finalized check point, range: %d, %d",
-						startSlot, endSlot)
-				}
-			}
-		}
 		fRoot := bytesutil.ToBytes32(postState.FinalizedCheckpoint().Root)
-
-		// Prune proto array fork choice nodes, all nodes before finalized check point will
-		// be pruned.
 		if err := s.forkChoiceStore.Prune(ctx, fRoot); err != nil {
-			return nil, errors.Wrap(err, "could not prune proto array fork choice nodes")
+			return errors.Wrap(err, "could not prune proto array fork choice nodes")
 		}
 
-		s.prevFinalizedCheckpt = s.finalizedCheckpt
-		s.finalizedCheckpt = postState.FinalizedCheckpoint()
-
 		if err := s.finalizedImpliesNewJustified(ctx, postState); err != nil {
-			return nil, errors.Wrap(err, "could not save new justified")
+			return errors.Wrap(err, "could not save new justified")
 		}
 
-		if !featureconfig.Get().DisableNewStateMgmt {
-			fRoot := bytesutil.ToBytes32(postState.FinalizedCheckpoint().Root)
-			fBlock, err := s.beaconDB.Block(ctx, fRoot)
-			if err != nil {
-				return nil, errors.Wrap(err, "could not get finalized block to migrate")
-			}
-			if err := s.stateGen.MigrateToCold(ctx, fBlock.Block.Slot, fRoot); err != nil {
-				return nil, errors.Wrap(err, "could not migrate to cold")
-			}
-		}
+		// Update deposit cache.
+		s.depositCache.InsertFinalizedDeposits(ctx, int64(postState.Eth1DepositIndex()))
 	}
 
-	// Epoch boundary bookkeeping such as logging epoch summaries.
-	if postState.Slot() >= s.nextEpochBoundarySlot {
-		logEpochData(postState)
-		reportEpochMetrics(postState)
+	defer reportAttestationInclusion(b)
 
-		// Update committees cache at epoch boundary slot.
-		if err := helpers.UpdateCommitteeCache(postState, helpers.CurrentEpoch(postState)); err != nil {
-			return nil, err
-		}
-		if err := helpers.UpdateProposerIndicesInCache(postState, helpers.CurrentEpoch(postState)); err != nil {
-			return nil, err
-		}
-
-		s.nextEpochBoundarySlot = helpers.StartSlot(helpers.NextEpoch(postState))
-	}
-
-	// Delete the processed block attestations from attestation pool.
-	if err := s.deletePoolAtts(b.Body.Attestations); err != nil {
-		return nil, err
-	}
-
-	// Delete the processed block attester slashings from slashings pool.
-	for i := 0; i < len(b.Body.AttesterSlashings); i++ {
-		s.slashingPool.MarkIncludedAttesterSlashing(b.Body.AttesterSlashings[i])
-	}
-
-	return postState, nil
+	return s.handleEpochBoundary(postState)
 }
 
 // onBlockInitialSyncStateTransition is called when an initial sync block is received.
-// It runs state transition on the block and without any BLS verification. The excluded BLS verification
-// includes attestation's aggregated signature. It also does not save attestations.
-func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed *ethpb.SignedBeaconBlock) error {
-	ctx, span := trace.StartSpan(ctx, "blockchain.onBlock")
+// It runs state transition on the block and without fork choice and post operation pool processes.
+// The block's signing root should be computed before calling this method to avoid redundant
+// computation in this method and methods it calls into.
+func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
+	ctx, span := trace.StartSpan(ctx, "blockChain.onBlockInitialSyncStateTransition")
 	defer span.End()
 
 	if signed == nil || signed.Block == nil {
@@ -201,123 +144,176 @@ func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed
 	b := signed.Block
 
 	// Retrieve incoming block's pre state.
-	preState, err := s.verifyBlkPreState(ctx, b)
+	if err := s.verifyBlkPreState(ctx, b); err != nil {
+		return err
+	}
+
+	preState, err := s.stateGen.StateByRootInitialSync(ctx, bytesutil.ToBytes32(signed.Block.ParentRoot))
 	if err != nil {
 		return err
 	}
+	if preState == nil {
+		return fmt.Errorf("nil pre state for slot %d", b.Slot)
+	}
 
 	// Exit early if the pre state slot is higher than incoming block's slot.
 	if preState.Slot() >= signed.Block.Slot {
 		return nil
 	}
 
-	postState, err := state.ExecuteStateTransitionNoVerifyAttSigs(ctx, preState, signed)
+	var postState *stateTrie.BeaconState
+	if featureconfig.Get().InitSyncNoVerify {
+		postState, err = state.ExecuteStateTransitionNoVerifyAttSigs(ctx, preState, signed)
+	} else {
+		postState, err = state.ExecuteStateTransition(ctx, preState, signed)
+	}
 	if err != nil {
 		return errors.Wrap(err, "could not execute state transition")
 	}
 
-	root, err := stateutil.BlockRoot(b)
-	if err != nil {
-		return errors.Wrapf(err, "could not get signing root of block %d", b.Slot)
+	if err := s.savePostStateInfo(ctx, blockRoot, signed, postState, true /* init sync */); err != nil {
+		return err
 	}
-	if !featureconfig.Get().NoInitSyncBatchSaveBlocks {
-		s.saveInitSyncBlock(root, signed)
-	} else {
-		if err := s.beaconDB.SaveBlock(ctx, signed); err != nil {
-			return errors.Wrapf(err, "could not save block from slot %d", b.Slot)
-		}
-	}
-
-	if err := s.insertBlockToForkChoiceStore(ctx, b, root, postState); err != nil {
-		return errors.Wrapf(err, "could not insert block %d to fork choice store", b.Slot)
-	}
-
-	if !featureconfig.Get().DisableNewStateMgmt {
-		if err := s.stateGen.SaveState(ctx, root, postState); err != nil {
-			return errors.Wrap(err, "could not save state")
-		}
-	} else {
-		s.initSyncStateLock.Lock()
-		defer s.initSyncStateLock.Unlock()
-		s.initSyncState[root] = postState.Copy()
-		s.filterBoundaryCandidates(ctx, root, postState)
-	}
-
-	// Update justified check point.
-	if postState.CurrentJustifiedCheckpoint().Epoch > s.justifiedCheckpt.Epoch {
-		if err := s.updateJustified(ctx, postState); err != nil {
-			return err
-		}
+	// Save the latest block as head in cache.
+	if err := s.saveHeadNoDB(ctx, signed, blockRoot, postState); err != nil {
+		return err
 	}
 
 	// Rate limit how many blocks (2 epochs worth of blocks) a node keeps in the memory.
-	if len(s.getInitSyncBlocks()) > int(initialSyncBlockCacheSize) {
+	if uint64(len(s.getInitSyncBlocks())) > initialSyncBlockCacheSize {
 		if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
 			return err
 		}
 		s.clearInitSyncBlocks()
 	}
 
+	if postState.CurrentJustifiedCheckpoint().Epoch > s.justifiedCheckpt.Epoch {
+		if err := s.updateJustifiedInitSync(ctx, postState.CurrentJustifiedCheckpoint()); err != nil {
+			return err
+		}
+	}
+
 	// Update finalized check point. Prune the block cache and helper caches on every new finalized epoch.
 	if postState.FinalizedCheckpointEpoch() > s.finalizedCheckpt.Epoch {
-		if featureconfig.Get().DisableNewStateMgmt {
-			startSlot := helpers.StartSlot(s.prevFinalizedCheckpt.Epoch)
-			endSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
-			if endSlot > startSlot {
-				if err := s.rmStatesOlderThanLastFinalized(ctx, startSlot, endSlot); err != nil {
-					return errors.Wrapf(err, "could not delete states prior to finalized check point, range: %d, %d",
-						startSlot, endSlot)
-				}
-			}
-
-			if err := s.saveInitState(ctx, postState); err != nil {
-				return errors.Wrap(err, "could not save init sync finalized state")
-			}
-		}
-
-		if !featureconfig.Get().NoInitSyncBatchSaveBlocks {
-			if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
-				return err
-			}
-			s.clearInitSyncBlocks()
-		}
-
-		if err := s.beaconDB.SaveFinalizedCheckpoint(ctx, postState.FinalizedCheckpoint()); err != nil {
-			return errors.Wrap(err, "could not save finalized checkpoint")
-		}
-
-		s.prevFinalizedCheckpt = s.finalizedCheckpt
-		s.finalizedCheckpt = postState.FinalizedCheckpoint()
-
-		if err := s.finalizedImpliesNewJustified(ctx, postState); err != nil {
-			return errors.Wrap(err, "could not save new justified")
-		}
-
-		if !featureconfig.Get().DisableNewStateMgmt {
-			fRoot := bytesutil.ToBytes32(postState.FinalizedCheckpoint().Root)
-			fBlock, err := s.beaconDB.Block(ctx, fRoot)
-			if err != nil {
-				return errors.Wrap(err, "could not get finalized block to migrate")
-			}
-			if err := s.stateGen.MigrateToCold(ctx, fBlock.Block.Slot, fRoot); err != nil {
-				return errors.Wrap(err, "could not migrate to cold")
-			}
+		if err := s.updateFinalized(ctx, postState.FinalizedCheckpoint()); err != nil {
+			return err
 		}
 	}
 
-	if featureconfig.Get().DisableNewStateMgmt {
-		numOfStates := len(s.boundaryRoots)
-		if numOfStates > initialSyncCacheSize {
-			if err = s.persistCachedStates(ctx, numOfStates); err != nil {
-				return err
+	return s.handleEpochBoundary(postState)
+}
+
+func (s *Service) onBlockBatch(ctx context.Context, blks []*ethpb.SignedBeaconBlock,
+	blockRoots [][32]byte) ([]*ethpb.Checkpoint, []*ethpb.Checkpoint, error) {
+	ctx, span := trace.StartSpan(ctx, "blockChain.onBlockBatch")
+	defer span.End()
+
+	if len(blks) == 0 || len(blockRoots) == 0 {
+		return nil, nil, errors.New("no blocks provided")
+	}
+	if blks[0] == nil || blks[0].Block == nil {
+		return nil, nil, errors.New("nil block")
+	}
+	b := blks[0].Block
+
+	// Retrieve incoming block's pre state.
+	if err := s.verifyBlkPreState(ctx, b); err != nil {
+		return nil, nil, err
+	}
+	preState, err := s.stateGen.StateByRootInitialSync(ctx, bytesutil.ToBytes32(b.ParentRoot))
+	if err != nil {
+		return nil, nil, err
+	}
+	if preState == nil {
+		return nil, nil, fmt.Errorf("nil pre state for slot %d", b.Slot)
+	}
+
+	jCheckpoints := make([]*ethpb.Checkpoint, len(blks))
+	fCheckpoints := make([]*ethpb.Checkpoint, len(blks))
+	sigSet := &bls.SignatureSet{
+		Signatures: []bls.Signature{},
+		PublicKeys: []bls.PublicKey{},
+		Messages:   [][32]byte{},
+	}
+	set := new(bls.SignatureSet)
+	boundaries := make(map[[32]byte]*stateTrie.BeaconState)
+	for i, b := range blks {
+		set, preState, err = state.ExecuteStateTransitionNoVerifyAnySig(ctx, preState, b)
+		if err != nil {
+			return nil, nil, err
+		}
+		// Save potential boundary states.
+		if helpers.IsEpochStart(preState.Slot()) {
+			boundaries[blockRoots[i]] = preState.Copy()
+			if err := s.handleEpochBoundary(preState); err != nil {
+				return nil, nil, fmt.Errorf("could not handle epoch boundary state")
 			}
 		}
-		if len(s.initSyncState) > maxCacheSize {
-			s.pruneOldNonFinalizedStates()
+		jCheckpoints[i] = preState.CurrentJustifiedCheckpoint()
+		fCheckpoints[i] = preState.FinalizedCheckpoint()
+		sigSet.Join(set)
+	}
+	verify, err := bls.VerifyMultipleSignatures(sigSet.Signatures, sigSet.Messages, sigSet.PublicKeys)
+	if err != nil {
+		return nil, nil, err
+	}
+	if !verify {
+		return nil, nil, errors.New("batch block signature verification failed")
+	}
+	for r, st := range boundaries {
+		if err := s.stateGen.SaveState(ctx, r, st); err != nil {
+			return nil, nil, err
+		}
+	}
+	// Also saves the last post state which to be used as pre state for the next batch.
+	lastB := blks[len(blks)-1]
+	lastBR := blockRoots[len(blockRoots)-1]
+	if err := s.stateGen.SaveState(ctx, lastBR, preState); err != nil {
+		return nil, nil, err
+	}
+	if err := s.saveHeadNoDB(ctx, lastB, lastBR, preState); err != nil {
+		return nil, nil, err
+	}
+	return fCheckpoints, jCheckpoints, nil
+}
+
+// handles a block after the block's batch has been verified, where we can save blocks
+// their state summaries and split them off to relative hot/cold storage.
+func (s *Service) handleBlockAfterBatchVerify(ctx context.Context, signed *ethpb.SignedBeaconBlock,
+	blockRoot [32]byte, fCheckpoint *ethpb.Checkpoint, jCheckpoint *ethpb.Checkpoint) error {
+	b := signed.Block
+
+	s.saveInitSyncBlock(blockRoot, signed)
+	if err := s.insertBlockToForkChoiceStore(ctx, b, blockRoot, fCheckpoint, jCheckpoint); err != nil {
+		return err
+	}
+	s.stateGen.SaveStateSummary(ctx, signed, blockRoot)
+
+	// Rate limit how many blocks (2 epochs worth of blocks) a node keeps in the memory.
+	if uint64(len(s.getInitSyncBlocks())) > initialSyncBlockCacheSize {
+		if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+			return err
+		}
+		s.clearInitSyncBlocks()
+	}
+
+	if jCheckpoint.Epoch > s.justifiedCheckpt.Epoch {
+		if err := s.updateJustifiedInitSync(ctx, jCheckpoint); err != nil {
+			return err
 		}
 	}
 
-	// Epoch boundary bookkeeping such as logging epoch summaries.
+	// Update finalized check point. Prune the block cache and helper caches on every new finalized epoch.
+	if fCheckpoint.Epoch > s.finalizedCheckpt.Epoch {
+		if err := s.updateFinalized(ctx, fCheckpoint); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Epoch boundary bookkeeping such as logging epoch summaries.
+func (s *Service) handleEpochBoundary(postState *stateTrie.BeaconState) error {
 	if postState.Slot() >= s.nextEpochBoundarySlot {
 		reportEpochMetrics(postState)
 		s.nextEpochBoundarySlot = helpers.StartSlot(helpers.NextEpoch(postState))
@@ -329,32 +325,19 @@ func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed
 		if err := helpers.UpdateProposerIndicesInCache(postState, helpers.CurrentEpoch(postState)); err != nil {
 			return err
 		}
-
-		if featureconfig.Get().DisableNewStateMgmt && helpers.IsEpochStart(postState.Slot()) {
-			if err := s.beaconDB.SaveState(ctx, postState, root); err != nil {
-				return errors.Wrap(err, "could not save state")
-			}
-		}
 	}
-
 	return nil
 }
 
 // This feeds in the block and block's attestations to fork choice store. It's allows fork choice store
 // to gain information on the most current chain.
-func (s *Service) insertBlockToForkChoiceStore(ctx context.Context, blk *ethpb.BeaconBlock, root [32]byte, state *stateTrie.BeaconState) error {
-	if err := s.fillInForkChoiceMissingBlocks(ctx, blk, state); err != nil {
+func (s *Service) insertBlockAndAttestationsToForkChoiceStore(ctx context.Context, blk *ethpb.BeaconBlock, root [32]byte,
+	state *stateTrie.BeaconState) error {
+	fCheckpoint := state.FinalizedCheckpoint()
+	jCheckpoint := state.CurrentJustifiedCheckpoint()
+	if err := s.insertBlockToForkChoiceStore(ctx, blk, root, fCheckpoint, jCheckpoint); err != nil {
 		return err
 	}
-
-	// Feed in block to fork choice store.
-	if err := s.forkChoiceStore.ProcessBlock(ctx,
-		blk.Slot, root, bytesutil.ToBytes32(blk.ParentRoot),
-		state.CurrentJustifiedCheckpoint().Epoch,
-		state.FinalizedCheckpointEpoch()); err != nil {
-		return errors.Wrap(err, "could not process block for proto array fork choice")
-	}
-
 	// Feed in block's attestations to fork choice store.
 	for _, a := range blk.Body.Attestations {
 		committee, err := helpers.BeaconCommitteeFromState(state, a.Data.Slot, a.Data.CommitteeIndex)
@@ -364,6 +347,39 @@ func (s *Service) insertBlockToForkChoiceStore(ctx context.Context, blk *ethpb.B
 		indices := attestationutil.AttestingIndices(a.AggregationBits, committee)
 		s.forkChoiceStore.ProcessAttestation(ctx, indices, bytesutil.ToBytes32(a.Data.BeaconBlockRoot), a.Data.Target.Epoch)
 	}
-
+	return nil
+}
+
+func (s *Service) insertBlockToForkChoiceStore(ctx context.Context, blk *ethpb.BeaconBlock,
+	root [32]byte, fCheckpoint *ethpb.Checkpoint, jCheckpoint *ethpb.Checkpoint) error {
+	if err := s.fillInForkChoiceMissingBlocks(ctx, blk, fCheckpoint, jCheckpoint); err != nil {
+		return err
+	}
+	// Feed in block to fork choice store.
+	if err := s.forkChoiceStore.ProcessBlock(ctx,
+		blk.Slot, root, bytesutil.ToBytes32(blk.ParentRoot), bytesutil.ToBytes32(blk.Body.Graffiti),
+		jCheckpoint.Epoch,
+		fCheckpoint.Epoch); err != nil {
+		return errors.Wrap(err, "could not process block for proto array fork choice")
+	}
+	return nil
+}
+
+// This saves post state info to DB or cache. This also saves post state info to fork choice store.
+// Post state info consists of processed block and state. Do not call this method unless the block and state are verified.
+func (s *Service) savePostStateInfo(ctx context.Context, r [32]byte, b *ethpb.SignedBeaconBlock, state *stateTrie.BeaconState, initSync bool) error {
+	ctx, span := trace.StartSpan(ctx, "blockChain.savePostStateInfo")
+	defer span.End()
+	if initSync {
+		s.saveInitSyncBlock(r, b)
+	} else if err := s.beaconDB.SaveBlock(ctx, b); err != nil {
+		return errors.Wrapf(err, "could not save block from slot %d", b.Block.Slot)
+	}
+	if err := s.stateGen.SaveState(ctx, r, state); err != nil {
+		return errors.Wrap(err, "could not save state")
+	}
+	if err := s.insertBlockAndAttestationsToForkChoiceStore(ctx, b.Block, r, state); err != nil {
+		return errors.Wrapf(err, "could not insert block %d to fork choice store", b.Block.Slot)
+	}
 	return nil
 }

beacon-chain/blockchain/process_block_helpers.go

@@ -7,12 +7,11 @@ import (
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
+	"github.com/prysmaticlabs/prysm/shared/attestationutil"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/roughtime"
 	"github.com/prysmaticlabs/prysm/shared/traceutil"
@@ -21,29 +20,41 @@ import (
 
 // CurrentSlot returns the current slot based on time.
 func (s *Service) CurrentSlot() uint64 {
-	return uint64(roughtime.Now().Unix()-s.genesisTime.Unix()) / params.BeaconConfig().SecondsPerSlot
+	now := roughtime.Now().Unix()
+	genesis := s.genesisTime.Unix()
+	if now < genesis {
+		return 0
+	}
+	return uint64(now-genesis) / params.BeaconConfig().SecondsPerSlot
 }
 
 // getBlockPreState returns the pre state of an incoming block. It uses the parent root of the block
 // to retrieve the state in DB. It verifies the pre state's validity and the incoming block
 // is in the correct time window.
 func (s *Service) getBlockPreState(ctx context.Context, b *ethpb.BeaconBlock) (*stateTrie.BeaconState, error) {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.getBlockPreState")
+	ctx, span := trace.StartSpan(ctx, "forkChoice.getBlockPreState")
 	defer span.End()
 
 	// Verify incoming block has a valid pre state.
-	preState, err := s.verifyBlkPreState(ctx, b)
-	if err != nil {
+	if err := s.verifyBlkPreState(ctx, b); err != nil {
 		return nil, err
 	}
 
-	// Verify block slot time is not from the feature.
-	if err := helpers.VerifySlotTime(preState.GenesisTime(), b.Slot, helpers.TimeShiftTolerance); err != nil {
+	preState, err := s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(b.ParentRoot))
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not get pre state for slot %d", b.Slot)
+	}
+	if preState == nil {
+		return nil, errors.Wrapf(err, "nil pre state for slot %d", b.Slot)
+	}
+
+	// Verify block slot time is not from the future.
+	if err := helpers.VerifySlotTime(helpers.GenesisTime(preState), b.Slot, params.BeaconNetworkConfig().MaximumGossipClockDisparity); err != nil {
 		return nil, err
 	}
 
 	// Verify block is a descendent of a finalized block.
-	if err := s.verifyBlkDescendant(ctx, bytesutil.ToBytes32(b.ParentRoot), b.Slot); err != nil {
+	if err := s.VerifyBlkDescendant(ctx, bytesutil.ToBytes32(b.ParentRoot)); err != nil {
 		return nil, err
 	}
 
@@ -56,78 +67,56 @@ func (s *Service) getBlockPreState(ctx context.Context, b *ethpb.BeaconBlock) (*
 }
 
 // verifyBlkPreState validates input block has a valid pre-state.
-func (s *Service) verifyBlkPreState(ctx context.Context, b *ethpb.BeaconBlock) (*stateTrie.BeaconState, error) {
+func (s *Service) verifyBlkPreState(ctx context.Context, b *ethpb.BeaconBlock) error {
 	ctx, span := trace.StartSpan(ctx, "chainService.verifyBlkPreState")
 	defer span.End()
 
-	if !featureconfig.Get().DisableNewStateMgmt {
-		parentRoot := bytesutil.ToBytes32(b.ParentRoot)
-		if !s.stateGen.StateSummaryExists(ctx, parentRoot) {
-			return nil, errors.New("provided block root does not have block saved in the db")
-		}
-		preState, err := s.stateGen.StateByRoot(ctx, parentRoot)
-		if err != nil {
-			return nil, errors.Wrapf(err, "could not get pre state for slot %d", b.Slot)
-		}
-		if preState == nil {
-			return nil, errors.Wrapf(err, "nil pre state for slot %d", b.Slot)
-		}
-		return preState, nil // No copy needed from newly hydrated state gen object.
+	parentRoot := bytesutil.ToBytes32(b.ParentRoot)
+	// Loosen the check to HasBlock because state summary gets saved in batches
+	// during initial syncing. There's no risk given a state summary object is just a
+	// a subset of the block object.
+	if !s.stateGen.StateSummaryExists(ctx, parentRoot) && !s.beaconDB.HasBlock(ctx, parentRoot) {
+		return errors.New("could not reconstruct parent state")
 	}
-
-	preState := s.initSyncState[bytesutil.ToBytes32(b.ParentRoot)]
-	var err error
-	if preState == nil {
-		if featureconfig.Get().CheckHeadState {
-			headRoot, err := s.HeadRoot(ctx)
-			if err != nil {
-				return nil, errors.Wrapf(err, "could not get head root")
-			}
-			if bytes.Equal(headRoot, b.ParentRoot) {
-				return s.HeadState(ctx)
-			}
-		}
-		preState, err = s.beaconDB.State(ctx, bytesutil.ToBytes32(b.ParentRoot))
-		if err != nil {
-			return nil, errors.Wrapf(err, "could not get pre state for slot %d", b.Slot)
-		}
-		if preState == nil {
-			if bytes.Equal(s.finalizedCheckpt.Root, b.ParentRoot) {
-				return nil, fmt.Errorf("pre state of slot %d does not exist", b.Slot)
-			}
-			preState, err = s.generateState(ctx, bytesutil.ToBytes32(s.finalizedCheckpt.Root), bytesutil.ToBytes32(b.ParentRoot))
-			if err != nil {
-				return nil, err
-			}
-		}
-		return preState, nil // No copy needed from newly hydrated DB object.
+	has, err := s.stateGen.HasState(ctx, parentRoot)
+	if err != nil {
+		return err
 	}
-	return preState.Copy(), nil
+	if !has {
+		if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+			return errors.Wrap(err, "could not save initial sync blocks")
+		}
+		s.clearInitSyncBlocks()
+	}
+	return nil
 }
 
-// verifyBlkDescendant validates input block root is a descendant of the
+// VerifyBlkDescendant validates input block root is a descendant of the
 // current finalized block root.
-func (s *Service) verifyBlkDescendant(ctx context.Context, root [32]byte, slot uint64) error {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.verifyBlkDescendant")
+func (s *Service) VerifyBlkDescendant(ctx context.Context, root [32]byte) error {
+	ctx, span := trace.StartSpan(ctx, "forkChoice.VerifyBlkDescendant")
 	defer span.End()
-
-	finalizedBlkSigned, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(s.finalizedCheckpt.Root))
-	if err != nil || finalizedBlkSigned == nil || finalizedBlkSigned.Block == nil {
-		return errors.Wrap(err, "could not get finalized block")
+	fRoot := s.ensureRootNotZeros(bytesutil.ToBytes32(s.finalizedCheckpt.Root))
+	finalizedBlkSigned, err := s.beaconDB.Block(ctx, fRoot)
+	if err != nil {
+		return err
+	}
+	if finalizedBlkSigned == nil || finalizedBlkSigned.Block == nil {
+		return errors.New("nil finalized block")
 	}
 	finalizedBlk := finalizedBlkSigned.Block
-
 	bFinalizedRoot, err := s.ancestor(ctx, root[:], finalizedBlk.Slot)
 	if err != nil {
 		return errors.Wrap(err, "could not get finalized block root")
 	}
 	if bFinalizedRoot == nil {
-		return fmt.Errorf("no finalized block known for block from slot %d", slot)
+		return fmt.Errorf("no finalized block known for block %#x", bytesutil.Trunc(root[:]))
 	}
 
-	if !bytes.Equal(bFinalizedRoot, s.finalizedCheckpt.Root) {
-		err := fmt.Errorf("block from slot %d is not a descendent of the current finalized block slot %d, %#x != %#x",
-			slot, finalizedBlk.Slot, bytesutil.Trunc(bFinalizedRoot), bytesutil.Trunc(s.finalizedCheckpt.Root))
+	if !bytes.Equal(bFinalizedRoot, fRoot[:]) {
+		err := fmt.Errorf("block %#x is not a descendent of the current finalized block slot %d, %#x != %#x",
+			bytesutil.Trunc(root[:]), finalizedBlk.Slot, bytesutil.Trunc(bFinalizedRoot),
+			bytesutil.Trunc(fRoot[:]))
 		traceutil.AnnotateError(span, err)
 		return err
 	}
@@ -144,64 +133,6 @@ func (s *Service) verifyBlkFinalizedSlot(b *ethpb.BeaconBlock) error {
 	return nil
 }
 
-// rmStatesOlderThanLastFinalized deletes the states in db since last finalized check point.
-func (s *Service) rmStatesOlderThanLastFinalized(ctx context.Context, startSlot uint64, endSlot uint64) error {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.rmStatesBySlots")
-	defer span.End()
-
-	// Make sure start slot is not a skipped slot
-	for i := startSlot; i > 0; i-- {
-		filter := filters.NewFilter().SetStartSlot(i).SetEndSlot(i)
-		b, err := s.beaconDB.Blocks(ctx, filter)
-		if err != nil {
-			return err
-		}
-		if len(b) > 0 {
-			startSlot = i
-			break
-		}
-	}
-
-	// Make sure finalized slot is not a skipped slot.
-	for i := endSlot; i > 0; i-- {
-		filter := filters.NewFilter().SetStartSlot(i).SetEndSlot(i)
-		b, err := s.beaconDB.Blocks(ctx, filter)
-		if err != nil {
-			return err
-		}
-		if len(b) > 0 {
-			endSlot = i - 1
-			break
-		}
-	}
-
-	// Do not remove genesis state
-	if startSlot == 0 {
-		startSlot++
-	}
-	// If end slot comes less than start slot
-	if endSlot < startSlot {
-		endSlot = startSlot
-	}
-
-	filter := filters.NewFilter().SetStartSlot(startSlot).SetEndSlot(endSlot)
-	roots, err := s.beaconDB.BlockRoots(ctx, filter)
-	if err != nil {
-		return err
-	}
-
-	roots, err = s.filterBlockRoots(ctx, roots)
-	if err != nil {
-		return err
-	}
-
-	if err := s.beaconDB.DeleteStates(ctx, roots); err != nil {
-		return err
-	}
-
-	return nil
-}
-
 // shouldUpdateCurrentJustified prevents bouncing attack, by only update conflicting justified
 // checkpoints in the fork choice if in the early slots of the epoch.
 // Otherwise, delay incorporation of new justified checkpoint until next epoch boundary.
@@ -211,9 +142,9 @@ func (s *Service) shouldUpdateCurrentJustified(ctx context.Context, newJustified
 		return true, nil
 	}
 	var newJustifiedBlockSigned *ethpb.SignedBeaconBlock
-	justifiedRoot := bytesutil.ToBytes32(newJustifiedCheckpt.Root)
+	justifiedRoot := s.ensureRootNotZeros(bytesutil.ToBytes32(newJustifiedCheckpt.Root))
 	var err error
-	if !featureconfig.Get().NoInitSyncBatchSaveBlocks && s.hasInitSyncBlock(justifiedRoot) {
+	if s.hasInitSyncBlock(justifiedRoot) {
 		newJustifiedBlockSigned = s.getInitSyncBlock(justifiedRoot)
 	} else {
 		newJustifiedBlockSigned, err = s.beaconDB.Block(ctx, justifiedRoot)
@@ -230,8 +161,8 @@ func (s *Service) shouldUpdateCurrentJustified(ctx context.Context, newJustified
 		return false, nil
 	}
 	var justifiedBlockSigned *ethpb.SignedBeaconBlock
-	cachedJustifiedRoot := bytesutil.ToBytes32(s.justifiedCheckpt.Root)
-	if !featureconfig.Get().NoInitSyncBatchSaveBlocks && s.hasInitSyncBlock(cachedJustifiedRoot) {
+	cachedJustifiedRoot := s.ensureRootNotZeros(bytesutil.ToBytes32(s.justifiedCheckpt.Root))
+	if s.hasInitSyncBlock(cachedJustifiedRoot) {
 		justifiedBlockSigned = s.getInitSyncBlock(cachedJustifiedRoot)
 	} else {
 		justifiedBlockSigned, err = s.beaconDB.Block(ctx, cachedJustifiedRoot)
@@ -244,7 +175,7 @@ func (s *Service) shouldUpdateCurrentJustified(ctx context.Context, newJustified
 		return false, errors.New("nil justified block")
 	}
 	justifiedBlock := justifiedBlockSigned.Block
-	b, err := s.ancestor(ctx, newJustifiedCheckpt.Root, justifiedBlock.Slot)
+	b, err := s.ancestor(ctx, justifiedRoot[:], justifiedBlock.Slot)
 	if err != nil {
 		return false, err
 	}
@@ -267,102 +198,62 @@ func (s *Service) updateJustified(ctx context.Context, state *stateTrie.BeaconSt
 	if canUpdate {
 		s.prevJustifiedCheckpt = s.justifiedCheckpt
 		s.justifiedCheckpt = cpt
-	}
-
-	if featureconfig.Get().DisableNewStateMgmt {
-		justifiedRoot := bytesutil.ToBytes32(cpt.Root)
-
-		justifiedState := s.initSyncState[justifiedRoot]
-		// If justified state is nil, resume back to normal syncing process and save
-		// justified check point.
-		var err error
-		if justifiedState == nil {
-			if s.beaconDB.HasState(ctx, justifiedRoot) {
-				return s.beaconDB.SaveJustifiedCheckpoint(ctx, cpt)
-			}
-			justifiedState, err = s.generateState(ctx, bytesutil.ToBytes32(s.finalizedCheckpt.Root), justifiedRoot)
-			if err != nil {
-				log.Error(err)
-				return s.beaconDB.SaveJustifiedCheckpoint(ctx, cpt)
-			}
-		}
-		if err := s.beaconDB.SaveState(ctx, justifiedState, justifiedRoot); err != nil {
-			return errors.Wrap(err, "could not save justified state")
+		if err := s.cacheJustifiedStateBalances(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root)); err != nil {
+			return err
 		}
 	}
 
 	return s.beaconDB.SaveJustifiedCheckpoint(ctx, cpt)
 }
 
-// This saves every finalized state in DB during initial sync, needed as part of optimization to
-// use cache state during initial sync in case of restart.
-func (s *Service) saveInitState(ctx context.Context, state *stateTrie.BeaconState) error {
-	cpt := state.FinalizedCheckpoint()
-	finalizedRoot := bytesutil.ToBytes32(cpt.Root)
-	fs := s.initSyncState[finalizedRoot]
-	if fs == nil {
-		var err error
-		fs, err = s.beaconDB.State(ctx, finalizedRoot)
-		if err != nil {
-			return err
-		}
-		if fs == nil {
-			fs, err = s.generateState(ctx, bytesutil.ToBytes32(s.prevFinalizedCheckpt.Root), finalizedRoot)
-			if err != nil {
-				// This might happen if the client was in sync and is now re-syncing for whatever reason.
-				log.Warn("Initial sync cache did not have finalized state root cached")
-				return err
-			}
-		}
+// This caches input checkpoint as justified for the service struct. It rotates current justified to previous justified,
+// caches justified checkpoint balances for fork choice and save justified checkpoint in DB.
+// This method does not have defense against fork choice bouncing attack, which is why it's only recommend to be used during initial syncing.
+func (s *Service) updateJustifiedInitSync(ctx context.Context, cp *ethpb.Checkpoint) error {
+	s.prevJustifiedCheckpt = s.justifiedCheckpt
+	s.justifiedCheckpt = cp
+	if err := s.cacheJustifiedStateBalances(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root)); err != nil {
+		return err
 	}
 
-	if err := s.beaconDB.SaveState(ctx, fs, finalizedRoot); err != nil {
-		return errors.Wrap(err, "could not save state")
-	}
-	return nil
+	return s.beaconDB.SaveJustifiedCheckpoint(ctx, cp)
 }
 
-// This filters block roots that are not known as head root and finalized root in DB.
-// It serves as the last line of defence before we prune states.
-func (s *Service) filterBlockRoots(ctx context.Context, roots [][32]byte) ([][32]byte, error) {
-	f, err := s.beaconDB.FinalizedCheckpoint(ctx)
-	if err != nil {
-		return nil, err
+func (s *Service) updateFinalized(ctx context.Context, cp *ethpb.Checkpoint) error {
+	// Blocks need to be saved so that we can retrieve finalized block from
+	// DB when migrating states.
+	if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+		return err
 	}
-	fRoot := f.Root
-	h, err := s.beaconDB.HeadBlock(ctx)
-	if err != nil {
-		return nil, err
-	}
-	hRoot, err := ssz.HashTreeRoot(h.Block)
-	if err != nil {
-		return nil, err
+	s.clearInitSyncBlocks()
+
+	s.prevFinalizedCheckpt = s.finalizedCheckpt
+	s.finalizedCheckpt = cp
+
+	fRoot := bytesutil.ToBytes32(cp.Root)
+	if err := s.stateGen.MigrateToCold(ctx, fRoot); err != nil {
+		return errors.Wrap(err, "could not migrate to cold")
 	}
 
-	filtered := make([][32]byte, 0, len(roots))
-	for _, root := range roots {
-		if bytes.Equal(root[:], fRoot[:]) || bytes.Equal(root[:], hRoot[:]) {
-			continue
-		}
-		filtered = append(filtered, root)
-	}
+	s.attPool.ClearSeenAtts()
 
-	return filtered, nil
+	return s.beaconDB.SaveFinalizedCheckpoint(ctx, cp)
 }
 
 // ancestor returns the block root of an ancestry block from the input block root.
 //
 // Spec pseudocode definition:
-//   def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash:
+//   def get_ancestor(store: Store, root: Root, slot: Slot) -> Root:
 //    block = store.blocks[root]
 //    if block.slot > slot:
-//      return get_ancestor(store, block.parent_root, slot)
+//        return get_ancestor(store, block.parent_root, slot)
 //    elif block.slot == slot:
-//      return root
+//        return root
 //    else:
-//      return Bytes32()  # root is older than queried slot: no results.
+//        # root is older than queried slot, thus a skip slot. Return most recent root prior to slot
+//        return root
 func (s *Service) ancestor(ctx context.Context, root []byte, slot uint64) ([]byte, error) {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.ancestor")
+	ctx, span := trace.StartSpan(ctx, "forkChoice.ancestor")
 	defer span.End()
 
 	// Stop recursive ancestry lookup if context is cancelled.
@@ -370,27 +261,27 @@ func (s *Service) ancestor(ctx context.Context, root []byte, slot uint64) ([]byt
 		return nil, ctx.Err()
 	}
 
-	signed, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(root))
+	r := bytesutil.ToBytes32(root)
+	// Get ancestor root from fork choice store instead of recursively looking up blocks in DB.
+	// This is most optimal outcome.
+	if s.forkChoiceStore.HasParent(r) {
+		return s.forkChoiceStore.AncestorRoot(ctx, r, slot)
+	}
+
+	signed, err := s.beaconDB.Block(ctx, r)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not get ancestor block")
 	}
 
-	if !featureconfig.Get().NoInitSyncBatchSaveBlocks && s.hasInitSyncBlock(bytesutil.ToBytes32(root)) {
-		signed = s.getInitSyncBlock(bytesutil.ToBytes32(root))
+	if s.hasInitSyncBlock(r) {
+		signed = s.getInitSyncBlock(r)
 	}
 
 	if signed == nil || signed.Block == nil {
 		return nil, errors.New("nil block")
 	}
 	b := signed.Block
-
-	// If we dont have the ancestor in the DB, simply return nil so rest of fork choice
-	// operation can proceed. This is not an error condition.
-	if b == nil || b.Slot < slot {
-		return nil, nil
-	}
-
-	if b.Slot == slot {
+	if b.Slot == slot || b.Slot < slot {
 		return root, nil
 	}
 
@@ -401,34 +292,49 @@ func (s *Service) ancestor(ctx context.Context, root []byte, slot uint64) ([]byt
 // the store's justified is not in chain with finalized check point.
 //
 // Spec definition:
-//   if (
-//            state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch
-//            or get_ancestor(store, store.justified_checkpoint.root, finalized_slot) != store.finalized_checkpoint.root
-//        ):
-//            store.justified_checkpoint = state.current_justified_checkpoint
+//   # Potentially update justified if different from store
+//        if store.justified_checkpoint != state.current_justified_checkpoint:
+//            # Update justified if new justified is later than store justified
+//            if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch:
+//                store.justified_checkpoint = state.current_justified_checkpoint
+//                return
+//            # Update justified if store justified is not in chain with finalized checkpoint
+//            finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)
+//            ancestor_at_finalized_slot = get_ancestor(store, store.justified_checkpoint.root, finalized_slot)
+//            if ancestor_at_finalized_slot != store.finalized_checkpoint.root:
+//                store.justified_checkpoint = state.current_justified_checkpoint
 func (s *Service) finalizedImpliesNewJustified(ctx context.Context, state *stateTrie.BeaconState) error {
-	finalizedBlkSigned, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(s.finalizedCheckpt.Root))
-	if err != nil || finalizedBlkSigned == nil || finalizedBlkSigned.Block == nil {
-		return errors.Wrap(err, "could not get finalized block")
-	}
-	finalizedBlk := finalizedBlkSigned.Block
+	// Update justified if it's different than the one cached in the store.
+	if !attestationutil.CheckPointIsEqual(s.justifiedCheckpt, state.CurrentJustifiedCheckpoint()) {
+		if state.CurrentJustifiedCheckpoint().Epoch > s.justifiedCheckpt.Epoch {
+			s.justifiedCheckpt = state.CurrentJustifiedCheckpoint()
+			if err := s.cacheJustifiedStateBalances(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root)); err != nil {
+				return err
+			}
+			return nil
+		}
 
-	anc, err := s.ancestor(ctx, s.justifiedCheckpt.Root, finalizedBlk.Slot)
-	if err != nil {
-		return err
+		// Update justified if store justified is not in chain with finalized check point.
+		finalizedSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
+		justifiedRoot := s.ensureRootNotZeros(bytesutil.ToBytes32(s.justifiedCheckpt.Root))
+		anc, err := s.ancestor(ctx, justifiedRoot[:], finalizedSlot)
+		if err != nil {
+			return err
+		}
+		if !bytes.Equal(anc, s.finalizedCheckpt.Root) {
+			s.justifiedCheckpt = state.CurrentJustifiedCheckpoint()
+			if err := s.cacheJustifiedStateBalances(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root)); err != nil {
+				return err
+			}
+		}
 	}
-
-	// Either the new justified is later than stored justified or not in chain with finalized check pint.
-	if cpt := state.CurrentJustifiedCheckpoint(); cpt != nil && cpt.Epoch > s.justifiedCheckpt.Epoch || !bytes.Equal(anc, s.finalizedCheckpt.Root) {
-		s.justifiedCheckpt = state.CurrentJustifiedCheckpoint()
-	}
-
 	return nil
 }
 
-// This retrieves missing blocks from DB (ie. the blocks that couldn't received over sync) and inserts them to fork choice store.
+// This retrieves missing blocks from DB (ie. the blocks that couldn't be received over sync) and inserts them to fork choice store.
 // This is useful for block tree visualizer and additional vote accounting.
-func (s *Service) fillInForkChoiceMissingBlocks(ctx context.Context, blk *ethpb.BeaconBlock, state *stateTrie.BeaconState) error {
+func (s *Service) fillInForkChoiceMissingBlocks(ctx context.Context, blk *ethpb.BeaconBlock,
+	fCheckpoint *ethpb.Checkpoint, jCheckpoint *ethpb.Checkpoint) error {
 	pendingNodes := make([]*ethpb.BeaconBlock, 0)
 
 	parentRoot := bytesutil.ToBytes32(blk.ParentRoot)
@@ -452,15 +358,15 @@ func (s *Service) fillInForkChoiceMissingBlocks(ctx context.Context, blk *ethpb.
 	// Lower slots should be at the end of the list.
 	for i := len(pendingNodes) - 1; i >= 0; i-- {
 		b := pendingNodes[i]
-		r, err := ssz.HashTreeRoot(b)
+		r, err := stateutil.BlockRoot(b)
 		if err != nil {
 			return err
 		}
 
 		if err := s.forkChoiceStore.ProcessBlock(ctx,
-			b.Slot, r, bytesutil.ToBytes32(b.ParentRoot),
-			state.CurrentJustifiedCheckpoint().Epoch,
-			state.FinalizedCheckpointEpoch()); err != nil {
+			b.Slot, r, bytesutil.ToBytes32(b.ParentRoot), bytesutil.ToBytes32(b.Body.Graffiti),
+			jCheckpoint.Epoch,
+			fCheckpoint.Epoch); err != nil {
 			return errors.Wrap(err, "could not process block for proto array fork choice")
 		}
 	}
@@ -484,3 +390,12 @@ func (s *Service) deletePoolAtts(atts []*ethpb.Attestation) error {
 
 	return nil
 }
+
+// This ensures that the input root defaults to using genesis root instead of zero hashes. This is needed for handling
+// fork choice justification routine.
+func (s *Service) ensureRootNotZeros(root [32]byte) [32]byte {
+	if root == params.BeaconConfig().ZeroHash {
+		return s.genesisRoot
+	}
+	return root
+}

beacon-chain/blockchain/receive_attestation.go

@@ -9,6 +9,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
@@ -22,9 +23,10 @@ import (
 type AttestationReceiver interface {
 	ReceiveAttestationNoPubsub(ctx context.Context, att *ethpb.Attestation) error
 	IsValidAttestation(ctx context.Context, att *ethpb.Attestation) bool
+	AttestationPreState(ctx context.Context, att *ethpb.Attestation) (*state.BeaconState, error)
 }
 
-// ReceiveAttestationNoPubsub is a function that defines the operations that are preformed on
+// ReceiveAttestationNoPubsub is a function that defines the operations that are performed on
 // attestation that is received from regular sync. The operations consist of:
 //  1. Validate attestation, update validator's latest vote
 //  2. Apply fork choice to the processed attestation
@@ -39,15 +41,10 @@ func (s *Service) ReceiveAttestationNoPubsub(ctx context.Context, att *ethpb.Att
 	}
 
 	if !featureconfig.Get().DisableUpdateHeadPerAttestation {
-		baseState, err := s.getAttPreState(ctx, att.Data.Target)
-		if err != nil {
-			return err
-		}
-
 		// This updates fork choice head, if a new head could not be updated due to
 		// long range or intermediate forking. It simply logs a warning and returns nil
 		// as that's more appropriate than returning errors.
-		if err := s.updateHead(ctx, baseState.Balances()); err != nil {
+		if err := s.updateHead(ctx, s.getJustifiedBalances()); err != nil {
 			log.Warnf("Resolving fork due to new attestation: %v", err)
 			return nil
 		}
@@ -58,9 +55,9 @@ func (s *Service) ReceiveAttestationNoPubsub(ctx context.Context, att *ethpb.Att
 
 // IsValidAttestation returns true if the attestation can be verified against its pre-state.
 func (s *Service) IsValidAttestation(ctx context.Context, att *ethpb.Attestation) bool {
-	baseState, err := s.getAttPreState(ctx, att.Data.Target)
+	baseState, err := s.AttestationPreState(ctx, att)
 	if err != nil {
-		log.WithError(err).Error("Failed to validate attestation")
+		log.WithError(err).Error("Failed to get attestation pre state")
 		return false
 	}
 
@@ -72,6 +69,11 @@ func (s *Service) IsValidAttestation(ctx context.Context, att *ethpb.Attestation
 	return true
 }
 
+// AttestationPreState returns the pre state of attestation.
+func (s *Service) AttestationPreState(ctx context.Context, att *ethpb.Attestation) (*state.BeaconState, error) {
+	return s.getAttPreState(ctx, att.Data.Target)
+}
+
 // This processes attestations from the attestation pool to account for validator votes and fork choice.
 func (s *Service) processAttestation(subscribedToStateEvents chan struct{}) {
 	// Wait for state to be initialized.
@@ -90,13 +92,15 @@ func (s *Service) processAttestation(subscribedToStateEvents chan struct{}) {
 			ctx := context.Background()
 			atts := s.attPool.ForkchoiceAttestations()
 			for _, a := range atts {
-				var hasState bool
-				if !featureconfig.Get().DisableNewStateMgmt {
-					hasState = s.stateGen.StateSummaryExists(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
-				} else {
-					hasState = s.beaconDB.HasState(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot)) && s.beaconDB.HasState(ctx, bytesutil.ToBytes32(a.Data.Target.Root))
+				// Based on the spec, don't process the attestation until the subsequent slot.
+				// This delays consideration in the fork choice until their slot is in the past.
+				// https://github.com/ethereum/eth2.0-specs/blob/dev/specs/phase0/fork-choice.md#validate_on_attestation
+				nextSlot := a.Data.Slot + 1
+				if err := helpers.VerifySlotTime(uint64(s.genesisTime.Unix()), nextSlot, params.BeaconNetworkConfig().MaximumGossipClockDisparity); err != nil {
+					continue
 				}
 
+				hasState := s.stateGen.StateSummaryExists(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
 				hasBlock := s.hasBlock(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
 				if !(hasState && hasBlock) {
 					continue

beacon-chain/blockchain/receive_attestation_test.go

@@ -5,21 +5,18 @@ import (
 	"time"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
 )
 
 func TestVerifyCheckpointEpoch_Ok(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	helpers.ClearCache()
+	db, sc := testDB.SetupDB(t)
 
-	chainService := setupBeaconChain(t, db)
+	chainService := setupBeaconChain(t, db, sc)
 	chainService.genesisTime = time.Now()
 
-	if !chainService.verifyCheckpointEpoch(&ethpb.Checkpoint{}) {
-		t.Error("Wanted true, got false")
-	}
-
-	if chainService.verifyCheckpointEpoch(&ethpb.Checkpoint{Epoch: 1}) {
-		t.Error("Wanted false, got true")
-	}
+	assert.Equal(t, true, chainService.verifyCheckpointEpoch(&ethpb.Checkpoint{}))
+	assert.Equal(t, false, chainService.verifyCheckpointEpoch(&ethpb.Checkpoint{Epoch: 1}))
 }

beacon-chain/blockchain/receive_block.go

@@ -1,20 +1,13 @@
 package blockchain
 
 import (
-	"bytes"
 	"context"
-	"encoding/hex"
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/traceutil"
 	"github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
@@ -22,87 +15,32 @@ import (
 
 // BlockReceiver interface defines the methods of chain service receive and processing new blocks.
 type BlockReceiver interface {
-	ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock) error
-	ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedBeaconBlock) error
-	ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *ethpb.SignedBeaconBlock) error
-	ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedBeaconBlock) error
+	ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error
+	ReceiveBlockInitialSync(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error
+	ReceiveBlockBatch(ctx context.Context, blocks []*ethpb.SignedBeaconBlock, blkRoots [][32]byte) error
 	HasInitSyncBlock(root [32]byte) bool
 }
 
-// ReceiveBlock is a function that defines the operations that are preformed on
-// blocks that is received from rpc service. The operations consists of:
-//   1. Gossip block to other peers
-//   2. Validate block, apply state transition and update check points
-//   3. Apply fork choice to the processed block
-//   4. Save latest head info
-func (s *Service) ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
-	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlock")
-	defer span.End()
-
-	root, err := ssz.HashTreeRoot(block.Block)
-	if err != nil {
-		return errors.Wrap(err, "could not get signing root on received block")
-	}
-
-	// Broadcast the new block to the network.
-	if err := s.p2p.Broadcast(ctx, block); err != nil {
-		return errors.Wrap(err, "could not broadcast block")
-	}
-	log.WithFields(logrus.Fields{
-		"blockRoot": hex.EncodeToString(root[:]),
-	}).Debug("Broadcasting block")
-
-	if err := s.ReceiveBlockNoPubsub(ctx, block); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// ReceiveBlockNoPubsub is a function that defines the the operations (minus pubsub)
-// that are preformed on blocks that is received from regular sync service. The operations consists of:
+// ReceiveBlock is a function that defines the the operations (minus pubsub)
+// that are performed on blocks that is received from regular sync service. The operations consists of:
 //   1. Validate block, apply state transition and update check points
 //   2. Apply fork choice to the processed block
 //   3. Save latest head info
-func (s *Service) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
-	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlockNoPubsub")
+func (s *Service) ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
+	ctx, span := trace.StartSpan(ctx, "blockChain.ReceiveBlock")
 	defer span.End()
 	blockCopy := stateTrie.CopySignedBeaconBlock(block)
 
 	// Apply state transition on the new block.
-	postState, err := s.onBlock(ctx, blockCopy)
-	if err != nil {
+	if err := s.onBlock(ctx, blockCopy, blockRoot); err != nil {
 		err := errors.Wrap(err, "could not process block")
 		traceutil.AnnotateError(span, err)
 		return err
 	}
 
-	// Add attestations from the block to the pool for fork choice.
-	if err := s.attPool.SaveBlockAttestations(blockCopy.Block.Body.Attestations); err != nil {
-		log.Errorf("Could not save attestation for fork choice: %v", err)
-		return nil
-	}
-	for _, exit := range block.Block.Body.VoluntaryExits {
-		s.exitPool.MarkIncluded(exit)
-	}
-
-	s.epochParticipationLock.Lock()
-	defer s.epochParticipationLock.Unlock()
-	s.epochParticipation[helpers.SlotToEpoch(blockCopy.Block.Slot)] = precompute.Balances
-
-	root, err := stateutil.BlockRoot(blockCopy.Block)
-	if err != nil {
-		return errors.Wrap(err, "could not get signing root on received block")
-	}
-
-	if featureconfig.Get().DisableForkChoice && block.Block.Slot > s.headSlot() {
-		if err := s.saveHead(ctx, root); err != nil {
-			return errors.Wrap(err, "could not save head")
-		}
-	} else {
-		if err := s.updateHead(ctx, postState.Balances()); err != nil {
-			return errors.Wrap(err, "could not save head")
-		}
+	// Update and save head block after fork choice.
+	if err := s.updateHead(ctx, s.getJustifiedBalances()); err != nil {
+		log.WithError(err).Warn("Could not update head")
 	}
 
 	// Send notification of the processed block to the state feed.
@@ -110,13 +48,21 @@ func (s *Service) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedB
 		Type: statefeed.BlockProcessed,
 		Data: &statefeed.BlockProcessedData{
 			Slot:      blockCopy.Block.Slot,
-			BlockRoot: root,
+			BlockRoot: blockRoot,
 			Verified:  true,
 		},
 	})
 
+	// Handle post block operations such as attestations and exits.
+	if err := s.handlePostBlockOperations(blockCopy.Block); err != nil {
+		return err
+	}
+
 	// Reports on block and fork choice metrics.
-	reportSlotMetrics(blockCopy.Block.Slot, s.headSlot(), s.finalizedCheckpt)
+	reportSlotMetrics(blockCopy.Block.Slot, s.headSlot(), s.CurrentSlot(), s.finalizedCheckpt)
+
+	// Log block sync status.
+	logBlockSyncStatus(blockCopy.Block, blockRoot, s.finalizedCheckpt)
 
 	// Log state transition data.
 	logStateTransitionData(blockCopy.Block)
@@ -124,105 +70,32 @@ func (s *Service) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedB
 	return nil
 }
 
-// ReceiveBlockNoPubsubForkchoice is a function that defines the all operations (minus pubsub and forkchoice)
-// that are preformed blocks that is received from initial sync service. The operations consists of:
-//   1. Validate block, apply state transition and update check points
-//   2. Save latest head info
-func (s *Service) ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
-	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlockNoForkchoice")
+// ReceiveBlockInitialSync processes the input block for the purpose of initial syncing.
+// This method should only be used on blocks during initial syncing phase.
+func (s *Service) ReceiveBlockInitialSync(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
+	ctx, span := trace.StartSpan(ctx, "blockChain.ReceiveBlockNoVerify")
 	defer span.End()
 	blockCopy := stateTrie.CopySignedBeaconBlock(block)
 
 	// Apply state transition on the new block.
-	_, err := s.onBlock(ctx, blockCopy)
-	if err != nil {
+	if err := s.onBlockInitialSyncStateTransition(ctx, blockCopy, blockRoot); err != nil {
 		err := errors.Wrap(err, "could not process block")
 		traceutil.AnnotateError(span, err)
 		return err
 	}
 
-	root, err := stateutil.BlockRoot(blockCopy.Block)
-	if err != nil {
-		return errors.Wrap(err, "could not get signing root on received block")
-	}
-	cachedHeadRoot, err := s.HeadRoot(ctx)
-	if err != nil {
-		return errors.Wrap(err, "could not get head root from cache")
-	}
-	if !bytes.Equal(root[:], cachedHeadRoot) {
-		if err := s.saveHead(ctx, root); err != nil {
-			return errors.Wrap(err, "could not save head")
-		}
-	}
-
 	// Send notification of the processed block to the state feed.
 	s.stateNotifier.StateFeed().Send(&feed.Event{
 		Type: statefeed.BlockProcessed,
 		Data: &statefeed.BlockProcessedData{
 			Slot:      blockCopy.Block.Slot,
-			BlockRoot: root,
+			BlockRoot: blockRoot,
 			Verified:  true,
 		},
 	})
 
-	// Reports on block and fork choice metrics.
-	reportSlotMetrics(blockCopy.Block.Slot, s.headSlot(), s.finalizedCheckpt)
-
-	// Log state transition data.
-	logStateTransitionData(blockCopy.Block)
-
-	s.epochParticipationLock.Lock()
-	defer s.epochParticipationLock.Unlock()
-	s.epochParticipation[helpers.SlotToEpoch(blockCopy.Block.Slot)] = precompute.Balances
-
-	return nil
-}
-
-// ReceiveBlockNoVerify runs state transition on a input block without verifying the block's BLS contents.
-// Depends on the security model, this is the "minimal" work a node can do to sync the chain.
-// It simulates light client behavior and assumes 100% trust with the syncing peer.
-func (s *Service) ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
-	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlockNoVerify")
-	defer span.End()
-	blockCopy := stateTrie.CopySignedBeaconBlock(block)
-
-	// Apply state transition on the incoming newly received blockCopy without verifying its BLS contents.
-	if err := s.onBlockInitialSyncStateTransition(ctx, blockCopy); err != nil {
-		err := errors.Wrap(err, "could not process block")
-		traceutil.AnnotateError(span, err)
-		return err
-	}
-
-	root, err := stateutil.BlockRoot(blockCopy.Block)
-	if err != nil {
-		return errors.Wrap(err, "could not get signing root on received blockCopy")
-	}
-
-	cachedHeadRoot, err := s.HeadRoot(ctx)
-	if err != nil {
-		return errors.Wrap(err, "could not get head root from cache")
-	}
-
-	if !bytes.Equal(root[:], cachedHeadRoot) {
-		if err := s.saveHeadNoDB(ctx, blockCopy, root); err != nil {
-			err := errors.Wrap(err, "could not save head")
-			traceutil.AnnotateError(span, err)
-			return err
-		}
-	}
-
-	// Send notification of the processed block to the state feed.
-	s.stateNotifier.StateFeed().Send(&feed.Event{
-		Type: statefeed.BlockProcessed,
-		Data: &statefeed.BlockProcessedData{
-			Slot:      blockCopy.Block.Slot,
-			BlockRoot: root,
-			Verified:  false,
-		},
-	})
-
 	// Reports on blockCopy and fork choice metrics.
-	reportSlotMetrics(blockCopy.Block.Slot, s.headSlot(), s.finalizedCheckpt)
+	reportSlotMetrics(blockCopy.Block.Slot, s.headSlot(), s.CurrentSlot(), s.finalizedCheckpt)
 
 	// Log state transition data.
 	log.WithFields(logrus.Fields{
@@ -231,9 +104,43 @@ func (s *Service) ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedB
 		"deposits":     len(blockCopy.Block.Body.Deposits),
 	}).Debug("Finished applying state transition")
 
-	s.epochParticipationLock.Lock()
-	defer s.epochParticipationLock.Unlock()
-	s.epochParticipation[helpers.SlotToEpoch(blockCopy.Block.Slot)] = precompute.Balances
+	return nil
+}
+
+// ReceiveBlockBatch processes the whole block batch at once, assuming the block batch is linear ,transitioning
+// the state, performing batch verification of all collected signatures and then performing the appropriate
+// actions for a block post-transition.
+func (s *Service) ReceiveBlockBatch(ctx context.Context, blocks []*ethpb.SignedBeaconBlock, blkRoots [][32]byte) error {
+	ctx, span := trace.StartSpan(ctx, "blockChain.ReceiveBlockBatch")
+	defer span.End()
+
+	// Apply state transition on the incoming newly received blockCopy without verifying its BLS contents.
+	fCheckpoints, jCheckpoints, err := s.onBlockBatch(ctx, blocks, blkRoots)
+	if err != nil {
+		err := errors.Wrap(err, "could not process block in batch")
+		traceutil.AnnotateError(span, err)
+		return err
+	}
+
+	for i, b := range blocks {
+		blockCopy := stateTrie.CopySignedBeaconBlock(b)
+		if err = s.handleBlockAfterBatchVerify(ctx, blockCopy, blkRoots[i], fCheckpoints[i], jCheckpoints[i]); err != nil {
+			traceutil.AnnotateError(span, err)
+			return err
+		}
+		// Send notification of the processed block to the state feed.
+		s.stateNotifier.StateFeed().Send(&feed.Event{
+			Type: statefeed.BlockProcessed,
+			Data: &statefeed.BlockProcessedData{
+				Slot:      blockCopy.Block.Slot,
+				BlockRoot: blkRoots[i],
+				Verified:  true,
+			},
+		})
+
+		// Reports on blockCopy and fork choice metrics.
+		reportSlotMetrics(blockCopy.Block.Slot, s.headSlot(), s.CurrentSlot(), s.finalizedCheckpt)
+	}
 
 	return nil
 }
@@ -242,3 +149,26 @@ func (s *Service) ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedB
 func (s *Service) HasInitSyncBlock(root [32]byte) bool {
 	return s.hasInitSyncBlock(root)
 }
+
+func (s *Service) handlePostBlockOperations(b *ethpb.BeaconBlock) error {
+	// Delete the processed block attestations from attestation pool.
+	if err := s.deletePoolAtts(b.Body.Attestations); err != nil {
+		return err
+	}
+
+	// Add block attestations to the fork choice pool to compute head.
+	if err := s.attPool.SaveBlockAttestations(b.Body.Attestations); err != nil {
+		log.Errorf("Could not save block attestations for fork choice: %v", err)
+		return nil
+	}
+	// Mark block exits as seen so we don't include same ones in future blocks.
+	for _, e := range b.Body.VoluntaryExits {
+		s.exitPool.MarkIncluded(e)
+	}
+
+	//  Mark attester slashings as seen so we don't include same ones in future blocks.
+	for _, as := range b.Body.AttesterSlashings {
+		s.slashingPool.MarkIncludedAttesterSlashing(as)
+	}
+	return nil
+}

beacon-chain/blockchain/receive_block_test.go

@@ -0,0 +1,370 @@
+package blockchain
+
+import (
+	"context"
+	"sync"
+	"testing"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	blockchainTesting "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
+	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	"github.com/prysmaticlabs/prysm/beacon-chain/operations/attestations"
+	"github.com/prysmaticlabs/prysm/beacon-chain/operations/voluntaryexits"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+)
+
+func TestService_ReceiveBlock(t *testing.T) {
+	ctx := context.Background()
+
+	genesis, keys := testutil.DeterministicGenesisState(t, 64)
+	genFullBlock := func(t *testing.T, conf *testutil.BlockGenConfig, slot uint64) *ethpb.SignedBeaconBlock {
+		blk, err := testutil.GenerateFullBlock(genesis, keys, conf, slot)
+		assert.NoError(t, err)
+		return blk
+	}
+	bc := params.BeaconConfig()
+	bc.ShardCommitteePeriod = 0 // Required for voluntary exits test in reasonable time.
+	params.OverrideBeaconConfig(bc)
+
+	type args struct {
+		block *ethpb.SignedBeaconBlock
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+		check   func(*testing.T, *Service)
+	}{
+		{
+			name: "applies block with state transition",
+			args: args{
+				block: genFullBlock(t, testutil.DefaultBlockGenConfig(), 2 /*slot*/),
+			},
+			check: func(t *testing.T, s *Service) {
+				if hs := s.head.state.Slot(); hs != 2 {
+					t.Errorf("Unexpected state slot. Got %d but wanted %d", hs, 2)
+				}
+				if bs := s.head.block.Block.Slot; bs != 2 {
+					t.Errorf("Unexpected head block slot. Got %d but wanted %d", bs, 2)
+				}
+			},
+		},
+		{
+			name: "saves attestations to pool",
+			args: args{
+				block: genFullBlock(t,
+					&testutil.BlockGenConfig{
+						NumProposerSlashings: 0,
+						NumAttesterSlashings: 0,
+						NumAttestations:      2,
+						NumDeposits:          0,
+						NumVoluntaryExits:    0,
+					},
+					1, /*slot*/
+				),
+			},
+			check: func(t *testing.T, s *Service) {
+				if baCount := len(s.attPool.BlockAttestations()); baCount != 2 {
+					t.Errorf("Did not get the correct number of block attestations saved to the pool. "+
+						"Got %d but wanted %d", baCount, 2)
+				}
+			},
+		},
+
+		{
+			name: "updates exit pool",
+			args: args{
+				block: genFullBlock(t, &testutil.BlockGenConfig{
+					NumProposerSlashings: 0,
+					NumAttesterSlashings: 0,
+					NumAttestations:      0,
+					NumDeposits:          0,
+					NumVoluntaryExits:    3,
+				},
+					1, /*slot*/
+				),
+			},
+			check: func(t *testing.T, s *Service) {
+				var n int
+				for i := uint64(0); int(i) < genesis.NumValidators(); i++ {
+					if s.exitPool.HasBeenIncluded(i) {
+						n++
+					}
+				}
+				if n != 3 {
+					t.Errorf("Did not mark the correct number of exits. Got %d but wanted %d", n, 3)
+				}
+			},
+		},
+		{
+			name: "notifies block processed on state feed",
+			args: args{
+				block: genFullBlock(t, testutil.DefaultBlockGenConfig(), 1 /*slot*/),
+			},
+			check: func(t *testing.T, s *Service) {
+				if recvd := len(s.stateNotifier.(*blockchainTesting.MockStateNotifier).ReceivedEvents()); recvd < 1 {
+					t.Errorf("Received %d state notifications, expected at least 1", recvd)
+				}
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			db, stateSummaryCache := testDB.SetupDB(t)
+			genesisBlockRoot := bytesutil.ToBytes32(nil)
+			require.NoError(t, db.SaveState(ctx, genesis, genesisBlockRoot))
+
+			cfg := &Config{
+				BeaconDB: db,
+				ForkChoiceStore: protoarray.New(
+					0, // justifiedEpoch
+					0, // finalizedEpoch
+					genesisBlockRoot,
+				),
+				AttPool:       attestations.NewPool(),
+				ExitPool:      voluntaryexits.NewPool(),
+				StateNotifier: &blockchainTesting.MockStateNotifier{RecordEvents: true},
+				StateGen:      stategen.New(db, stateSummaryCache),
+			}
+			s, err := NewService(ctx, cfg)
+			require.NoError(t, err)
+			require.NoError(t, s.saveGenesisData(ctx, genesis))
+			gBlk, err := s.beaconDB.GenesisBlock(ctx)
+			require.NoError(t, err)
+			gRoot, err := stateutil.BlockRoot(gBlk.Block)
+			s.finalizedCheckpt = &ethpb.Checkpoint{Root: gRoot[:]}
+			root, err := stateutil.BlockRoot(tt.args.block.Block)
+			require.NoError(t, err)
+			if err := s.ReceiveBlock(ctx, tt.args.block, root); (err != nil) != tt.wantErr {
+				t.Errorf("ReceiveBlock() error = %v, wantErr %v", err, tt.wantErr)
+			} else {
+				tt.check(t, s)
+			}
+		})
+	}
+}
+
+func TestService_ReceiveBlockUpdateHead(t *testing.T) {
+	ctx := context.Background()
+	genesis, keys := testutil.DeterministicGenesisState(t, 64)
+	b, err := testutil.GenerateFullBlock(genesis, keys, testutil.DefaultBlockGenConfig(), 1)
+	assert.NoError(t, err)
+	db, stateSummaryCache := testDB.SetupDB(t)
+	genesisBlockRoot := bytesutil.ToBytes32(nil)
+	require.NoError(t, db.SaveState(ctx, genesis, genesisBlockRoot))
+	cfg := &Config{
+		BeaconDB: db,
+		ForkChoiceStore: protoarray.New(
+			0, // justifiedEpoch
+			0, // finalizedEpoch
+			genesisBlockRoot,
+		),
+		AttPool:       attestations.NewPool(),
+		ExitPool:      voluntaryexits.NewPool(),
+		StateNotifier: &blockchainTesting.MockStateNotifier{RecordEvents: true},
+		StateGen:      stategen.New(db, stateSummaryCache),
+	}
+	s, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+	require.NoError(t, s.saveGenesisData(ctx, genesis))
+	gBlk, err := s.beaconDB.GenesisBlock(ctx)
+	require.NoError(t, err)
+	gRoot, err := stateutil.BlockRoot(gBlk.Block)
+	s.finalizedCheckpt = &ethpb.Checkpoint{Root: gRoot[:]}
+	root, err := stateutil.BlockRoot(b.Block)
+	require.NoError(t, err)
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		require.NoError(t, s.ReceiveBlock(ctx, b, root))
+		wg.Done()
+	}()
+	wg.Wait()
+	if recvd := len(s.stateNotifier.(*blockchainTesting.MockStateNotifier).ReceivedEvents()); recvd < 1 {
+		t.Errorf("Received %d state notifications, expected at least 1", recvd)
+	}
+	// Verify fork choice has processed the block. (Genesis block and the new block)
+	assert.Equal(t, 2, len(s.forkChoiceStore.Nodes()))
+}
+
+func TestService_ReceiveBlockInitialSync(t *testing.T) {
+	ctx := context.Background()
+
+	genesis, keys := testutil.DeterministicGenesisState(t, 64)
+	genFullBlock := func(t *testing.T, conf *testutil.BlockGenConfig, slot uint64) *ethpb.SignedBeaconBlock {
+		blk, err := testutil.GenerateFullBlock(genesis, keys, conf, slot)
+		if err != nil {
+			t.Error(err)
+		}
+		return blk
+	}
+
+	type args struct {
+		block *ethpb.SignedBeaconBlock
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+		check   func(*testing.T, *Service)
+	}{
+		{
+			name: "applies block with state transition",
+			args: args{
+				block: genFullBlock(t, testutil.DefaultBlockGenConfig(), 2 /*slot*/),
+			},
+			check: func(t *testing.T, s *Service) {
+				assert.Equal(t, uint64(2), s.head.state.Slot(), "Incorrect head state slot")
+				assert.Equal(t, uint64(2), s.head.block.Block.Slot, "Incorrect head block slot")
+			},
+		},
+		{
+			name: "notifies block processed on state feed",
+			args: args{
+				block: genFullBlock(t, testutil.DefaultBlockGenConfig(), 1 /*slot*/),
+			},
+			check: func(t *testing.T, s *Service) {
+				if recvd := len(s.stateNotifier.(*blockchainTesting.MockStateNotifier).ReceivedEvents()); recvd < 1 {
+					t.Errorf("Received %d state notifications, expected at least 1", recvd)
+				}
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			db, stateSummaryCache := testDB.SetupDB(t)
+			genesisBlockRoot := bytesutil.ToBytes32(nil)
+
+			cfg := &Config{
+				BeaconDB: db,
+				ForkChoiceStore: protoarray.New(
+					0, // justifiedEpoch
+					0, // finalizedEpoch
+					genesisBlockRoot,
+				),
+				StateNotifier: &blockchainTesting.MockStateNotifier{RecordEvents: true},
+				StateGen:      stategen.New(db, stateSummaryCache),
+			}
+			s, err := NewService(ctx, cfg)
+			require.NoError(t, err)
+			err = s.saveGenesisData(ctx, genesis)
+			require.NoError(t, err)
+			gBlk, err := s.beaconDB.GenesisBlock(ctx)
+			require.NoError(t, err)
+
+			gRoot, err := stateutil.BlockRoot(gBlk.Block)
+			s.finalizedCheckpt = &ethpb.Checkpoint{Root: gRoot[:]}
+			root, err := stateutil.BlockRoot(tt.args.block.Block)
+			require.NoError(t, err)
+
+			if err := s.ReceiveBlockInitialSync(ctx, tt.args.block, root); (err != nil) != tt.wantErr {
+				t.Errorf("ReceiveBlockInitialSync() error = %v, wantErr %v", err, tt.wantErr)
+			} else {
+				tt.check(t, s)
+			}
+		})
+	}
+}
+
+func TestService_ReceiveBlockBatch(t *testing.T) {
+	ctx := context.Background()
+
+	genesis, keys := testutil.DeterministicGenesisState(t, 64)
+	genFullBlock := func(t *testing.T, conf *testutil.BlockGenConfig, slot uint64) *ethpb.SignedBeaconBlock {
+		blk, err := testutil.GenerateFullBlock(genesis, keys, conf, slot)
+		if err != nil {
+			t.Error(err)
+		}
+		return blk
+	}
+
+	type args struct {
+		block *ethpb.SignedBeaconBlock
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+		check   func(*testing.T, *Service)
+	}{
+		{
+			name: "applies block with state transition",
+			args: args{
+				block: genFullBlock(t, testutil.DefaultBlockGenConfig(), 2 /*slot*/),
+			},
+			check: func(t *testing.T, s *Service) {
+				assert.Equal(t, uint64(2), s.head.state.Slot(), "Incorrect head state slot")
+				assert.Equal(t, uint64(2), s.head.block.Block.Slot, "Incorrect head block slot")
+			},
+		},
+		{
+			name: "notifies block processed on state feed",
+			args: args{
+				block: genFullBlock(t, testutil.DefaultBlockGenConfig(), 1 /*slot*/),
+			},
+			check: func(t *testing.T, s *Service) {
+				if recvd := len(s.stateNotifier.(*blockchainTesting.MockStateNotifier).ReceivedEvents()); recvd < 1 {
+					t.Errorf("Received %d state notifications, expected at least 1", recvd)
+				}
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			db, stateSummaryCache := testDB.SetupDB(t)
+			genesisBlockRoot := bytesutil.ToBytes32(nil)
+
+			cfg := &Config{
+				BeaconDB: db,
+				ForkChoiceStore: protoarray.New(
+					0, // justifiedEpoch
+					0, // finalizedEpoch
+					genesisBlockRoot,
+				),
+				StateNotifier: &blockchainTesting.MockStateNotifier{RecordEvents: true},
+				StateGen:      stategen.New(db, stateSummaryCache),
+			}
+			s, err := NewService(ctx, cfg)
+			require.NoError(t, err)
+			err = s.saveGenesisData(ctx, genesis)
+			require.NoError(t, err)
+			gBlk, err := s.beaconDB.GenesisBlock(ctx)
+			require.NoError(t, err)
+
+			gRoot, err := stateutil.BlockRoot(gBlk.Block)
+			s.finalizedCheckpt = &ethpb.Checkpoint{Root: gRoot[:]}
+			root, err := stateutil.BlockRoot(tt.args.block.Block)
+			require.NoError(t, err)
+			blks := []*ethpb.SignedBeaconBlock{tt.args.block}
+			roots := [][32]byte{root}
+			if err := s.ReceiveBlockBatch(ctx, blks, roots); (err != nil) != tt.wantErr {
+				t.Errorf("ReceiveBlockBatch() error = %v, wantErr %v", err, tt.wantErr)
+			} else {
+				tt.check(t, s)
+			}
+		})
+	}
+}
+
+func TestService_HasInitSyncBlock(t *testing.T) {
+	s, err := NewService(context.Background(), &Config{})
+	require.NoError(t, err)
+	r := [32]byte{'a'}
+	if s.HasInitSyncBlock(r) {
+		t.Error("Should not have block")
+	}
+	s.saveInitSyncBlock(r, testutil.NewBeaconBlock())
+	if !s.HasInitSyncBlock(r) {
+		t.Error("Should have block")
+	}
+}

beacon-chain/blockchain/service.go

@@ -1,6 +1,6 @@
-// Package blockchain defines the life-cycle and status of the beacon chain
-// as well as the Ethereum Serenity beacon chain fork-choice rule based on
-// Casper Proof of Stake finality.
+// Package blockchain defines the life-cycle of the blockchain at the core of
+// eth2, including processing of new blocks and attestations using casper
+// proof of stake.
 package blockchain
 
 import (
@@ -12,17 +12,14 @@ import (
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
 	"github.com/prysmaticlabs/prysm/beacon-chain/flags"
 	f "github.com/prysmaticlabs/prysm/beacon-chain/forkchoice"
 	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
@@ -33,50 +30,51 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/slotutil"
 	"go.opencensus.io/trace"
 )
 
 // Service represents a service that handles the internal
 // logic of managing the full PoS beacon chain.
 type Service struct {
-	ctx                    context.Context
-	cancel                 context.CancelFunc
-	beaconDB               db.HeadAccessDatabase
-	depositCache           *depositcache.DepositCache
-	chainStartFetcher      powchain.ChainStartFetcher
-	attPool                attestations.Pool
-	slashingPool           *slashings.Pool
-	exitPool               *voluntaryexits.Pool
-	genesisTime            time.Time
-	p2p                    p2p.Broadcaster
-	maxRoutines            int64
-	head                   *head
-	headLock               sync.RWMutex
-	stateNotifier          statefeed.Notifier
-	genesisRoot            [32]byte
-	epochParticipation     map[uint64]*precompute.Balance
-	epochParticipationLock sync.RWMutex
-	forkChoiceStore        f.ForkChoicer
-	justifiedCheckpt       *ethpb.Checkpoint
-	prevJustifiedCheckpt   *ethpb.Checkpoint
-	bestJustifiedCheckpt   *ethpb.Checkpoint
-	finalizedCheckpt       *ethpb.Checkpoint
-	prevFinalizedCheckpt   *ethpb.Checkpoint
-	nextEpochBoundarySlot  uint64
-	voteLock               sync.RWMutex
-	initSyncState          map[[32]byte]*stateTrie.BeaconState
-	boundaryRoots          [][32]byte
-	initSyncStateLock      sync.RWMutex
-	checkpointState        *cache.CheckpointStateCache
-	checkpointStateLock    sync.Mutex
-	stateGen               *stategen.State
-	opsService             *attestations.Service
-	initSyncBlocks         map[[32]byte]*ethpb.SignedBeaconBlock
-	initSyncBlocksLock     sync.RWMutex
+	ctx                       context.Context
+	cancel                    context.CancelFunc
+	beaconDB                  db.HeadAccessDatabase
+	depositCache              *depositcache.DepositCache
+	chainStartFetcher         powchain.ChainStartFetcher
+	attPool                   attestations.Pool
+	slashingPool              *slashings.Pool
+	exitPool                  *voluntaryexits.Pool
+	genesisTime               time.Time
+	p2p                       p2p.Broadcaster
+	maxRoutines               int
+	head                      *head
+	headLock                  sync.RWMutex
+	stateNotifier             statefeed.Notifier
+	genesisRoot               [32]byte
+	forkChoiceStore           f.ForkChoicer
+	justifiedCheckpt          *ethpb.Checkpoint
+	prevJustifiedCheckpt      *ethpb.Checkpoint
+	bestJustifiedCheckpt      *ethpb.Checkpoint
+	finalizedCheckpt          *ethpb.Checkpoint
+	prevFinalizedCheckpt      *ethpb.Checkpoint
+	nextEpochBoundarySlot     uint64
+	initSyncState             map[[32]byte]*stateTrie.BeaconState
+	boundaryRoots             [][32]byte
+	checkpointState           *cache.CheckpointStateCache
+	checkpointStateLock       sync.Mutex
+	stateGen                  *stategen.State
+	opsService                *attestations.Service
+	initSyncBlocks            map[[32]byte]*ethpb.SignedBeaconBlock
+	initSyncBlocksLock        sync.RWMutex
+	recentCanonicalBlocks     map[[32]byte]bool
+	recentCanonicalBlocksLock sync.RWMutex
+	justifiedBalances         []uint64
+	justifiedBalancesLock     sync.RWMutex
 }
 
 // Config options for the service.
@@ -89,7 +87,7 @@ type Config struct {
 	ExitPool          *voluntaryexits.Pool
 	SlashingPool      *slashings.Pool
 	P2p               p2p.Broadcaster
-	MaxRoutines       int64
+	MaxRoutines       int
 	StateNotifier     statefeed.Notifier
 	ForkChoiceStore   f.ForkChoicer
 	OpsService        *attestations.Service
@@ -101,32 +99,32 @@ type Config struct {
 func NewService(ctx context.Context, cfg *Config) (*Service, error) {
 	ctx, cancel := context.WithCancel(ctx)
 	return &Service{
-		ctx:                ctx,
-		cancel:             cancel,
-		beaconDB:           cfg.BeaconDB,
-		depositCache:       cfg.DepositCache,
-		chainStartFetcher:  cfg.ChainStartFetcher,
-		attPool:            cfg.AttPool,
-		exitPool:           cfg.ExitPool,
-		slashingPool:       cfg.SlashingPool,
-		p2p:                cfg.P2p,
-		maxRoutines:        cfg.MaxRoutines,
-		stateNotifier:      cfg.StateNotifier,
-		epochParticipation: make(map[uint64]*precompute.Balance),
-		forkChoiceStore:    cfg.ForkChoiceStore,
-		initSyncState:      make(map[[32]byte]*stateTrie.BeaconState),
-		boundaryRoots:      [][32]byte{},
-		checkpointState:    cache.NewCheckpointStateCache(),
-		opsService:         cfg.OpsService,
-		stateGen:           cfg.StateGen,
-		initSyncBlocks:     make(map[[32]byte]*ethpb.SignedBeaconBlock),
+		ctx:                   ctx,
+		cancel:                cancel,
+		beaconDB:              cfg.BeaconDB,
+		depositCache:          cfg.DepositCache,
+		chainStartFetcher:     cfg.ChainStartFetcher,
+		attPool:               cfg.AttPool,
+		exitPool:              cfg.ExitPool,
+		slashingPool:          cfg.SlashingPool,
+		p2p:                   cfg.P2p,
+		maxRoutines:           cfg.MaxRoutines,
+		stateNotifier:         cfg.StateNotifier,
+		forkChoiceStore:       cfg.ForkChoiceStore,
+		initSyncState:         make(map[[32]byte]*stateTrie.BeaconState),
+		boundaryRoots:         [][32]byte{},
+		checkpointState:       cache.NewCheckpointStateCache(),
+		opsService:            cfg.OpsService,
+		stateGen:              cfg.StateGen,
+		initSyncBlocks:        make(map[[32]byte]*ethpb.SignedBeaconBlock),
+		recentCanonicalBlocks: make(map[[32]byte]bool),
+		justifiedBalances:     make([]uint64, 0),
 	}, nil
 }
 
 // Start a blockchain service's main event loop.
 func (s *Service) Start() {
-	ctx := context.TODO()
-	beaconState, err := s.beaconDB.HeadState(ctx)
+	beaconState, err := s.beaconDB.HeadState(s.ctx)
 	if err != nil {
 		log.Fatalf("Could not fetch beacon state: %v", err)
 	}
@@ -134,22 +132,15 @@ func (s *Service) Start() {
 	// For running initial sync with state cache, in an event of restart, we use
 	// last finalized check point as start point to sync instead of head
 	// state. This is because we no longer save state every slot during sync.
-	cp, err := s.beaconDB.FinalizedCheckpoint(ctx)
+	cp, err := s.beaconDB.FinalizedCheckpoint(s.ctx)
 	if err != nil {
 		log.Fatalf("Could not fetch finalized cp: %v", err)
 	}
 
 	if beaconState == nil {
-		if !featureconfig.Get().DisableNewStateMgmt {
-			beaconState, err = s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(cp.Root))
-			if err != nil {
-				log.Fatalf("Could not fetch beacon state by root: %v", err)
-			}
-		} else {
-			beaconState, err = s.beaconDB.State(ctx, bytesutil.ToBytes32(cp.Root))
-			if err != nil {
-				log.Fatalf("Could not fetch beacon state: %v", err)
-			}
+		beaconState, err = s.stateGen.StateByRoot(s.ctx, bytesutil.ToBytes32(cp.Root))
+		if err != nil {
+			log.Fatalf("Could not fetch beacon state by root: %v", err)
 		}
 	}
 
@@ -159,36 +150,39 @@ func (s *Service) Start() {
 	// If the chain has already been initialized, simply start the block processing routine.
 	if beaconState != nil {
 		log.Info("Blockchain data already exists in DB, initializing...")
-		s.genesisTime = time.Unix(int64(beaconState.GenesisTime()), 0)
-		s.opsService.SetGenesisTime(beaconState.GenesisTime())
-		if err := s.initializeChainInfo(ctx); err != nil {
+		s.genesisTime = time.Unix(int64(helpers.GenesisTime(beaconState)), 0)
+		s.opsService.SetGenesisTime(helpers.GenesisTime(beaconState))
+		if err := s.initializeChainInfo(s.ctx); err != nil {
 			log.Fatalf("Could not set up chain info: %v", err)
 		}
-		justifiedCheckpoint, err := s.beaconDB.JustifiedCheckpoint(ctx)
+
+		// We start a counter to genesis, if needed.
+		gState, err := s.beaconDB.GenesisState(s.ctx)
+		if err != nil {
+			log.Fatalf("Could not retrieve genesis state: %v", err)
+		}
+		go slotutil.CountdownToGenesis(s.ctx, s.genesisTime, uint64(gState.NumValidators()))
+
+		justifiedCheckpoint, err := s.beaconDB.JustifiedCheckpoint(s.ctx)
 		if err != nil {
 			log.Fatalf("Could not get justified checkpoint: %v", err)
 		}
-		finalizedCheckpoint, err := s.beaconDB.FinalizedCheckpoint(ctx)
+		finalizedCheckpoint, err := s.beaconDB.FinalizedCheckpoint(s.ctx)
 		if err != nil {
 			log.Fatalf("Could not get finalized checkpoint: %v", err)
 		}
 
 		// Resume fork choice.
 		s.justifiedCheckpt = stateTrie.CopyCheckpoint(justifiedCheckpoint)
+		if err := s.cacheJustifiedStateBalances(s.ctx, s.ensureRootNotZeros(bytesutil.ToBytes32(s.justifiedCheckpt.Root))); err != nil {
+			log.Fatalf("Could not cache justified state balances: %v", err)
+		}
 		s.prevJustifiedCheckpt = stateTrie.CopyCheckpoint(justifiedCheckpoint)
 		s.bestJustifiedCheckpt = stateTrie.CopyCheckpoint(justifiedCheckpoint)
 		s.finalizedCheckpt = stateTrie.CopyCheckpoint(finalizedCheckpoint)
 		s.prevFinalizedCheckpt = stateTrie.CopyCheckpoint(finalizedCheckpoint)
 		s.resumeForkChoice(justifiedCheckpoint, finalizedCheckpoint)
 
-		if featureconfig.Get().DisableNewStateMgmt {
-			if finalizedCheckpoint.Epoch > 1 {
-				if err := s.pruneGarbageState(ctx, helpers.StartSlot(finalizedCheckpoint.Epoch)-params.BeaconConfig().SlotsPerEpoch); err != nil {
-					log.WithError(err).Warn("Could not prune old states")
-				}
-			}
-		}
-
 		s.stateNotifier.StateFeed().Send(&feed.Event{
 			Type: statefeed.Initialized,
 			Data: &statefeed.InitializedData{
@@ -217,7 +211,7 @@ func (s *Service) Start() {
 							return
 						}
 						log.WithField("starttime", data.StartTime).Debug("Received chain start event")
-						s.processChainStartTime(ctx, data.StartTime)
+						s.processChainStartTime(s.ctx, data.StartTime)
 						return
 					}
 				case <-s.ctx.Done():
@@ -242,6 +236,11 @@ func (s *Service) processChainStartTime(ctx context.Context, genesisTime time.Ti
 	if err != nil {
 		log.Fatalf("Could not initialize beacon chain: %v", err)
 	}
+	// We start a counter to genesis, if needed.
+	go slotutil.CountdownToGenesis(ctx, genesisTime, uint64(initializedState.NumValidators()))
+
+	// We send out a state initialized event to the rest of the services
+	// running in the beacon node.
 	s.stateNotifier.StateFeed().Send(&feed.Event{
 		Type: statefeed.Initialized,
 		Data: &statefeed.InitializedData{
@@ -286,7 +285,7 @@ func (s *Service) initializeBeaconChain(
 		return nil, err
 	}
 
-	s.opsService.SetGenesisTime(genesisState.GenesisTime())
+	s.opsService.SetGenesisTime(helpers.GenesisTime(genesisState))
 
 	return genesisState, nil
 }
@@ -294,13 +293,17 @@ func (s *Service) initializeBeaconChain(
 // Stop the blockchain service's main event loop and associated goroutines.
 func (s *Service) Stop() error {
 	defer s.cancel()
+
+	if s.stateGen != nil && s.head != nil && s.head.state != nil {
+		return s.stateGen.ForceCheckpoint(s.ctx, s.head.state.FinalizedCheckpoint().Root)
+	}
 	return nil
 }
 
 // Status always returns nil unless there is an error condition that causes
 // this service to be unhealthy.
 func (s *Service) Status() error {
-	if runtime.NumGoroutine() > int(s.maxRoutines) {
+	if runtime.NumGoroutine() > s.maxRoutines {
 		return fmt.Errorf("too many goroutines %d", runtime.NumGoroutine())
 	}
 	return nil
@@ -319,40 +322,41 @@ func (s *Service) saveGenesisData(ctx context.Context, genesisState *stateTrie.B
 		return err
 	}
 	genesisBlk := blocks.NewGenesisBlock(stateRoot[:])
-	genesisBlkRoot, err := ssz.HashTreeRoot(genesisBlk.Block)
+	genesisBlkRoot, err := stateutil.BlockRoot(genesisBlk.Block)
 	if err != nil {
 		return errors.Wrap(err, "could not get genesis block root")
 	}
+	s.genesisRoot = genesisBlkRoot
 
 	if err := s.beaconDB.SaveBlock(ctx, genesisBlk); err != nil {
 		return errors.Wrap(err, "could not save genesis block")
 	}
-	if !featureconfig.Get().DisableNewStateMgmt {
-		if err := s.stateGen.SaveState(ctx, genesisBlkRoot, genesisState); err != nil {
-			return errors.Wrap(err, "could not save genesis state")
-		}
-		if err := s.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{
-			Slot: 0,
-			Root: genesisBlkRoot[:],
-		}); err != nil {
-			return err
-		}
-	} else {
-		if err := s.beaconDB.SaveState(ctx, genesisState, genesisBlkRoot); err != nil {
-			return errors.Wrap(err, "could not save genesis state")
-		}
+	if err := s.beaconDB.SaveState(ctx, genesisState, genesisBlkRoot); err != nil {
+		return errors.Wrap(err, "could not save genesis state")
 	}
+	if err := s.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{
+		Slot: 0,
+		Root: genesisBlkRoot[:],
+	}); err != nil {
+		return err
+	}
+
+	s.stateGen.SaveFinalizedState(0, genesisBlkRoot, genesisState)
+
 	if err := s.beaconDB.SaveHeadBlockRoot(ctx, genesisBlkRoot); err != nil {
 		return errors.Wrap(err, "could not save head block root")
 	}
 	if err := s.beaconDB.SaveGenesisBlockRoot(ctx, genesisBlkRoot); err != nil {
-		return errors.Wrap(err, "could save genesis block root")
+		return errors.Wrap(err, "could not save genesis block root")
 	}
 
-	genesisCheckpoint := &ethpb.Checkpoint{Root: genesisBlkRoot[:]}
+	// Finalized checkpoint at genesis is a zero hash.
+	genesisCheckpoint := genesisState.FinalizedCheckpoint()
 
-	// Add the genesis block to the fork choice store.
 	s.justifiedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
+	if err := s.cacheJustifiedStateBalances(ctx, genesisBlkRoot); err != nil {
+		return err
+	}
 	s.prevJustifiedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
 	s.bestJustifiedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
 	s.finalizedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
@@ -362,6 +366,7 @@ func (s *Service) saveGenesisData(ctx context.Context, genesisState *stateTrie.B
 		genesisBlk.Block.Slot,
 		genesisBlkRoot,
 		params.BeaconConfig().ZeroHash,
+		[32]byte{},
 		genesisCheckpoint.Epoch,
 		genesisCheckpoint.Epoch); err != nil {
 		log.Fatalf("Could not process genesis block for fork choice: %v", err)
@@ -381,7 +386,7 @@ func (s *Service) initializeChainInfo(ctx context.Context) error {
 	if genesisBlock == nil {
 		return errors.New("no genesis block in db")
 	}
-	genesisBlkRoot, err := ssz.HashTreeRoot(genesisBlock.Block)
+	genesisBlkRoot, err := stateutil.BlockRoot(genesisBlock.Block)
 	if err != nil {
 		return errors.Wrap(err, "could not get signing root of genesis block")
 	}
@@ -392,7 +397,7 @@ func (s *Service) initializeChainInfo(ctx context.Context) error {
 		if err != nil {
 			return errors.Wrap(err, "could not retrieve head block")
 		}
-		headRoot, err := ssz.HashTreeRoot(headBlock.Block)
+		headRoot, err := stateutil.BlockRoot(headBlock.Block)
 		if err != nil {
 			return errors.Wrap(err, "could not hash head block")
 		}
@@ -415,20 +420,14 @@ func (s *Service) initializeChainInfo(ctx context.Context) error {
 	}
 	finalizedRoot := bytesutil.ToBytes32(finalized.Root)
 	var finalizedState *stateTrie.BeaconState
-	if !featureconfig.Get().DisableNewStateMgmt {
-		finalizedRoot = s.beaconDB.LastArchivedIndexRoot(ctx)
-		finalizedState, err = s.stateGen.Resume(ctx)
-		if err != nil {
-			return errors.Wrap(err, "could not get finalized state from db")
-		}
-		if finalizedRoot == params.BeaconConfig().ZeroHash {
-			finalizedRoot = bytesutil.ToBytes32(finalized.Root)
-		}
-	} else {
-		finalizedState, err = s.beaconDB.State(ctx, bytesutil.ToBytes32(finalized.Root))
-		if err != nil {
-			return errors.Wrap(err, "could not get finalized state from db")
-		}
+
+	finalizedState, err = s.stateGen.Resume(ctx)
+	if err != nil {
+		return errors.Wrap(err, "could not get finalized state from db")
+	}
+	finalizedRoot = s.beaconDB.LastArchivedRoot(ctx)
+	if finalizedRoot == params.BeaconConfig().ZeroHash {
+		finalizedRoot = bytesutil.ToBytes32(finalized.Root)
 	}
 
 	finalizedBlock, err := s.beaconDB.Block(ctx, finalizedRoot)
@@ -444,29 +443,6 @@ func (s *Service) initializeChainInfo(ctx context.Context) error {
 	return nil
 }
 
-// This is called when a client starts from a non-genesis slot. It deletes the states in DB
-// from slot 1 (avoid genesis state) to `slot`.
-func (s *Service) pruneGarbageState(ctx context.Context, slot uint64) error {
-	if featureconfig.Get().DontPruneStateStartUp {
-		return nil
-	}
-
-	filter := filters.NewFilter().SetStartSlot(1).SetEndSlot(slot)
-	roots, err := s.beaconDB.BlockRoots(ctx, filter)
-	if err != nil {
-		return err
-	}
-	if err := s.beaconDB.DeleteStates(ctx, roots); err != nil {
-		return err
-	}
-
-	if err := s.beaconDB.SaveLastArchivedIndex(ctx, 0); err != nil {
-		return err
-	}
-
-	return nil
-}
-
 // This is called when a client starts from non-genesis slot. This passes last justified and finalized
 // information to fork choice service to initializes fork choice store.
 func (s *Service) resumeForkChoice(justifiedCheckpoint *ethpb.Checkpoint, finalizedCheckpoint *ethpb.Checkpoint) {

beacon-chain/blockchain/service_norace_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 	"github.com/sirupsen/logrus"
 )
 
@@ -15,17 +16,12 @@ func init() {
 }
 
 func TestChainService_SaveHead_DataRace(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 	s := &Service{
 		beaconDB: db,
 	}
 	go func() {
-		if err := s.saveHead(context.Background(), [32]byte{}, ); err != nil {
-			t.Fatal(err)
-		}
+		require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
 	}()
-	if err := s.saveHead(context.Background(), [32]byte{}, ); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
 }

beacon-chain/blockchain/service_test.go

@@ -11,16 +11,13 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	ssz "github.com/prysmaticlabs/go-ssz"
+	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache"
 	b "github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
-	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/attestations"
@@ -28,11 +25,15 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
 	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	protodb "github.com/prysmaticlabs/prysm/proto/beacon/db"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/event"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 	"github.com/sirupsen/logrus"
 	logTest "github.com/sirupsen/logrus/hooks/test"
 )
@@ -42,42 +43,6 @@ func init() {
 	logrus.SetOutput(ioutil.Discard)
 }
 
-type store struct {
-	headRoot []byte
-}
-
-func (s *store) OnBlock(ctx context.Context, b *ethpb.SignedBeaconBlock) (*beaconstate.BeaconState, error) {
-	return nil, nil
-}
-
-func (s *store) OnBlockCacheFilteredTree(ctx context.Context, b *ethpb.SignedBeaconBlock) (*beaconstate.BeaconState, error) {
-	return nil, nil
-}
-
-func (s *store) OnBlockInitialSyncStateTransition(ctx context.Context, b *ethpb.SignedBeaconBlock) (*beaconstate.BeaconState, error) {
-	return nil, nil
-}
-
-func (s *store) OnAttestation(ctx context.Context, a *ethpb.Attestation) ([]uint64, error) {
-	return nil, nil
-}
-
-func (s *store) GenesisStore(ctx context.Context, justifiedCheckpoint *ethpb.Checkpoint, finalizedCheckpoint *ethpb.Checkpoint) error {
-	return nil
-}
-
-func (s *store) FinalizedCheckpt() *ethpb.Checkpoint {
-	return nil
-}
-
-func (s *store) JustifiedCheckpt() *ethpb.Checkpoint {
-	return nil
-}
-
-func (s *store) Head(ctx context.Context) ([]byte, error) {
-	return s.headRoot, nil
-}
-
 type mockBeaconNode struct {
 	stateFeed *event.Feed
 }
@@ -99,10 +64,15 @@ func (mb *mockBroadcaster) Broadcast(_ context.Context, _ proto.Message) error {
 	return nil
 }
 
+func (mb *mockBroadcaster) BroadcastAttestation(_ context.Context, _ uint64, _ *ethpb.Attestation) error {
+	mb.broadcastCalled = true
+	return nil
+}
+
 var _ = p2p.Broadcaster(&mockBroadcaster{})
 
-func setupBeaconChain(t *testing.T, beaconDB db.Database) *Service {
-	endpoint := "ws://127.0.0.1"
+func setupBeaconChain(t *testing.T, beaconDB db.Database, sc *cache.StateSummaryCache) *Service {
+	endpoint := "http://127.0.0.1"
 	ctx := context.Background()
 	var web3Service *powchain.Service
 	var err error
@@ -122,193 +92,104 @@ func setupBeaconChain(t *testing.T, beaconDB db.Database) *Service {
 		},
 		DepositContainers: []*protodb.DepositContainer{},
 	})
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	web3Service, err = powchain.NewService(ctx, &powchain.Web3ServiceConfig{
 		BeaconDB:        beaconDB,
-		ETH1Endpoint:    endpoint,
+		HTTPEndPoint:    endpoint,
 		DepositContract: common.Address{},
 	})
-	if err != nil {
-		t.Fatalf("unable to set up web3 service: %v", err)
-	}
+	require.NoError(t, err, "Unable to set up web3 service")
 
 	opsService, err := attestations.NewService(ctx, &attestations.Config{Pool: attestations.NewPool()})
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
+
+	depositCache, err := depositcache.NewDepositCache()
+	require.NoError(t, err)
+
 	cfg := &Config{
 		BeaconBlockBuf:    0,
 		BeaconDB:          beaconDB,
-		DepositCache:      depositcache.NewDepositCache(),
+		DepositCache:      depositCache,
 		ChainStartFetcher: web3Service,
 		P2p:               &mockBroadcaster{},
 		StateNotifier:     &mockBeaconNode{},
 		AttPool:           attestations.NewPool(),
-		StateGen:          stategen.New(beaconDB, cache.NewStateSummaryCache()),
+		StateGen:          stategen.New(beaconDB, sc),
 		ForkChoiceStore:   protoarray.New(0, 0, params.BeaconConfig().ZeroHash),
 		OpsService:        opsService,
 	}
 
-	chainService, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatalf("unable to setup chain service: %v", err)
+	// Safe a state in stategen to purposes of testing a service stop / shutdown.
+	if err := cfg.StateGen.SaveState(ctx, bytesutil.ToBytes32(bState.FinalizedCheckpoint().Root), bState); err != nil {
+		t.Fatal(err)
 	}
+
+	chainService, err := NewService(ctx, cfg)
+	require.NoError(t, err, "Unable to setup chain service")
 	chainService.genesisTime = time.Unix(1, 0) // non-zero time
 
 	return chainService
 }
 
-func TestChainStartStop_Uninitialized(t *testing.T) {
-	hook := logTest.NewGlobal()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	chainService := setupBeaconChain(t, db)
-
-	// Listen for state events.
-	stateSubChannel := make(chan *feed.Event, 1)
-	stateSub := chainService.stateNotifier.StateFeed().Subscribe(stateSubChannel)
-
-	// Test the chain start state notifier.
-	genesisTime := time.Unix(1, 0)
-	chainService.Start()
-	event := &feed.Event{
-		Type: statefeed.ChainStarted,
-		Data: &statefeed.ChainStartedData{
-			StartTime: genesisTime,
-		},
-	}
-	// Send in a loop to ensure it is delivered (busy wait for the service to subscribe to the state feed).
-	for sent := 1; sent == 1; {
-		sent = chainService.stateNotifier.StateFeed().Send(event)
-		if sent == 1 {
-			// Flush our local subscriber.
-			<-stateSubChannel
-		}
-	}
-
-	// Now wait for notification the state is ready.
-	for stateInitialized := false; stateInitialized == false; {
-		recv := <-stateSubChannel
-		if recv.Type == statefeed.Initialized {
-			stateInitialized = true
-		}
-	}
-	stateSub.Unsubscribe()
-
-	beaconState, err := db.HeadState(context.Background())
-	if err != nil {
-		t.Fatal(err)
-	}
-	if beaconState == nil || beaconState.Slot() != 0 {
-		t.Error("Expected canonical state feed to send a state with genesis block")
-	}
-	if err := chainService.Stop(); err != nil {
-		t.Fatalf("Unable to stop chain service: %v", err)
-	}
-	// The context should have been canceled.
-	if chainService.ctx.Err() != context.Canceled {
-		t.Error("Context was not canceled")
-	}
-	testutil.AssertLogsContain(t, hook, "Waiting")
-	testutil.AssertLogsContain(t, hook, "Initialized beacon chain genesis state")
-}
-
 func TestChainStartStop_Initialized(t *testing.T) {
 	hook := logTest.NewGlobal()
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
-	chainService := setupBeaconChain(t, db)
+	chainService := setupBeaconChain(t, db, sc)
 
-	genesisBlk := b.NewGenesisBlock([]byte{})
-	blkRoot, err := ssz.HashTreeRoot(genesisBlk.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveBlock(ctx, genesisBlk); err != nil {
-		t.Fatal(err)
-	}
+	genesisBlk := testutil.NewBeaconBlock()
+	blkRoot, err := stateutil.BlockRoot(genesisBlk.Block)
+	require.NoError(t, err)
+	require.NoError(t, db.SaveBlock(ctx, genesisBlk))
 	s := testutil.NewBeaconState()
-	if err := s.SetSlot(1); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveState(ctx, s, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveHeadBlockRoot(ctx, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveGenesisBlockRoot(ctx, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveJustifiedCheckpoint(ctx, &ethpb.Checkpoint{Root: blkRoot[:]}); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, s.SetSlot(1))
+	require.NoError(t, db.SaveState(ctx, s, blkRoot))
+	require.NoError(t, db.SaveHeadBlockRoot(ctx, blkRoot))
+	require.NoError(t, db.SaveGenesisBlockRoot(ctx, blkRoot))
+	require.NoError(t, db.SaveJustifiedCheckpoint(ctx, &ethpb.Checkpoint{Root: blkRoot[:]}))
 
 	// Test the start function.
 	chainService.Start()
 
-	if err := chainService.Stop(); err != nil {
-		t.Fatalf("unable to stop chain service: %v", err)
-	}
+	require.NoError(t, chainService.Stop(), "Unable to stop chain service")
 
 	// The context should have been canceled.
-	if chainService.ctx.Err() != context.Canceled {
-		t.Error("context was not canceled")
-	}
-	testutil.AssertLogsContain(t, hook, "data already exists")
+	assert.Equal(t, context.Canceled, chainService.ctx.Err(), "Context was not canceled")
+	require.LogsContain(t, hook, "data already exists")
 }
 
 func TestChainService_InitializeBeaconChain(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	helpers.ClearCache()
+	db, sc := testDB.SetupDB(t)
 	ctx := context.Background()
 
-	bc := setupBeaconChain(t, db)
+	bc := setupBeaconChain(t, db, sc)
 	var err error
 
 	// Set up 10 deposits pre chain start for validators to register
 	count := uint64(10)
 	deposits, _, err := testutil.DeterministicDepositsAndKeys(count)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	trie, _, err := testutil.DepositTrieFromDeposits(deposits)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	hashTreeRoot := trie.HashTreeRoot()
 	genState, err := state.EmptyGenesisState()
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	err = genState.SetEth1Data(&ethpb.Eth1Data{
 		DepositRoot:  hashTreeRoot[:],
 		DepositCount: uint64(len(deposits)),
 	})
-	for _, deposit := range deposits {
-		genState, err = b.ProcessPreGenesisDeposit(ctx, genState, deposit)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
+	genState, err = b.ProcessPreGenesisDeposits(ctx, genState, deposits)
+	require.NoError(t, err)
 
-	if _, err := bc.initializeBeaconChain(ctx, time.Unix(0, 0), genState, &ethpb.Eth1Data{
-		DepositRoot: hashTreeRoot[:],
-	}); err != nil {
-		t.Fatal(err)
-	}
+	_, err = bc.initializeBeaconChain(ctx, time.Unix(0, 0), genState, &ethpb.Eth1Data{DepositRoot: hashTreeRoot[:]})
+	require.NoError(t, err)
 
-	if _, err := bc.HeadState(ctx); err != nil {
-		t.Error(err)
-	}
+	_, err = bc.HeadState(ctx)
+	assert.NoError(t, err)
 	headBlk, err := bc.HeadBlock(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	if headBlk == nil {
 		t.Error("Head state can't be nil after initialize beacon chain")
 	}
@@ -317,249 +198,152 @@ func TestChainService_InitializeBeaconChain(t *testing.T) {
 	}
 }
 
+func TestChainService_CorrectGenesisRoots(t *testing.T) {
+	ctx := context.Background()
+	db, sc := testDB.SetupDB(t)
+
+	chainService := setupBeaconChain(t, db, sc)
+
+	genesisBlk := testutil.NewBeaconBlock()
+	blkRoot, err := stateutil.BlockRoot(genesisBlk.Block)
+	require.NoError(t, err)
+	require.NoError(t, db.SaveBlock(ctx, genesisBlk))
+	s := testutil.NewBeaconState()
+	require.NoError(t, s.SetSlot(0))
+	require.NoError(t, db.SaveState(ctx, s, blkRoot))
+	require.NoError(t, db.SaveHeadBlockRoot(ctx, blkRoot))
+	require.NoError(t, db.SaveGenesisBlockRoot(ctx, blkRoot))
+	// Test the start function.
+	chainService.Start()
+
+	require.DeepEqual(t, params.BeaconConfig().ZeroHash[:], chainService.finalizedCheckpt.Root, "Finalize Checkpoint root is incorrect")
+	require.DeepEqual(t, params.BeaconConfig().ZeroHash[:], chainService.justifiedCheckpt.Root, "Justified Checkpoint root is incorrect")
+
+	require.NoError(t, chainService.Stop(), "Unable to stop chain service")
+
+}
+
 func TestChainService_InitializeChainInfo(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 	ctx := context.Background()
 
-	genesis := b.NewGenesisBlock([]byte{})
-	genesisRoot, err := ssz.HashTreeRoot(genesis.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveGenesisBlockRoot(ctx, genesisRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveBlock(ctx, genesis); err != nil {
-		t.Fatal(err)
-	}
+	genesis := testutil.NewBeaconBlock()
+	genesisRoot, err := stateutil.BlockRoot(genesis.Block)
+	require.NoError(t, err)
+	require.NoError(t, db.SaveGenesisBlockRoot(ctx, genesisRoot))
+	require.NoError(t, db.SaveBlock(ctx, genesis))
 
 	finalizedSlot := params.BeaconConfig().SlotsPerEpoch*2 + 1
-	headBlock := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: finalizedSlot, ParentRoot: genesisRoot[:]}}
+	headBlock := testutil.NewBeaconBlock()
+	headBlock.Block.Slot = finalizedSlot
+	headBlock.Block.ParentRoot = bytesutil.PadTo(genesisRoot[:], 32)
 	headState := testutil.NewBeaconState()
-	if err := headState.SetSlot(finalizedSlot); err != nil {
-		t.Fatal(err)
-	}
-	if err := headState.SetGenesisValidatorRoot(params.BeaconConfig().ZeroHash[:]); err != nil {
-		t.Fatal(err)
-	}
-	headRoot, err := ssz.HashTreeRoot(headBlock.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveState(ctx, headState, headRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveState(ctx, headState, genesisRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveBlock(ctx, headBlock); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, headState.SetSlot(finalizedSlot))
+	require.NoError(t, headState.SetGenesisValidatorRoot(params.BeaconConfig().ZeroHash[:]))
+	headRoot, err := stateutil.BlockRoot(headBlock.Block)
+	require.NoError(t, err)
+	require.NoError(t, db.SaveState(ctx, headState, headRoot))
+	require.NoError(t, db.SaveState(ctx, headState, genesisRoot))
+	require.NoError(t, db.SaveBlock(ctx, headBlock))
 	if err := db.SaveFinalizedCheckpoint(ctx, &ethpb.Checkpoint{
 		Epoch: helpers.SlotToEpoch(finalizedSlot),
 		Root:  headRoot[:],
 	}); err != nil {
 		t.Fatal(err)
 	}
-	if err := db.SaveBlock(ctx, headBlock); err != nil {
-		t.Fatal(err)
-	}
-	c := &Service{beaconDB: db, stateGen: stategen.New(db, cache.NewStateSummaryCache())}
-	if err := c.initializeChainInfo(ctx); err != nil {
-		t.Fatal(err)
-	}
+	c := &Service{beaconDB: db, stateGen: stategen.New(db, sc)}
+	require.NoError(t, c.initializeChainInfo(ctx))
 	headBlk, err := c.HeadBlock(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(headBlk, headBlock) {
-		t.Error("head block incorrect")
-	}
+	require.NoError(t, err)
+	assert.DeepEqual(t, headBlock, headBlk, "Head block incorrect")
 	s, err := c.HeadState(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(s.InnerStateUnsafe(), headState.InnerStateUnsafe()) {
-		t.Error("head state incorrect")
-	}
-	if headBlock.Block.Slot != c.HeadSlot() {
-		t.Error("head slot incorrect")
-	}
+	require.NoError(t, err)
+	assert.DeepEqual(t, headState.InnerStateUnsafe(), s.InnerStateUnsafe(), "Head state incorrect")
+	assert.Equal(t, c.HeadSlot(), headBlock.Block.Slot, "Head slot incorrect")
 	r, err := c.HeadRoot(context.Background())
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	if !bytes.Equal(headRoot[:], r) {
 		t.Error("head slot incorrect")
 	}
-	if c.genesisRoot != genesisRoot {
-		t.Error("genesis block root incorrect")
-	}
+	assert.Equal(t, genesisRoot, c.genesisRoot, "Genesis block root incorrect")
 }
 
 func TestChainService_SaveHeadNoDB(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 	ctx := context.Background()
 	s := &Service{
 		beaconDB: db,
-		stateGen: stategen.New(db, cache.NewStateSummaryCache()),
+		stateGen: stategen.New(db, sc),
 	}
 	b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1}}
 	r, err := ssz.HashTreeRoot(b)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	newState := testutil.NewBeaconState()
-	if err := s.stateGen.SaveState(ctx, r, newState); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := s.saveHeadNoDB(ctx, b, r); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, s.stateGen.SaveState(ctx, r, newState))
+	require.NoError(t, s.saveHeadNoDB(ctx, b, r, newState))
 
 	newB, err := s.beaconDB.HeadBlock(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	if reflect.DeepEqual(newB, b) {
 		t.Error("head block should not be equal")
 	}
 }
 
-func TestChainService_PruneOldStates(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	ctx := context.Background()
-	s := &Service{
-		beaconDB: db,
-	}
-
-	for i := 0; i < 100; i++ {
-		block := &ethpb.BeaconBlock{Slot: uint64(i)}
-		if err := s.beaconDB.SaveBlock(ctx, &ethpb.SignedBeaconBlock{Block: block}); err != nil {
-			t.Fatal(err)
-		}
-		r, err := ssz.HashTreeRoot(block)
-		if err != nil {
-			t.Fatal(err)
-		}
-		newState := testutil.NewBeaconState()
-		if err := newState.SetSlot(uint64(i)); err != nil {
-			t.Fatal(err)
-		}
-		if err := s.beaconDB.SaveState(ctx, newState, r); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// Delete half of the states.
-	if err := s.pruneGarbageState(ctx, 50); err != nil {
-		t.Fatal(err)
-	}
-
-	filter := filters.NewFilter().SetStartSlot(1).SetEndSlot(100)
-	roots, err := s.beaconDB.BlockRoots(ctx, filter)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := 1; i < 50; i++ {
-		s, err := s.beaconDB.State(ctx, roots[i])
-		if err != nil {
-			t.Fatal(err)
-		}
-		if s != nil {
-			t.Errorf("wanted nil for slot %d", i)
-		}
-	}
-}
-
 func TestHasBlock_ForkChoiceAndDB(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 	s := &Service{
 		forkChoiceStore:  protoarray.New(0, 0, [32]byte{}),
 		finalizedCheckpt: &ethpb.Checkpoint{},
 		beaconDB:         db,
 	}
 	block := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Body: &ethpb.BeaconBlockBody{}}}
-	r, err := ssz.HashTreeRoot(block.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
+	r, err := stateutil.BlockRoot(block.Block)
+	require.NoError(t, err)
 	bs := &pb.BeaconState{FinalizedCheckpoint: &ethpb.Checkpoint{}, CurrentJustifiedCheckpoint: &ethpb.Checkpoint{}}
 	state, err := beaconstate.InitializeFromProto(bs)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := s.insertBlockToForkChoiceStore(ctx, block.Block, r, state); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
+	require.NoError(t, s.insertBlockAndAttestationsToForkChoiceStore(ctx, block.Block, r, state))
 
-	if s.hasBlock(ctx, [32]byte{}) {
-		t.Error("Should not have block")
-	}
-
-	if !s.hasBlock(ctx, r) {
-		t.Error("Should have block")
-	}
+	assert.Equal(t, false, s.hasBlock(ctx, [32]byte{}), "Should not have block")
+	assert.Equal(t, true, s.hasBlock(ctx, r), "Should have block")
 }
 
 func BenchmarkHasBlockDB(b *testing.B) {
-	db := testDB.SetupDB(b)
-	defer testDB.TeardownDB(b, db)
+	db, _ := testDB.SetupDB(b)
 	ctx := context.Background()
 	s := &Service{
 		beaconDB: db,
 	}
 	block := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{}}
-	if err := s.beaconDB.SaveBlock(ctx, block); err != nil {
-		b.Fatal(err)
-	}
-	r, err := ssz.HashTreeRoot(block.Block)
-	if err != nil {
-		b.Fatal(err)
-	}
+	require.NoError(b, s.beaconDB.SaveBlock(ctx, block))
+	r, err := stateutil.BlockRoot(block.Block)
+	require.NoError(b, err)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		if !s.beaconDB.HasBlock(ctx, r) {
-			b.Fatal("Block is not in DB")
-		}
+		require.Equal(b, true, s.beaconDB.HasBlock(ctx, r), "Block is not in DB")
 	}
 }
 
 func BenchmarkHasBlockForkChoiceStore(b *testing.B) {
 	ctx := context.Background()
-	db := testDB.SetupDB(b)
-	defer testDB.TeardownDB(b, db)
+	db, _ := testDB.SetupDB(b)
 	s := &Service{
 		forkChoiceStore:  protoarray.New(0, 0, [32]byte{}),
 		finalizedCheckpt: &ethpb.Checkpoint{},
 		beaconDB:         db,
 	}
 	block := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Body: &ethpb.BeaconBlockBody{}}}
-	r, err := ssz.HashTreeRoot(block.Block)
-	if err != nil {
-		b.Fatal(err)
-	}
+	r, err := stateutil.BlockRoot(block.Block)
+	require.NoError(b, err)
 	bs := &pb.BeaconState{FinalizedCheckpoint: &ethpb.Checkpoint{}, CurrentJustifiedCheckpoint: &ethpb.Checkpoint{}}
 	state, err := beaconstate.InitializeFromProto(bs)
-	if err != nil {
-		b.Fatal(err)
-	}
-	if err := s.insertBlockToForkChoiceStore(ctx, block.Block, r, state); err != nil {
-		b.Fatal(err)
-	}
+	require.NoError(b, err)
+	require.NoError(b, s.insertBlockAndAttestationsToForkChoiceStore(ctx, block.Block, r, state))
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		if !s.forkChoiceStore.HasNode(r) {
-			b.Fatal("Block is not in fork choice store")
-		}
+		require.Equal(b, true, s.forkChoiceStore.HasNode(r), "Block is not in fork choice store")
 	}
 }

beacon-chain/blockchain/testing/BUILD.bazel

@@ -1,25 +1,30 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@prysm//tools/go:def.bzl", "go_library")
 
 go_library(
     name = "go_default_library",
     testonly = True,
     srcs = ["mock.go"],
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing",
-    visibility = ["//beacon-chain:__subpackages__"],
+    visibility = [
+        "//beacon-chain:__subpackages__",
+        "//fuzz:__pkg__",
+    ],
     deps = [
         "//beacon-chain/core/epoch/precompute:go_default_library",
+        "//beacon-chain/core/feed:go_default_library",
         "//beacon-chain/core/feed/block:go_default_library",
         "//beacon-chain/core/feed/operation:go_default_library",
         "//beacon-chain/core/feed/state:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/db:go_default_library",
+        "//beacon-chain/forkchoice/protoarray:go_default_library",
         "//beacon-chain/state:go_default_library",
+        "//beacon-chain/state/stateutil:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/event:go_default_library",
         "//shared/params:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
-        "@com_github_prysmaticlabs_go_ssz//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
     ],
 )

beacon-chain/blockchain/testing/mock.go

@@ -1,20 +1,25 @@
+// Package testing includes useful mocks for writing unit
+// tests which depend on logic from the blockchain package.
 package testing
 
 import (
 	"bytes"
 	"context"
+	"sync"
 	"time"
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
 	blockfeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/block"
 	opfeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/operation"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/event"
 	"github.com/prysmaticlabs/prysm/shared/params"
@@ -32,12 +37,16 @@ type ChainService struct {
 	BlocksReceived              []*ethpb.SignedBeaconBlock
 	Balance                     *precompute.Balance
 	Genesis                     time.Time
+	ValidatorsRoot              [32]byte
 	Fork                        *pb.Fork
+	ETH1Data                    *ethpb.Eth1Data
 	DB                          db.Database
 	stateNotifier               statefeed.Notifier
 	blockNotifier               blockfeed.Notifier
 	opNotifier                  opfeed.Notifier
 	ValidAttestation            bool
+	ForkChoiceStore             *protoarray.Store
+	VerifyBlkDescendantErr      error
 }
 
 // StateNotifier mocks the same method in the chain service.
@@ -72,12 +81,40 @@ func (msn *MockBlockNotifier) BlockFeed() *event.Feed {
 // MockStateNotifier mocks the state notifier.
 type MockStateNotifier struct {
 	feed *event.Feed
+
+	recv     []*feed.Event
+	recvLock sync.Mutex
+	recvCh   chan *feed.Event
+
+	RecordEvents bool
+}
+
+// ReceivedEvents returns the events received by the state feed in this mock.
+func (msn *MockStateNotifier) ReceivedEvents() []*feed.Event {
+	msn.recvLock.Lock()
+	defer msn.recvLock.Unlock()
+	return msn.recv
 }
 
 // StateFeed returns a state feed.
 func (msn *MockStateNotifier) StateFeed() *event.Feed {
-	if msn.feed == nil {
+	if msn.feed == nil && msn.recvCh == nil {
 		msn.feed = new(event.Feed)
+		if msn.RecordEvents {
+			msn.recvCh = make(chan *feed.Event)
+			sub := msn.feed.Subscribe(msn.recvCh)
+
+			go func() {
+				select {
+				case evt := <-msn.recvCh:
+					msn.recvLock.Lock()
+					msn.recv = append(msn.recv, evt)
+					msn.recvLock.Unlock()
+				case <-sub.Err():
+					sub.Unsubscribe()
+				}
+			}()
+		}
 	}
 	return msn.feed
 }
@@ -103,23 +140,8 @@ func (mon *MockOperationNotifier) OperationFeed() *event.Feed {
 	return mon.feed
 }
 
-// ReceiveBlock mocks ReceiveBlock method in chain service.
-func (ms *ChainService) ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
-	return nil
-}
-
-// ReceiveBlockNoVerify mocks ReceiveBlockNoVerify method in chain service.
-func (ms *ChainService) ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
-	return nil
-}
-
-// ReceiveBlockNoPubsub mocks ReceiveBlockNoPubsub method in chain service.
-func (ms *ChainService) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
-	return nil
-}
-
-// ReceiveBlockNoPubsubForkchoice mocks ReceiveBlockNoPubsubForkchoice method in chain service.
-func (ms *ChainService) ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
+// ReceiveBlockInitialSync mocks ReceiveBlockInitialSync method in chain service.
+func (ms *ChainService) ReceiveBlockInitialSync(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
 	if ms.State == nil {
 		ms.State = &stateTrie.BeaconState{}
 	}
@@ -130,7 +152,63 @@ func (ms *ChainService) ReceiveBlockNoPubsubForkchoice(ctx context.Context, bloc
 		return err
 	}
 	ms.BlocksReceived = append(ms.BlocksReceived, block)
-	signingRoot, err := ssz.HashTreeRoot(block.Block)
+	signingRoot, err := stateutil.BlockRoot(block.Block)
+	if err != nil {
+		return err
+	}
+	if ms.DB != nil {
+		if err := ms.DB.SaveBlock(ctx, block); err != nil {
+			return err
+		}
+		logrus.Infof("Saved block with root: %#x at slot %d", signingRoot, block.Block.Slot)
+	}
+	ms.Root = signingRoot[:]
+	ms.Block = block
+	return nil
+}
+
+// ReceiveBlockBatch processes blocks in batches from initial-sync.
+func (ms *ChainService) ReceiveBlockBatch(ctx context.Context, blks []*ethpb.SignedBeaconBlock, roots [][32]byte) error {
+	if ms.State == nil {
+		ms.State = &stateTrie.BeaconState{}
+	}
+	for _, block := range blks {
+		if !bytes.Equal(ms.Root, block.Block.ParentRoot) {
+			return errors.Errorf("wanted %#x but got %#x", ms.Root, block.Block.ParentRoot)
+		}
+		if err := ms.State.SetSlot(block.Block.Slot); err != nil {
+			return err
+		}
+		ms.BlocksReceived = append(ms.BlocksReceived, block)
+		signingRoot, err := stateutil.BlockRoot(block.Block)
+		if err != nil {
+			return err
+		}
+		if ms.DB != nil {
+			if err := ms.DB.SaveBlock(ctx, block); err != nil {
+				return err
+			}
+			logrus.Infof("Saved block with root: %#x at slot %d", signingRoot, block.Block.Slot)
+		}
+		ms.Root = signingRoot[:]
+		ms.Block = block
+	}
+	return nil
+}
+
+// ReceiveBlock mocks ReceiveBlock method in chain service.
+func (ms *ChainService) ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
+	if ms.State == nil {
+		ms.State = &stateTrie.BeaconState{}
+	}
+	if !bytes.Equal(ms.Root, block.Block.ParentRoot) {
+		return errors.Errorf("wanted %#x but got %#x", ms.Root, block.Block.ParentRoot)
+	}
+	if err := ms.State.SetSlot(block.Block.Slot); err != nil {
+		return err
+	}
+	ms.BlocksReceived = append(ms.BlocksReceived, block)
+	signingRoot, err := stateutil.BlockRoot(block.Block)
 	if err != nil {
 		return err
 	}
@@ -199,8 +277,13 @@ func (ms *ChainService) ReceiveAttestationNoPubsub(context.Context, *ethpb.Attes
 	return nil
 }
 
+// AttestationPreState mocks AttestationPreState method in chain service.
+func (ms *ChainService) AttestationPreState(ctx context.Context, att *ethpb.Attestation) (*stateTrie.BeaconState, error) {
+	return ms.State, nil
+}
+
 // HeadValidatorsIndices mocks the same method in the chain service.
-func (ms *ChainService) HeadValidatorsIndices(epoch uint64) ([]uint64, error) {
+func (ms *ChainService) HeadValidatorsIndices(ctx context.Context, epoch uint64) ([]uint64, error) {
 	if ms.State == nil {
 		return []uint64{}, nil
 	}
@@ -208,15 +291,30 @@ func (ms *ChainService) HeadValidatorsIndices(epoch uint64) ([]uint64, error) {
 }
 
 // HeadSeed mocks the same method in the chain service.
-func (ms *ChainService) HeadSeed(epoch uint64) ([32]byte, error) {
+func (ms *ChainService) HeadSeed(ctx context.Context, epoch uint64) ([32]byte, error) {
 	return helpers.Seed(ms.State, epoch, params.BeaconConfig().DomainBeaconAttester)
 }
 
+// HeadETH1Data provides the current ETH1Data of the head state.
+func (ms *ChainService) HeadETH1Data() *ethpb.Eth1Data {
+	return ms.ETH1Data
+}
+
+// ProtoArrayStore mocks the same method in the chain service.
+func (ms *ChainService) ProtoArrayStore() *protoarray.Store {
+	return ms.ForkChoiceStore
+}
+
 // GenesisTime mocks the same method in the chain service.
 func (ms *ChainService) GenesisTime() time.Time {
 	return ms.Genesis
 }
 
+// GenesisValidatorRoot mocks the same method in the chain service.
+func (ms *ChainService) GenesisValidatorRoot() [32]byte {
+	return ms.ValidatorsRoot
+}
+
 // CurrentSlot mocks the same method in the chain service.
 func (ms *ChainService) CurrentSlot() uint64 {
 	return uint64(time.Now().Unix()-ms.Genesis.Unix()) / params.BeaconConfig().SecondsPerSlot
@@ -232,6 +330,12 @@ func (ms *ChainService) IsValidAttestation(ctx context.Context, att *ethpb.Attes
 	return ms.ValidAttestation
 }
 
+// IsCanonical returns and determines whether a block with the provided root is part of
+// the canonical chain.
+func (ms *ChainService) IsCanonical(ctx context.Context, blockRoot [32]byte) (bool, error) {
+	return true, nil
+}
+
 // ClearCachedStates does nothing.
 func (ms *ChainService) ClearCachedStates() {}
 
@@ -239,3 +343,13 @@ func (ms *ChainService) ClearCachedStates() {}
 func (ms *ChainService) HasInitSyncBlock(root [32]byte) bool {
 	return false
 }
+
+// HeadGenesisValidatorRoot mocks HeadGenesisValidatorRoot method in chain service.
+func (ms *ChainService) HeadGenesisValidatorRoot() [32]byte {
+	return [32]byte{}
+}
+
+// VerifyBlkDescendant mocks VerifyBlkDescendant and always returns nil.
+func (ms *ChainService) VerifyBlkDescendant(ctx context.Context, root [32]byte) error {
+	return ms.VerifyBlkDescendantErr
+}

beacon-chain/cache/BUILD.bazel

@@ -1,4 +1,5 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -6,16 +7,17 @@ go_library(
         "attestation_data.go",
         "checkpoint_state.go",
         "committee.go",
-        "committee_ids.go",
         "common.go",
-        "eth1_data.go",
+        "doc.go",
         "hot_state_cache.go",
         "skip_slot_cache.go",
         "state_summary.go",
+        "subnet_ids.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/cache",
     visibility = [
         "//beacon-chain:__subpackages__",
+        "//fuzz:__pkg__",
         "//tools:__subpackages__",
     ],
     deps = [
@@ -26,6 +28,7 @@ go_library(
         "//shared/params:go_default_library",
         "//shared/sliceutil:go_default_library",
         "@com_github_hashicorp_golang_lru//:go_default_library",
+        "@com_github_patrickmn_go_cache//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
@@ -41,12 +44,11 @@ go_test(
         "attestation_data_test.go",
         "checkpoint_state_test.go",
         "committee_fuzz_test.go",
-        "committee_ids_test.go",
         "committee_test.go",
-        "eth1_data_test.go",
         "feature_flag_test.go",
         "hot_state_cache_test.go",
         "skip_slot_cache_test.go",
+        "subnet_ids_test.go",
     ],
     embed = [":go_default_library"],
     deps = [
@@ -54,8 +56,9 @@ go_test(
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/bytesutil:go_default_library",
         "//shared/featureconfig:go_default_library",
-        "//shared/hashutil:go_default_library",
         "//shared/params:go_default_library",
+        "//shared/testutil/assert:go_default_library",
+        "//shared/testutil/require:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
         "@com_github_google_gofuzz//:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",

beacon-chain/cache/attestation_data.go

@@ -11,6 +11,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"k8s.io/client-go/tools/cache"
 )
 
@@ -97,6 +99,9 @@ func (c *AttestationCache) Get(ctx context.Context, req *ethpb.AttestationDataRe
 
 	if exists && item != nil && item.(*attestationReqResWrapper).res != nil {
 		attestationCacheHit.Inc()
+		if featureconfig.Get().ReduceAttesterStateCopy {
+			return state.CopyAttestationData(item.(*attestationReqResWrapper).res), nil
+		}
 		return item.(*attestationReqResWrapper).res, nil
 	}
 	attestationCacheMiss.Inc()
@@ -162,6 +167,9 @@ func wrapperToKey(i interface{}) (string, error) {
 }
 
 func reqToKey(req *ethpb.AttestationDataRequest) (string, error) {
+	if featureconfig.Get().ReduceAttesterStateCopy {
+		return fmt.Sprintf("%d", req.Slot), nil
+	}
 	return fmt.Sprintf("%d-%d", req.CommitteeIndex, req.Slot), nil
 }
 

beacon-chain/cache/attestation_data_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
 )
 
 func TestAttestationCache_RoundTrip(t *testing.T) {
@@ -19,34 +20,20 @@ func TestAttestationCache_RoundTrip(t *testing.T) {
 	}
 
 	response, err := c.Get(ctx, req)
-	if err != nil {
-		t.Error(err)
-	}
+	assert.NoError(t, err)
+	assert.Equal(t, (*ethpb.AttestationData)(nil), response)
 
-	if response != nil {
-		t.Errorf("Empty cache returned an object: %v", response)
-	}
-
-	if err := c.MarkInProgress(req); err != nil {
-		t.Error(err)
-	}
+	assert.NoError(t, c.MarkInProgress(req))
 
 	res := &ethpb.AttestationData{
 		Target: &ethpb.Checkpoint{Epoch: 5},
 	}
 
-	if err = c.Put(ctx, req, res); err != nil {
-		t.Error(err)
-	}
-
-	if err := c.MarkNotInProgress(req); err != nil {
-		t.Error(err)
-	}
+	assert.NoError(t, c.Put(ctx, req, res))
+	assert.NoError(t, c.MarkNotInProgress(req))
 
 	response, err = c.Get(ctx, req)
-	if err != nil {
-		t.Error(err)
-	}
+	assert.NoError(t, err)
 
 	if !proto.Equal(response, res) {
 		t.Error("Expected equal protos to return from cache")

beacon-chain/cache/checkpoint_state.go

@@ -4,12 +4,12 @@ import (
 	"errors"
 	"sync"
 
+	lru "github.com/hashicorp/golang-lru"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/hashutil"
-	"k8s.io/client-go/tools/cache"
 )
 
 var (
@@ -24,7 +24,7 @@ var (
 
 	// Metrics.
 	checkpointStateMiss = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "check_point_statecache_miss",
+		Name: "check_point_state_cache_miss",
 		Help: "The number of check point state requests that aren't present in the cache.",
 	})
 	checkpointStateHit = promauto.NewCounter(prometheus.CounterOpts{
@@ -33,36 +33,20 @@ var (
 	})
 )
 
-// CheckpointState defines the active validator indices per epoch.
-type CheckpointState struct {
-	Checkpoint *ethpb.Checkpoint
-	State      *stateTrie.BeaconState
-}
-
 // CheckpointStateCache is a struct with 1 queue for looking up state by checkpoint.
 type CheckpointStateCache struct {
-	cache *cache.FIFO
+	cache *lru.Cache
 	lock  sync.RWMutex
 }
 
-// checkpointState takes the checkpoint as the key of the resulting state.
-func checkpointState(obj interface{}) (string, error) {
-	info, ok := obj.(*CheckpointState)
-	if !ok {
-		return "", ErrNotCheckpointState
-	}
-
-	h, err := hashutil.HashProto(info.Checkpoint)
-	if err != nil {
-		return "", err
-	}
-	return string(h[:]), nil
-}
-
 // NewCheckpointStateCache creates a new checkpoint state cache for storing/accessing processed state.
 func NewCheckpointStateCache() *CheckpointStateCache {
+	cache, err := lru.New(maxCheckpointStateSize)
+	if err != nil {
+		panic(err)
+	}
 	return &CheckpointStateCache{
-		cache: cache.NewFIFO(checkpointState),
+		cache: cache,
 	}
 }
 
@@ -76,43 +60,27 @@ func (c *CheckpointStateCache) StateByCheckpoint(cp *ethpb.Checkpoint) (*stateTr
 		return nil, err
 	}
 
-	obj, exists, err := c.cache.GetByKey(string(h[:]))
-	if err != nil {
-		return nil, err
-	}
+	item, exists := c.cache.Get(h)
 
-	if exists {
+	if exists && item != nil {
 		checkpointStateHit.Inc()
-	} else {
-		checkpointStateMiss.Inc()
-		return nil, nil
+		// Copy here is unnecessary since the return will only be used to verify attestation signature.
+		return item.(*stateTrie.BeaconState), nil
 	}
 
-	info, ok := obj.(*CheckpointState)
-	if !ok {
-		return nil, ErrNotCheckpointState
-	}
-
-	return info.State.Copy(), nil
+	checkpointStateMiss.Inc()
+	return nil, nil
 }
 
 // AddCheckpointState adds CheckpointState object to the cache. This method also trims the least
 // recently added CheckpointState object if the cache size has ready the max cache size limit.
-func (c *CheckpointStateCache) AddCheckpointState(cp *CheckpointState) error {
+func (c *CheckpointStateCache) AddCheckpointState(cp *ethpb.Checkpoint, s *stateTrie.BeaconState) error {
 	c.lock.Lock()
 	defer c.lock.Unlock()
-	if err := c.cache.AddIfNotPresent(&CheckpointState{
-		Checkpoint: stateTrie.CopyCheckpoint(cp.Checkpoint),
-		State:      cp.State.Copy(),
-	}); err != nil {
+	h, err := hashutil.HashProto(cp)
+	if err != nil {
 		return err
 	}
-
-	trim(c.cache, maxCheckpointStateSize)
+	c.cache.Add(h, s)
 	return nil
 }
-
-// CheckpointStateKeys returns the keys of the state in cache.
-func (c *CheckpointStateCache) CheckpointStateKeys() []string {
-	return c.cache.ListKeys()
-}

beacon-chain/cache/checkpoint_state_test.go

@@ -1,50 +1,18 @@
 package cache
 
 import (
-	"reflect"
 	"testing"
 
 	"github.com/gogo/protobuf/proto"
-	"github.com/prysmaticlabs/prysm/shared/params"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/hashutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 )
 
-func TestCheckpointStateCacheKeyFn_OK(t *testing.T) {
-	cp := &ethpb.Checkpoint{Epoch: 1, Root: bytesutil.PadTo([]byte{'A'}, 32)}
-	st, err := stateTrie.InitializeFromProto(&pb.BeaconState{
-		Slot: 64,
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	info := &CheckpointState{
-		Checkpoint: cp,
-		State:      st,
-	}
-	key, err := checkpointState(info)
-	if err != nil {
-		t.Fatal(err)
-	}
-	wantedKey, err := hashutil.HashProto(cp)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if key != string(wantedKey[:]) {
-		t.Errorf("Incorrect hash key: %s, expected %s", key, string(wantedKey[:]))
-	}
-}
-
-func TestCheckpointStateCacheKeyFn_InvalidObj(t *testing.T) {
-	_, err := checkpointState("bad")
-	if err != ErrNotCheckpointState {
-		t.Errorf("Expected error %v, got %v", ErrNotCheckpointState, err)
-	}
-}
-
 func TestCheckpointStateCache_StateByCheckpoint(t *testing.T) {
 	cache := NewCheckpointStateCache()
 
@@ -53,29 +21,18 @@ func TestCheckpointStateCache_StateByCheckpoint(t *testing.T) {
 		GenesisValidatorsRoot: params.BeaconConfig().ZeroHash[:],
 		Slot:                  64,
 	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	info1 := &CheckpointState{
-		Checkpoint: cp1,
-		State:      st,
-	}
-	state, err := cache.StateByCheckpoint(cp1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if state != nil {
-		t.Error("Expected state not to exist in empty cache")
-	}
+	require.NoError(t, err)
+
+	state, err := cache.StateByCheckpoint(cp1)
+	require.NoError(t, err)
+	assert.Equal(t, (*stateTrie.BeaconState)(nil), state, "Expected state not to exist in empty cache")
+
+	require.NoError(t, cache.AddCheckpointState(cp1, st))
 
-	if err := cache.AddCheckpointState(info1); err != nil {
-		t.Fatal(err)
-	}
 	state, err = cache.StateByCheckpoint(cp1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !proto.Equal(state.InnerStateUnsafe(), info1.State.InnerStateUnsafe()) {
+	require.NoError(t, err)
+
+	if !proto.Equal(state.InnerStateUnsafe(), st.InnerStateUnsafe()) {
 		t.Error("incorrectly cached state")
 	}
 
@@ -83,31 +40,16 @@ func TestCheckpointStateCache_StateByCheckpoint(t *testing.T) {
 	st2, err := stateTrie.InitializeFromProto(&pb.BeaconState{
 		Slot: 128,
 	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	info2 := &CheckpointState{
-		Checkpoint: cp2,
-		State:      st2,
-	}
-	if err := cache.AddCheckpointState(info2); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
+	require.NoError(t, cache.AddCheckpointState(cp2, st2))
+
 	state, err = cache.StateByCheckpoint(cp2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(state.CloneInnerState(), info2.State.CloneInnerState()) {
-		t.Error("incorrectly cached state")
-	}
+	require.NoError(t, err)
+	assert.DeepEqual(t, st2.CloneInnerState(), state.CloneInnerState(), "incorrectly cached state")
 
 	state, err = cache.StateByCheckpoint(cp1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(state.CloneInnerState(), info1.State.CloneInnerState()) {
-		t.Error("incorrectly cached state")
-	}
+	require.NoError(t, err)
+	assert.DeepEqual(t, st.CloneInnerState(), state.CloneInnerState(), "incorrectly cached state")
 }
 
 func TestCheckpointStateCache_MaxSize(t *testing.T) {
@@ -115,27 +57,12 @@ func TestCheckpointStateCache_MaxSize(t *testing.T) {
 	st, err := stateTrie.InitializeFromProto(&pb.BeaconState{
 		Slot: 0,
 	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	for i := 0; i < maxCheckpointStateSize+100; i++ {
-		if err := st.SetSlot(uint64(i)); err != nil {
-			t.Fatal(err)
-		}
-		info := &CheckpointState{
-			Checkpoint: &ethpb.Checkpoint{Epoch: uint64(i)},
-			State:      st,
-		}
-		if err := c.AddCheckpointState(info); err != nil {
-			t.Fatal(err)
-		}
+	require.NoError(t, err)
+
+	for i := uint64(0); i < uint64(maxCheckpointStateSize+100); i++ {
+		require.NoError(t, st.SetSlot(i))
+		require.NoError(t, c.AddCheckpointState(&ethpb.Checkpoint{Epoch: i}, st))
 	}
 
-	if len(c.cache.ListKeys()) != maxCheckpointStateSize {
-		t.Errorf(
-			"Expected hash cache key size to be %d, got %d",
-			maxCheckpointStateSize,
-			len(c.cache.ListKeys()),
-		)
-	}
+	assert.Equal(t, maxCheckpointStateSize, len(c.cache.Keys()))
 }

beacon-chain/cache/committee.go

@@ -17,9 +17,8 @@ var (
 	ErrNotCommittee = errors.New("object is not a committee struct")
 
 	// maxCommitteesCacheSize defines the max number of shuffled committees on per randao basis can cache.
-	// Due to reorgs, it's good to keep the old cache around for quickly switch over. 10 is a generous
-	// cache size as it considers 3 concurrent branches over 3 epochs.
-	maxCommitteesCacheSize = 10
+	// Due to reorgs and long finality, it's good to keep the old cache around for quickly switch over.
+	maxCommitteesCacheSize = uint64(32)
 
 	// CommitteeCacheMiss tracks the number of committee requests that aren't present in the cache.
 	CommitteeCacheMiss = promauto.NewCounter(prometheus.CounterOpts{
@@ -96,7 +95,7 @@ func (c *CommitteeCache) Committee(slot uint64, seed [32]byte, index uint64) ([]
 	indexOffSet := index + (slot%params.BeaconConfig().SlotsPerEpoch)*committeeCountPerSlot
 	start, end := startEndIndices(item, indexOffSet)
 
-	if int(end) > len(item.ShuffledIndices) || end < start {
+	if end > uint64(len(item.ShuffledIndices)) || end < start {
 		return nil, errors.New("requested index out of bound")
 	}
 
@@ -169,6 +168,30 @@ func (c *CommitteeCache) ActiveIndices(seed [32]byte) ([]uint64, error) {
 	return item.SortedIndices, nil
 }
 
+// ActiveIndicesCount returns the active indices count of a given seed stored in cache.
+func (c *CommitteeCache) ActiveIndicesCount(seed [32]byte) (int, error) {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	obj, exists, err := c.CommitteeCache.GetByKey(key(seed))
+	if err != nil {
+		return 0, err
+	}
+
+	if exists {
+		CommitteeCacheHit.Inc()
+	} else {
+		CommitteeCacheMiss.Inc()
+		return 0, nil
+	}
+
+	item, ok := obj.(*Committees)
+	if !ok {
+		return 0, ErrNotCommittee
+	}
+
+	return len(item.SortedIndices), nil
+}
+
 // ProposerIndices returns the proposer indices of a given seed.
 func (c *CommitteeCache) ProposerIndices(seed [32]byte) ([]uint64, error) {
 	c.lock.RLock()

beacon-chain/cache/committee_fuzz_test.go

@@ -1,10 +1,11 @@
 package cache
 
 import (
-	"reflect"
 	"testing"
 
 	fuzz "github.com/google/gofuzz"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 )
 
 func TestCommitteeKeyFuzz_OK(t *testing.T) {
@@ -14,12 +15,8 @@ func TestCommitteeKeyFuzz_OK(t *testing.T) {
 	for i := 0; i < 100000; i++ {
 		fuzzer.Fuzz(c)
 		k, err := committeeKeyFn(c)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if k != key(c.Seed) {
-			t.Errorf("Incorrect hash k: %s, expected %s", k, key(c.Seed))
-		}
+		require.NoError(t, err)
+		assert.Equal(t, key(c.Seed), k)
 	}
 }
 
@@ -30,17 +27,12 @@ func TestCommitteeCache_FuzzCommitteesByEpoch(t *testing.T) {
 
 	for i := 0; i < 100000; i++ {
 		fuzzer.Fuzz(c)
-		if err := cache.AddCommitteeShuffledList(c); err != nil {
-			t.Fatal(err)
-		}
-		if _, err := cache.Committee(0, c.Seed, 0); err != nil {
-			t.Fatal(err)
-		}
+		require.NoError(t, cache.AddCommitteeShuffledList(c))
+		_, err := cache.Committee(0, c.Seed, 0)
+		require.NoError(t, err)
 	}
 
-	if len(cache.CommitteeCache.ListKeys()) != maxCommitteesCacheSize {
-		t.Error("Incorrect key size")
-	}
+	assert.Equal(t, maxCommitteesCacheSize, uint64(len(cache.CommitteeCache.ListKeys())), "Incorrect key size")
 }
 
 func TestCommitteeCache_FuzzActiveIndices(t *testing.T) {
@@ -50,19 +42,12 @@ func TestCommitteeCache_FuzzActiveIndices(t *testing.T) {
 
 	for i := 0; i < 100000; i++ {
 		fuzzer.Fuzz(c)
-		if err := cache.AddCommitteeShuffledList(c); err != nil {
-			t.Fatal(err)
-		}
+		require.NoError(t, cache.AddCommitteeShuffledList(c))
+
 		indices, err := cache.ActiveIndices(c.Seed)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if !reflect.DeepEqual(indices, c.SortedIndices) {
-			t.Error("Saved indices not the same")
-		}
+		require.NoError(t, err)
+		assert.DeepEqual(t, c.SortedIndices, indices)
 	}
 
-	if len(cache.CommitteeCache.ListKeys()) != maxCommitteesCacheSize {
-		t.Error("Incorrect key size")
-	}
+	assert.Equal(t, maxCommitteesCacheSize, uint64(len(cache.CommitteeCache.ListKeys())), "Incorrect key size")
 }

beacon-chain/cache/committee_ids.go

@@ -1,87 +0,0 @@
-package cache
-
-import (
-	"sync"
-
-	lru "github.com/hashicorp/golang-lru"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/prysmaticlabs/prysm/shared/sliceutil"
-)
-
-type committeeIDs struct {
-	attester       *lru.Cache
-	attesterLock   sync.RWMutex
-	aggregator     *lru.Cache
-	aggregatorLock sync.RWMutex
-}
-
-// CommitteeIDs for attester and aggregator.
-var CommitteeIDs = newCommitteeIDs()
-
-func newCommitteeIDs() *committeeIDs {
-	// Given a node can calculate committee assignments of current epoch and next epoch.
-	// Max size is set to 2 epoch length.
-	cacheSize := int(params.BeaconConfig().MaxCommitteesPerSlot * params.BeaconConfig().SlotsPerEpoch * 2)
-	attesterCache, err := lru.New(cacheSize)
-	if err != nil {
-		panic(err)
-	}
-	aggregatorCache, err := lru.New(cacheSize)
-	if err != nil {
-		panic(err)
-	}
-	return &committeeIDs{attester: attesterCache, aggregator: aggregatorCache}
-}
-
-// AddAttesterCommiteeID adds committee ID for subscribing subnet for the attester of a given slot.
-func (c *committeeIDs) AddAttesterCommiteeID(slot uint64, committeeID uint64) {
-	c.attesterLock.Lock()
-	defer c.attesterLock.Unlock()
-
-	ids := []uint64{committeeID}
-	val, exists := c.attester.Get(slot)
-	if exists {
-		ids = sliceutil.UnionUint64(append(val.([]uint64), ids...))
-	}
-	c.attester.Add(slot, ids)
-}
-
-// GetAttesterCommitteeIDs gets the committee ID for subscribing subnet for attester of the slot.
-func (c *committeeIDs) GetAttesterCommitteeIDs(slot uint64) []uint64 {
-	c.attesterLock.RLock()
-	defer c.attesterLock.RUnlock()
-
-	val, exists := c.attester.Get(slot)
-	if !exists {
-		return nil
-	}
-	if v, ok := val.([]uint64); ok {
-	   return v
-	}
-	return nil
-}
-
-// AddAggregatorCommiteeID adds committee ID for subscribing subnet for the aggregator of a given slot.
-func (c *committeeIDs) AddAggregatorCommiteeID(slot uint64, committeeID uint64) {
-	c.aggregatorLock.Lock()
-	defer c.aggregatorLock.Unlock()
-
-	ids := []uint64{committeeID}
-	val, exists := c.aggregator.Get(slot)
-	if exists {
-		ids = sliceutil.UnionUint64(append(val.([]uint64), ids...))
-	}
-	c.aggregator.Add(slot, ids)
-}
-
-// GetAggregatorCommitteeIDs gets the committee ID for subscribing subnet for aggregator of the slot.
-func (c *committeeIDs) GetAggregatorCommitteeIDs(slot uint64) []uint64 {
-	c.aggregatorLock.RLock()
-	defer c.aggregatorLock.RUnlock()
-
-	val, exists := c.aggregator.Get(slot)
-	if !exists {
-		return []uint64{}
-	}
-	return val.([]uint64)
-}

beacon-chain/cache/committee_ids_test.go

@@ -1,56 +0,0 @@
-package cache
-
-import (
-	"reflect"
-	"testing"
-)
-
-func TestCommitteeIDCache_RoundTrip(t *testing.T) {
-	c := newCommitteeIDs()
-	slot := uint64(100)
-	committeeIDs := c.GetAggregatorCommitteeIDs(slot)
-	if len(committeeIDs) != 0 {
-		t.Errorf("Empty cache returned an object: %v", committeeIDs)
-	}
-
-	c.AddAggregatorCommiteeID(slot, 1)
-	res := c.GetAggregatorCommitteeIDs(slot)
-	if !reflect.DeepEqual(res, []uint64{1}) {
-		t.Error("Expected equal value to return from cache")
-	}
-
-	c.AddAggregatorCommiteeID(slot, 2)
-	res = c.GetAggregatorCommitteeIDs(slot)
-	if !reflect.DeepEqual(res, []uint64{1, 2}) {
-		t.Error("Expected equal value to return from cache")
-	}
-
-	c.AddAggregatorCommiteeID(slot, 3)
-	res = c.GetAggregatorCommitteeIDs(slot)
-	if !reflect.DeepEqual(res, []uint64{1, 2, 3}) {
-		t.Error("Expected equal value to return from cache")
-	}
-
-	committeeIDs = c.GetAttesterCommitteeIDs(slot)
-	if len(committeeIDs) != 0 {
-		t.Errorf("Empty cache returned an object: %v", committeeIDs)
-	}
-
-	c.AddAttesterCommiteeID(slot, 11)
-	res = c.GetAttesterCommitteeIDs(slot)
-	if !reflect.DeepEqual(res, []uint64{11}) {
-		t.Error("Expected equal value to return from cache")
-	}
-
-	c.AddAttesterCommiteeID(slot, 22)
-	res = c.GetAttesterCommitteeIDs(slot)
-	if !reflect.DeepEqual(res, []uint64{11, 22}) {
-		t.Error("Expected equal value to return from cache")
-	}
-
-	c.AddAttesterCommiteeID(slot, 33)
-	res = c.GetAttesterCommitteeIDs(slot)
-	if !reflect.DeepEqual(res, []uint64{11, 22, 33}) {
-		t.Error("Expected equal value to return from cache")
-	}
-}

beacon-chain/cache/committee_test.go

@@ -2,13 +2,14 @@ package cache
 
 import (
 	"math"
-	"reflect"
 	"sort"
 	"strconv"
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 )
 
 func TestCommitteeKeyFn_OK(t *testing.T) {
@@ -19,19 +20,13 @@ func TestCommitteeKeyFn_OK(t *testing.T) {
 	}
 
 	k, err := committeeKeyFn(item)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if k != key(item.Seed) {
-		t.Errorf("Incorrect hash k: %s, expected %s", k, key(item.Seed))
-	}
+	require.NoError(t, err)
+	assert.Equal(t, key(item.Seed), k)
 }
 
 func TestCommitteeKeyFn_InvalidObj(t *testing.T) {
 	_, err := committeeKeyFn("bad")
-	if err != ErrNotCommittee {
-		t.Errorf("Expected error %v, got %v", ErrNotCommittee, err)
-	}
+	assert.Equal(t, ErrNotCommittee, err)
 }
 
 func TestCommitteeCache_CommitteesByEpoch(t *testing.T) {
@@ -46,30 +41,18 @@ func TestCommitteeCache_CommitteesByEpoch(t *testing.T) {
 	slot := params.BeaconConfig().SlotsPerEpoch
 	committeeIndex := uint64(1)
 	indices, err := cache.Committee(slot, item.Seed, committeeIndex)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	if indices != nil {
 		t.Error("Expected committee not to exist in empty cache")
 	}
+	require.NoError(t, cache.AddCommitteeShuffledList(item))
 
-	if err := cache.AddCommitteeShuffledList(item); err != nil {
-		t.Fatal(err)
-	}
 	wantedIndex := uint64(0)
 	indices, err = cache.Committee(slot, item.Seed, wantedIndex)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	start, end := startEndIndices(item, wantedIndex)
-	if !reflect.DeepEqual(indices, item.ShuffledIndices[start:end]) {
-		t.Errorf(
-			"Expected fetched active indices to be %v, got %v",
-			indices,
-			item.ShuffledIndices[start:end],
-		)
-	}
+	assert.DeepEqual(t, item.ShuffledIndices[start:end], indices)
 }
 
 func TestCommitteeCache_ActiveIndices(t *testing.T) {
@@ -77,24 +60,31 @@ func TestCommitteeCache_ActiveIndices(t *testing.T) {
 
 	item := &Committees{Seed: [32]byte{'A'}, SortedIndices: []uint64{1, 2, 3, 4, 5, 6}}
 	indices, err := cache.ActiveIndices(item.Seed)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	if indices != nil {
-		t.Error("Expected committee count not to exist in empty cache")
+		t.Error("Expected committee not to exist in empty cache")
 	}
 
-	if err := cache.AddCommitteeShuffledList(item); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, cache.AddCommitteeShuffledList(item))
 
 	indices, err = cache.ActiveIndices(item.Seed)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(indices, item.SortedIndices) {
-		t.Error("Did not receive correct active indices from cache")
-	}
+	require.NoError(t, err)
+	assert.DeepEqual(t, item.SortedIndices, indices)
+}
+
+func TestCommitteeCache_ActiveCount(t *testing.T) {
+	cache := NewCommitteesCache()
+
+	item := &Committees{Seed: [32]byte{'A'}, SortedIndices: []uint64{1, 2, 3, 4, 5, 6}}
+	count, err := cache.ActiveIndicesCount(item.Seed)
+	require.NoError(t, err)
+	assert.Equal(t, 0, count, "Expected active count not to exist in empty cache")
+
+	require.NoError(t, cache.AddCommitteeShuffledList(item))
+
+	count, err = cache.ActiveIndicesCount(item.Seed)
+	require.NoError(t, err)
+	assert.Equal(t, len(item.SortedIndices), count)
 }
 
 func TestCommitteeCache_AddProposerIndicesList(t *testing.T) {
@@ -103,75 +93,55 @@ func TestCommitteeCache_AddProposerIndicesList(t *testing.T) {
 	seed := [32]byte{'A'}
 	indices := []uint64{1, 2, 3, 4, 5}
 	indices, err := cache.ProposerIndices(seed)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	if indices != nil {
 		t.Error("Expected committee count not to exist in empty cache")
 	}
-	if err := cache.AddProposerIndicesList(seed, indices); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, cache.AddProposerIndicesList(seed, indices))
+
 	received, err := cache.ProposerIndices(seed)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(indices, received) {
-		t.Error("Did not receive correct proposer indices from cache")
-	}
+	require.NoError(t, err)
+	assert.DeepEqual(t, received, indices)
 
 	item := &Committees{Seed: [32]byte{'B'}, SortedIndices: []uint64{1, 2, 3, 4, 5, 6}}
-	if err := cache.AddCommitteeShuffledList(item); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, cache.AddCommitteeShuffledList(item))
+
 	indices, err = cache.ProposerIndices(item.Seed)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	if indices != nil {
 		t.Error("Expected committee count not to exist in empty cache")
 	}
-	if err := cache.AddProposerIndicesList(item.Seed, indices); err != nil {
-		t.Fatal(err)
-	}
-	received, err = cache.ProposerIndices(item.Seed)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(indices, received) {
-		t.Error("Did not receive correct proposer indices from cache")
-	}
+	require.NoError(t, cache.AddProposerIndicesList(item.Seed, indices))
 
+	received, err = cache.ProposerIndices(item.Seed)
+	require.NoError(t, err)
+	assert.DeepEqual(t, received, indices)
 }
 
 func TestCommitteeCache_CanRotate(t *testing.T) {
 	cache := NewCommitteesCache()
 
 	// Should rotate out all the epochs except 190 through 199.
-	for i := 100; i < 200; i++ {
+	start := 100
+	end := 200
+	for i := start; i < end; i++ {
 		s := []byte(strconv.Itoa(i))
 		item := &Committees{Seed: bytesutil.ToBytes32(s)}
-		if err := cache.AddCommitteeShuffledList(item); err != nil {
-			t.Fatal(err)
-		}
+		require.NoError(t, cache.AddCommitteeShuffledList(item))
 	}
 
 	k := cache.CommitteeCache.ListKeys()
-	if len(k) != maxCommitteesCacheSize {
-		t.Errorf("wanted: %d, got: %d", maxCommitteesCacheSize, len(k))
-	}
+	assert.Equal(t, maxCommitteesCacheSize, uint64(len(k)))
 
 	sort.Slice(k, func(i, j int) bool {
 		return k[i] < k[j]
 	})
-	s := bytesutil.ToBytes32([]byte(strconv.Itoa(190)))
-	if k[0] != key(s) {
-		t.Error("incorrect key received for slot 190")
-	}
+	wanted := end - int(maxCommitteesCacheSize)
+	s := bytesutil.ToBytes32([]byte(strconv.Itoa(wanted)))
+	assert.Equal(t, key(s), k[0], "incorrect key received for slot 190")
+
 	s = bytesutil.ToBytes32([]byte(strconv.Itoa(199)))
-	if k[len(k)-1] != key(s) {
-		t.Error("incorrect key received for slot 199")
-	}
+	assert.Equal(t, key(s), k[len(k)-1], "incorrect key received for slot 199")
 }
 
 func TestCommitteeCacheOutOfRange(t *testing.T) {
@@ -184,11 +154,8 @@ func TestCommitteeCacheOutOfRange(t *testing.T) {
 		SortedIndices:   []uint64{},
 		ProposerIndices: []uint64{},
 	})
-	if err != nil {
-		t.Error(err)
-	}
+	require.NoError(t, err)
+
 	_, err = cache.Committee(0, seed, math.MaxUint64) // Overflow!
-	if err == nil {
-		t.Fatal("Did not fail as expected")
-	}
+	require.NotNil(t, err, "Did not fail as expected")
 }

beacon-chain/cache/common.go

@@ -8,12 +8,12 @@ import (
 var (
 	// maxCacheSize is 4x of the epoch length for additional cache padding.
 	// Requests should be only accessing committees within defined epoch length.
-	maxCacheSize = int(4 * params.BeaconConfig().SlotsPerEpoch)
+	maxCacheSize = 4 * params.BeaconConfig().SlotsPerEpoch
 )
 
 // trim the FIFO queue to the maxSize.
-func trim(queue *cache.FIFO, maxSize int) {
-	for s := len(queue.ListKeys()); s > maxSize; s-- {
+func trim(queue *cache.FIFO, maxSize uint64) {
+	for s := uint64(len(queue.ListKeys())); s > maxSize; s-- {
 		_, err := queue.Pop(popProcessNoopFunc)
 		if err != nil {
 			// popProcessNoopFunc never returns an error, but we handle this anyway to make linter

beacon-chain/cache/depositcache/BUILD.bazel

@@ -1,4 +1,5 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -12,9 +13,12 @@ go_library(
         "//proto/beacon/db:go_default_library",
         "//shared/bytesutil:go_default_library",
         "//shared/hashutil:go_default_library",
+        "//shared/params:go_default_library",
+        "//shared/trieutil:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
+        "@com_github_prysmaticlabs_go_ssz//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@io_opencensus_go//trace:go_default_library",
     ],
@@ -23,15 +27,20 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = [
-        "deposits_test.go",
+        "deposits_cache_test.go",
         "pending_deposits_test.go",
     ],
     embed = [":go_default_library"],
     deps = [
         "//proto/beacon/db:go_default_library",
         "//shared/bytesutil:go_default_library",
+        "//shared/params:go_default_library",
+        "//shared/testutil/assert:go_default_library",
+        "//shared/testutil/require:go_default_library",
+        "//shared/trieutil:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
+        "@com_github_prysmaticlabs_go_ssz//:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",
     ],
 )

beacon-chain/cache/depositcache/deposits_cache.go

@@ -1,3 +1,7 @@
+// Package depositcache is the source of validator deposits maintained
+// in-memory by the beacon node – deposits processed from the
+// eth1 powchain are then stored in this cache to be accessed by
+// any other service during a beacon node's runtime.
 package depositcache
 
 import (
@@ -11,8 +15,11 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/go-ssz"
 	dbpb "github.com/prysmaticlabs/prysm/proto/beacon/db"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/trieutil"
 	log "github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
 )
@@ -26,31 +33,48 @@ var (
 
 // DepositFetcher defines a struct which can retrieve deposit information from a store.
 type DepositFetcher interface {
-	AllDeposits(ctx context.Context, beforeBlk *big.Int) []*ethpb.Deposit
+	AllDeposits(ctx context.Context, untilBlk *big.Int) []*ethpb.Deposit
 	DepositByPubkey(ctx context.Context, pubKey []byte) (*ethpb.Deposit, *big.Int)
 	DepositsNumberAndRootAtHeight(ctx context.Context, blockHeight *big.Int) (uint64, [32]byte)
+	FinalizedDeposits(ctx context.Context) *FinalizedDeposits
+	NonFinalizedDeposits(ctx context.Context, untilBlk *big.Int) []*ethpb.Deposit
+}
+
+// FinalizedDeposits stores the trie of deposits that have been included
+// in the beacon state up to the latest finalized checkpoint.
+type FinalizedDeposits struct {
+	Deposits        *trieutil.SparseMerkleTrie
+	MerkleTrieIndex int64
 }
 
 // DepositCache stores all in-memory deposit objects. This
 // stores all the deposit related data that is required by the beacon-node.
 type DepositCache struct {
 	// Beacon chain deposits in memory.
-	pendingDeposits       []*dbpb.DepositContainer
-	deposits              []*dbpb.DepositContainer
-	depositsLock          sync.RWMutex
-	chainStartDeposits    []*ethpb.Deposit
-	chainstartPubkeys     map[string]bool
-	chainstartPubkeysLock sync.RWMutex
+	pendingDeposits    []*dbpb.DepositContainer
+	deposits           []*dbpb.DepositContainer
+	finalizedDeposits  *FinalizedDeposits
+	depositsLock       sync.RWMutex
+	chainStartDeposits []*ethpb.Deposit
+	chainStartPubkeys  map[string]bool
 }
 
 // NewDepositCache instantiates a new deposit cache
-func NewDepositCache() *DepositCache {
+func NewDepositCache() (*DepositCache, error) {
+	finalizedDepositsTrie, err := trieutil.NewTrie(int(params.BeaconConfig().DepositContractTreeDepth))
+	if err != nil {
+		return nil, err
+	}
+
+	// finalizedDeposits.MerkleTrieIndex is initialized to -1 because it represents the index of the last trie item.
+	// Inserting the first item into the trie will set the value of the index to 0.
 	return &DepositCache{
 		pendingDeposits:    []*dbpb.DepositContainer{},
 		deposits:           []*dbpb.DepositContainer{},
-		chainstartPubkeys:  make(map[string]bool),
+		finalizedDeposits:  &FinalizedDeposits{Deposits: finalizedDepositsTrie, MerkleTrieIndex: -1},
+		chainStartPubkeys:  make(map[string]bool),
 		chainStartDeposits: make([]*ethpb.Deposit, 0),
-	}
+	}, nil
 }
 
 // InsertDeposit into the database. If deposit or block number are nil
@@ -69,7 +93,7 @@ func (dc *DepositCache) InsertDeposit(ctx context.Context, d *ethpb.Deposit, blo
 	}
 	dc.depositsLock.Lock()
 	defer dc.depositsLock.Unlock()
-	// keep the slice sorted on insertion in order to avoid costly sorting on retrival.
+	// Keep the slice sorted on insertion in order to avoid costly sorting on retrieval.
 	heightIdx := sort.Search(len(dc.deposits), func(i int) bool { return dc.deposits[i].Index >= index })
 	newDeposits := append([]*dbpb.DepositContainer{{Deposit: d, Eth1BlockHeight: blockNum, DepositRoot: depositRoot[:], Index: index}}, dc.deposits[heightIdx:]...)
 	dc.deposits = append(dc.deposits[:heightIdx], newDeposits...)
@@ -88,9 +112,40 @@ func (dc *DepositCache) InsertDepositContainers(ctx context.Context, ctrs []*dbp
 	historicalDepositsCount.Add(float64(len(ctrs)))
 }
 
-// AllDepositContainers returns a list of deposits all historical deposit containers until the given block number.
+// InsertFinalizedDeposits inserts deposits up to eth1DepositIndex (inclusive) into the finalized deposits cache.
+func (dc *DepositCache) InsertFinalizedDeposits(ctx context.Context, eth1DepositIndex int64) {
+	ctx, span := trace.StartSpan(ctx, "DepositsCache.InsertFinalizedDeposits")
+	defer span.End()
+	dc.depositsLock.Lock()
+	defer dc.depositsLock.Unlock()
+
+	depositTrie := dc.finalizedDeposits.Deposits
+	insertIndex := dc.finalizedDeposits.MerkleTrieIndex + 1
+	for _, d := range dc.deposits {
+		if d.Index <= dc.finalizedDeposits.MerkleTrieIndex {
+			continue
+		}
+		if d.Index > eth1DepositIndex {
+			break
+		}
+		depHash, err := ssz.HashTreeRoot(d.Deposit.Data)
+		if err != nil {
+			log.WithError(err).Error("Could not hash deposit data. Finalized deposit cache not updated.")
+			return
+		}
+		depositTrie.Insert(depHash[:], int(insertIndex))
+		insertIndex++
+	}
+
+	dc.finalizedDeposits = &FinalizedDeposits{
+		Deposits:        depositTrie,
+		MerkleTrieIndex: eth1DepositIndex,
+	}
+}
+
+// AllDepositContainers returns all historical deposit containers.
 func (dc *DepositCache) AllDepositContainers(ctx context.Context) []*dbpb.DepositContainer {
-	ctx, span := trace.StartSpan(ctx, "BeaconDB.AllDepositContainers")
+	ctx, span := trace.StartSpan(ctx, "DepositsCache.AllDepositContainers")
 	defer span.End()
 	dc.depositsLock.RLock()
 	defer dc.depositsLock.RUnlock()
@@ -102,27 +157,23 @@ func (dc *DepositCache) AllDepositContainers(ctx context.Context) []*dbpb.Deposi
 func (dc *DepositCache) MarkPubkeyForChainstart(ctx context.Context, pubkey string) {
 	ctx, span := trace.StartSpan(ctx, "DepositsCache.MarkPubkeyForChainstart")
 	defer span.End()
-	dc.chainstartPubkeysLock.Lock()
-	defer dc.chainstartPubkeysLock.Unlock()
-	dc.chainstartPubkeys[pubkey] = true
+	dc.chainStartPubkeys[pubkey] = true
 }
 
 // PubkeyInChainstart returns bool for whether the pubkey passed in has deposited.
 func (dc *DepositCache) PubkeyInChainstart(ctx context.Context, pubkey string) bool {
 	ctx, span := trace.StartSpan(ctx, "DepositsCache.PubkeyInChainstart")
 	defer span.End()
-	dc.chainstartPubkeysLock.Lock()
-	defer dc.chainstartPubkeysLock.Unlock()
-	if dc.chainstartPubkeys != nil {
-		return dc.chainstartPubkeys[pubkey]
+	if dc.chainStartPubkeys != nil {
+		return dc.chainStartPubkeys[pubkey]
 	}
-	dc.chainstartPubkeys = make(map[string]bool)
+	dc.chainStartPubkeys = make(map[string]bool)
 	return false
 }
 
-// AllDeposits returns a list of deposits all historical deposits until the given block number
+// AllDeposits returns a list of historical deposits until the given block number
 // (inclusive). If no block is specified then this method returns all historical deposits.
-func (dc *DepositCache) AllDeposits(ctx context.Context, beforeBlk *big.Int) []*ethpb.Deposit {
+func (dc *DepositCache) AllDeposits(ctx context.Context, untilBlk *big.Int) []*ethpb.Deposit {
 	ctx, span := trace.StartSpan(ctx, "DepositsCache.AllDeposits")
 	defer span.End()
 	dc.depositsLock.RLock()
@@ -130,7 +181,7 @@ func (dc *DepositCache) AllDeposits(ctx context.Context, beforeBlk *big.Int) []*
 
 	var deposits []*ethpb.Deposit
 	for _, ctnr := range dc.deposits {
-		if beforeBlk == nil || beforeBlk.Uint64() >= ctnr.Eth1BlockHeight {
+		if untilBlk == nil || untilBlk.Uint64() >= ctnr.Eth1BlockHeight {
 			deposits = append(deposits, ctnr.Deposit)
 		}
 	}
@@ -172,3 +223,39 @@ func (dc *DepositCache) DepositByPubkey(ctx context.Context, pubKey []byte) (*et
 	}
 	return deposit, blockNum
 }
+
+// FinalizedDeposits returns the finalized deposits trie.
+func (dc *DepositCache) FinalizedDeposits(ctx context.Context) *FinalizedDeposits {
+	ctx, span := trace.StartSpan(ctx, "DepositsCache.FinalizedDeposits")
+	defer span.End()
+	dc.depositsLock.RLock()
+	defer dc.depositsLock.RUnlock()
+
+	return &FinalizedDeposits{
+		Deposits:        dc.finalizedDeposits.Deposits.Copy(),
+		MerkleTrieIndex: dc.finalizedDeposits.MerkleTrieIndex,
+	}
+}
+
+// NonFinalizedDeposits returns the list of non-finalized deposits until the given block number (inclusive).
+// If no block is specified then this method returns all non-finalized deposits.
+func (dc *DepositCache) NonFinalizedDeposits(ctx context.Context, untilBlk *big.Int) []*ethpb.Deposit {
+	ctx, span := trace.StartSpan(ctx, "DepositsCache.NonFinalizedDeposits")
+	defer span.End()
+	dc.depositsLock.RLock()
+	defer dc.depositsLock.RUnlock()
+
+	if dc.finalizedDeposits == nil {
+		return dc.AllDeposits(ctx, untilBlk)
+	}
+
+	lastFinalizedDepositIndex := dc.finalizedDeposits.MerkleTrieIndex
+	var deposits []*ethpb.Deposit
+	for _, d := range dc.deposits {
+		if (d.Index > lastFinalizedDepositIndex) && (untilBlk == nil || untilBlk.Uint64() >= d.Eth1BlockHeight) {
+			deposits = append(deposits, d.Deposit)
+		}
+	}
+
+	return deposits
+}

beacon-chain/cache/depositcache/deposits_cache_test.go

@@ -0,0 +1,483 @@
+package depositcache
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"math/big"
+	"testing"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/go-ssz"
+	dbpb "github.com/prysmaticlabs/prysm/proto/beacon/db"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+	"github.com/prysmaticlabs/prysm/shared/trieutil"
+	logTest "github.com/sirupsen/logrus/hooks/test"
+)
+
+const nilDepositErr = "Ignoring nil deposit insertion"
+
+var _ = DepositFetcher(&DepositCache{})
+
+func TestInsertDeposit_LogsOnNilDepositInsertion(t *testing.T) {
+	hook := logTest.NewGlobal()
+	dc, err := NewDepositCache()
+	require.NoError(t, err)
+
+	dc.InsertDeposit(context.Background(), nil, 1, 0, [32]byte{})
+
+	require.Equal(t, 0, len(dc.deposits), "Number of deposits changed")
+	assert.Equal(t, nilDepositErr, hook.LastEntry().Message)
+}
+
+func TestInsertDeposit_MaintainsSortedOrderByIndex(t *testing.T) {
+	dc, err := NewDepositCache()
+	require.NoError(t, err)
+
+	insertions := []struct {
+		blkNum  uint64
+		deposit *ethpb.Deposit
+		index   int64
+	}{
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{},
+			index:   0,
+		},
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{},
+			index:   3,
+		},
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{},
+			index:   1,
+		},
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{},
+			index:   4,
+		},
+	}
+
+	for _, ins := range insertions {
+		dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{})
+	}
+
+	expectedIndices := []int64{0, 1, 3, 4}
+	for i, ei := range expectedIndices {
+		assert.Equal(t, ei, dc.deposits[i].Index,
+			fmt.Sprintf("dc.deposits[%d].Index = %d, wanted %d", i, dc.deposits[i].Index, ei))
+	}
+}
+
+func TestAllDeposits_ReturnsAllDeposits(t *testing.T) {
+	dc, err := NewDepositCache()
+	require.NoError(t, err)
+
+	deposits := []*dbpb.DepositContainer{
+		{
+			Eth1BlockHeight: 10,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 10,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 10,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 11,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 11,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 12,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 12,
+			Deposit:         &ethpb.Deposit{},
+		},
+	}
+	dc.deposits = deposits
+
+	d := dc.AllDeposits(context.Background(), nil)
+	assert.Equal(t, len(deposits), len(d))
+}
+
+func TestAllDeposits_FiltersDepositUpToAndIncludingBlockNumber(t *testing.T) {
+	dc, err := NewDepositCache()
+	require.NoError(t, err)
+
+	deposits := []*dbpb.DepositContainer{
+		{
+			Eth1BlockHeight: 10,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 10,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 10,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 11,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 11,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 12,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 12,
+			Deposit:         &ethpb.Deposit{},
+		},
+	}
+	dc.deposits = deposits
+
+	d := dc.AllDeposits(context.Background(), big.NewInt(11))
+	assert.Equal(t, 5, len(d))
+}
+
+func TestDepositsNumberAndRootAtHeight_ReturnsAppropriateCountAndRoot(t *testing.T) {
+	dc, err := NewDepositCache()
+	require.NoError(t, err)
+
+	dc.deposits = []*dbpb.DepositContainer{
+		{
+			Eth1BlockHeight: 10,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 10,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 10,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 10,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 11,
+			Deposit:         &ethpb.Deposit{},
+			DepositRoot:     []byte("root"),
+		},
+		{
+			Eth1BlockHeight: 12,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 12,
+			Deposit:         &ethpb.Deposit{},
+		},
+	}
+
+	n, root := dc.DepositsNumberAndRootAtHeight(context.Background(), big.NewInt(11))
+	assert.Equal(t, 5, int(n))
+	assert.Equal(t, bytesutil.ToBytes32([]byte("root")), root)
+}
+
+func TestDepositsNumberAndRootAtHeight_ReturnsEmptyTrieIfBlockHeightLessThanOldestDeposit(t *testing.T) {
+	dc, err := NewDepositCache()
+	require.NoError(t, err)
+
+	dc.deposits = []*dbpb.DepositContainer{
+		{
+			Eth1BlockHeight: 10,
+			Deposit:         &ethpb.Deposit{},
+			DepositRoot:     []byte("root"),
+		},
+		{
+			Eth1BlockHeight: 11,
+			Deposit:         &ethpb.Deposit{},
+			DepositRoot:     []byte("root"),
+		},
+	}
+
+	n, root := dc.DepositsNumberAndRootAtHeight(context.Background(), big.NewInt(2))
+	assert.Equal(t, 0, int(n))
+	assert.Equal(t, [32]byte{}, root)
+}
+
+func TestDepositByPubkey_ReturnsFirstMatchingDeposit(t *testing.T) {
+	dc, err := NewDepositCache()
+	require.NoError(t, err)
+
+	dc.deposits = []*dbpb.DepositContainer{
+		{
+			Eth1BlockHeight: 9,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey: []byte("pk0"),
+				},
+			},
+		},
+		{
+			Eth1BlockHeight: 10,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey: []byte("pk1"),
+				},
+			},
+		},
+		{
+			Eth1BlockHeight: 11,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey: []byte("pk1"),
+				},
+			},
+		},
+		{
+			Eth1BlockHeight: 12,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey: []byte("pk2"),
+				},
+			},
+		},
+	}
+
+	dep, blkNum := dc.DepositByPubkey(context.Background(), []byte("pk1"))
+
+	if !bytes.Equal(dep.Data.PublicKey, []byte("pk1")) {
+		t.Error("Returned wrong deposit")
+	}
+	assert.Equal(t, 0, blkNum.Cmp(big.NewInt(10)),
+		fmt.Sprintf("Returned wrong block number %v", blkNum))
+}
+
+func TestFinalizedDeposits_DepositsCachedCorrectly(t *testing.T) {
+	dc, err := NewDepositCache()
+	require.NoError(t, err)
+
+	finalizedDeposits := []*dbpb.DepositContainer{
+		{
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey: []byte{0},
+				},
+			},
+			Index: 0,
+		},
+		{
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey: []byte{1},
+				},
+			},
+			Index: 1,
+		},
+		{
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey: []byte{2},
+				},
+			},
+			Index: 2,
+		},
+	}
+	dc.deposits = append(finalizedDeposits, &dbpb.DepositContainer{
+		Deposit: &ethpb.Deposit{
+			Data: &ethpb.Deposit_Data{
+				PublicKey: []byte{3},
+			},
+		},
+		Index: 3,
+	})
+
+	dc.InsertFinalizedDeposits(context.Background(), 2)
+
+	cachedDeposits := dc.FinalizedDeposits(context.Background())
+	require.NotNil(t, cachedDeposits, "Deposits not cached")
+	assert.Equal(t, int64(2), cachedDeposits.MerkleTrieIndex)
+
+	var deps [][]byte
+	for _, d := range finalizedDeposits {
+		hash, err := ssz.HashTreeRoot(d.Deposit.Data)
+		require.NoError(t, err, "Could not hash deposit data")
+		deps = append(deps, hash[:])
+	}
+	trie, err := trieutil.GenerateTrieFromItems(deps, int(params.BeaconConfig().DepositContractTreeDepth))
+	require.NoError(t, err, "Could not generate deposit trie")
+	assert.Equal(t, trie.HashTreeRoot(), cachedDeposits.Deposits.HashTreeRoot())
+}
+
+func TestFinalizedDeposits_UtilizesPreviouslyCachedDeposits(t *testing.T) {
+	dc, err := NewDepositCache()
+	require.NoError(t, err)
+
+	oldFinalizedDeposits := []*dbpb.DepositContainer{
+		{
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey: []byte{0},
+				},
+			},
+			Index: 0,
+		},
+		{
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey: []byte{1},
+				},
+			},
+			Index: 1,
+		},
+	}
+	newFinalizedDeposit := dbpb.DepositContainer{
+		Deposit: &ethpb.Deposit{
+			Data: &ethpb.Deposit_Data{
+				PublicKey: []byte{2},
+			},
+		},
+		Index: 2,
+	}
+	dc.deposits = oldFinalizedDeposits
+	dc.InsertFinalizedDeposits(context.Background(), 1)
+	// Artificially exclude old deposits so that they can only be retrieved from previously finalized deposits.
+	dc.deposits = []*dbpb.DepositContainer{&newFinalizedDeposit}
+
+	dc.InsertFinalizedDeposits(context.Background(), 2)
+
+	cachedDeposits := dc.FinalizedDeposits(context.Background())
+	require.NotNil(t, cachedDeposits, "Deposits not cached")
+	assert.Equal(t, int64(2), cachedDeposits.MerkleTrieIndex)
+
+	var deps [][]byte
+	for _, d := range append(oldFinalizedDeposits, &newFinalizedDeposit) {
+		hash, err := ssz.HashTreeRoot(d.Deposit.Data)
+		require.NoError(t, err, "Could not hash deposit data")
+		deps = append(deps, hash[:])
+	}
+	trie, err := trieutil.GenerateTrieFromItems(deps, int(params.BeaconConfig().DepositContractTreeDepth))
+	require.NoError(t, err, "Could not generate deposit trie")
+	assert.Equal(t, trie.HashTreeRoot(), cachedDeposits.Deposits.HashTreeRoot())
+}
+
+func TestFinalizedDeposits_InitializedCorrectly(t *testing.T) {
+	dc, err := NewDepositCache()
+	require.NoError(t, err)
+
+	finalizedDeposits := dc.finalizedDeposits
+	assert.NotNil(t, finalizedDeposits)
+	assert.NotNil(t, finalizedDeposits.Deposits)
+	assert.Equal(t, int64(-1), finalizedDeposits.MerkleTrieIndex)
+}
+
+func TestNonFinalizedDeposits_ReturnsAllNonFinalizedDeposits(t *testing.T) {
+	dc, err := NewDepositCache()
+	require.NoError(t, err)
+
+	finalizedDeposits := []*dbpb.DepositContainer{
+		{
+			Eth1BlockHeight: 10,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey: []byte{0},
+				},
+			},
+			Index: 0,
+		},
+		{
+			Eth1BlockHeight: 10,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey: []byte{1},
+				},
+			},
+			Index: 1,
+		},
+	}
+	dc.deposits = append(finalizedDeposits,
+		&dbpb.DepositContainer{
+			Eth1BlockHeight: 10,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey: []byte{2},
+				},
+			},
+			Index: 2,
+		},
+		&dbpb.DepositContainer{
+			Eth1BlockHeight: 11,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey: []byte{3},
+				},
+			},
+			Index: 3,
+		})
+	dc.InsertFinalizedDeposits(context.Background(), 1)
+
+	deps := dc.NonFinalizedDeposits(context.Background(), nil)
+	assert.Equal(t, 2, len(deps))
+}
+
+func TestNonFinalizedDeposits_ReturnsNonFinalizedDepositsUpToBlockNumber(t *testing.T) {
+	dc, err := NewDepositCache()
+	require.NoError(t, err)
+
+	finalizedDeposits := []*dbpb.DepositContainer{
+		{
+			Eth1BlockHeight: 10,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey: []byte{0},
+				},
+			},
+			Index: 0,
+		},
+		{
+			Eth1BlockHeight: 10,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey: []byte{1},
+				},
+			},
+			Index: 1,
+		},
+	}
+	dc.deposits = append(finalizedDeposits,
+		&dbpb.DepositContainer{
+			Eth1BlockHeight: 10,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey: []byte{2},
+				},
+			},
+			Index: 2,
+		},
+		&dbpb.DepositContainer{
+			Eth1BlockHeight: 11,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey: []byte{3},
+				},
+			},
+			Index: 3,
+		})
+	dc.InsertFinalizedDeposits(context.Background(), 1)
+
+	deps := dc.NonFinalizedDeposits(context.Background(), big.NewInt(10))
+	assert.Equal(t, 1, len(deps))
+}

beacon-chain/cache/depositcache/deposits_test.go

@@ -1,275 +0,0 @@
-package depositcache
-
-import (
-	"bytes"
-	"context"
-	"math/big"
-	"testing"
-
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	dbpb "github.com/prysmaticlabs/prysm/proto/beacon/db"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	logTest "github.com/sirupsen/logrus/hooks/test"
-)
-
-const nilDepositErr = "Ignoring nil deposit insertion"
-
-var _ = DepositFetcher(&DepositCache{})
-
-func TestBeaconDB_InsertDeposit_LogsOnNilDepositInsertion(t *testing.T) {
-	hook := logTest.NewGlobal()
-	dc := DepositCache{}
-
-	dc.InsertDeposit(context.Background(), nil, 1, 0, [32]byte{})
-
-	if len(dc.deposits) != 0 {
-		t.Fatal("Number of deposits changed")
-	}
-	if hook.LastEntry().Message != nilDepositErr {
-		t.Errorf("Did not log correct message, wanted \"Ignoring nil deposit insertion\", got \"%s\"", hook.LastEntry().Message)
-	}
-}
-
-func TestBeaconDB_InsertDeposit_MaintainsSortedOrderByIndex(t *testing.T) {
-	dc := DepositCache{}
-
-	insertions := []struct {
-		blkNum  uint64
-		deposit *ethpb.Deposit
-		index   int64
-	}{
-		{
-			blkNum:  0,
-			deposit: &ethpb.Deposit{},
-			index:   0,
-		},
-		{
-			blkNum:  0,
-			deposit: &ethpb.Deposit{},
-			index:   3,
-		},
-		{
-			blkNum:  0,
-			deposit: &ethpb.Deposit{},
-			index:   1,
-		},
-		{
-			blkNum:  0,
-			deposit: &ethpb.Deposit{},
-			index:   4,
-		},
-	}
-
-	for _, ins := range insertions {
-		dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{})
-	}
-
-	expectedIndices := []int64{0, 1, 3, 4}
-	for i, ei := range expectedIndices {
-		if dc.deposits[i].Index != ei {
-			t.Errorf("dc.deposits[%d].Index = %d, wanted %d", i, dc.deposits[i].Index, ei)
-		}
-	}
-}
-
-func TestBeaconDB_AllDeposits_ReturnsAllDeposits(t *testing.T) {
-	dc := DepositCache{}
-
-	deposits := []*dbpb.DepositContainer{
-		{
-			Eth1BlockHeight: 10,
-			Deposit:         &ethpb.Deposit{},
-		},
-		{
-			Eth1BlockHeight: 10,
-			Deposit:         &ethpb.Deposit{},
-		},
-		{
-			Eth1BlockHeight: 10,
-			Deposit:         &ethpb.Deposit{},
-		},
-		{
-			Eth1BlockHeight: 11,
-			Deposit:         &ethpb.Deposit{},
-		},
-		{
-			Eth1BlockHeight: 11,
-			Deposit:         &ethpb.Deposit{},
-		},
-		{
-			Eth1BlockHeight: 12,
-			Deposit:         &ethpb.Deposit{},
-		},
-		{
-			Eth1BlockHeight: 12,
-			Deposit:         &ethpb.Deposit{},
-		},
-	}
-	dc.deposits = deposits
-
-	d := dc.AllDeposits(context.Background(), nil)
-	if len(d) != len(deposits) {
-		t.Errorf("Return the wrong number of deposits (%d) wanted %d", len(d), len(deposits))
-	}
-}
-
-func TestBeaconDB_AllDeposits_FiltersDepositUpToAndIncludingBlockNumber(t *testing.T) {
-	dc := DepositCache{}
-
-	deposits := []*dbpb.DepositContainer{
-		{
-			Eth1BlockHeight: 10,
-			Deposit:         &ethpb.Deposit{},
-		},
-		{
-			Eth1BlockHeight: 10,
-			Deposit:         &ethpb.Deposit{},
-		},
-		{
-			Eth1BlockHeight: 10,
-			Deposit:         &ethpb.Deposit{},
-		},
-		{
-			Eth1BlockHeight: 11,
-			Deposit:         &ethpb.Deposit{},
-		},
-		{
-			Eth1BlockHeight: 11,
-			Deposit:         &ethpb.Deposit{},
-		},
-		{
-			Eth1BlockHeight: 12,
-			Deposit:         &ethpb.Deposit{},
-		},
-		{
-			Eth1BlockHeight: 12,
-			Deposit:         &ethpb.Deposit{},
-		},
-	}
-	dc.deposits = deposits
-
-	d := dc.AllDeposits(context.Background(), big.NewInt(11))
-	expected := 5
-	if len(d) != expected {
-		t.Errorf("Return the wrong number of deposits (%d) wanted %d", len(d), expected)
-	}
-}
-
-func TestBeaconDB_DepositsNumberAndRootAtHeight_ReturnsAppropriateCountAndRoot(t *testing.T) {
-	dc := DepositCache{}
-
-	dc.deposits = []*dbpb.DepositContainer{
-		{
-			Eth1BlockHeight: 10,
-			Deposit:         &ethpb.Deposit{},
-		},
-		{
-			Eth1BlockHeight: 10,
-			Deposit:         &ethpb.Deposit{},
-		},
-		{
-			Eth1BlockHeight: 10,
-			Deposit:         &ethpb.Deposit{},
-		},
-		{
-			Eth1BlockHeight: 10,
-			Deposit:         &ethpb.Deposit{},
-		},
-		{
-			Eth1BlockHeight: 11,
-			Deposit:         &ethpb.Deposit{},
-			DepositRoot:     []byte("root"),
-		},
-		{
-			Eth1BlockHeight: 12,
-			Deposit:         &ethpb.Deposit{},
-		},
-		{
-			Eth1BlockHeight: 12,
-			Deposit:         &ethpb.Deposit{},
-		},
-	}
-
-	n, root := dc.DepositsNumberAndRootAtHeight(context.Background(), big.NewInt(11))
-	if int(n) != 5 {
-		t.Errorf("Returned unexpected deposits number %d wanted %d", n, 5)
-	}
-
-	if root != bytesutil.ToBytes32([]byte("root")) {
-		t.Errorf("Returned unexpected root: %v", root)
-	}
-}
-
-func TestBeaconDB_DepositsNumberAndRootAtHeight_ReturnsEmptyTrieIfBlockHeightLessThanOldestDeposit(t *testing.T) {
-	dc := DepositCache{}
-
-	dc.deposits = []*dbpb.DepositContainer{
-		{
-			Eth1BlockHeight: 10,
-			Deposit:         &ethpb.Deposit{},
-			DepositRoot:     []byte("root"),
-		},
-		{
-			Eth1BlockHeight: 11,
-			Deposit:         &ethpb.Deposit{},
-			DepositRoot:     []byte("root"),
-		},
-	}
-
-	n, root := dc.DepositsNumberAndRootAtHeight(context.Background(), big.NewInt(2))
-	if int(n) != 0 {
-		t.Errorf("Returned unexpected deposits number %d wanted %d", n, 0)
-	}
-
-	if root != [32]byte{} {
-		t.Errorf("Returned unexpected root: %v", root)
-	}
-}
-
-func TestBeaconDB_DepositByPubkey_ReturnsFirstMatchingDeposit(t *testing.T) {
-	dc := DepositCache{}
-
-	dc.deposits = []*dbpb.DepositContainer{
-		{
-			Eth1BlockHeight: 9,
-			Deposit: &ethpb.Deposit{
-				Data: &ethpb.Deposit_Data{
-					PublicKey: []byte("pk0"),
-				},
-			},
-		},
-		{
-			Eth1BlockHeight: 10,
-			Deposit: &ethpb.Deposit{
-				Data: &ethpb.Deposit_Data{
-					PublicKey: []byte("pk1"),
-				},
-			},
-		},
-		{
-			Eth1BlockHeight: 11,
-			Deposit: &ethpb.Deposit{
-				Data: &ethpb.Deposit_Data{
-					PublicKey: []byte("pk1"),
-				},
-			},
-		},
-		{
-			Eth1BlockHeight: 12,
-			Deposit: &ethpb.Deposit{
-				Data: &ethpb.Deposit_Data{
-					PublicKey: []byte("pk2"),
-				},
-			},
-		},
-	}
-
-	dep, blkNum := dc.DepositByPubkey(context.Background(), []byte("pk1"))
-
-	if !bytes.Equal(dep.Data.PublicKey, []byte("pk1")) {
-		t.Error("Returned wrong deposit")
-	}
-	if blkNum.Cmp(big.NewInt(10)) != 0 {
-		t.Errorf("Returned wrong block number %v", blkNum)
-	}
-}

beacon-chain/cache/depositcache/pending_deposits.go

@@ -24,7 +24,7 @@ var (
 // PendingDepositsFetcher specifically outlines a struct that can retrieve deposits
 // which have not yet been included in the chain.
 type PendingDepositsFetcher interface {
-	PendingContainers(ctx context.Context, beforeBlk *big.Int) []*dbpb.DepositContainer
+	PendingContainers(ctx context.Context, untilBlk *big.Int) []*dbpb.DepositContainer
 }
 
 // InsertPendingDeposit into the database. If deposit or block number are nil
@@ -50,36 +50,23 @@ func (dc *DepositCache) InsertPendingDeposit(ctx context.Context, d *ethpb.Depos
 // PendingDeposits returns a list of deposits until the given block number
 // (inclusive). If no block is specified then this method returns all pending
 // deposits.
-func (dc *DepositCache) PendingDeposits(ctx context.Context, beforeBlk *big.Int) []*ethpb.Deposit {
+func (dc *DepositCache) PendingDeposits(ctx context.Context, untilBlk *big.Int) []*ethpb.Deposit {
 	ctx, span := trace.StartSpan(ctx, "DepositsCache.PendingDeposits")
 	defer span.End()
-	dc.depositsLock.RLock()
-	defer dc.depositsLock.RUnlock()
 
-	var depositCntrs []*dbpb.DepositContainer
-	for _, ctnr := range dc.pendingDeposits {
-		if beforeBlk == nil || beforeBlk.Uint64() >= ctnr.Eth1BlockHeight {
-			depositCntrs = append(depositCntrs, ctnr)
-		}
-	}
-	// Sort the deposits by Merkle index.
-	sort.SliceStable(depositCntrs, func(i, j int) bool {
-		return depositCntrs[i].Index < depositCntrs[j].Index
-	})
+	depositCntrs := dc.PendingContainers(ctx, untilBlk)
 
 	var deposits []*ethpb.Deposit
 	for _, dep := range depositCntrs {
 		deposits = append(deposits, dep.Deposit)
 	}
 
-	span.AddAttributes(trace.Int64Attribute("count", int64(len(deposits))))
-
 	return deposits
 }
 
 // PendingContainers returns a list of deposit containers until the given block number
 // (inclusive).
-func (dc *DepositCache) PendingContainers(ctx context.Context, beforeBlk *big.Int) []*dbpb.DepositContainer {
+func (dc *DepositCache) PendingContainers(ctx context.Context, untilBlk *big.Int) []*dbpb.DepositContainer {
 	ctx, span := trace.StartSpan(ctx, "DepositsCache.PendingDeposits")
 	defer span.End()
 	dc.depositsLock.RLock()
@@ -87,7 +74,7 @@ func (dc *DepositCache) PendingContainers(ctx context.Context, beforeBlk *big.In
 
 	var depositCntrs []*dbpb.DepositContainer
 	for _, ctnr := range dc.pendingDeposits {
-		if beforeBlk == nil || beforeBlk.Uint64() >= ctnr.Eth1BlockHeight {
+		if untilBlk == nil || untilBlk.Uint64() >= ctnr.Eth1BlockHeight {
 			depositCntrs = append(depositCntrs, ctnr)
 		}
 	}
@@ -141,7 +128,7 @@ func (dc *DepositCache) RemovePendingDeposit(ctx context.Context, d *ethpb.Depos
 }
 
 // PrunePendingDeposits removes any deposit which is older than the given deposit merkle tree index.
-func (dc *DepositCache) PrunePendingDeposits(ctx context.Context, merkleTreeIndex int) {
+func (dc *DepositCache) PrunePendingDeposits(ctx context.Context, merkleTreeIndex int64) {
 	ctx, span := trace.StartSpan(ctx, "DepositsCache.PrunePendingDeposits")
 	defer span.End()
 
@@ -155,7 +142,7 @@ func (dc *DepositCache) PrunePendingDeposits(ctx context.Context, merkleTreeInde
 
 	var cleanDeposits []*dbpb.DepositContainer
 	for _, dp := range dc.pendingDeposits {
-		if dp.Index >= int64(merkleTreeIndex) {
+		if dp.Index >= merkleTreeIndex {
 			cleanDeposits = append(cleanDeposits, dp)
 		}
 	}

beacon-chain/cache/depositcache/pending_deposits_test.go

@@ -3,13 +3,14 @@ package depositcache
 import (
 	"context"
 	"math/big"
-	"reflect"
 	"testing"
 
 	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	dbpb "github.com/prysmaticlabs/prysm/proto/beacon/db"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
 )
 
 var _ = PendingDepositsFetcher(&DepositCache{})
@@ -18,25 +19,21 @@ func TestInsertPendingDeposit_OK(t *testing.T) {
 	dc := DepositCache{}
 	dc.InsertPendingDeposit(context.Background(), &ethpb.Deposit{}, 111, 100, [32]byte{})
 
-	if len(dc.pendingDeposits) != 1 {
-		t.Error("Deposit not inserted")
-	}
+	assert.Equal(t, 1, len(dc.pendingDeposits), "Deposit not inserted")
 }
 
 func TestInsertPendingDeposit_ignoresNilDeposit(t *testing.T) {
 	dc := DepositCache{}
 	dc.InsertPendingDeposit(context.Background(), nil /*deposit*/, 0 /*blockNum*/, 0, [32]byte{})
 
-	if len(dc.pendingDeposits) > 0 {
-		t.Error("Unexpected deposit insertion")
-	}
+	assert.Equal(t, 0, len(dc.pendingDeposits))
 }
 
 func TestRemovePendingDeposit_OK(t *testing.T) {
 	db := DepositCache{}
-	proof1 := make([][]byte, 33)
+	proof1 := make([][]byte, int(params.BeaconConfig().DepositContractTreeDepth)+1)
 	proof1[0] = bytesutil.PadTo([]byte{'A'}, 32)
-	proof2 := make([][]byte, 33)
+	proof2 := make([][]byte, int(params.BeaconConfig().DepositContractTreeDepth)+1)
 	proof2[0] = bytesutil.PadTo([]byte{'A'}, 32)
 	depToRemove := &ethpb.Deposit{Proof: proof1}
 	otherDep := &ethpb.Deposit{Proof: proof2}
@@ -55,21 +52,17 @@ func TestRemovePendingDeposit_IgnoresNilDeposit(t *testing.T) {
 	dc := DepositCache{}
 	dc.pendingDeposits = []*dbpb.DepositContainer{{Deposit: &ethpb.Deposit{}}}
 	dc.RemovePendingDeposit(context.Background(), nil /*deposit*/)
-	if len(dc.pendingDeposits) != 1 {
-		t.Errorf("Deposit unexpectedly removed")
-	}
+	assert.Equal(t, 1, len(dc.pendingDeposits), "Deposit unexpectedly removed")
 }
 
 func TestPendingDeposit_RoundTrip(t *testing.T) {
 	dc := DepositCache{}
-	proof := make([][]byte, 33)
+	proof := make([][]byte, int(params.BeaconConfig().DepositContractTreeDepth)+1)
 	proof[0] = bytesutil.PadTo([]byte{'A'}, 32)
 	dep := &ethpb.Deposit{Proof: proof}
 	dc.InsertPendingDeposit(context.Background(), dep, 111, 100, [32]byte{})
 	dc.RemovePendingDeposit(context.Background(), dep)
-	if len(dc.pendingDeposits) != 0 {
-		t.Error("Failed to insert & delete a pending deposit")
-	}
+	assert.Equal(t, 0, len(dc.pendingDeposits), "Failed to insert & delete a pending deposit")
 }
 
 func TestPendingDeposits_OK(t *testing.T) {
@@ -86,15 +79,10 @@ func TestPendingDeposits_OK(t *testing.T) {
 		{Proof: [][]byte{[]byte("A")}},
 		{Proof: [][]byte{[]byte("B")}},
 	}
-
-	if !reflect.DeepEqual(deposits, expected) {
-		t.Errorf("Unexpected deposits. got=%+v want=%+v", deposits, expected)
-	}
+	assert.DeepEqual(t, expected, deposits)
 
 	all := dc.PendingDeposits(context.Background(), nil)
-	if len(all) != len(dc.pendingDeposits) {
-		t.Error("PendingDeposits(ctx, nil) did not return all deposits")
-	}
+	assert.Equal(t, len(dc.pendingDeposits), len(all), "PendingDeposits(ctx, nil) did not return all deposits")
 }
 
 func TestPrunePendingDeposits_ZeroMerkleIndex(t *testing.T) {
@@ -118,9 +106,7 @@ func TestPrunePendingDeposits_ZeroMerkleIndex(t *testing.T) {
 		{Eth1BlockHeight: 10, Index: 10},
 		{Eth1BlockHeight: 12, Index: 12},
 	}
-	if !reflect.DeepEqual(dc.pendingDeposits, expected) {
-		t.Errorf("Unexpected deposits. got=%+v want=%+v", dc.pendingDeposits, expected)
-	}
+	assert.DeepEqual(t, expected, dc.pendingDeposits)
 }
 
 func TestPrunePendingDeposits_OK(t *testing.T) {
@@ -143,9 +129,7 @@ func TestPrunePendingDeposits_OK(t *testing.T) {
 		{Eth1BlockHeight: 12, Index: 12},
 	}
 
-	if !reflect.DeepEqual(dc.pendingDeposits, expected) {
-		t.Errorf("Unexpected deposits. got=%+v want=%+v", dc.pendingDeposits, expected)
-	}
+	assert.DeepEqual(t, expected, dc.pendingDeposits)
 
 	dc.pendingDeposits = []*dbpb.DepositContainer{
 		{Eth1BlockHeight: 2, Index: 2},
@@ -162,8 +146,5 @@ func TestPrunePendingDeposits_OK(t *testing.T) {
 		{Eth1BlockHeight: 12, Index: 12},
 	}
 
-	if !reflect.DeepEqual(dc.pendingDeposits, expected) {
-		t.Errorf("Unexpected deposits. got=%+v want=%+v", dc.pendingDeposits, expected)
-	}
-
+	assert.DeepEqual(t, expected, dc.pendingDeposits)
 }

beacon-chain/cache/doc.go

@@ -0,0 +1,5 @@
+// Package cache includes all important caches for the runtime
+// of an eth2 beacon node, ensuring the node does not spend
+// resources computing duplicate operations such as committee
+// calculations for validators during the same epoch, etc.
+package cache

beacon-chain/cache/eth1_data.go

@@ -1,139 +0,0 @@
-package cache
-
-import (
-	"errors"
-	"sync"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"k8s.io/client-go/tools/cache"
-)
-
-var (
-	// ErrNotEth1DataVote will be returned when a cache object is not a pointer to
-	// a Eth1DataVote struct.
-	ErrNotEth1DataVote = errors.New("object is not a eth1 data vote obj")
-
-	// maxEth1DataVoteSize defines the max number of eth1 data votes can cache.
-	maxEth1DataVoteSize = 1000
-
-	// Metrics.
-	eth1DataVoteCacheMiss = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "eth1_data_vote_cache_miss",
-		Help: "The number of eth1 data vote count requests that aren't present in the cache.",
-	})
-	eth1DataVoteCacheHit = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "eth1_data_vote_cache_hit",
-		Help: "The number of eth1 data vote count requests that are present in the cache.",
-	})
-)
-
-// Eth1DataVote defines the struct which keeps track of the vote count of individual deposit root.
-type Eth1DataVote struct {
-	Eth1DataHash [32]byte
-	VoteCount    uint64
-}
-
-// Eth1DataVoteCache is a struct with 1 queue for looking up eth1 data vote count by deposit root.
-type Eth1DataVoteCache struct {
-	eth1DataVoteCache *cache.FIFO
-	lock              sync.RWMutex
-}
-
-// eth1DataVoteKeyFn takes the eth1data hash as the key for the eth1 data vote count of a given eth1data object.
-func eth1DataVoteKeyFn(obj interface{}) (string, error) {
-	eInfo, ok := obj.(*Eth1DataVote)
-	if !ok {
-		return "", ErrNotEth1DataVote
-	}
-
-	return string(eInfo.Eth1DataHash[:]), nil
-}
-
-// NewEth1DataVoteCache creates a new eth1 data vote count cache for storing/accessing Eth1DataVote.
-func NewEth1DataVoteCache() *Eth1DataVoteCache {
-	return &Eth1DataVoteCache{
-		eth1DataVoteCache: cache.NewFIFO(eth1DataVoteKeyFn),
-	}
-}
-
-// Eth1DataVote fetches eth1 data vote count by the eth1data hash. Returns vote count,
-// if exists. Otherwise returns false, nil.
-func (c *Eth1DataVoteCache) Eth1DataVote(eth1DataHash [32]byte) (uint64, error) {
-	if !featureconfig.Get().EnableEth1DataVoteCache {
-		// Return a miss result if cache is not enabled.
-		eth1DataVoteCacheMiss.Inc()
-		return 0, nil
-	}
-
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-	obj, exists, err := c.eth1DataVoteCache.GetByKey(string(eth1DataHash[:]))
-	if err != nil {
-		return 0, err
-	}
-
-	if exists {
-		eth1DataVoteCacheHit.Inc()
-	} else {
-		eth1DataVoteCacheMiss.Inc()
-		return 0, nil
-	}
-
-	eInfo, ok := obj.(*Eth1DataVote)
-	if !ok {
-		return 0, ErrNotEth1DataVote
-	}
-
-	return eInfo.VoteCount, nil
-}
-
-// AddEth1DataVote adds eth1 data vote object to the cache. This method also trims the least
-// recently added Eth1DataVoteByEpoch object if the cache size has ready the max cache size limit.
-func (c *Eth1DataVoteCache) AddEth1DataVote(eth1DataVote *Eth1DataVote) error {
-	if !featureconfig.Get().EnableEth1DataVoteCache {
-		return nil
-	}
-
-	c.lock.Lock()
-	defer c.lock.Unlock()
-	if err := c.eth1DataVoteCache.Add(eth1DataVote); err != nil {
-		return err
-	}
-
-	trim(c.eth1DataVoteCache, maxEth1DataVoteSize)
-	return nil
-}
-
-// IncrementEth1DataVote increments the existing eth1 data object's vote count by 1,
-// and returns the vote count.
-func (c *Eth1DataVoteCache) IncrementEth1DataVote(eth1DataHash [32]byte) (uint64, error) {
-	if !featureconfig.Get().EnableEth1DataVoteCache {
-		return 0, nil
-	}
-
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-	obj, exists, err := c.eth1DataVoteCache.GetByKey(string(eth1DataHash[:]))
-	if err != nil {
-		return 0, err
-	}
-	if !exists {
-		return 0, errors.New("eth1 data vote object does not exist")
-	}
-
-	eth1DataVoteCacheHit.Inc()
-
-	eInfo, ok := obj.(*Eth1DataVote)
-	if !ok {
-		return 0, errors.New("cached value is not of type *Eth1DataVote")
-	}
-	eInfo.VoteCount++
-
-	if err := c.eth1DataVoteCache.Add(eInfo); err != nil {
-		return 0, err
-	}
-
-	return eInfo.VoteCount, nil
-}

beacon-chain/cache/eth1_data_test.go

@@ -1,113 +0,0 @@
-package cache
-
-import (
-	"strconv"
-	"testing"
-)
-
-func TestEth1DataVoteKeyFn_OK(t *testing.T) {
-	eInfo := &Eth1DataVote{
-		VoteCount:    44,
-		Eth1DataHash: [32]byte{'A'},
-	}
-
-	key, err := eth1DataVoteKeyFn(eInfo)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if key != string(eInfo.Eth1DataHash[:]) {
-		t.Errorf("Incorrect hash key: %s, expected %s", key, string(eInfo.Eth1DataHash[:]))
-	}
-}
-
-func TestEth1DataVoteKeyFn_InvalidObj(t *testing.T) {
-	_, err := eth1DataVoteKeyFn("bad")
-	if err != ErrNotEth1DataVote {
-		t.Errorf("Expected error %v, got %v", ErrNotEth1DataVote, err)
-	}
-}
-
-func TestEth1DataVoteCache_CanAdd(t *testing.T) {
-	cache := NewEth1DataVoteCache()
-
-	eInfo := &Eth1DataVote{
-		VoteCount:    55,
-		Eth1DataHash: [32]byte{'B'},
-	}
-	count, err := cache.Eth1DataVote(eInfo.Eth1DataHash)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if count != 0 {
-		t.Error("Expected seed not to exist in empty cache")
-	}
-
-	if err := cache.AddEth1DataVote(eInfo); err != nil {
-		t.Fatal(err)
-	}
-	count, err = cache.Eth1DataVote(eInfo.Eth1DataHash)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if count != eInfo.VoteCount {
-		t.Errorf(
-			"Expected vote count to be %d, got %d",
-			eInfo.VoteCount,
-			count,
-		)
-	}
-}
-
-func TestEth1DataVoteCache_CanIncrement(t *testing.T) {
-	cache := NewEth1DataVoteCache()
-
-	eInfo := &Eth1DataVote{
-		VoteCount:    55,
-		Eth1DataHash: [32]byte{'B'},
-	}
-
-	if err := cache.AddEth1DataVote(eInfo); err != nil {
-		t.Fatal(err)
-	}
-
-	_, err := cache.IncrementEth1DataVote(eInfo.Eth1DataHash)
-	if err != nil {
-		t.Fatal(err)
-	}
-	_, err = cache.IncrementEth1DataVote(eInfo.Eth1DataHash)
-	if err != nil {
-		t.Error(err)
-	}
-	count, err := cache.IncrementEth1DataVote(eInfo.Eth1DataHash)
-
-	if count != 58 {
-		t.Errorf(
-			"Expected vote count to be %d, got %d",
-			58,
-			count,
-		)
-	}
-}
-
-func TestEth1Data_MaxSize(t *testing.T) {
-	cache := NewEth1DataVoteCache()
-
-	for i := 0; i < maxEth1DataVoteSize+1; i++ {
-		var hash [32]byte
-		copy(hash[:], strconv.Itoa(i))
-		eInfo := &Eth1DataVote{
-			Eth1DataHash: hash,
-		}
-		if err := cache.AddEth1DataVote(eInfo); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	if len(cache.eth1DataVoteCache.ListKeys()) != maxEth1DataVoteSize {
-		t.Errorf(
-			"Expected hash cache key size to be %d, got %d",
-			maxEth1DataVoteSize,
-			len(cache.eth1DataVoteCache.ListKeys()),
-		)
-	}
-}

beacon-chain/cache/feature_flag_test.go

@@ -1,9 +1,14 @@
 package cache
 
-import "github.com/prysmaticlabs/prysm/shared/featureconfig"
+import (
+	"os"
+	"testing"
 
-func init() {
-	featureconfig.Init(&featureconfig.Flags{
-		EnableEth1DataVoteCache: true,
-	})
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+)
+
+func TestMain(m *testing.M) {
+	resetCfg := featureconfig.InitWithReset(&featureconfig.Flags{EnableEth1DataVoteCache: true})
+	defer resetCfg()
+	os.Exit(m.Run())
 }

beacon-chain/cache/hot_state_cache.go

@@ -1,6 +1,8 @@
 package cache
 
 import (
+	"sync"
+
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
@@ -9,7 +11,7 @@ import (
 
 var (
 	// hotStateCacheSize defines the max number of hot state this can cache.
-	hotStateCacheSize = 16
+	hotStateCacheSize = 32
 	// Metrics
 	hotStateCacheHit = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "hot_state_cache_hit",
@@ -24,6 +26,7 @@ var (
 // HotStateCache is used to store the processed beacon state after finalized check point..
 type HotStateCache struct {
 	cache *lru.Cache
+	lock  sync.RWMutex
 }
 
 // NewHotStateCache initializes the map and underlying cache.
@@ -40,6 +43,8 @@ func NewHotStateCache() *HotStateCache {
 // Get returns a cached response via input block root, if any.
 // The response is copied by default.
 func (c *HotStateCache) Get(root [32]byte) *stateTrie.BeaconState {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
 	item, exists := c.cache.Get(root)
 
 	if exists && item != nil {
@@ -50,12 +55,36 @@ func (c *HotStateCache) Get(root [32]byte) *stateTrie.BeaconState {
 	return nil
 }
 
+// GetWithoutCopy returns a non-copied cached response via input block root.
+func (c *HotStateCache) GetWithoutCopy(root [32]byte) *stateTrie.BeaconState {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	item, exists := c.cache.Get(root)
+	if exists && item != nil {
+		hotStateCacheHit.Inc()
+		return item.(*stateTrie.BeaconState)
+	}
+	hotStateCacheMiss.Inc()
+	return nil
+}
+
 // Put the response in the cache.
 func (c *HotStateCache) Put(root [32]byte, state *stateTrie.BeaconState) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
 	c.cache.Add(root, state)
 }
 
 // Has returns true if the key exists in the cache.
 func (c *HotStateCache) Has(root [32]byte) bool {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
 	return c.cache.Contains(root)
 }
+
+// Delete deletes the key exists in the cache.
+func (c *HotStateCache) Delete(root [32]byte) bool {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return c.cache.Remove(root)
+}

beacon-chain/cache/hot_state_cache_test.go

@@ -1,41 +1,34 @@
 package cache_test
 
 import (
-	"reflect"
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 )
 
 func TestHotStateCache_RoundTrip(t *testing.T) {
 	c := cache.NewHotStateCache()
 	root := [32]byte{'A'}
 	state := c.Get(root)
-	if state != nil {
-		t.Errorf("Empty cache returned an object: %v", state)
-	}
-	if c.Has(root) {
-		t.Error("Empty cache has an object")
-	}
+	assert.Equal(t, (*stateTrie.BeaconState)(nil), state)
+	assert.Equal(t, false, c.Has(root), "Empty cache has an object")
 
 	state, err := stateTrie.InitializeFromProto(&pb.BeaconState{
 		Slot: 10,
 	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	c.Put(root, state)
+	require.NoError(t, err)
+
+	c.Put(root, state)
+	assert.Equal(t, true, c.Has(root), "Empty cache does not have an object")
 
-	if !c.Has(root) {
-		t.Error("Empty cache does not have an object")
-	}
 	res := c.Get(root)
-	if state == nil {
-		t.Errorf("Empty cache returned an object: %v", state)
-	}
-	if !reflect.DeepEqual(state.CloneInnerState(), res.CloneInnerState()) {
-		t.Error("Expected equal protos to return from cache")
-	}
+	assert.NotNil(t, state)
+	assert.DeepEqual(t, res.CloneInnerState(), state.CloneInnerState(), "Expected equal protos to return from cache")
+
+	c.Delete(root)
+	assert.Equal(t, false, c.Has(root), "Cache not supposed to have the object")
 }

beacon-chain/cache/skip_slot_cache_test.go

@@ -2,12 +2,13 @@ package cache_test
 
 import (
 	"context"
-	"reflect"
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 )
 
 func TestSkipSlotCache_RoundTrip(t *testing.T) {
@@ -15,39 +16,20 @@ func TestSkipSlotCache_RoundTrip(t *testing.T) {
 	c := cache.NewSkipSlotCache()
 
 	state, err := c.Get(ctx, 5)
-	if err != nil {
-		t.Error(err)
-	}
+	require.NoError(t, err)
+	assert.Equal(t, (*stateTrie.BeaconState)(nil), state, "Empty cache returned an object")
 
-	if state != nil {
-		t.Errorf("Empty cache returned an object: %v", state)
-	}
-
-	if err := c.MarkInProgress(5); err != nil {
-		t.Error(err)
-	}
+	require.NoError(t, c.MarkInProgress(5))
 
 	state, err = stateTrie.InitializeFromProto(&pb.BeaconState{
 		Slot: 10,
 	})
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
-	if err = c.Put(ctx, 5, state); err != nil {
-		t.Error(err)
-	}
-
-	if err := c.MarkNotInProgress(5); err != nil {
-		t.Error(err)
-	}
+	require.NoError(t, c.Put(ctx, 5, state))
+	require.NoError(t, c.MarkNotInProgress(5))
 
 	res, err := c.Get(ctx, 5)
-	if err != nil {
-		t.Error(err)
-	}
-
-	if !reflect.DeepEqual(state.CloneInnerState(), res.CloneInnerState()) {
-		t.Error("Expected equal protos to return from cache")
-	}
+	require.NoError(t, err)
+	assert.DeepEqual(t, res.CloneInnerState(), state.CloneInnerState(), "Expected equal protos to return from cache")
 }

beacon-chain/cache/subnet_ids.go

@@ -0,0 +1,134 @@
+package cache
+
+import (
+	"sync"
+	"time"
+
+	lru "github.com/hashicorp/golang-lru"
+	"github.com/patrickmn/go-cache"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/sliceutil"
+)
+
+type subnetIDs struct {
+	attester          *lru.Cache
+	attesterLock      sync.RWMutex
+	aggregator        *lru.Cache
+	aggregatorLock    sync.RWMutex
+	persistentSubnets *cache.Cache
+	subnetsLock       sync.RWMutex
+}
+
+// SubnetIDs for attester and aggregator.
+var SubnetIDs = newSubnetIDs()
+
+func newSubnetIDs() *subnetIDs {
+	// Given a node can calculate committee assignments of current epoch and next epoch.
+	// Max size is set to 2 epoch length.
+	cacheSize := params.BeaconConfig().MaxCommitteesPerSlot * params.BeaconConfig().SlotsPerEpoch * 2
+	attesterCache, err := lru.New(int(cacheSize))
+	if err != nil {
+		panic(err)
+	}
+	aggregatorCache, err := lru.New(int(cacheSize))
+	if err != nil {
+		panic(err)
+	}
+	epochDuration := time.Duration(params.BeaconConfig().SlotsPerEpoch * params.BeaconConfig().SecondsPerSlot)
+	subLength := epochDuration * time.Duration(params.BeaconNetworkConfig().EpochsPerRandomSubnetSubscription)
+	persistentCache := cache.New(subLength*time.Second, epochDuration*time.Second)
+	return &subnetIDs{attester: attesterCache, aggregator: aggregatorCache, persistentSubnets: persistentCache}
+}
+
+// AddAttesterSubnetID adds the subnet index for subscribing subnet for the attester of a given slot.
+func (c *subnetIDs) AddAttesterSubnetID(slot uint64, subnetID uint64) {
+	c.attesterLock.Lock()
+	defer c.attesterLock.Unlock()
+
+	ids := []uint64{subnetID}
+	val, exists := c.attester.Get(slot)
+	if exists {
+		ids = sliceutil.UnionUint64(append(val.([]uint64), ids...))
+	}
+	c.attester.Add(slot, ids)
+}
+
+// GetAttesterSubnetIDs gets the subnet IDs for subscribed subnets for attesters of the slot.
+func (c *subnetIDs) GetAttesterSubnetIDs(slot uint64) []uint64 {
+	c.attesterLock.RLock()
+	defer c.attesterLock.RUnlock()
+
+	val, exists := c.attester.Get(slot)
+	if !exists {
+		return nil
+	}
+	if v, ok := val.([]uint64); ok {
+		return v
+	}
+	return nil
+}
+
+// AddAggregatorSubnetID adds the subnet ID for subscribing subnet for the aggregator of a given slot.
+func (c *subnetIDs) AddAggregatorSubnetID(slot uint64, subnetID uint64) {
+	c.aggregatorLock.Lock()
+	defer c.aggregatorLock.Unlock()
+
+	ids := []uint64{subnetID}
+	val, exists := c.aggregator.Get(slot)
+	if exists {
+		ids = sliceutil.UnionUint64(append(val.([]uint64), ids...))
+	}
+	c.aggregator.Add(slot, ids)
+}
+
+// GetAggregatorSubnetIDs gets the subnet IDs for subscribing subnet for aggregator of the slot.
+func (c *subnetIDs) GetAggregatorSubnetIDs(slot uint64) []uint64 {
+	c.aggregatorLock.RLock()
+	defer c.aggregatorLock.RUnlock()
+
+	val, exists := c.aggregator.Get(slot)
+	if !exists {
+		return []uint64{}
+	}
+	return val.([]uint64)
+}
+
+// GetPersistentSubnets retrieves the persistent subnet and expiration time of that validator's
+// subscription.
+func (c *subnetIDs) GetPersistentSubnets(pubkey []byte) ([]uint64, bool, time.Time) {
+	c.subnetsLock.RLock()
+	defer c.subnetsLock.RUnlock()
+
+	id, duration, ok := c.persistentSubnets.GetWithExpiration(string(pubkey))
+	if !ok {
+		return []uint64{}, ok, time.Time{}
+	}
+	return id.([]uint64), ok, duration
+}
+
+// GetAllSubnets retrieves all the non-expired subscribed subnets of all the validators
+// in the cache.
+func (c *subnetIDs) GetAllSubnets() []uint64 {
+	c.subnetsLock.RLock()
+	defer c.subnetsLock.RUnlock()
+
+	itemsMap := c.persistentSubnets.Items()
+	committees := []uint64{}
+
+	for _, v := range itemsMap {
+		if v.Expired() {
+			continue
+		}
+		committees = append(committees, v.Object.([]uint64)...)
+	}
+	return sliceutil.SetUint64(committees)
+}
+
+// AddPersistentCommittee adds the relevant committee for that particular validator along with its
+// expiration period.
+func (c *subnetIDs) AddPersistentCommittee(pubkey []byte, comIndex []uint64, duration time.Duration) {
+	c.subnetsLock.Lock()
+	defer c.subnetsLock.Unlock()
+
+	c.persistentSubnets.Set(string(pubkey), comIndex, duration)
+}

beacon-chain/cache/subnet_ids_test.go

@@ -0,0 +1,66 @@
+package cache
+
+import (
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+)
+
+func TestSubnetIDsCache_RoundTrip(t *testing.T) {
+	c := newSubnetIDs()
+	slot := uint64(100)
+	committeeIDs := c.GetAggregatorSubnetIDs(slot)
+	assert.Equal(t, 0, len(committeeIDs), "Empty cache returned an object")
+
+	c.AddAggregatorSubnetID(slot, 1)
+	res := c.GetAggregatorSubnetIDs(slot)
+	assert.DeepEqual(t, []uint64{1}, res)
+
+	c.AddAggregatorSubnetID(slot, 2)
+	res = c.GetAggregatorSubnetIDs(slot)
+	assert.DeepEqual(t, []uint64{1, 2}, res)
+
+	c.AddAggregatorSubnetID(slot, 3)
+	res = c.GetAggregatorSubnetIDs(slot)
+	assert.DeepEqual(t, []uint64{1, 2, 3}, res)
+
+	committeeIDs = c.GetAttesterSubnetIDs(slot)
+	assert.Equal(t, 0, len(committeeIDs), "Empty cache returned an object")
+
+	c.AddAttesterSubnetID(slot, 11)
+	res = c.GetAttesterSubnetIDs(slot)
+	assert.DeepEqual(t, []uint64{11}, res)
+
+	c.AddAttesterSubnetID(slot, 22)
+	res = c.GetAttesterSubnetIDs(slot)
+	assert.DeepEqual(t, []uint64{11, 22}, res)
+
+	c.AddAttesterSubnetID(slot, 33)
+	res = c.GetAttesterSubnetIDs(slot)
+	assert.DeepEqual(t, []uint64{11, 22, 33}, res)
+}
+
+func TestSubnetIDsCache_PersistentCommitteeRoundtrip(t *testing.T) {
+	pubkeySet := [][48]byte{}
+	c := newSubnetIDs()
+
+	for i := 0; i < 20; i++ {
+		pubkey := [48]byte{byte(i)}
+		pubkeySet = append(pubkeySet, pubkey)
+		c.AddPersistentCommittee(pubkey[:], []uint64{uint64(i)}, 0)
+	}
+
+	for i := uint64(0); i < 20; i++ {
+		pubkey := [48]byte{byte(i)}
+
+		idxs, ok, _ := c.GetPersistentSubnets(pubkey[:])
+		if !ok {
+			t.Errorf("Couldn't find entry in cache for pubkey %#x", pubkey)
+			continue
+		}
+		require.Equal(t, i, idxs[0])
+	}
+	coms := c.GetAllSubnets()
+	assert.Equal(t, 20, len(coms))
+}

beacon-chain/core/blocks/BUILD.bazel

@@ -1,18 +1,28 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_library(
     name = "go_default_library",
     srcs = [
-        "block.go",
-        "block_operations.go",
+        "attestation.go",
+        "attester_slashing.go",
+        "deposit.go",
+        "eth1_data.go",
+        "exit.go",
+        "genesis.go",
+        "header.go",
+        "log.go",
+        "proposer_slashing.go",
+        "randao.go",
+        "signature.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/core/blocks",
     visibility = [
         "//beacon-chain:__subpackages__",
+        "//fuzz:__pkg__",
         "//shared/testutil:__pkg__",
     ],
     deps = [
-        "//beacon-chain/cache:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/validators:go_default_library",
         "//beacon-chain/state:go_default_library",
@@ -21,7 +31,7 @@ go_library(
         "//shared/attestationutil:go_default_library",
         "//shared/bls:go_default_library",
         "//shared/bytesutil:go_default_library",
-        "//shared/featureconfig:go_default_library",
+        "//shared/depositutil:go_default_library",
         "//shared/hashutil:go_default_library",
         "//shared/mathutil:go_default_library",
         "//shared/params:go_default_library",
@@ -40,11 +50,17 @@ go_test(
     name = "go_default_test",
     size = "medium",
     srcs = [
+        "attestation_test.go",
+        "attester_slashing_test.go",
         "block_operations_fuzz_test.go",
-        "block_operations_test.go",
         "block_regression_test.go",
-        "block_test.go",
+        "deposit_test.go",
         "eth1_data_test.go",
+        "exit_test.go",
+        "genesis_test.go",
+        "header_test.go",
+        "proposer_slashing_test.go",
+        "randao_test.go",
     ],
     embed = [":go_default_library"],
     deps = [
@@ -52,11 +68,15 @@ go_test(
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/stateutil:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
+        "//shared/aggregation:go_default_library",
+        "//shared/aggregation/attestations:go_default_library",
         "//shared/attestationutil:go_default_library",
         "//shared/bls:go_default_library",
         "//shared/bytesutil:go_default_library",
         "//shared/params:go_default_library",
         "//shared/testutil:go_default_library",
+        "//shared/testutil/assert:go_default_library",
+        "//shared/testutil/require:go_default_library",
         "//shared/trieutil:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
         "@com_github_google_gofuzz//:go_default_library",
@@ -64,6 +84,5 @@ go_test(
         "@com_github_prysmaticlabs_go_bitfield//:go_default_library",
         "@com_github_prysmaticlabs_go_ssz//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
-        "@in_gopkg_d4l3k_messagediff_v1//:go_default_library",
     ],
 )

beacon-chain/core/blocks/attestation.go

@@ -0,0 +1,327 @@
+package blocks
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/attestationutil"
+	"github.com/prysmaticlabs/prysm/shared/bls"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"go.opencensus.io/trace"
+)
+
+// ProcessAttestations applies processing operations to a block's inner attestation
+// records.
+func ProcessAttestations(
+	ctx context.Context,
+	beaconState *stateTrie.BeaconState,
+	body *ethpb.BeaconBlockBody,
+) (*stateTrie.BeaconState, error) {
+	var err error
+	for idx, attestation := range body.Attestations {
+		beaconState, err = ProcessAttestation(ctx, beaconState, attestation)
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not verify attestation at index %d in block", idx)
+		}
+	}
+	return beaconState, nil
+}
+
+// ProcessAttestation verifies an input attestation can pass through processing using the given beacon state.
+//
+// Spec pseudocode definition:
+//  def process_attestation(state: BeaconState, attestation: Attestation) -> None:
+//    data = attestation.data
+//    assert data.target.epoch in (get_previous_epoch(state), get_current_epoch(state))
+//    assert data.target.epoch == compute_epoch_at_slot(data.slot)
+//    assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= data.slot + SLOTS_PER_EPOCH
+//    assert data.index < get_committee_count_per_slot(state, data.target.epoch)
+//
+//    committee = get_beacon_committee(state, data.slot, data.index)
+//    assert len(attestation.aggregation_bits) == len(committee)
+//
+//    pending_attestation = PendingAttestation(
+//        data=data,
+//        aggregation_bits=attestation.aggregation_bits,
+//        inclusion_delay=state.slot - data.slot,
+//        proposer_index=get_beacon_proposer_index(state),
+//    )
+//
+//    if data.target.epoch == get_current_epoch(state):
+//        assert data.source == state.current_justified_checkpoint
+//        state.current_epoch_attestations.append(pending_attestation)
+//    else:
+//        assert data.source == state.previous_justified_checkpoint
+//        state.previous_epoch_attestations.append(pending_attestation)
+//
+//    # Check signature
+//    assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation))
+func ProcessAttestation(
+	ctx context.Context,
+	beaconState *stateTrie.BeaconState,
+	att *ethpb.Attestation,
+) (*stateTrie.BeaconState, error) {
+	beaconState, err := ProcessAttestationNoVerify(ctx, beaconState, att)
+	if err != nil {
+		return nil, err
+	}
+	return beaconState, VerifyAttestation(ctx, beaconState, att)
+}
+
+// ProcessAttestationsNoVerify applies processing operations to a block's inner attestation
+// records. The only difference would be that the attestation signature would not be verified.
+func ProcessAttestationsNoVerify(
+	ctx context.Context,
+	beaconState *stateTrie.BeaconState,
+	body *ethpb.BeaconBlockBody,
+) (*stateTrie.BeaconState, error) {
+	var err error
+	for idx, attestation := range body.Attestations {
+		beaconState, err = ProcessAttestationNoVerify(ctx, beaconState, attestation)
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not verify attestation at index %d in block", idx)
+		}
+	}
+	return beaconState, nil
+}
+
+// ProcessAttestationNoVerify processes the attestation without verifying the attestation signature. This
+// method is used to validate attestations whose signatures have already been verified.
+func ProcessAttestationNoVerify(
+	ctx context.Context,
+	beaconState *stateTrie.BeaconState,
+	att *ethpb.Attestation,
+) (*stateTrie.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "core.ProcessAttestationNoVerify")
+	defer span.End()
+
+	if att == nil || att.Data == nil || att.Data.Target == nil {
+		return nil, errors.New("nil attestation data target")
+	}
+
+	currEpoch := helpers.SlotToEpoch(beaconState.Slot())
+	var prevEpoch uint64
+	if currEpoch == 0 {
+		prevEpoch = 0
+	} else {
+		prevEpoch = currEpoch - 1
+	}
+	data := att.Data
+	if data.Target.Epoch != prevEpoch && data.Target.Epoch != currEpoch {
+		return nil, fmt.Errorf(
+			"expected target epoch (%d) to be the previous epoch (%d) or the current epoch (%d)",
+			data.Target.Epoch,
+			prevEpoch,
+			currEpoch,
+		)
+	}
+	if helpers.SlotToEpoch(data.Slot) != data.Target.Epoch {
+		return nil, fmt.Errorf("data slot is not in the same epoch as target %d != %d", helpers.SlotToEpoch(data.Slot), data.Target.Epoch)
+	}
+
+	s := att.Data.Slot
+	minInclusionCheck := s+params.BeaconConfig().MinAttestationInclusionDelay <= beaconState.Slot()
+	epochInclusionCheck := beaconState.Slot() <= s+params.BeaconConfig().SlotsPerEpoch
+	if !minInclusionCheck {
+		return nil, fmt.Errorf(
+			"attestation slot %d + inclusion delay %d > state slot %d",
+			s,
+			params.BeaconConfig().MinAttestationInclusionDelay,
+			beaconState.Slot(),
+		)
+	}
+	if !epochInclusionCheck {
+		return nil, fmt.Errorf(
+			"state slot %d > attestation slot %d + SLOTS_PER_EPOCH %d",
+			beaconState.Slot(),
+			s,
+			params.BeaconConfig().SlotsPerEpoch,
+		)
+	}
+	activeValidatorCount, err := helpers.ActiveValidatorCount(beaconState, att.Data.Target.Epoch)
+	if err != nil {
+		return nil, err
+	}
+	c := helpers.SlotCommitteeCount(activeValidatorCount)
+	if att.Data.CommitteeIndex > c {
+		return nil, fmt.Errorf("committee index %d >= committee count %d", att.Data.CommitteeIndex, c)
+	}
+
+	if err := helpers.VerifyAttestationBitfieldLengths(beaconState, att); err != nil {
+		return nil, errors.Wrap(err, "could not verify attestation bitfields")
+	}
+
+	proposerIndex, err := helpers.BeaconProposerIndex(beaconState)
+	if err != nil {
+		return nil, err
+	}
+	pendingAtt := &pb.PendingAttestation{
+		Data:            data,
+		AggregationBits: att.AggregationBits,
+		InclusionDelay:  beaconState.Slot() - s,
+		ProposerIndex:   proposerIndex,
+	}
+
+	var ffgSourceEpoch uint64
+	var ffgSourceRoot []byte
+	var ffgTargetEpoch uint64
+	if data.Target.Epoch == currEpoch {
+		ffgSourceEpoch = beaconState.CurrentJustifiedCheckpoint().Epoch
+		ffgSourceRoot = beaconState.CurrentJustifiedCheckpoint().Root
+		ffgTargetEpoch = currEpoch
+		if err := beaconState.AppendCurrentEpochAttestations(pendingAtt); err != nil {
+			return nil, err
+		}
+	} else {
+		ffgSourceEpoch = beaconState.PreviousJustifiedCheckpoint().Epoch
+		ffgSourceRoot = beaconState.PreviousJustifiedCheckpoint().Root
+		ffgTargetEpoch = prevEpoch
+		if err := beaconState.AppendPreviousEpochAttestations(pendingAtt); err != nil {
+			return nil, err
+		}
+	}
+	if data.Source.Epoch != ffgSourceEpoch {
+		return nil, fmt.Errorf("expected source epoch %d, received %d", ffgSourceEpoch, data.Source.Epoch)
+	}
+	if !bytes.Equal(data.Source.Root, ffgSourceRoot) {
+		return nil, fmt.Errorf("expected source root %#x, received %#x", ffgSourceRoot, data.Source.Root)
+	}
+	if data.Target.Epoch != ffgTargetEpoch {
+		return nil, fmt.Errorf("expected target epoch %d, received %d", ffgTargetEpoch, data.Target.Epoch)
+	}
+
+	return beaconState, nil
+}
+
+// VerifyAttestations will verify the signatures of the provided attestations. This method performs
+// a single BLS verification call to verify the signatures of all of the provided attestations. All
+// of the provided attestations must have valid signatures or this method will return an error.
+// This method does not determine which attestation signature is invalid, only that one or more
+// attestation signatures were not valid.
+func VerifyAttestations(ctx context.Context, beaconState *stateTrie.BeaconState, atts []*ethpb.Attestation) error {
+	ctx, span := trace.StartSpan(ctx, "core.VerifyAttestations")
+	defer span.End()
+	span.AddAttributes(trace.Int64Attribute("attestations", int64(len(atts))))
+
+	if len(atts) == 0 {
+		return nil
+	}
+
+	fork := beaconState.Fork()
+	gvr := beaconState.GenesisValidatorRoot()
+	dt := params.BeaconConfig().DomainBeaconAttester
+
+	// Split attestations by fork. Note: the signature domain will differ based on the fork.
+	var preForkAtts []*ethpb.Attestation
+	var postForkAtts []*ethpb.Attestation
+	for _, a := range atts {
+		if helpers.SlotToEpoch(a.Data.Slot) < fork.Epoch {
+			preForkAtts = append(preForkAtts, a)
+		} else {
+			postForkAtts = append(postForkAtts, a)
+		}
+	}
+
+	// Check attestations from before the fork.
+	if fork.Epoch > 0 { // Check to prevent underflow.
+		prevDomain, err := helpers.Domain(fork, fork.Epoch-1, dt, gvr)
+		if err != nil {
+			return err
+		}
+		if err := verifyAttestationsWithDomain(ctx, beaconState, preForkAtts, prevDomain); err != nil {
+			return err
+		}
+	} else if len(preForkAtts) > 0 {
+		// This is a sanity check that preForkAtts were not ignored when fork.Epoch == 0. This
+		// condition is not possible, but it doesn't hurt to check anyway.
+		return errors.New("some attestations were not verified from previous fork before genesis")
+	}
+
+	// Then check attestations from after the fork.
+	currDomain, err := helpers.Domain(fork, fork.Epoch, dt, gvr)
+	if err != nil {
+		return err
+	}
+
+	return verifyAttestationsWithDomain(ctx, beaconState, postForkAtts, currDomain)
+}
+
+// VerifyAttestation converts and attestation into an indexed attestation and verifies
+// the signature in that attestation.
+func VerifyAttestation(ctx context.Context, beaconState *stateTrie.BeaconState, att *ethpb.Attestation) error {
+	if att == nil || att.Data == nil || att.AggregationBits.Count() == 0 {
+		return fmt.Errorf("nil or missing attestation data: %v", att)
+	}
+	committee, err := helpers.BeaconCommitteeFromState(beaconState, att.Data.Slot, att.Data.CommitteeIndex)
+	if err != nil {
+		return err
+	}
+	indexedAtt := attestationutil.ConvertToIndexed(ctx, att, committee)
+	return VerifyIndexedAttestation(ctx, beaconState, indexedAtt)
+}
+
+// VerifyIndexedAttestation determines the validity of an indexed attestation.
+//
+// Spec pseudocode definition:
+//  def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool:
+//    """
+//    Check if ``indexed_attestation`` is not empty, has sorted and unique indices and has a valid aggregate signature.
+//    """
+//    # Verify indices are sorted and unique
+//    indices = indexed_attestation.attesting_indices
+//    if len(indices) == 0 or not indices == sorted(set(indices)):
+//        return False
+//    # Verify aggregate signature
+//    pubkeys = [state.validators[i].pubkey for i in indices]
+//    domain = get_domain(state, DOMAIN_BEACON_ATTESTER, indexed_attestation.data.target.epoch)
+//    signing_root = compute_signing_root(indexed_attestation.data, domain)
+//    return bls.FastAggregateVerify(pubkeys, signing_root, indexed_attestation.signature)
+func VerifyIndexedAttestation(ctx context.Context, beaconState *stateTrie.BeaconState, indexedAtt *ethpb.IndexedAttestation) error {
+	ctx, span := trace.StartSpan(ctx, "core.VerifyIndexedAttestation")
+	defer span.End()
+
+	if err := attestationutil.IsValidAttestationIndices(ctx, indexedAtt); err != nil {
+		return err
+	}
+	domain, err := helpers.Domain(beaconState.Fork(), indexedAtt.Data.Target.Epoch, params.BeaconConfig().DomainBeaconAttester, beaconState.GenesisValidatorRoot())
+	if err != nil {
+		return err
+	}
+	indices := indexedAtt.AttestingIndices
+	pubkeys := []bls.PublicKey{}
+	for i := 0; i < len(indices); i++ {
+		pubkeyAtIdx := beaconState.PubkeyAtIndex(indices[i])
+		pk, err := bls.PublicKeyFromBytes(pubkeyAtIdx[:])
+		if err != nil {
+			return errors.Wrap(err, "could not deserialize validator public key")
+		}
+		pubkeys = append(pubkeys, pk)
+	}
+	return attestationutil.VerifyIndexedAttestationSig(ctx, indexedAtt, pubkeys, domain)
+}
+
+// Inner method to verify attestations. This abstraction allows for the domain to be provided as an
+// argument.
+func verifyAttestationsWithDomain(ctx context.Context, beaconState *stateTrie.BeaconState, atts []*ethpb.Attestation, domain []byte) error {
+	if len(atts) == 0 {
+		return nil
+	}
+	set, err := createAttestationSignatureSet(ctx, beaconState, atts, domain)
+	if err != nil {
+		return err
+	}
+	verify, err := bls.VerifyMultipleSignatures(set.Signatures, set.Messages, set.PublicKeys)
+	if err != nil {
+		return errors.Errorf("got error in multiple verification: %v", err)
+	}
+	if !verify {
+		return errors.New("one or more attestation signatures did not verify")
+	}
+	return nil
+}

beacon-chain/core/blocks/attestation_test.go

@@ -0,0 +1,859 @@
+package blocks_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/go-bitfield"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/aggregation"
+	attaggregation "github.com/prysmaticlabs/prysm/shared/aggregation/attestations"
+	"github.com/prysmaticlabs/prysm/shared/attestationutil"
+	"github.com/prysmaticlabs/prysm/shared/bls"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+)
+
+func TestProcessAttestations_InclusionDelayFailure(t *testing.T) {
+	attestations := []*ethpb.Attestation{
+		{
+			Data: &ethpb.AttestationData{
+				Target: &ethpb.Checkpoint{Epoch: 0},
+				Slot:   5,
+			},
+		},
+	}
+	block := &ethpb.BeaconBlock{
+		Body: &ethpb.BeaconBlockBody{
+			Attestations: attestations,
+		},
+	}
+	beaconState, _ := testutil.DeterministicGenesisState(t, 100)
+
+	want := fmt.Sprintf(
+		"attestation slot %d + inclusion delay %d > state slot %d",
+		attestations[0].Data.Slot,
+		params.BeaconConfig().MinAttestationInclusionDelay,
+		beaconState.Slot(),
+	)
+	_, err := blocks.ProcessAttestations(context.Background(), beaconState, block.Body)
+	assert.ErrorContains(t, want, err)
+}
+
+func TestProcessAttestations_NeitherCurrentNorPrevEpoch(t *testing.T) {
+	att := &ethpb.Attestation{
+		Data: &ethpb.AttestationData{
+			Source: &ethpb.Checkpoint{Epoch: 0, Root: []byte("hello-world")},
+			Target: &ethpb.Checkpoint{Epoch: 0}}}
+
+	block := &ethpb.BeaconBlock{
+		Body: &ethpb.BeaconBlockBody{
+			Attestations: []*ethpb.Attestation{att},
+		},
+	}
+	beaconState, _ := testutil.DeterministicGenesisState(t, 100)
+	err := beaconState.SetSlot(beaconState.Slot() + params.BeaconConfig().SlotsPerEpoch*4 + params.BeaconConfig().MinAttestationInclusionDelay)
+	require.NoError(t, err)
+	pfc := beaconState.PreviousJustifiedCheckpoint()
+	pfc.Root = []byte("hello-world")
+	require.NoError(t, beaconState.SetPreviousJustifiedCheckpoint(pfc))
+	require.NoError(t, beaconState.SetPreviousEpochAttestations([]*pb.PendingAttestation{}))
+
+	want := fmt.Sprintf(
+		"expected target epoch (%d) to be the previous epoch (%d) or the current epoch (%d)",
+		att.Data.Target.Epoch,
+		helpers.PrevEpoch(beaconState),
+		helpers.CurrentEpoch(beaconState),
+	)
+	_, err = blocks.ProcessAttestations(context.Background(), beaconState, block.Body)
+	assert.ErrorContains(t, want, err)
+}
+
+func TestProcessAttestations_CurrentEpochFFGDataMismatches(t *testing.T) {
+	aggBits := bitfield.NewBitlist(3)
+	attestations := []*ethpb.Attestation{
+		{
+			Data: &ethpb.AttestationData{
+				Target: &ethpb.Checkpoint{Epoch: 0},
+				Source: &ethpb.Checkpoint{Epoch: 1},
+			},
+			AggregationBits: aggBits,
+		},
+	}
+	block := &ethpb.BeaconBlock{
+		Body: &ethpb.BeaconBlockBody{
+			Attestations: attestations,
+		},
+	}
+	beaconState, _ := testutil.DeterministicGenesisState(t, 100)
+	require.NoError(t, beaconState.SetSlot(beaconState.Slot()+params.BeaconConfig().MinAttestationInclusionDelay))
+	cfc := beaconState.CurrentJustifiedCheckpoint()
+	cfc.Root = []byte("hello-world")
+	require.NoError(t, beaconState.SetCurrentJustifiedCheckpoint(cfc))
+	require.NoError(t, beaconState.SetCurrentEpochAttestations([]*pb.PendingAttestation{}))
+
+	want := fmt.Sprintf(
+		"expected source epoch %d, received %d",
+		helpers.CurrentEpoch(beaconState),
+		attestations[0].Data.Source.Epoch,
+	)
+	_, err := blocks.ProcessAttestations(context.Background(), beaconState, block.Body)
+	assert.ErrorContains(t, want, err)
+
+	block.Body.Attestations[0].Data.Source.Epoch = helpers.CurrentEpoch(beaconState)
+	block.Body.Attestations[0].Data.Source.Root = []byte{}
+
+	want = fmt.Sprintf(
+		"expected source root %#x, received %#x",
+		beaconState.CurrentJustifiedCheckpoint().Root,
+		attestations[0].Data.Source.Root,
+	)
+	_, err = blocks.ProcessAttestations(context.Background(), beaconState, block.Body)
+	assert.ErrorContains(t, want, err)
+}
+
+func TestProcessAttestations_PrevEpochFFGDataMismatches(t *testing.T) {
+	beaconState, _ := testutil.DeterministicGenesisState(t, 100)
+
+	aggBits := bitfield.NewBitlist(3)
+	aggBits.SetBitAt(0, true)
+	attestations := []*ethpb.Attestation{
+		{
+			Data: &ethpb.AttestationData{
+				Source: &ethpb.Checkpoint{Epoch: 1},
+				Target: &ethpb.Checkpoint{Epoch: 1},
+				Slot:   params.BeaconConfig().SlotsPerEpoch,
+			},
+			AggregationBits: aggBits,
+		},
+	}
+	block := &ethpb.BeaconBlock{
+		Body: &ethpb.BeaconBlockBody{
+			Attestations: attestations,
+		},
+	}
+
+	err := beaconState.SetSlot(beaconState.Slot() + params.BeaconConfig().SlotsPerEpoch + params.BeaconConfig().MinAttestationInclusionDelay)
+	require.NoError(t, err)
+	pfc := beaconState.PreviousJustifiedCheckpoint()
+	pfc.Root = []byte("hello-world")
+	require.NoError(t, beaconState.SetPreviousJustifiedCheckpoint(pfc))
+	require.NoError(t, beaconState.SetPreviousEpochAttestations([]*pb.PendingAttestation{}))
+
+	want := fmt.Sprintf(
+		"expected source epoch %d, received %d",
+		helpers.PrevEpoch(beaconState),
+		attestations[0].Data.Source.Epoch,
+	)
+	_, err = blocks.ProcessAttestations(context.Background(), beaconState, block.Body)
+	assert.ErrorContains(t, want, err)
+
+	block.Body.Attestations[0].Data.Source.Epoch = helpers.PrevEpoch(beaconState)
+	block.Body.Attestations[0].Data.Target.Epoch = helpers.CurrentEpoch(beaconState)
+	block.Body.Attestations[0].Data.Source.Root = []byte{}
+
+	want = fmt.Sprintf(
+		"expected source root %#x, received %#x",
+		beaconState.CurrentJustifiedCheckpoint().Root,
+		attestations[0].Data.Source.Root,
+	)
+	_, err = blocks.ProcessAttestations(context.Background(), beaconState, block.Body)
+	assert.ErrorContains(t, want, err)
+}
+
+func TestProcessAttestations_InvalidAggregationBitsLength(t *testing.T) {
+	beaconState, _ := testutil.DeterministicGenesisState(t, 100)
+
+	aggBits := bitfield.NewBitlist(4)
+	att := &ethpb.Attestation{
+		Data: &ethpb.AttestationData{
+			Source: &ethpb.Checkpoint{Epoch: 0, Root: []byte("hello-world")},
+			Target: &ethpb.Checkpoint{Epoch: 0}},
+		AggregationBits: aggBits,
+	}
+
+	block := &ethpb.BeaconBlock{
+		Body: &ethpb.BeaconBlockBody{
+			Attestations: []*ethpb.Attestation{att},
+		},
+	}
+
+	err := beaconState.SetSlot(beaconState.Slot() + params.BeaconConfig().MinAttestationInclusionDelay)
+	require.NoError(t, err)
+
+	cfc := beaconState.CurrentJustifiedCheckpoint()
+	cfc.Root = []byte("hello-world")
+	require.NoError(t, beaconState.SetCurrentJustifiedCheckpoint(cfc))
+	require.NoError(t, beaconState.SetCurrentEpochAttestations([]*pb.PendingAttestation{}))
+
+	expected := "failed to verify aggregation bitfield: wanted participants bitfield length 3, got: 4"
+	_, err = blocks.ProcessAttestations(context.Background(), beaconState, block.Body)
+	assert.ErrorContains(t, expected, err)
+}
+
+func TestProcessAttestations_OK(t *testing.T) {
+	beaconState, privKeys := testutil.DeterministicGenesisState(t, 100)
+
+	aggBits := bitfield.NewBitlist(3)
+	aggBits.SetBitAt(0, true)
+	var mockRoot [32]byte
+	copy(mockRoot[:], "hello-world")
+	att := &ethpb.Attestation{
+		Data: &ethpb.AttestationData{
+			Source: &ethpb.Checkpoint{Epoch: 0, Root: mockRoot[:]},
+			Target: &ethpb.Checkpoint{Epoch: 0, Root: mockRoot[:]},
+		},
+		AggregationBits: aggBits,
+	}
+
+	cfc := beaconState.CurrentJustifiedCheckpoint()
+	cfc.Root = mockRoot[:]
+	require.NoError(t, beaconState.SetCurrentJustifiedCheckpoint(cfc))
+	require.NoError(t, beaconState.SetCurrentEpochAttestations([]*pb.PendingAttestation{}))
+
+	committee, err := helpers.BeaconCommitteeFromState(beaconState, att.Data.Slot, att.Data.CommitteeIndex)
+	require.NoError(t, err)
+	attestingIndices := attestationutil.AttestingIndices(att.AggregationBits, committee)
+	require.NoError(t, err)
+	sigs := make([]bls.Signature, len(attestingIndices))
+	for i, indice := range attestingIndices {
+		sb, err := helpers.ComputeDomainAndSign(beaconState, 0, att.Data, params.BeaconConfig().DomainBeaconAttester, privKeys[indice])
+		require.NoError(t, err)
+		sig, err := bls.SignatureFromBytes(sb)
+		require.NoError(t, err)
+		sigs[i] = sig
+	}
+	att.Signature = bls.AggregateSignatures(sigs).Marshal()[:]
+
+	block := &ethpb.BeaconBlock{
+		Body: &ethpb.BeaconBlockBody{
+			Attestations: []*ethpb.Attestation{att},
+		},
+	}
+
+	err = beaconState.SetSlot(beaconState.Slot() + params.BeaconConfig().MinAttestationInclusionDelay)
+	require.NoError(t, err)
+	_, err = blocks.ProcessAttestations(context.Background(), beaconState, block.Body)
+	assert.NoError(t, err)
+}
+
+func TestProcessAggregatedAttestation_OverlappingBits(t *testing.T) {
+	beaconState, privKeys := testutil.DeterministicGenesisState(t, 100)
+	data := &ethpb.AttestationData{
+		Source: &ethpb.Checkpoint{Epoch: 0, Root: []byte("hello-world")},
+		Target: &ethpb.Checkpoint{Epoch: 0, Root: []byte("hello-world")},
+	}
+	aggBits1 := bitfield.NewBitlist(4)
+	aggBits1.SetBitAt(0, true)
+	aggBits1.SetBitAt(1, true)
+	aggBits1.SetBitAt(2, true)
+	att1 := &ethpb.Attestation{
+		Data:            data,
+		AggregationBits: aggBits1,
+	}
+
+	cfc := beaconState.CurrentJustifiedCheckpoint()
+	cfc.Root = []byte("hello-world")
+	require.NoError(t, beaconState.SetCurrentJustifiedCheckpoint(cfc))
+	require.NoError(t, beaconState.SetCurrentEpochAttestations([]*pb.PendingAttestation{}))
+
+	committee, err := helpers.BeaconCommitteeFromState(beaconState, att1.Data.Slot, att1.Data.CommitteeIndex)
+	require.NoError(t, err)
+	attestingIndices1 := attestationutil.AttestingIndices(att1.AggregationBits, committee)
+	require.NoError(t, err)
+	sigs := make([]bls.Signature, len(attestingIndices1))
+	for i, indice := range attestingIndices1 {
+		sb, err := helpers.ComputeDomainAndSign(beaconState, 0, att1.Data, params.BeaconConfig().DomainBeaconAttester, privKeys[indice])
+		require.NoError(t, err)
+		sig, err := bls.SignatureFromBytes(sb)
+		require.NoError(t, err)
+		sigs[i] = sig
+	}
+	att1.Signature = bls.AggregateSignatures(sigs).Marshal()[:]
+
+	aggBits2 := bitfield.NewBitlist(4)
+	aggBits2.SetBitAt(1, true)
+	aggBits2.SetBitAt(2, true)
+	aggBits2.SetBitAt(3, true)
+	att2 := &ethpb.Attestation{
+		Data:            data,
+		AggregationBits: aggBits2,
+	}
+
+	committee, err = helpers.BeaconCommitteeFromState(beaconState, att2.Data.Slot, att2.Data.CommitteeIndex)
+	require.NoError(t, err)
+	attestingIndices2 := attestationutil.AttestingIndices(att2.AggregationBits, committee)
+	require.NoError(t, err)
+	sigs = make([]bls.Signature, len(attestingIndices2))
+	for i, indice := range attestingIndices2 {
+		sb, err := helpers.ComputeDomainAndSign(beaconState, 0, att2.Data, params.BeaconConfig().DomainBeaconAttester, privKeys[indice])
+		require.NoError(t, err)
+		sig, err := bls.SignatureFromBytes(sb)
+		require.NoError(t, err)
+		sigs[i] = sig
+	}
+	att2.Signature = bls.AggregateSignatures(sigs).Marshal()[:]
+
+	_, err = attaggregation.AggregatePair(att1, att2)
+	assert.ErrorContains(t, aggregation.ErrBitsOverlap.Error(), err)
+}
+
+func TestProcessAggregatedAttestation_NoOverlappingBits(t *testing.T) {
+	beaconState, privKeys := testutil.DeterministicGenesisState(t, 300)
+
+	var mockRoot [32]byte
+	copy(mockRoot[:], "hello-world")
+	data := &ethpb.AttestationData{
+		Source: &ethpb.Checkpoint{Epoch: 0, Root: mockRoot[:]},
+		Target: &ethpb.Checkpoint{Epoch: 0, Root: mockRoot[:]},
+	}
+	aggBits1 := bitfield.NewBitlist(9)
+	aggBits1.SetBitAt(0, true)
+	aggBits1.SetBitAt(1, true)
+	att1 := &ethpb.Attestation{
+		Data:            data,
+		AggregationBits: aggBits1,
+	}
+
+	cfc := beaconState.CurrentJustifiedCheckpoint()
+	cfc.Root = mockRoot[:]
+	require.NoError(t, beaconState.SetCurrentJustifiedCheckpoint(cfc))
+	require.NoError(t, beaconState.SetCurrentEpochAttestations([]*pb.PendingAttestation{}))
+
+	committee, err := helpers.BeaconCommitteeFromState(beaconState, att1.Data.Slot, att1.Data.CommitteeIndex)
+	require.NoError(t, err)
+	attestingIndices1 := attestationutil.AttestingIndices(att1.AggregationBits, committee)
+	require.NoError(t, err)
+	sigs := make([]bls.Signature, len(attestingIndices1))
+	for i, indice := range attestingIndices1 {
+		sb, err := helpers.ComputeDomainAndSign(beaconState, 0, data, params.BeaconConfig().DomainBeaconAttester, privKeys[indice])
+		require.NoError(t, err)
+		sig, err := bls.SignatureFromBytes(sb)
+		require.NoError(t, err)
+		sigs[i] = sig
+	}
+	att1.Signature = bls.AggregateSignatures(sigs).Marshal()[:]
+
+	aggBits2 := bitfield.NewBitlist(9)
+	aggBits2.SetBitAt(2, true)
+	aggBits2.SetBitAt(3, true)
+	att2 := &ethpb.Attestation{
+		Data:            data,
+		AggregationBits: aggBits2,
+	}
+
+	committee, err = helpers.BeaconCommitteeFromState(beaconState, att2.Data.Slot, att2.Data.CommitteeIndex)
+	require.NoError(t, err)
+	attestingIndices2 := attestationutil.AttestingIndices(att2.AggregationBits, committee)
+	require.NoError(t, err)
+	sigs = make([]bls.Signature, len(attestingIndices2))
+	for i, indice := range attestingIndices2 {
+		sb, err := helpers.ComputeDomainAndSign(beaconState, 0, att2.Data, params.BeaconConfig().DomainBeaconAttester, privKeys[indice])
+		require.NoError(t, err)
+		sig, err := bls.SignatureFromBytes(sb)
+		require.NoError(t, err)
+		sigs[i] = sig
+	}
+	att2.Signature = bls.AggregateSignatures(sigs).Marshal()[:]
+
+	aggregatedAtt, err := attaggregation.AggregatePair(att1, att2)
+	require.NoError(t, err)
+	block := &ethpb.BeaconBlock{
+		Body: &ethpb.BeaconBlockBody{
+			Attestations: []*ethpb.Attestation{aggregatedAtt},
+		},
+	}
+
+	err = beaconState.SetSlot(beaconState.Slot() + params.BeaconConfig().MinAttestationInclusionDelay)
+	require.NoError(t, err)
+
+	_, err = blocks.ProcessAttestations(context.Background(), beaconState, block.Body)
+	assert.NoError(t, err)
+}
+
+func TestProcessAttestationsNoVerify_IncorrectSlotTargetEpoch(t *testing.T) {
+	beaconState, _ := testutil.DeterministicGenesisState(t, 1)
+
+	att := &ethpb.Attestation{
+		Data: &ethpb.AttestationData{
+			Slot:   params.BeaconConfig().SlotsPerEpoch,
+			Target: &ethpb.Checkpoint{},
+		},
+	}
+	wanted := fmt.Sprintf("data slot is not in the same epoch as target %d != %d", helpers.SlotToEpoch(att.Data.Slot), att.Data.Target.Epoch)
+	_, err := blocks.ProcessAttestationNoVerify(context.TODO(), beaconState, att)
+	assert.ErrorContains(t, wanted, err)
+}
+
+func TestProcessAttestationsNoVerify_OK(t *testing.T) {
+	// Attestation with an empty signature
+
+	beaconState, _ := testutil.DeterministicGenesisState(t, 100)
+
+	aggBits := bitfield.NewBitlist(3)
+	aggBits.SetBitAt(1, true)
+	var mockRoot [32]byte
+	copy(mockRoot[:], "hello-world")
+	att := &ethpb.Attestation{
+		Data: &ethpb.AttestationData{
+			Source: &ethpb.Checkpoint{Epoch: 0, Root: mockRoot[:]},
+			Target: &ethpb.Checkpoint{Epoch: 0},
+		},
+		AggregationBits: aggBits,
+	}
+
+	zeroSig := [96]byte{}
+	att.Signature = zeroSig[:]
+
+	err := beaconState.SetSlot(beaconState.Slot() + params.BeaconConfig().MinAttestationInclusionDelay)
+	require.NoError(t, err)
+	ckp := beaconState.CurrentJustifiedCheckpoint()
+	copy(ckp.Root, "hello-world")
+	require.NoError(t, beaconState.SetCurrentJustifiedCheckpoint(ckp))
+	require.NoError(t, beaconState.SetCurrentEpochAttestations([]*pb.PendingAttestation{}))
+
+	_, err = blocks.ProcessAttestationNoVerify(context.TODO(), beaconState, att)
+	assert.NoError(t, err)
+}
+
+func TestProcessAttestationsNoVerify_BadAttIdx(t *testing.T) {
+	beaconState, _ := testutil.DeterministicGenesisState(t, 100)
+	aggBits := bitfield.NewBitlist(3)
+	aggBits.SetBitAt(1, true)
+	var mockRoot [32]byte
+	copy(mockRoot[:], "hello-world")
+	att := &ethpb.Attestation{
+		Data: &ethpb.AttestationData{
+			CommitteeIndex: 100,
+			Source:         &ethpb.Checkpoint{Epoch: 0, Root: mockRoot[:]},
+			Target:         &ethpb.Checkpoint{Epoch: 0},
+		},
+		AggregationBits: aggBits,
+	}
+	zeroSig := [96]byte{}
+	att.Signature = zeroSig[:]
+	require.NoError(t, beaconState.SetSlot(beaconState.Slot()+params.BeaconConfig().MinAttestationInclusionDelay))
+	ckp := beaconState.CurrentJustifiedCheckpoint()
+	copy(ckp.Root, "hello-world")
+	require.NoError(t, beaconState.SetCurrentJustifiedCheckpoint(ckp))
+	require.NoError(t, beaconState.SetCurrentEpochAttestations([]*pb.PendingAttestation{}))
+	_, err := blocks.ProcessAttestationNoVerify(context.TODO(), beaconState, att)
+	require.ErrorContains(t, "committee index 100 >= committee count 1", err)
+}
+
+func TestConvertToIndexed_OK(t *testing.T) {
+	helpers.ClearCache()
+	validators := make([]*ethpb.Validator, 2*params.BeaconConfig().SlotsPerEpoch)
+	for i := 0; i < len(validators); i++ {
+		validators[i] = &ethpb.Validator{
+			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
+		}
+	}
+
+	state, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot:        5,
+		Validators:  validators,
+		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+	})
+	require.NoError(t, err)
+	tests := []struct {
+		aggregationBitfield    bitfield.Bitlist
+		wantedAttestingIndices []uint64
+	}{
+		{
+			aggregationBitfield:    bitfield.Bitlist{0x07},
+			wantedAttestingIndices: []uint64{43, 47},
+		},
+		{
+			aggregationBitfield:    bitfield.Bitlist{0x03},
+			wantedAttestingIndices: []uint64{47},
+		},
+		{
+			aggregationBitfield:    bitfield.Bitlist{0x01},
+			wantedAttestingIndices: []uint64{},
+		},
+	}
+
+	var sig [96]byte
+	copy(sig[:], "signed")
+	attestation := &ethpb.Attestation{
+		Signature: sig[:],
+		Data: &ethpb.AttestationData{
+			Source: &ethpb.Checkpoint{Epoch: 0},
+			Target: &ethpb.Checkpoint{Epoch: 0},
+		},
+	}
+	for _, tt := range tests {
+		attestation.AggregationBits = tt.aggregationBitfield
+		wanted := &ethpb.IndexedAttestation{
+			AttestingIndices: tt.wantedAttestingIndices,
+			Data:             attestation.Data,
+			Signature:        attestation.Signature,
+		}
+
+		committee, err := helpers.BeaconCommitteeFromState(state, attestation.Data.Slot, attestation.Data.CommitteeIndex)
+		require.NoError(t, err)
+		ia := attestationutil.ConvertToIndexed(context.Background(), attestation, committee)
+		assert.DeepEqual(t, wanted, ia, "Convert attestation to indexed attestation didn't result as wanted")
+	}
+}
+
+func TestVerifyIndexedAttestation_OK(t *testing.T) {
+	numOfValidators := 4 * params.BeaconConfig().SlotsPerEpoch
+	validators := make([]*ethpb.Validator, numOfValidators)
+	_, keys, err := testutil.DeterministicDepositsAndKeys(numOfValidators)
+	require.NoError(t, err)
+	for i := 0; i < len(validators); i++ {
+		validators[i] = &ethpb.Validator{
+			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
+			PublicKey: keys[i].PublicKey().Marshal(),
+		}
+	}
+
+	state, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot:       5,
+		Validators: validators,
+		Fork: &pb.Fork{
+			Epoch:           0,
+			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
+			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
+		},
+		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+	})
+	require.NoError(t, err)
+	tests := []struct {
+		attestation *ethpb.IndexedAttestation
+	}{
+		{attestation: &ethpb.IndexedAttestation{
+			Data: &ethpb.AttestationData{
+				Target: &ethpb.Checkpoint{
+					Epoch: 2,
+				},
+			},
+			AttestingIndices: []uint64{1},
+		}},
+		{attestation: &ethpb.IndexedAttestation{
+			Data: &ethpb.AttestationData{
+				Target: &ethpb.Checkpoint{
+					Epoch: 1,
+				},
+			},
+			AttestingIndices: []uint64{47, 99, 101},
+		}},
+		{attestation: &ethpb.IndexedAttestation{
+			Data: &ethpb.AttestationData{
+				Target: &ethpb.Checkpoint{
+					Epoch: 4,
+				},
+			},
+			AttestingIndices: []uint64{21, 72},
+		}},
+		{attestation: &ethpb.IndexedAttestation{
+			Data: &ethpb.AttestationData{
+				Target: &ethpb.Checkpoint{
+					Epoch: 7,
+				},
+			},
+			AttestingIndices: []uint64{100, 121, 122},
+		}},
+	}
+
+	for _, tt := range tests {
+		var sig []bls.Signature
+		for _, idx := range tt.attestation.AttestingIndices {
+			sb, err := helpers.ComputeDomainAndSign(state, tt.attestation.Data.Target.Epoch, tt.attestation.Data, params.BeaconConfig().DomainBeaconAttester, keys[idx])
+			require.NoError(t, err)
+			validatorSig, err := bls.SignatureFromBytes(sb)
+			require.NoError(t, err)
+			sig = append(sig, validatorSig)
+		}
+		aggSig := bls.AggregateSignatures(sig)
+		marshalledSig := aggSig.Marshal()
+
+		tt.attestation.Signature = marshalledSig
+
+		err = blocks.VerifyIndexedAttestation(context.Background(), state, tt.attestation)
+		assert.NoError(t, err, "Failed to verify indexed attestation")
+	}
+}
+
+func TestValidateIndexedAttestation_AboveMaxLength(t *testing.T) {
+	indexedAtt1 := &ethpb.IndexedAttestation{
+		AttestingIndices: make([]uint64, params.BeaconConfig().MaxValidatorsPerCommittee+5),
+	}
+
+	for i := uint64(0); i < params.BeaconConfig().MaxValidatorsPerCommittee+5; i++ {
+		indexedAtt1.AttestingIndices[i] = i
+		indexedAtt1.Data = &ethpb.AttestationData{
+			Target: &ethpb.Checkpoint{
+				Epoch: i,
+			},
+		}
+	}
+
+	want := "validator indices count exceeds MAX_VALIDATORS_PER_COMMITTEE"
+	err := blocks.VerifyIndexedAttestation(context.Background(), &stateTrie.BeaconState{}, indexedAtt1)
+	assert.ErrorContains(t, want, err)
+}
+
+func TestValidateIndexedAttestation_BadAttestationsSignatureSet(t *testing.T) {
+	beaconState, keys := testutil.DeterministicGenesisState(t, 1000)
+
+	sig := keys[0].Sign([]byte{'t', 'e', 's', 't'})
+	list := bitfield.Bitlist{0b11111111}
+	atts := []*ethpb.Attestation{}
+	for i := uint64(0); i < 1000; i++ {
+		atts = append(atts, &ethpb.Attestation{
+			Data: &ethpb.AttestationData{
+				CommitteeIndex: 1,
+				Slot:           1,
+			},
+			Signature:       sig.Marshal(),
+			AggregationBits: list,
+		})
+	}
+
+	want := "nil or missing indexed attestation data"
+	_, err := blocks.AttestationSignatureSet(context.Background(), beaconState, atts)
+	assert.ErrorContains(t, want, err)
+
+	atts = []*ethpb.Attestation{}
+	list = bitfield.Bitlist{0b00000000}
+	for i := uint64(0); i < 1000; i++ {
+		atts = append(atts, &ethpb.Attestation{
+			Data: &ethpb.AttestationData{
+				CommitteeIndex: 1,
+				Slot:           1,
+				Target: &ethpb.Checkpoint{
+					Root: []byte{},
+				},
+			},
+			Signature:       sig.Marshal(),
+			AggregationBits: list,
+		})
+	}
+
+	want = "expected non-empty attesting indices"
+	_, err = blocks.AttestationSignatureSet(context.Background(), beaconState, atts)
+	assert.ErrorContains(t, want, err)
+}
+
+func TestVerifyAttestations_VerifiesMultipleAttestations(t *testing.T) {
+	ctx := context.Background()
+	numOfValidators := 4 * params.BeaconConfig().SlotsPerEpoch
+	validators := make([]*ethpb.Validator, numOfValidators)
+	_, keys, err := testutil.DeterministicDepositsAndKeys(numOfValidators)
+	require.NoError(t, err)
+	for i := 0; i < len(validators); i++ {
+		validators[i] = &ethpb.Validator{
+			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
+			PublicKey: keys[i].PublicKey().Marshal(),
+		}
+	}
+
+	st, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot:       5,
+		Validators: validators,
+		Fork: &pb.Fork{
+			Epoch:           0,
+			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
+			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
+		},
+		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+	})
+
+	comm1, err := helpers.BeaconCommitteeFromState(st, 1 /*slot*/, 0 /*committeeIndex*/)
+	require.NoError(t, err)
+	att1 := &ethpb.Attestation{
+		AggregationBits: bitfield.NewBitlist(uint64(len(comm1))),
+		Data: &ethpb.AttestationData{
+			Slot:           1,
+			CommitteeIndex: 0,
+			Target:         new(ethpb.Checkpoint),
+		},
+		Signature: nil,
+	}
+	domain, err := helpers.Domain(st.Fork(), st.Fork().Epoch, params.BeaconConfig().DomainBeaconAttester, st.GenesisValidatorRoot())
+	require.NoError(t, err)
+	root, err := helpers.ComputeSigningRoot(att1.Data, domain)
+	require.NoError(t, err)
+	var sigs []bls.Signature
+	for i, u := range comm1 {
+		att1.AggregationBits.SetBitAt(uint64(i), true)
+		sigs = append(sigs, keys[u].Sign(root[:]))
+	}
+	att1.Signature = bls.AggregateSignatures(sigs).Marshal()
+
+	comm2, err := helpers.BeaconCommitteeFromState(st, 1 /*slot*/, 1 /*committeeIndex*/)
+	require.NoError(t, err)
+	att2 := &ethpb.Attestation{
+		AggregationBits: bitfield.NewBitlist(uint64(len(comm2))),
+		Data: &ethpb.AttestationData{
+			Slot:           1,
+			CommitteeIndex: 1,
+			Target:         new(ethpb.Checkpoint),
+		},
+		Signature: nil,
+	}
+	root, err = helpers.ComputeSigningRoot(att2.Data, domain)
+	require.NoError(t, err)
+	sigs = nil
+	for i, u := range comm2 {
+		att2.AggregationBits.SetBitAt(uint64(i), true)
+		sigs = append(sigs, keys[u].Sign(root[:]))
+	}
+	att2.Signature = bls.AggregateSignatures(sigs).Marshal()
+
+	require.NoError(t, blocks.VerifyAttestations(ctx, st, []*ethpb.Attestation{att1, att2}))
+}
+
+func TestVerifyAttestations_HandlesPlannedFork(t *testing.T) {
+	// In this test, att1 is from the prior fork and att2 is from the new fork.
+	ctx := context.Background()
+	numOfValidators := 4 * params.BeaconConfig().SlotsPerEpoch
+	validators := make([]*ethpb.Validator, numOfValidators)
+	_, keys, err := testutil.DeterministicDepositsAndKeys(numOfValidators)
+	require.NoError(t, err)
+	for i := 0; i < len(validators); i++ {
+		validators[i] = &ethpb.Validator{
+			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
+			PublicKey: keys[i].PublicKey().Marshal(),
+		}
+	}
+
+	st, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot:       35,
+		Validators: validators,
+		Fork: &pb.Fork{
+			Epoch:           1,
+			CurrentVersion:  []byte{0, 1, 2, 3},
+			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
+		},
+		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+	})
+
+	comm1, err := helpers.BeaconCommitteeFromState(st, 1 /*slot*/, 0 /*committeeIndex*/)
+	require.NoError(t, err)
+	att1 := &ethpb.Attestation{
+		AggregationBits: bitfield.NewBitlist(uint64(len(comm1))),
+		Data: &ethpb.AttestationData{
+			Slot:           1,
+			CommitteeIndex: 0,
+			Target:         new(ethpb.Checkpoint),
+		},
+		Signature: nil,
+	}
+	prevDomain, err := helpers.Domain(st.Fork(), st.Fork().Epoch-1, params.BeaconConfig().DomainBeaconAttester, st.GenesisValidatorRoot())
+	require.NoError(t, err)
+	root, err := helpers.ComputeSigningRoot(att1.Data, prevDomain)
+	require.NoError(t, err)
+	var sigs []bls.Signature
+	for i, u := range comm1 {
+		att1.AggregationBits.SetBitAt(uint64(i), true)
+		sigs = append(sigs, keys[u].Sign(root[:]))
+	}
+	att1.Signature = bls.AggregateSignatures(sigs).Marshal()
+
+	comm2, err := helpers.BeaconCommitteeFromState(st, 1*params.BeaconConfig().SlotsPerEpoch+1 /*slot*/, 1 /*committeeIndex*/)
+	require.NoError(t, err)
+	att2 := &ethpb.Attestation{
+		AggregationBits: bitfield.NewBitlist(uint64(len(comm2))),
+		Data: &ethpb.AttestationData{
+			Slot:           1*params.BeaconConfig().SlotsPerEpoch + 1,
+			CommitteeIndex: 1,
+			Target:         new(ethpb.Checkpoint),
+		},
+		Signature: nil,
+	}
+	currDomain, err := helpers.Domain(st.Fork(), st.Fork().Epoch, params.BeaconConfig().DomainBeaconAttester, st.GenesisValidatorRoot())
+	root, err = helpers.ComputeSigningRoot(att2.Data, currDomain)
+	require.NoError(t, err)
+	sigs = nil
+	for i, u := range comm2 {
+		att2.AggregationBits.SetBitAt(uint64(i), true)
+		sigs = append(sigs, keys[u].Sign(root[:]))
+	}
+	att2.Signature = bls.AggregateSignatures(sigs).Marshal()
+
+	require.NoError(t, blocks.VerifyAttestations(ctx, st, []*ethpb.Attestation{att1, att2}))
+}
+
+func TestRetrieveAttestationSignatureSet_VerifiesMultipleAttestations(t *testing.T) {
+	ctx := context.Background()
+	numOfValidators := 4 * params.BeaconConfig().SlotsPerEpoch
+	validators := make([]*ethpb.Validator, numOfValidators)
+	_, keys, err := testutil.DeterministicDepositsAndKeys(numOfValidators)
+	require.NoError(t, err)
+	for i := 0; i < len(validators); i++ {
+		validators[i] = &ethpb.Validator{
+			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
+			PublicKey: keys[i].PublicKey().Marshal(),
+		}
+	}
+
+	st, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot:       5,
+		Validators: validators,
+		Fork: &pb.Fork{
+			Epoch:           0,
+			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
+			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
+		},
+		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+	})
+
+	comm1, err := helpers.BeaconCommitteeFromState(st, 1 /*slot*/, 0 /*committeeIndex*/)
+	require.NoError(t, err)
+	att1 := &ethpb.Attestation{
+		AggregationBits: bitfield.NewBitlist(uint64(len(comm1))),
+		Data: &ethpb.AttestationData{
+			Slot:           1,
+			CommitteeIndex: 0,
+			Target:         new(ethpb.Checkpoint),
+		},
+		Signature: nil,
+	}
+	domain, err := helpers.Domain(st.Fork(), st.Fork().Epoch, params.BeaconConfig().DomainBeaconAttester, st.GenesisValidatorRoot())
+	require.NoError(t, err)
+	root, err := helpers.ComputeSigningRoot(att1.Data, domain)
+	require.NoError(t, err)
+	var sigs []bls.Signature
+	for i, u := range comm1 {
+		att1.AggregationBits.SetBitAt(uint64(i), true)
+		sigs = append(sigs, keys[u].Sign(root[:]))
+	}
+	att1.Signature = bls.AggregateSignatures(sigs).Marshal()
+
+	comm2, err := helpers.BeaconCommitteeFromState(st, 1 /*slot*/, 1 /*committeeIndex*/)
+	require.NoError(t, err)
+	att2 := &ethpb.Attestation{
+		AggregationBits: bitfield.NewBitlist(uint64(len(comm2))),
+		Data: &ethpb.AttestationData{
+			Slot:           1,
+			CommitteeIndex: 1,
+			Target:         new(ethpb.Checkpoint),
+		},
+		Signature: nil,
+	}
+	root, err = helpers.ComputeSigningRoot(att2.Data, domain)
+	require.NoError(t, err)
+	sigs = nil
+	for i, u := range comm2 {
+		att2.AggregationBits.SetBitAt(uint64(i), true)
+		sigs = append(sigs, keys[u].Sign(root[:]))
+	}
+	att2.Signature = bls.AggregateSignatures(sigs).Marshal()
+
+	set, err := blocks.AttestationSignatureSet(ctx, st, []*ethpb.Attestation{att1, att2})
+	require.NoError(t, err)
+	verified, err := set.Verify()
+	require.NoError(t, err)
+	assert.Equal(t, true, verified, "Multiple signatures were unable to be verified.")
+}

beacon-chain/core/blocks/attester_slashing.go

@@ -0,0 +1,129 @@
+package blocks
+
+import (
+	"context"
+	"sort"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	v "github.com/prysmaticlabs/prysm/beacon-chain/core/validators"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/attestationutil"
+	"github.com/prysmaticlabs/prysm/shared/sliceutil"
+)
+
+// ProcessAttesterSlashings is one of the operations performed
+// on each processed beacon block to slash attesters based on
+// Casper FFG slashing conditions if any slashable events occurred.
+//
+// Spec pseudocode definition:
+//   def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSlashing) -> None:
+//    attestation_1 = attester_slashing.attestation_1
+//    attestation_2 = attester_slashing.attestation_2
+//    assert is_slashable_attestation_data(attestation_1.data, attestation_2.data)
+//    assert is_valid_indexed_attestation(state, attestation_1)
+//    assert is_valid_indexed_attestation(state, attestation_2)
+//
+//    slashed_any = False
+//    indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)
+//    for index in sorted(indices):
+//        if is_slashable_validator(state.validators[index], get_current_epoch(state)):
+//            slash_validator(state, index)
+//            slashed_any = True
+//    assert slashed_any
+func ProcessAttesterSlashings(
+	ctx context.Context,
+	beaconState *stateTrie.BeaconState,
+	body *ethpb.BeaconBlockBody,
+) (*stateTrie.BeaconState, error) {
+	for idx, slashing := range body.AttesterSlashings {
+		if err := VerifyAttesterSlashing(ctx, beaconState, slashing); err != nil {
+			return nil, errors.Wrapf(err, "could not verify attester slashing %d", idx)
+		}
+		slashableIndices := slashableAttesterIndices(slashing)
+		sort.SliceStable(slashableIndices, func(i, j int) bool {
+			return slashableIndices[i] < slashableIndices[j]
+		})
+		currentEpoch := helpers.SlotToEpoch(beaconState.Slot())
+		var err error
+		var slashedAny bool
+		var val *stateTrie.ReadOnlyValidator
+		for _, validatorIndex := range slashableIndices {
+			val, err = beaconState.ValidatorAtIndexReadOnly(validatorIndex)
+			if err != nil {
+				return nil, err
+			}
+			if helpers.IsSlashableValidator(val.ActivationEpoch(), val.WithdrawableEpoch(), val.Slashed(), currentEpoch) {
+				beaconState, err = v.SlashValidator(beaconState, validatorIndex)
+				if err != nil {
+					return nil, errors.Wrapf(err, "could not slash validator index %d",
+						validatorIndex)
+				}
+				slashedAny = true
+			}
+		}
+		if !slashedAny {
+			return nil, errors.New("unable to slash any validator despite confirmed attester slashing")
+		}
+	}
+	return beaconState, nil
+}
+
+// VerifyAttesterSlashing validates the attestation data in both attestations in the slashing object.
+func VerifyAttesterSlashing(ctx context.Context, beaconState *stateTrie.BeaconState, slashing *ethpb.AttesterSlashing) error {
+	if slashing == nil {
+		return errors.New("nil slashing")
+	}
+	if slashing.Attestation_1 == nil || slashing.Attestation_2 == nil {
+		return errors.New("nil attestation")
+	}
+	if slashing.Attestation_1.Data == nil || slashing.Attestation_2.Data == nil {
+		return errors.New("nil attestation data")
+	}
+	att1 := slashing.Attestation_1
+	att2 := slashing.Attestation_2
+	data1 := att1.Data
+	data2 := att2.Data
+	if !IsSlashableAttestationData(data1, data2) {
+		return errors.New("attestations are not slashable")
+	}
+	if err := VerifyIndexedAttestation(ctx, beaconState, att1); err != nil {
+		return errors.Wrap(err, "could not validate indexed attestation")
+	}
+	if err := VerifyIndexedAttestation(ctx, beaconState, att2); err != nil {
+		return errors.Wrap(err, "could not validate indexed attestation")
+	}
+	return nil
+}
+
+// IsSlashableAttestationData verifies a slashing against the Casper Proof of Stake FFG rules.
+//
+// Spec pseudocode definition:
+//   def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationData) -> bool:
+//    """
+//    Check if ``data_1`` and ``data_2`` are slashable according to Casper FFG rules.
+//    """
+//    return (
+//        # Double vote
+//        (data_1 != data_2 and data_1.target.epoch == data_2.target.epoch) or
+//        # Surround vote
+//        (data_1.source.epoch < data_2.source.epoch and data_2.target.epoch < data_1.target.epoch)
+//    )
+func IsSlashableAttestationData(data1 *ethpb.AttestationData, data2 *ethpb.AttestationData) bool {
+	if data1 == nil || data2 == nil || data1.Target == nil || data2.Target == nil || data1.Source == nil || data2.Source == nil {
+		return false
+	}
+	isDoubleVote := !attestationutil.AttDataIsEqual(data1, data2) && data1.Target.Epoch == data2.Target.Epoch
+	isSurroundVote := data1.Source.Epoch < data2.Source.Epoch && data2.Target.Epoch < data1.Target.Epoch
+	return isDoubleVote || isSurroundVote
+}
+
+func slashableAttesterIndices(slashing *ethpb.AttesterSlashing) []uint64 {
+	if slashing == nil || slashing.Attestation_1 == nil || slashing.Attestation_2 == nil {
+		return nil
+	}
+	indices1 := slashing.Attestation_1.AttestingIndices
+	indices2 := slashing.Attestation_2.AttestingIndices
+	return sliceutil.IntersectionUint64(indices1, indices2)
+}

beacon-chain/core/blocks/attester_slashing_test.go

@@ -0,0 +1,179 @@
+package blocks_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/bls"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+)
+
+func TestSlashableAttestationData_CanSlash(t *testing.T) {
+	att1 := &ethpb.AttestationData{
+		Target: &ethpb.Checkpoint{Epoch: 1},
+		Source: &ethpb.Checkpoint{Root: []byte{'A'}},
+	}
+	att2 := &ethpb.AttestationData{
+		Target: &ethpb.Checkpoint{Epoch: 1},
+		Source: &ethpb.Checkpoint{Root: []byte{'B'}},
+	}
+	assert.Equal(t, true, blocks.IsSlashableAttestationData(att1, att2), "Atts should have been slashable")
+	att1.Target.Epoch = 4
+	att1.Source.Epoch = 2
+	att2.Source.Epoch = 3
+	assert.Equal(t, true, blocks.IsSlashableAttestationData(att1, att2), "Atts should have been slashable")
+}
+
+func TestProcessAttesterSlashings_DataNotSlashable(t *testing.T) {
+	slashings := []*ethpb.AttesterSlashing{
+		{
+			Attestation_1: &ethpb.IndexedAttestation{
+				Data: &ethpb.AttestationData{
+					Source: &ethpb.Checkpoint{Epoch: 0},
+					Target: &ethpb.Checkpoint{Epoch: 0},
+				},
+			},
+			Attestation_2: &ethpb.IndexedAttestation{
+				Data: &ethpb.AttestationData{
+					Source: &ethpb.Checkpoint{Epoch: 1},
+					Target: &ethpb.Checkpoint{Epoch: 1},
+				},
+			},
+		},
+	}
+	registry := []*ethpb.Validator{}
+	currentSlot := uint64(0)
+
+	beaconState, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Validators: registry,
+		Slot:       currentSlot,
+	})
+	require.NoError(t, err)
+	block := &ethpb.BeaconBlock{
+		Body: &ethpb.BeaconBlockBody{
+			AttesterSlashings: slashings,
+		},
+	}
+	want := fmt.Sprint("attestations are not slashable")
+	_, err = blocks.ProcessAttesterSlashings(context.Background(), beaconState, block.Body)
+	assert.ErrorContains(t, want, err)
+}
+
+func TestProcessAttesterSlashings_IndexedAttestationFailedToVerify(t *testing.T) {
+	registry := []*ethpb.Validator{}
+	currentSlot := uint64(0)
+
+	beaconState, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Validators: registry,
+		Slot:       currentSlot,
+	})
+	require.NoError(t, err)
+
+	slashings := []*ethpb.AttesterSlashing{
+		{
+			Attestation_1: &ethpb.IndexedAttestation{
+				Data: &ethpb.AttestationData{
+					Source: &ethpb.Checkpoint{Epoch: 1},
+					Target: &ethpb.Checkpoint{Epoch: 0},
+				},
+				AttestingIndices: make([]uint64, params.BeaconConfig().MaxValidatorsPerCommittee+1),
+			},
+			Attestation_2: &ethpb.IndexedAttestation{
+				Data: &ethpb.AttestationData{
+					Source: &ethpb.Checkpoint{Epoch: 0},
+					Target: &ethpb.Checkpoint{Epoch: 0},
+				},
+				AttestingIndices: make([]uint64, params.BeaconConfig().MaxValidatorsPerCommittee+1),
+			},
+		},
+	}
+
+	block := &ethpb.BeaconBlock{
+		Body: &ethpb.BeaconBlockBody{
+			AttesterSlashings: slashings,
+		},
+	}
+
+	want := fmt.Sprint("validator indices count exceeds MAX_VALIDATORS_PER_COMMITTEE")
+	_, err = blocks.ProcessAttesterSlashings(context.Background(), beaconState, block.Body)
+	assert.ErrorContains(t, want, err)
+}
+
+func TestProcessAttesterSlashings_AppliesCorrectStatus(t *testing.T) {
+	beaconState, privKeys := testutil.DeterministicGenesisState(t, 100)
+	for _, vv := range beaconState.Validators() {
+		vv.WithdrawableEpoch = 1 * params.BeaconConfig().SlotsPerEpoch
+	}
+
+	att1 := &ethpb.IndexedAttestation{
+		Data: &ethpb.AttestationData{
+			Source: &ethpb.Checkpoint{Epoch: 1},
+			Target: &ethpb.Checkpoint{Epoch: 0},
+		},
+		AttestingIndices: []uint64{0, 1},
+	}
+	domain, err := helpers.Domain(beaconState.Fork(), 0, params.BeaconConfig().DomainBeaconAttester, beaconState.GenesisValidatorRoot())
+	require.NoError(t, err)
+	signingRoot, err := helpers.ComputeSigningRoot(att1.Data, domain)
+	assert.NoError(t, err, "Could not get signing root of beacon block header")
+	sig0 := privKeys[0].Sign(signingRoot[:])
+	sig1 := privKeys[1].Sign(signingRoot[:])
+	aggregateSig := bls.AggregateSignatures([]bls.Signature{sig0, sig1})
+	att1.Signature = aggregateSig.Marshal()[:]
+
+	att2 := &ethpb.IndexedAttestation{
+		Data: &ethpb.AttestationData{
+			Source: &ethpb.Checkpoint{Epoch: 0},
+			Target: &ethpb.Checkpoint{Epoch: 0},
+		},
+		AttestingIndices: []uint64{0, 1},
+	}
+	signingRoot, err = helpers.ComputeSigningRoot(att2.Data, domain)
+	assert.NoError(t, err, "Could not get signing root of beacon block header")
+	sig0 = privKeys[0].Sign(signingRoot[:])
+	sig1 = privKeys[1].Sign(signingRoot[:])
+	aggregateSig = bls.AggregateSignatures([]bls.Signature{sig0, sig1})
+	att2.Signature = aggregateSig.Marshal()[:]
+
+	slashings := []*ethpb.AttesterSlashing{
+		{
+			Attestation_1: att1,
+			Attestation_2: att2,
+		},
+	}
+
+	currentSlot := 2 * params.BeaconConfig().SlotsPerEpoch
+	require.NoError(t, beaconState.SetSlot(currentSlot))
+
+	block := &ethpb.BeaconBlock{
+		Body: &ethpb.BeaconBlockBody{
+			AttesterSlashings: slashings,
+		},
+	}
+
+	newState, err := blocks.ProcessAttesterSlashings(context.Background(), beaconState, block.Body)
+	require.NoError(t, err)
+	newRegistry := newState.Validators()
+
+	// Given the intersection of slashable indices is [1], only validator
+	// at index 1 should be slashed and exited. We confirm this below.
+	if newRegistry[1].ExitEpoch != beaconState.Validators()[1].ExitEpoch {
+		t.Errorf(
+			`
+			Expected validator at index 1's exit epoch to match
+			%d, received %d instead
+			`,
+			beaconState.Validators()[1].ExitEpoch,
+			newRegistry[1].ExitEpoch,
+		)
+	}
+}

beacon-chain/core/blocks/block_operations_fuzz_test.go

@@ -25,12 +25,8 @@ func TestFuzzProcessAttestationNoVerify_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(att)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
-		if _, err = ProcessAttestationNoVerify(ctx, s, att); err != nil {
-			t.Log(err)
-		}
+		_, err = ProcessAttestationNoVerify(ctx, s, att)
+		_ = err
 	}
 }
 
@@ -44,12 +40,8 @@ func TestFuzzProcessBlockHeader_10000(t *testing.T) {
 		fuzzer.Fuzz(block)
 
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
-		if _, err = ProcessBlockHeader(s, block); err != nil {
-			t.Log(err)
-		}
+		_, err = ProcessBlockHeader(s, block)
+		_ = err
 	}
 }
 
@@ -70,13 +62,9 @@ func TestFuzzverifyDepositDataSigningRoot_10000(t *testing.T) {
 		fuzzer.Fuzz(&p)
 		fuzzer.Fuzz(&s)
 		fuzzer.Fuzz(&d)
-		if err := verifySignature(ba, pubkey[:], sig[:], domain[:]); err != nil {
-			t.Log(err)
-		}
-		if err := verifySignature(ba, p, s, d); err != nil {
-			t.Log(err)
-		}
-
+		err := verifySignature(ba, pubkey[:], sig[:], domain[:])
+		err = verifySignature(ba, p, s, d)
+		_ = err
 	}
 }
 
@@ -102,8 +90,8 @@ func TestFuzzareEth1DataEqual_10000(t *testing.T) {
 	for i := 0; i < 10000; i++ {
 		fuzzer.Fuzz(eth1data)
 		fuzzer.Fuzz(eth1data2)
-		areEth1DataEqual(eth1data, eth1data2)
-		areEth1DataEqual(eth1data, eth1data)
+		AreEth1DataEqual(eth1data, eth1data2)
+		AreEth1DataEqual(eth1data, eth1data)
 	}
 }
 
@@ -117,12 +105,8 @@ func TestFuzzEth1DataHasEnoughSupport_10000(t *testing.T) {
 		s, err := beaconstate.InitializeFromProto(&ethereum_beacon_p2p_v1.BeaconState{
 			Eth1DataVotes: stateVotes,
 		})
-		if err != nil {
-			t.Log(err)
-		}
-		if _, err := Eth1DataHasEnoughSupport(s, eth1data); err != nil {
-			t.Log(err)
-		}
+		_, err = Eth1DataHasEnoughSupport(s, eth1data)
+		_ = err
 	}
 
 }
@@ -136,12 +120,8 @@ func TestFuzzProcessBlockHeaderNoVerify_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(block)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
-		if _, err = ProcessBlockHeaderNoVerify(s, block); err != nil {
-			t.Log(err)
-		}
+		_, err = ProcessBlockHeaderNoVerify(s, block)
+		_ = err
 	}
 }
 
@@ -154,9 +134,6 @@ func TestFuzzProcessRandao_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(blockBody)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessRandao(s, blockBody)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
@@ -173,9 +150,6 @@ func TestFuzzProcessRandaoNoVerify_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(blockBody)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessRandaoNoVerify(s, blockBody)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
@@ -192,9 +166,6 @@ func TestFuzzProcessProposerSlashings_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(blockBody)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessProposerSlashings(ctx, s, blockBody)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
@@ -210,12 +181,8 @@ func TestFuzzVerifyProposerSlashing_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(proposerSlashing)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
-		if err := VerifyProposerSlashing(s, proposerSlashing); err != nil {
-			t.Log(err)
-		}
+		err = VerifyProposerSlashing(s, proposerSlashing)
+		_ = err
 	}
 }
 
@@ -228,9 +195,6 @@ func TestFuzzProcessAttesterSlashings_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(blockBody)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessAttesterSlashings(ctx, s, blockBody)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
@@ -247,12 +211,8 @@ func TestFuzzVerifyAttesterSlashing_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(attesterSlashing)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
-		if err := VerifyAttesterSlashing(ctx, s, attesterSlashing); err != nil {
-			t.Log(err)
-		}
+		err = VerifyAttesterSlashing(ctx, s, attesterSlashing)
+		_ = err
 	}
 }
 
@@ -287,9 +247,6 @@ func TestFuzzProcessAttestations_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(blockBody)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessAttestations(ctx, s, blockBody)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
@@ -306,9 +263,6 @@ func TestFuzzProcessAttestationsNoVerify_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(blockBody)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessAttestationsNoVerify(ctx, s, blockBody)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
@@ -325,9 +279,6 @@ func TestFuzzProcessAttestation_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(attestation)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessAttestation(ctx, s, attestation)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, attestation)
@@ -344,12 +295,8 @@ func TestFuzzVerifyIndexedAttestationn_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(idxAttestation)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
-		if err := VerifyIndexedAttestation(ctx, s, idxAttestation); err != nil {
-			t.Log(err)
-		}
+		err = VerifyIndexedAttestation(ctx, s, idxAttestation)
+		_ = err
 	}
 }
 
@@ -362,12 +309,8 @@ func TestFuzzVerifyAttestation_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(attestation)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
-		if err := VerifyAttestation(ctx, s, attestation); err != nil {
-			t.Log(err)
-		}
+		err = VerifyAttestation(ctx, s, attestation)
+		_ = err
 	}
 }
 
@@ -380,10 +323,7 @@ func TestFuzzProcessDeposits_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(blockBody)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
-		r, err := ProcessDeposits(ctx, s, blockBody)
+		r, err := ProcessDeposits(ctx, s, blockBody.Deposits)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
 		}
@@ -400,10 +340,7 @@ func TestFuzzProcessPreGenesisDeposit_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(deposit)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
-		r, err := ProcessPreGenesisDeposit(ctx, s, deposit)
+		r, err := ProcessPreGenesisDeposits(ctx, s, []*eth.Deposit{deposit})
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, deposit)
 		}
@@ -419,10 +356,7 @@ func TestFuzzProcessDeposit_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(deposit)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
-		r, err := ProcessDeposit(s, deposit)
+		r, err := ProcessDeposit(s, deposit, true)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, deposit)
 		}
@@ -437,12 +371,8 @@ func TestFuzzverifyDeposit_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(deposit)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
-		if err := verifyDeposit(s, deposit); err != nil {
-			t.Log(err)
-		}
+		err = verifyDeposit(s, deposit)
+		_ = err
 	}
 }
 
@@ -455,9 +385,6 @@ func TestFuzzProcessVoluntaryExits_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(blockBody)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessVoluntaryExits(ctx, s, blockBody)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
@@ -473,9 +400,6 @@ func TestFuzzProcessVoluntaryExitsNoVerify_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(blockBody)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessVoluntaryExitsNoVerify(s, blockBody)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
@@ -486,7 +410,7 @@ func TestFuzzProcessVoluntaryExitsNoVerify_10000(t *testing.T) {
 func TestFuzzVerifyExit_10000(t *testing.T) {
 	fuzzer := fuzz.NewWithSeed(0)
 	ve := &eth.SignedVoluntaryExit{}
-	val := &eth.Validator{}
+	val := &beaconstate.ReadOnlyValidator{}
 	fork := &pb.Fork{}
 	var slot uint64
 
@@ -495,8 +419,7 @@ func TestFuzzVerifyExit_10000(t *testing.T) {
 		fuzzer.Fuzz(val)
 		fuzzer.Fuzz(fork)
 		fuzzer.Fuzz(&slot)
-		if err := VerifyExit(val, slot, fork, ve, params.BeaconConfig().ZeroHash[:]); err != nil {
-			t.Log(err)
-		}
+		err := VerifyExit(val, slot, fork, ve, params.BeaconConfig().ZeroHash[:])
+		_ = err
 	}
 }

beacon-chain/core/blocks/block_regression_test.go

@@ -10,6 +10,8 @@ import (
 	"github.com/prysmaticlabs/prysm/shared/bls"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 )
 
 func TestProcessAttesterSlashings_RegressionSlashableIndices(t *testing.T) {
@@ -43,14 +45,10 @@ func TestProcessAttesterSlashings_RegressionSlashableIndices(t *testing.T) {
 		AttestingIndices: setA,
 	}
 	domain, err := helpers.Domain(beaconState.Fork(), 0, params.BeaconConfig().DomainBeaconAttester, beaconState.GenesisValidatorRoot())
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	signingRoot, err := helpers.ComputeSigningRoot(att1.Data, domain)
-	if err != nil {
-		t.Errorf("Could not get signing root of beacon block header: %v", err)
-	}
-	aggSigs := []*bls.Signature{}
+	require.NoError(t, err, "Could not get signing root of beacon block header")
+	aggSigs := []bls.Signature{}
 	for _, index := range setA {
 		sig := privKeys[index].Sign(signingRoot[:])
 		aggSigs = append(aggSigs, sig)
@@ -67,10 +65,8 @@ func TestProcessAttesterSlashings_RegressionSlashableIndices(t *testing.T) {
 		AttestingIndices: setB,
 	}
 	signingRoot, err = helpers.ComputeSigningRoot(att2.Data, domain)
-	if err != nil {
-		t.Errorf("Could not get signing root of beacon block header: %v", err)
-	}
-	aggSigs = []*bls.Signature{}
+	assert.NoError(t, err, "Could not get signing root of beacon block header")
+	aggSigs = []bls.Signature{}
 	for _, index := range setB {
 		sig := privKeys[index].Sign(signingRoot[:])
 		aggSigs = append(aggSigs, sig)
@@ -86,9 +82,7 @@ func TestProcessAttesterSlashings_RegressionSlashableIndices(t *testing.T) {
 	}
 
 	currentSlot := 2 * params.BeaconConfig().SlotsPerEpoch
-	if err := beaconState.SetSlot(currentSlot); err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, beaconState.SetSlot(currentSlot))
 
 	block := &ethpb.BeaconBlock{
 		Body: &ethpb.BeaconBlockBody{
@@ -97,9 +91,7 @@ func TestProcessAttesterSlashings_RegressionSlashableIndices(t *testing.T) {
 	}
 
 	newState, err := blocks.ProcessAttesterSlashings(context.Background(), beaconState, block.Body)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	newRegistry := newState.Validators()
 	if !newRegistry[expectedSlashedVal].Slashed {
 		t.Errorf("Validator with index %d was not slashed despite performing a double vote", expectedSlashedVal)

beacon-chain/core/blocks/block_test.go

@@ -1,21 +0,0 @@
-package blocks_test
-
-import (
-	"bytes"
-	"testing"
-
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-)
-
-func TestGenesisBlock_InitializedCorrectly(t *testing.T) {
-	stateHash := []byte{0}
-	b1 := blocks.NewGenesisBlock(stateHash)
-
-	if b1.Block.ParentRoot == nil {
-		t.Error("genesis block missing ParentHash field")
-	}
-
-	if !bytes.Equal(b1.Block.StateRoot, stateHash) {
-		t.Error("genesis block StateRootHash32 isn't initialized correctly")
-	}
-}

beacon-chain/core/blocks/deposit.go

@@ -0,0 +1,270 @@
+package blocks
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/go-ssz"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/bls"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/depositutil"
+	"github.com/prysmaticlabs/prysm/shared/mathutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/trieutil"
+)
+
+// ProcessPreGenesisDeposits processes a deposit for the beacon state before chainstart.
+func ProcessPreGenesisDeposits(
+	ctx context.Context,
+	beaconState *stateTrie.BeaconState,
+	deposits []*ethpb.Deposit,
+) (*stateTrie.BeaconState, error) {
+	var err error
+	beaconState, err = ProcessDeposits(ctx, beaconState, deposits)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not process deposit")
+	}
+	for _, deposit := range deposits {
+		pubkey := deposit.Data.PublicKey
+		index, ok := beaconState.ValidatorIndexByPubkey(bytesutil.ToBytes48(pubkey))
+		if !ok {
+			return beaconState, nil
+		}
+		balance, err := beaconState.BalanceAtIndex(index)
+		if err != nil {
+			return nil, err
+		}
+		validator, err := beaconState.ValidatorAtIndex(index)
+		if err != nil {
+			return nil, err
+		}
+		validator.EffectiveBalance = mathutil.Min(balance-balance%params.BeaconConfig().EffectiveBalanceIncrement, params.BeaconConfig().MaxEffectiveBalance)
+		if validator.EffectiveBalance ==
+			params.BeaconConfig().MaxEffectiveBalance {
+			validator.ActivationEligibilityEpoch = 0
+			validator.ActivationEpoch = 0
+		}
+		if err := beaconState.UpdateValidatorAtIndex(index, validator); err != nil {
+			return nil, err
+		}
+	}
+	return beaconState, nil
+}
+
+// ProcessDeposits is one of the operations performed on each processed
+// beacon block to verify queued validators from the Ethereum 1.0 Deposit Contract
+// into the beacon chain.
+//
+// Spec pseudocode definition:
+//   For each deposit in block.body.deposits:
+//     process_deposit(state, deposit)
+func ProcessDeposits(
+	ctx context.Context,
+	beaconState *stateTrie.BeaconState,
+	deposits []*ethpb.Deposit,
+) (*stateTrie.BeaconState, error) {
+	var err error
+
+	domain, err := helpers.ComputeDomain(params.BeaconConfig().DomainDeposit, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// Attempt to verify all deposit signatures at once, if this fails then fall back to processing
+	// individual deposits with signature verification enabled.
+	var verifySignature bool
+	if err := verifyDepositDataWithDomain(ctx, deposits, domain); err != nil {
+		log.WithError(err).Debug("Failed to verify deposit data, verifying signatures individually")
+		verifySignature = true
+	}
+
+	for _, deposit := range deposits {
+		if deposit == nil || deposit.Data == nil {
+			return nil, errors.New("got a nil deposit in block")
+		}
+		beaconState, err = ProcessDeposit(beaconState, deposit, verifySignature)
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not process deposit from %#x", bytesutil.Trunc(deposit.Data.PublicKey))
+		}
+	}
+	return beaconState, nil
+}
+
+// ProcessDeposit takes in a deposit object and inserts it
+// into the registry as a new validator or balance change.
+//
+// Spec pseudocode definition:
+// def process_deposit(state: BeaconState, deposit: Deposit) -> None:
+//    # Verify the Merkle branch
+//    assert is_valid_merkle_branch(
+//        leaf=hash_tree_root(deposit.data),
+//        branch=deposit.proof,
+//        depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1,  # Add 1 for the List length mix-in
+//        index=state.eth1_deposit_index,
+//        root=state.eth1_data.deposit_root,
+//    )
+//
+//    # Deposits must be processed in order
+//    state.eth1_deposit_index += 1
+//
+//    pubkey = deposit.data.pubkey
+//    amount = deposit.data.amount
+//    validator_pubkeys = [v.pubkey for v in state.validators]
+//    if pubkey not in validator_pubkeys:
+//        # Verify the deposit signature (proof of possession) which is not checked by the deposit contract
+//        deposit_message = DepositMessage(
+//            pubkey=deposit.data.pubkey,
+//            withdrawal_credentials=deposit.data.withdrawal_credentials,
+//            amount=deposit.data.amount,
+//        )
+//        domain = compute_domain(DOMAIN_DEPOSIT)  # Fork-agnostic domain since deposits are valid across forks
+//        signing_root = compute_signing_root(deposit_message, domain)
+//        if not bls.Verify(pubkey, signing_root, deposit.data.signature):
+//            return
+//
+//        # Add validator and balance entries
+//        state.validators.append(get_validator_from_deposit(state, deposit))
+//        state.balances.append(amount)
+//    else:
+//        # Increase balance by deposit amount
+//        index = ValidatorIndex(validator_pubkeys.index(pubkey))
+//        increase_balance(state, index, amount)
+func ProcessDeposit(beaconState *stateTrie.BeaconState, deposit *ethpb.Deposit, verifySignature bool) (*stateTrie.BeaconState, error) {
+	if err := verifyDeposit(beaconState, deposit); err != nil {
+		if deposit == nil || deposit.Data == nil {
+			return nil, err
+		}
+		return nil, errors.Wrapf(err, "could not verify deposit from %#x", bytesutil.Trunc(deposit.Data.PublicKey))
+	}
+	if err := beaconState.SetEth1DepositIndex(beaconState.Eth1DepositIndex() + 1); err != nil {
+		return nil, err
+	}
+	pubKey := deposit.Data.PublicKey
+	amount := deposit.Data.Amount
+	index, ok := beaconState.ValidatorIndexByPubkey(bytesutil.ToBytes48(pubKey))
+	if !ok {
+		if verifySignature {
+			domain, err := helpers.ComputeDomain(params.BeaconConfig().DomainDeposit, nil, nil)
+			if err != nil {
+				return nil, err
+			}
+			depositSig := deposit.Data.Signature
+			if err := verifyDepositDataSigningRoot(deposit.Data, pubKey, depositSig, domain); err != nil {
+				// Ignore this error as in the spec pseudo code.
+				log.Debugf("Skipping deposit: could not verify deposit data signature: %v", err)
+				return beaconState, nil
+			}
+		}
+
+		effectiveBalance := amount - (amount % params.BeaconConfig().EffectiveBalanceIncrement)
+		if params.BeaconConfig().MaxEffectiveBalance < effectiveBalance {
+			effectiveBalance = params.BeaconConfig().MaxEffectiveBalance
+		}
+		if err := beaconState.AppendValidator(&ethpb.Validator{
+			PublicKey:                  pubKey,
+			WithdrawalCredentials:      deposit.Data.WithdrawalCredentials,
+			ActivationEligibilityEpoch: params.BeaconConfig().FarFutureEpoch,
+			ActivationEpoch:            params.BeaconConfig().FarFutureEpoch,
+			ExitEpoch:                  params.BeaconConfig().FarFutureEpoch,
+			WithdrawableEpoch:          params.BeaconConfig().FarFutureEpoch,
+			EffectiveBalance:           effectiveBalance,
+		}); err != nil {
+			return nil, err
+		}
+		if err := beaconState.AppendBalance(amount); err != nil {
+			return nil, err
+		}
+	} else if err := helpers.IncreaseBalance(beaconState, index, amount); err != nil {
+		return nil, err
+	}
+
+	return beaconState, nil
+}
+
+func verifyDeposit(beaconState *stateTrie.BeaconState, deposit *ethpb.Deposit) error {
+	// Verify Merkle proof of deposit and deposit trie root.
+	if deposit == nil || deposit.Data == nil {
+		return errors.New("received nil deposit or nil deposit data")
+	}
+	eth1Data := beaconState.Eth1Data()
+	if eth1Data == nil {
+		return errors.New("received nil eth1data in the beacon state")
+	}
+
+	receiptRoot := eth1Data.DepositRoot
+	leaf, err := ssz.HashTreeRoot(deposit.Data)
+	if err != nil {
+		return errors.Wrap(err, "could not tree hash deposit data")
+	}
+	if ok := trieutil.VerifyMerkleBranch(
+		receiptRoot,
+		leaf[:],
+		int(beaconState.Eth1DepositIndex()),
+		deposit.Proof,
+	); !ok {
+		return fmt.Errorf(
+			"deposit merkle branch of deposit root did not verify for root: %#x",
+			receiptRoot,
+		)
+	}
+	return nil
+}
+
+// Deprecated: This method uses deprecated ssz.SigningRoot.
+func verifyDepositDataSigningRoot(obj *ethpb.Deposit_Data, pub []byte, signature []byte, domain []byte) error {
+	return depositutil.VerifyDepositSignature(obj, domain)
+}
+
+func verifyDepositDataWithDomain(ctx context.Context, deps []*ethpb.Deposit, domain []byte) error {
+	if len(deps) == 0 {
+		return nil
+	}
+	pks := make([]bls.PublicKey, len(deps))
+	sigs := make([]bls.Signature, len(deps))
+	msgs := make([][32]byte, len(deps))
+	for i, dep := range deps {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		if dep == nil || dep.Data == nil {
+			return errors.New("nil deposit")
+		}
+
+		dpk, err := bls.PublicKeyFromBytes(dep.Data.PublicKey)
+		if err != nil {
+			return err
+		}
+		pks[i] = dpk
+		dsig, err := bls.SignatureFromBytes(dep.Data.Signature)
+		if err != nil {
+			return err
+		}
+		sigs[i] = dsig
+		root, err := ssz.SigningRoot(dep.Data)
+		if err != nil {
+			return errors.Wrap(err, "could not get signing root")
+		}
+		signingData := &pb.SigningData{
+			ObjectRoot: root[:],
+			Domain:     domain,
+		}
+		ctrRoot, err := ssz.HashTreeRoot(signingData)
+		if err != nil {
+			return errors.Wrap(err, "could not get container root")
+		}
+		msgs[i] = ctrRoot
+	}
+	verify, err := bls.VerifyMultipleSignatures(sigs, msgs, pks)
+	if err != nil {
+		return errors.Errorf("could not verify multiple signatures: %v", err)
+	}
+	if !verify {
+		return errors.New("one or more deposit signatures did not verify")
+	}
+	return nil
+}

beacon-chain/core/blocks/deposit_test.go

@@ -0,0 +1,267 @@
+package blocks_test
+
+import (
+	"context"
+	"testing"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/go-ssz"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/bls"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+	"github.com/prysmaticlabs/prysm/shared/trieutil"
+)
+
+func TestProcessDeposits_SameValidatorMultipleDepositsSameBlock(t *testing.T) {
+	// Same validator created 3 valid deposits within the same block
+	testutil.ResetCache()
+	dep, _, err := testutil.DeterministicDepositsAndKeysSameValidator(3)
+	require.NoError(t, err)
+	eth1Data, err := testutil.DeterministicEth1Data(len(dep))
+	require.NoError(t, err)
+	block := &ethpb.BeaconBlock{
+		Body: &ethpb.BeaconBlockBody{
+			// 3 deposits from the same validator
+			Deposits: []*ethpb.Deposit{dep[0], dep[1], dep[2]},
+		},
+	}
+	registry := []*ethpb.Validator{
+		{
+			PublicKey:             []byte{1},
+			WithdrawalCredentials: []byte{1, 2, 3},
+		},
+	}
+	balances := []uint64{0}
+	beaconState, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Validators: registry,
+		Balances:   balances,
+		Eth1Data:   eth1Data,
+		Fork: &pb.Fork{
+			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
+			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
+		},
+	})
+	require.NoError(t, err)
+	newState, err := blocks.ProcessDeposits(context.Background(), beaconState, block.Body.Deposits)
+	require.NoError(t, err, "Expected block deposits to process correctly")
+
+	assert.Equal(t, 2, len(newState.Validators()), "Incorrect validator count")
+}
+
+func TestProcessDeposits_MerkleBranchFailsVerification(t *testing.T) {
+	deposit := &ethpb.Deposit{
+		Data: &ethpb.Deposit_Data{
+			PublicKey: []byte{1, 2, 3},
+			Signature: make([]byte, 96),
+		},
+	}
+	leaf, err := ssz.HashTreeRoot(deposit.Data)
+	require.NoError(t, err)
+
+	// We then create a merkle branch for the test.
+	depositTrie, err := trieutil.GenerateTrieFromItems([][]byte{leaf[:]}, int(params.BeaconConfig().DepositContractTreeDepth))
+	require.NoError(t, err, "Could not generate trie")
+	proof, err := depositTrie.MerkleProof(0)
+	require.NoError(t, err, "Could not generate proof")
+
+	deposit.Proof = proof
+	block := &ethpb.BeaconBlock{
+		Body: &ethpb.BeaconBlockBody{
+			Deposits: []*ethpb.Deposit{deposit},
+		},
+	}
+	beaconState, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Eth1Data: &ethpb.Eth1Data{
+			DepositRoot: []byte{0},
+			BlockHash:   []byte{1},
+		},
+	})
+	require.NoError(t, err)
+	want := "deposit root did not verify"
+	_, err = blocks.ProcessDeposits(context.Background(), beaconState, block.Body.Deposits)
+	assert.ErrorContains(t, want, err)
+}
+
+func TestProcessDeposits_AddsNewValidatorDeposit(t *testing.T) {
+	dep, _, err := testutil.DeterministicDepositsAndKeys(1)
+	require.NoError(t, err)
+	eth1Data, err := testutil.DeterministicEth1Data(len(dep))
+	require.NoError(t, err)
+
+	block := &ethpb.BeaconBlock{
+		Body: &ethpb.BeaconBlockBody{
+			Deposits: []*ethpb.Deposit{dep[0]},
+		},
+	}
+	registry := []*ethpb.Validator{
+		{
+			PublicKey:             []byte{1},
+			WithdrawalCredentials: []byte{1, 2, 3},
+		},
+	}
+	balances := []uint64{0}
+	beaconState, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Validators: registry,
+		Balances:   balances,
+		Eth1Data:   eth1Data,
+		Fork: &pb.Fork{
+			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
+			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
+		},
+	})
+	require.NoError(t, err)
+	newState, err := blocks.ProcessDeposits(context.Background(), beaconState, block.Body.Deposits)
+	require.NoError(t, err, "Expected block deposits to process correctly")
+	if newState.Balances()[1] != dep[0].Data.Amount {
+		t.Errorf(
+			"Expected state validator balances index 0 to equal %d, received %d",
+			dep[0].Data.Amount,
+			newState.Balances()[1],
+		)
+	}
+}
+
+func TestProcessDeposits_RepeatedDeposit_IncreasesValidatorBalance(t *testing.T) {
+	sk := bls.RandKey()
+	deposit := &ethpb.Deposit{
+		Data: &ethpb.Deposit_Data{
+			PublicKey: sk.PublicKey().Marshal(),
+			Amount:    1000,
+		},
+	}
+	sr, err := helpers.ComputeSigningRoot(deposit.Data, bytesutil.ToBytes(3, 8))
+	require.NoError(t, err)
+	sig := sk.Sign(sr[:])
+	deposit.Data.Signature = sig.Marshal()
+	leaf, err := ssz.HashTreeRoot(deposit.Data)
+	require.NoError(t, err)
+
+	// We then create a merkle branch for the test.
+	depositTrie, err := trieutil.GenerateTrieFromItems([][]byte{leaf[:]}, int(params.BeaconConfig().DepositContractTreeDepth))
+	require.NoError(t, err, "Could not generate trie")
+	proof, err := depositTrie.MerkleProof(0)
+	require.NoError(t, err, "Could not generate proof")
+
+	deposit.Proof = proof
+	block := &ethpb.BeaconBlock{
+		Body: &ethpb.BeaconBlockBody{
+			Deposits: []*ethpb.Deposit{deposit},
+		},
+	}
+	registry := []*ethpb.Validator{
+		{
+			PublicKey: []byte{1, 2, 3},
+		},
+		{
+			PublicKey:             sk.PublicKey().Marshal(),
+			WithdrawalCredentials: []byte{1},
+		},
+	}
+	balances := []uint64{0, 50}
+	root := depositTrie.Root()
+	beaconState, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Validators: registry,
+		Balances:   balances,
+		Eth1Data: &ethpb.Eth1Data{
+			DepositRoot: root[:],
+			BlockHash:   root[:],
+		},
+	})
+	require.NoError(t, err)
+	newState, err := blocks.ProcessDeposits(context.Background(), beaconState, block.Body.Deposits)
+	require.NoError(t, err, "Process deposit failed")
+	assert.Equal(t, uint64(1000+50), newState.Balances()[1], "Expected balance at index 1 to be 1050")
+}
+
+func TestProcessDeposit_AddsNewValidatorDeposit(t *testing.T) {
+	//Similar to TestProcessDeposits_AddsNewValidatorDeposit except that this test directly calls ProcessDeposit
+	dep, _, err := testutil.DeterministicDepositsAndKeys(1)
+	require.NoError(t, err)
+	eth1Data, err := testutil.DeterministicEth1Data(len(dep))
+	require.NoError(t, err)
+
+	registry := []*ethpb.Validator{
+		{
+			PublicKey:             []byte{1},
+			WithdrawalCredentials: []byte{1, 2, 3},
+		},
+	}
+	balances := []uint64{0}
+	beaconState, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Validators: registry,
+		Balances:   balances,
+		Eth1Data:   eth1Data,
+		Fork: &pb.Fork{
+			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
+			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
+		},
+	})
+	require.NoError(t, err)
+	newState, err := blocks.ProcessDeposit(beaconState, dep[0], true)
+	require.NoError(t, err, "Process deposit failed")
+	assert.Equal(t, 2, len(newState.Validators()), "Expected validator list to have length 2")
+	assert.Equal(t, 2, len(newState.Balances()), "Expected validator balances list to have length 2")
+	if newState.Balances()[1] != dep[0].Data.Amount {
+		t.Errorf(
+			"Expected state validator balances index 1 to equal %d, received %d",
+			dep[0].Data.Amount,
+			newState.Balances()[1],
+		)
+	}
+}
+
+func TestProcessDeposit_SkipsInvalidDeposit(t *testing.T) {
+	// Same test settings as in TestProcessDeposit_AddsNewValidatorDeposit, except that we use an invalid signature
+	dep, _, err := testutil.DeterministicDepositsAndKeys(1)
+	require.NoError(t, err)
+	dep[0].Data.Signature = make([]byte, 96)
+	trie, _, err := testutil.DepositTrieFromDeposits(dep)
+	require.NoError(t, err)
+	root := trie.Root()
+	eth1Data := &ethpb.Eth1Data{
+		DepositRoot:  root[:],
+		DepositCount: 1,
+	}
+	registry := []*ethpb.Validator{
+		{
+			PublicKey:             []byte{1},
+			WithdrawalCredentials: []byte{1, 2, 3},
+		},
+	}
+	balances := []uint64{0}
+	beaconState, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Validators: registry,
+		Balances:   balances,
+		Eth1Data:   eth1Data,
+		Fork: &pb.Fork{
+			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
+			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
+		},
+	})
+	require.NoError(t, err)
+	newState, err := blocks.ProcessDeposit(beaconState, dep[0], true)
+	require.NoError(t, err, "Expected invalid block deposit to be ignored without error")
+
+	if newState.Eth1DepositIndex() != 1 {
+		t.Errorf(
+			"Expected Eth1DepositIndex to be increased by 1 after processing an invalid deposit, received change: %v",
+			newState.Eth1DepositIndex(),
+		)
+	}
+	if len(newState.Validators()) != 1 {
+		t.Errorf("Expected validator list to have length 1, received: %v", len(newState.Validators()))
+	}
+	if len(newState.Balances()) != 1 {
+		t.Errorf("Expected validator balances list to have length 1, received: %v", len(newState.Balances()))
+	}
+	if newState.Balances()[0] != 0 {
+		t.Errorf("Expected validator balance at index 0 to stay 0, received: %v", newState.Balances()[0])
+	}
+}

beacon-chain/core/blocks/eth1_data.go

@@ -0,0 +1,74 @@
+package blocks
+
+import (
+	"bytes"
+	"errors"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/params"
+)
+
+// ProcessEth1DataInBlock is an operation performed on each
+// beacon block to ensure the ETH1 data votes are processed
+// into the beacon state.
+//
+// Official spec definition:
+//   def process_eth1_data(state: BeaconState, body: BeaconBlockBody) -> None:
+//    state.eth1_data_votes.append(body.eth1_data)
+//    if state.eth1_data_votes.count(body.eth1_data) * 2 > EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH:
+//        state.latest_eth1_data = body.eth1_data
+func ProcessEth1DataInBlock(beaconState *stateTrie.BeaconState, block *ethpb.BeaconBlock) (*stateTrie.BeaconState, error) {
+	if beaconState == nil {
+		return nil, errors.New("nil state")
+	}
+	if block == nil || block.Body == nil {
+		return nil, errors.New("nil block or block withought body")
+	}
+	if err := beaconState.AppendEth1DataVotes(block.Body.Eth1Data); err != nil {
+		return nil, err
+	}
+	hasSupport, err := Eth1DataHasEnoughSupport(beaconState, block.Body.Eth1Data)
+	if err != nil {
+		return nil, err
+	}
+	if hasSupport {
+		if err := beaconState.SetEth1Data(block.Body.Eth1Data); err != nil {
+			return nil, err
+		}
+	}
+	return beaconState, nil
+}
+
+// AreEth1DataEqual checks equality between two eth1 data objects.
+func AreEth1DataEqual(a, b *ethpb.Eth1Data) bool {
+	if a == nil && b == nil {
+		return true
+	}
+	if a == nil || b == nil {
+		return false
+	}
+	return a.DepositCount == b.DepositCount &&
+		bytes.Equal(a.BlockHash, b.BlockHash) &&
+		bytes.Equal(a.DepositRoot, b.DepositRoot)
+}
+
+// Eth1DataHasEnoughSupport returns true when the given eth1data has more than 50% votes in the
+// eth1 voting period. A vote is cast by including eth1data in a block and part of state processing
+// appends eth1data to the state in the Eth1DataVotes list. Iterating through this list checks the
+// votes to see if they match the eth1data.
+func Eth1DataHasEnoughSupport(beaconState *stateTrie.BeaconState, data *ethpb.Eth1Data) (bool, error) {
+	voteCount := uint64(0)
+	data = stateTrie.CopyETH1Data(data)
+
+	for _, vote := range beaconState.Eth1DataVotes() {
+		if AreEth1DataEqual(vote, data) {
+			voteCount++
+		}
+	}
+
+	// If 50+% majority converged on the same eth1data, then it has enough support to update the
+	// state.
+	support := params.BeaconConfig().EpochsPerEth1VotingPeriod * params.BeaconConfig().SlotsPerEpoch
+	return voteCount*2 > support, nil
+}

beacon-chain/core/blocks/eth1_data_test.go

@@ -4,16 +4,19 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 )
 
-func FakeDeposits(n int) []*ethpb.Eth1Data {
+func FakeDeposits(n uint64) []*ethpb.Eth1Data {
 	deposits := make([]*ethpb.Eth1Data, n)
-	for i := 0; i < n; i++ {
+	for i := uint64(0); i < n; i++ {
 		deposits[i] = &ethpb.Eth1Data{
 			DepositCount: 1,
 			DepositRoot:  []byte("root"),
@@ -30,7 +33,7 @@ func TestEth1DataHasEnoughSupport(t *testing.T) {
 		votingPeriodLength uint64
 	}{
 		{
-			stateVotes: FakeDeposits(4 * int(params.BeaconConfig().SlotsPerEpoch)),
+			stateVotes: FakeDeposits(4 * params.BeaconConfig().SlotsPerEpoch),
 			data: &ethpb.Eth1Data{
 				DepositCount: 1,
 				DepositRoot:  []byte("root"),
@@ -38,7 +41,7 @@ func TestEth1DataHasEnoughSupport(t *testing.T) {
 			hasSupport:         true,
 			votingPeriodLength: 7,
 		}, {
-			stateVotes: FakeDeposits(4 * int(params.BeaconConfig().SlotsPerEpoch)),
+			stateVotes: FakeDeposits(4 * params.BeaconConfig().SlotsPerEpoch),
 			data: &ethpb.Eth1Data{
 				DepositCount: 1,
 				DepositRoot:  []byte("root"),
@@ -46,7 +49,7 @@ func TestEth1DataHasEnoughSupport(t *testing.T) {
 			hasSupport:         false,
 			votingPeriodLength: 8,
 		}, {
-			stateVotes: FakeDeposits(4 * int(params.BeaconConfig().SlotsPerEpoch)),
+			stateVotes: FakeDeposits(4 * params.BeaconConfig().SlotsPerEpoch),
 			data: &ethpb.Eth1Data{
 				DepositCount: 1,
 				DepositRoot:  []byte("root"),
@@ -56,6 +59,7 @@ func TestEth1DataHasEnoughSupport(t *testing.T) {
 		},
 	}
 
+	params.SetupTestConfigCleanup(t)
 	for i, tt := range tests {
 		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
 			c := params.BeaconConfig()
@@ -65,13 +69,9 @@ func TestEth1DataHasEnoughSupport(t *testing.T) {
 			s, err := beaconstate.InitializeFromProto(&pb.BeaconState{
 				Eth1DataVotes: tt.stateVotes,
 			})
-			if err != nil {
-				t.Fatal(err)
-			}
+			require.NoError(t, err)
 			result, err := blocks.Eth1DataHasEnoughSupport(s, tt.data)
-			if err != nil {
-				t.Fatal(err)
-			}
+			require.NoError(t, err)
 
 			if result != tt.hasSupport {
 				t.Errorf(
@@ -84,3 +84,107 @@ func TestEth1DataHasEnoughSupport(t *testing.T) {
 		})
 	}
 }
+
+func TestAreEth1DataEqual(t *testing.T) {
+	type args struct {
+		a *ethpb.Eth1Data
+		b *ethpb.Eth1Data
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "true when both are nil",
+			args: args{
+				a: nil,
+				b: nil,
+			},
+			want: true,
+		},
+		{
+			name: "false when only one is nil",
+			args: args{
+				a: nil,
+				b: &ethpb.Eth1Data{
+					DepositRoot:  make([]byte, 32),
+					DepositCount: 0,
+					BlockHash:    make([]byte, 32),
+				},
+			},
+			want: false,
+		},
+		{
+			name: "true when real equality",
+			args: args{
+				a: &ethpb.Eth1Data{
+					DepositRoot:  make([]byte, 32),
+					DepositCount: 0,
+					BlockHash:    make([]byte, 32),
+				},
+				b: &ethpb.Eth1Data{
+					DepositRoot:  make([]byte, 32),
+					DepositCount: 0,
+					BlockHash:    make([]byte, 32),
+				},
+			},
+			want: true,
+		},
+		{
+			name: "false is field value differs",
+			args: args{
+				a: &ethpb.Eth1Data{
+					DepositRoot:  make([]byte, 32),
+					DepositCount: 0,
+					BlockHash:    make([]byte, 32),
+				},
+				b: &ethpb.Eth1Data{
+					DepositRoot:  make([]byte, 32),
+					DepositCount: 64,
+					BlockHash:    make([]byte, 32),
+				},
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, blocks.AreEth1DataEqual(tt.args.a, tt.args.b))
+		})
+	}
+}
+
+func TestProcessEth1Data_SetsCorrectly(t *testing.T) {
+	beaconState, err := beaconstate.InitializeFromProto(&pb.BeaconState{
+		Eth1DataVotes: []*ethpb.Eth1Data{},
+	})
+	require.NoError(t, err)
+
+	block := &ethpb.BeaconBlock{
+		Body: &ethpb.BeaconBlockBody{
+			Eth1Data: &ethpb.Eth1Data{
+				DepositRoot: []byte{2},
+				BlockHash:   []byte{3},
+			},
+		},
+	}
+
+	period := params.BeaconConfig().EpochsPerEth1VotingPeriod * params.BeaconConfig().SlotsPerEpoch
+	for i := uint64(0); i < period; i++ {
+		beaconState, err = blocks.ProcessEth1DataInBlock(beaconState, block)
+		require.NoError(t, err)
+	}
+
+	newETH1DataVotes := beaconState.Eth1DataVotes()
+	if len(newETH1DataVotes) <= 1 {
+		t.Error("Expected new ETH1 data votes to have length > 1")
+	}
+	if !proto.Equal(beaconState.Eth1Data(), beaconstate.CopyETH1Data(block.Body.Eth1Data)) {
+		t.Errorf(
+			"Expected latest eth1 data to have been set to %v, received %v",
+			block.Body.Eth1Data,
+			beaconState.Eth1Data(),
+		)
+	}
+}

beacon-chain/core/blocks/exit.go

@@ -0,0 +1,174 @@
+package blocks
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	v "github.com/prysmaticlabs/prysm/beacon-chain/core/validators"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/params"
+)
+
+// ProcessVoluntaryExits is one of the operations performed
+// on each processed beacon block to determine which validators
+// should exit the state's validator registry.
+//
+// Spec pseudocode definition:
+//   def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None:
+//    """
+//    Process ``VoluntaryExit`` operation.
+//    """
+//    validator = state.validator_registry[exit.validator_index]
+//    # Verify the validator is active
+//    assert is_active_validator(validator, get_current_epoch(state))
+//    # Verify the validator has not yet exited
+//    assert validator.exit_epoch == FAR_FUTURE_EPOCH
+//    # Exits must specify an epoch when they become valid; they are not valid before then
+//    assert get_current_epoch(state) >= exit.epoch
+//    # Verify the validator has been active long enough
+//    assert get_current_epoch(state) >= validator.activation_epoch + PERSISTENT_COMMITTEE_PERIOD
+//    # Verify signature
+//    domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, exit.epoch)
+//    assert bls_verify(validator.pubkey, signing_root(exit), exit.signature, domain)
+//    # Initiate exit
+//    initiate_validator_exit(state, exit.validator_index)
+func ProcessVoluntaryExits(
+	ctx context.Context,
+	beaconState *stateTrie.BeaconState,
+	body *ethpb.BeaconBlockBody,
+) (*stateTrie.BeaconState, error) {
+	exits := body.VoluntaryExits
+	for idx, exit := range exits {
+		if exit == nil || exit.Exit == nil {
+			return nil, errors.New("nil voluntary exit in block body")
+		}
+		val, err := beaconState.ValidatorAtIndexReadOnly(exit.Exit.ValidatorIndex)
+		if err != nil {
+			return nil, err
+		}
+		if err := VerifyExit(val, beaconState.Slot(), beaconState.Fork(), exit, beaconState.GenesisValidatorRoot()); err != nil {
+			return nil, errors.Wrapf(err, "could not verify exit %d", idx)
+		}
+		beaconState, err = v.InitiateValidatorExit(beaconState, exit.Exit.ValidatorIndex)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return beaconState, nil
+}
+
+// ProcessVoluntaryExitsNoVerify processes all the voluntary exits in
+// a block body, without verifying their BLS signatures.
+func ProcessVoluntaryExitsNoVerify(
+	beaconState *stateTrie.BeaconState,
+	body *ethpb.BeaconBlockBody,
+) (*stateTrie.BeaconState, error) {
+	exits := body.VoluntaryExits
+
+	for idx, exit := range exits {
+		if exit == nil || exit.Exit == nil {
+			return nil, errors.New("nil exit")
+		}
+		val, err := beaconState.ValidatorAtIndexReadOnly(exit.Exit.ValidatorIndex)
+		if err != nil {
+			return nil, err
+		}
+		if err := verifyExitConditions(val, beaconState.Slot(), exit.Exit); err != nil {
+			return nil, err
+		}
+		// Validate that fork and genesis root are valid.
+		_, err = helpers.Domain(beaconState.Fork(), exit.Exit.Epoch, params.BeaconConfig().DomainVoluntaryExit, beaconState.GenesisValidatorRoot())
+		if err != nil {
+			return nil, err
+		}
+		beaconState, err = v.InitiateValidatorExit(beaconState, exit.Exit.ValidatorIndex)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to process voluntary exit at index %d", idx)
+		}
+	}
+	return beaconState, nil
+}
+
+// VerifyExit implements the spec defined validation for voluntary exits.
+//
+// Spec pseudocode definition:
+//   def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None:
+//    """
+//    Process ``VoluntaryExit`` operation.
+//    """
+//    validator = state.validator_registry[exit.validator_index]
+//    # Verify the validator is active
+//    assert is_active_validator(validator, get_current_epoch(state))
+//    # Verify the validator has not yet exited
+//    assert validator.exit_epoch == FAR_FUTURE_EPOCH
+//    # Exits must specify an epoch when they become valid; they are not valid before then
+//    assert get_current_epoch(state) >= exit.epoch
+//    # Verify the validator has been active long enough
+//    assert get_current_epoch(state) >= validator.activation_epoch + PERSISTENT_COMMITTEE_PERIOD
+//    # Verify signature
+//    domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, exit.epoch)
+//    assert bls_verify(validator.pubkey, signing_root(exit), exit.signature, domain)
+func VerifyExit(validator *stateTrie.ReadOnlyValidator, currentSlot uint64, fork *pb.Fork, signed *ethpb.SignedVoluntaryExit, genesisRoot []byte) error {
+	if signed == nil || signed.Exit == nil {
+		return errors.New("nil exit")
+	}
+
+	exit := signed.Exit
+	if err := verifyExitConditions(validator, currentSlot, exit); err != nil {
+		return err
+	}
+	domain, err := helpers.Domain(fork, exit.Epoch, params.BeaconConfig().DomainVoluntaryExit, genesisRoot)
+	if err != nil {
+		return err
+	}
+	valPubKey := validator.PublicKey()
+	if err := helpers.VerifySigningRoot(exit, valPubKey[:], signed.Signature, domain); err != nil {
+		return helpers.ErrSigFailedToVerify
+	}
+	return nil
+}
+
+// verifyExitConditions implements the spec defined validation for voluntary exits(excluding signatures).
+//
+// Spec pseudocode definition:
+//   def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None:
+//    """
+//    Process ``VoluntaryExit`` operation.
+//    """
+//    validator = state.validator_registry[exit.validator_index]
+//    # Verify the validator is active
+//    assert is_active_validator(validator, get_current_epoch(state))
+//    # Verify the validator has not yet exited
+//    assert validator.exit_epoch == FAR_FUTURE_EPOCH
+//    # Exits must specify an epoch when they become valid; they are not valid before then
+//    assert get_current_epoch(state) >= exit.epoch
+//    # Verify the validator has been active long enough
+//    assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD
+func verifyExitConditions(validator *stateTrie.ReadOnlyValidator, currentSlot uint64, exit *ethpb.VoluntaryExit) error {
+	currentEpoch := helpers.SlotToEpoch(currentSlot)
+	// Verify the validator is active.
+	if !helpers.IsActiveValidatorUsingTrie(validator, currentEpoch) {
+		return errors.New("non-active validator cannot exit")
+	}
+	// Verify the validator has not yet exited.
+	if validator.ExitEpoch() != params.BeaconConfig().FarFutureEpoch {
+		return fmt.Errorf("validator has already exited at epoch: %v", validator.ExitEpoch())
+	}
+	// Exits must specify an epoch when they become valid; they are not valid before then.
+	if currentEpoch < exit.Epoch {
+		return fmt.Errorf("expected current epoch >= exit epoch, received %d < %d", currentEpoch, exit.Epoch)
+	}
+	// Verify the validator has been active long enough.
+	if currentEpoch < validator.ActivationEpoch()+params.BeaconConfig().ShardCommitteePeriod {
+		return fmt.Errorf(
+			"validator has not been active long enough to exit, wanted epoch %d >= %d",
+			currentEpoch,
+			validator.ActivationEpoch()+params.BeaconConfig().ShardCommitteePeriod,
+		)
+	}
+	return nil
+}

beacon-chain/core/blocks/exit_test.go

@@ -0,0 +1,173 @@
+package blocks_test
+
+import (
+	"context"
+	"testing"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/bls"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+)
+
+func TestProcessVoluntaryExits_ValidatorNotActive(t *testing.T) {
+	exits := []*ethpb.SignedVoluntaryExit{
+		{
+			Exit: &ethpb.VoluntaryExit{
+				ValidatorIndex: 0,
+			},
+		},
+	}
+	registry := []*ethpb.Validator{
+		{
+			ExitEpoch: 0,
+		},
+	}
+	state, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Validators: registry,
+	})
+	require.NoError(t, err)
+	block := &ethpb.BeaconBlock{
+		Body: &ethpb.BeaconBlockBody{
+			VoluntaryExits: exits,
+		},
+	}
+
+	want := "non-active validator cannot exit"
+	_, err = blocks.ProcessVoluntaryExits(context.Background(), state, block.Body)
+	assert.ErrorContains(t, want, err)
+
+	// Check conformance of no verify method.
+	_, err = blocks.ProcessVoluntaryExitsNoVerify(state, block.Body)
+	assert.ErrorContains(t, want, err)
+}
+
+func TestProcessVoluntaryExits_InvalidExitEpoch(t *testing.T) {
+	exits := []*ethpb.SignedVoluntaryExit{
+		{
+			Exit: &ethpb.VoluntaryExit{
+				Epoch: 10,
+			},
+		},
+	}
+	registry := []*ethpb.Validator{
+		{
+			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
+		},
+	}
+	state, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Validators: registry,
+		Slot:       0,
+	})
+	require.NoError(t, err)
+	block := &ethpb.BeaconBlock{
+		Body: &ethpb.BeaconBlockBody{
+			VoluntaryExits: exits,
+		},
+	}
+
+	want := "expected current epoch >= exit epoch"
+	_, err = blocks.ProcessVoluntaryExits(context.Background(), state, block.Body)
+	assert.ErrorContains(t, want, err)
+
+	// Check conformance of no verify method.
+	_, err = blocks.ProcessVoluntaryExitsNoVerify(state, block.Body)
+	assert.ErrorContains(t, want, err)
+
+}
+
+func TestProcessVoluntaryExits_NotActiveLongEnoughToExit(t *testing.T) {
+	exits := []*ethpb.SignedVoluntaryExit{
+		{
+			Exit: &ethpb.VoluntaryExit{
+				ValidatorIndex: 0,
+				Epoch:          0,
+			},
+		},
+	}
+	registry := []*ethpb.Validator{
+		{
+			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
+		},
+	}
+	state, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Validators: registry,
+		Slot:       10,
+	})
+	require.NoError(t, err)
+	block := &ethpb.BeaconBlock{
+		Body: &ethpb.BeaconBlockBody{
+			VoluntaryExits: exits,
+		},
+	}
+
+	want := "validator has not been active long enough to exit"
+	_, err = blocks.ProcessVoluntaryExits(context.Background(), state, block.Body)
+	assert.ErrorContains(t, want, err)
+}
+
+func TestProcessVoluntaryExits_AppliesCorrectStatus(t *testing.T) {
+	exits := []*ethpb.SignedVoluntaryExit{
+		{
+			Exit: &ethpb.VoluntaryExit{
+				ValidatorIndex: 0,
+				Epoch:          0,
+			},
+		},
+	}
+	registry := []*ethpb.Validator{
+		{
+			ExitEpoch:       params.BeaconConfig().FarFutureEpoch,
+			ActivationEpoch: 0,
+		},
+	}
+	state, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Validators: registry,
+		Fork: &pb.Fork{
+			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
+			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
+		},
+		Slot: params.BeaconConfig().SlotsPerEpoch * 5,
+	})
+	require.NoError(t, err)
+	err = state.SetSlot(state.Slot() + (params.BeaconConfig().ShardCommitteePeriod * params.BeaconConfig().SlotsPerEpoch))
+	require.NoError(t, err)
+
+	priv := bls.RandKey()
+	val, err := state.ValidatorAtIndex(0)
+	require.NoError(t, err)
+	val.PublicKey = priv.PublicKey().Marshal()[:]
+	require.NoError(t, state.UpdateValidatorAtIndex(0, val))
+	exits[0].Signature, err = helpers.ComputeDomainAndSign(state, helpers.CurrentEpoch(state), exits[0].Exit, params.BeaconConfig().DomainVoluntaryExit, priv)
+	require.NoError(t, err)
+
+	block := &ethpb.BeaconBlock{
+		Body: &ethpb.BeaconBlockBody{
+			VoluntaryExits: exits,
+		},
+	}
+
+	stateCopy := state.Copy()
+	newState, err := blocks.ProcessVoluntaryExits(context.Background(), state, block.Body)
+	require.NoError(t, err, "Could not process exits")
+	newRegistry := newState.Validators()
+	if newRegistry[0].ExitEpoch != helpers.ActivationExitEpoch(state.Slot()/params.BeaconConfig().SlotsPerEpoch) {
+		t.Errorf("Expected validator exit epoch to be %d, got %d",
+			helpers.ActivationExitEpoch(state.Slot()/params.BeaconConfig().SlotsPerEpoch), newRegistry[0].ExitEpoch)
+	}
+
+	// Check conformance with NoVerify Exit Method.
+	newState, err = blocks.ProcessVoluntaryExitsNoVerify(stateCopy, block.Body)
+	require.NoError(t, err, "Could not process exits")
+	newRegistry = newState.Validators()
+	if newRegistry[0].ExitEpoch != helpers.ActivationExitEpoch(stateCopy.Slot()/params.BeaconConfig().SlotsPerEpoch) {
+		t.Errorf("Expected validator exit epoch to be %d, got %d",
+			helpers.ActivationExitEpoch(stateCopy.Slot()/params.BeaconConfig().SlotsPerEpoch), newRegistry[0].ExitEpoch)
+	}
+
+}

beacon-chain/core/blocks/genesis.go

@@ -1,6 +1,5 @@
-// Package blocks contains block processing libraries. These libraries
-// process and verify block specific messages such as PoW receipt root,
-// RANDAO, validator deposits, exits and slashing proofs.
+// Package blocks contains block processing libraries according to
+// the eth2spec.
 package blocks
 
 import (

beacon-chain/core/blocks/genesis_test.go

@@ -0,0 +1,16 @@
+package blocks_test
+
+import (
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+)
+
+func TestGenesisBlock_InitializedCorrectly(t *testing.T) {
+	stateHash := []byte{0}
+	b1 := blocks.NewGenesisBlock(stateHash)
+
+	assert.NotNil(t, b1.Block.ParentRoot, "Genesis block missing ParentHash field")
+	assert.DeepEqual(t, b1.Block.StateRoot, stateHash, "Genesis block StateRootHash32 isn't initialized correctly")
+}

beacon-chain/core/blocks/header.go

@@ -0,0 +1,135 @@
+package blocks
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+)
+
+// ProcessBlockHeader validates a block by its header.
+//
+// Spec pseudocode definition:
+//
+//  def process_block_header(state: BeaconState, block: BeaconBlock) -> None:
+//    # Verify that the slots match
+//    assert block.slot == state.slot
+//     # Verify that proposer index is the correct index
+//    assert block.proposer_index == get_beacon_proposer_index(state)
+//    # Verify that the parent matches
+//    assert block.parent_root == hash_tree_root(state.latest_block_header)
+//    # Save current block as the new latest block
+//    state.latest_block_header = BeaconBlockHeader(
+//        slot=block.slot,
+//        parent_root=block.parent_root,
+//        # state_root: zeroed, overwritten in the next `process_slot` call
+//        body_root=hash_tree_root(block.body),
+//		  # signature is always zeroed
+//    )
+//    # Verify proposer is not slashed
+//    proposer = state.validators[get_beacon_proposer_index(state)]
+//    assert not proposer.slashed
+//    # Verify proposer signature
+//    assert bls_verify(proposer.pubkey, signing_root(block), block.signature, get_domain(state, DOMAIN_BEACON_PROPOSER))
+func ProcessBlockHeader(
+	beaconState *stateTrie.BeaconState,
+	block *ethpb.SignedBeaconBlock,
+) (*stateTrie.BeaconState, error) {
+	beaconState, err := ProcessBlockHeaderNoVerify(beaconState, block.Block)
+	if err != nil {
+		return nil, err
+	}
+
+	// Verify proposer signature.
+	if err := VerifyBlockSignature(beaconState, block); err != nil {
+		return nil, err
+	}
+
+	return beaconState, nil
+}
+
+// ProcessBlockHeaderNoVerify validates a block by its header but skips proposer
+// signature verification.
+//
+// WARNING: This method does not verify proposer signature. This is used for proposer to compute state root
+// using a unsigned block.
+//
+// Spec pseudocode definition:
+//  def process_block_header(state: BeaconState, block: BeaconBlock) -> None:
+//    # Verify that the slots match
+//    assert block.slot == state.slot
+//     # Verify that proposer index is the correct index
+//    assert block.proposer_index == get_beacon_proposer_index(state)
+//    # Verify that the parent matches
+//    assert block.parent_root == hash_tree_root(state.latest_block_header)
+//    # Save current block as the new latest block
+//    state.latest_block_header = BeaconBlockHeader(
+//        slot=block.slot,
+//        parent_root=block.parent_root,
+//        # state_root: zeroed, overwritten in the next `process_slot` call
+//        body_root=hash_tree_root(block.body),
+//		  # signature is always zeroed
+//    )
+//    # Verify proposer is not slashed
+//    proposer = state.validators[get_beacon_proposer_index(state)]
+//    assert not proposer.slashed
+func ProcessBlockHeaderNoVerify(
+	beaconState *stateTrie.BeaconState,
+	block *ethpb.BeaconBlock,
+) (*stateTrie.BeaconState, error) {
+	if block == nil {
+		return nil, errors.New("nil block")
+	}
+	if beaconState.Slot() != block.Slot {
+		return nil, fmt.Errorf("state slot: %d is different than block slot: %d", beaconState.Slot(), block.Slot)
+	}
+	idx, err := helpers.BeaconProposerIndex(beaconState)
+	if err != nil {
+		return nil, err
+	}
+	if block.ProposerIndex != idx {
+		return nil, fmt.Errorf("proposer index: %d is different than calculated: %d", block.ProposerIndex, idx)
+	}
+	parentHeader := beaconState.LatestBlockHeader()
+	if parentHeader.Slot >= block.Slot {
+		return nil, fmt.Errorf("block.Slot %d must be greater than state.LatestBlockHeader.Slot %d", block.Slot, parentHeader.Slot)
+	}
+	parentRoot, err := stateutil.BlockHeaderRoot(parentHeader)
+	if err != nil {
+		return nil, err
+	}
+
+	if !bytes.Equal(block.ParentRoot, parentRoot[:]) {
+		return nil, fmt.Errorf(
+			"parent root %#x does not match the latest block header signing root in state %#x",
+			block.ParentRoot, parentRoot)
+	}
+
+	proposer, err := beaconState.ValidatorAtIndexReadOnly(idx)
+	if err != nil {
+		return nil, err
+	}
+	if proposer.Slashed() {
+		return nil, fmt.Errorf("proposer at index %d was previously slashed", idx)
+	}
+
+	bodyRoot, err := stateutil.BlockBodyRoot(block.Body)
+	if err != nil {
+		return nil, err
+	}
+	if err := beaconState.SetLatestBlockHeader(&ethpb.BeaconBlockHeader{
+		Slot:          block.Slot,
+		ProposerIndex: block.ProposerIndex,
+		ParentRoot:    block.ParentRoot,
+		StateRoot:     params.BeaconConfig().ZeroHash[:],
+		BodyRoot:      bodyRoot[:],
+	}); err != nil {
+		return nil, err
+	}
+	return beaconState, nil
+}

beacon-chain/core/blocks/header_test.go

@@ -0,0 +1,363 @@
+package blocks_test
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/gogo/protobuf/proto"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/go-ssz"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/bls"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+	"github.com/sirupsen/logrus"
+)
+
+func init() {
+	logrus.SetOutput(ioutil.Discard) // Ignore "validator activated" logs
+}
+
+func TestProcessBlockHeader_ImproperBlockSlot(t *testing.T) {
+	validators := make([]*ethpb.Validator, params.BeaconConfig().MinGenesisActiveValidatorCount)
+	for i := 0; i < len(validators); i++ {
+		validators[i] = &ethpb.Validator{
+			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
+			Slashed:   true,
+		}
+	}
+
+	state, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Validators:        validators,
+		Slot:              10,
+		LatestBlockHeader: &ethpb.BeaconBlockHeader{Slot: 10}, // Must be less than block.Slot
+		Fork: &pb.Fork{
+			PreviousVersion: []byte{0, 0, 0, 0},
+			CurrentVersion:  []byte{0, 0, 0, 0},
+		},
+		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+	})
+	require.NoError(t, err)
+
+	latestBlockSignedRoot, err := stateutil.BlockHeaderRoot(state.LatestBlockHeader())
+	require.NoError(t, err)
+
+	currentEpoch := helpers.CurrentEpoch(state)
+	priv := bls.RandKey()
+	pID, err := helpers.BeaconProposerIndex(state)
+	require.NoError(t, err)
+	block := &ethpb.SignedBeaconBlock{
+		Block: &ethpb.BeaconBlock{
+			ProposerIndex: pID,
+			Slot:          10,
+			Body: &ethpb.BeaconBlockBody{
+				RandaoReveal: []byte{'A', 'B', 'C'},
+			},
+			ParentRoot: latestBlockSignedRoot[:],
+		},
+	}
+	block.Signature, err = helpers.ComputeDomainAndSign(state, currentEpoch, block.Block, params.BeaconConfig().DomainBeaconProposer, priv)
+	require.NoError(t, err)
+
+	proposerIdx, err := helpers.BeaconProposerIndex(state)
+	require.NoError(t, err)
+	validators[proposerIdx].Slashed = false
+	validators[proposerIdx].PublicKey = priv.PublicKey().Marshal()
+	err = state.UpdateValidatorAtIndex(proposerIdx, validators[proposerIdx])
+	require.NoError(t, err)
+
+	_, err = blocks.ProcessBlockHeader(state, block)
+	assert.ErrorContains(t, "block.Slot 10 must be greater than state.LatestBlockHeader.Slot 10", err)
+}
+
+func TestProcessBlockHeader_WrongProposerSig(t *testing.T) {
+	testutil.ResetCache()
+	beaconState, privKeys := testutil.DeterministicGenesisState(t, 100)
+	require.NoError(t, beaconState.SetLatestBlockHeader(&ethpb.BeaconBlockHeader{Slot: 9}))
+	require.NoError(t, beaconState.SetSlot(10))
+
+	lbhdr, err := stateutil.BlockHeaderRoot(beaconState.LatestBlockHeader())
+	require.NoError(t, err)
+
+	proposerIdx, err := helpers.BeaconProposerIndex(beaconState)
+	require.NoError(t, err)
+
+	block := &ethpb.SignedBeaconBlock{
+		Block: &ethpb.BeaconBlock{
+			ProposerIndex: proposerIdx,
+			Slot:          10,
+			Body: &ethpb.BeaconBlockBody{
+				RandaoReveal: []byte{'A', 'B', 'C'},
+			},
+			ParentRoot: lbhdr[:],
+		},
+	}
+	block.Signature, err = helpers.ComputeDomainAndSign(beaconState, 0, block.Block, params.BeaconConfig().DomainBeaconProposer, privKeys[proposerIdx+1])
+	require.NoError(t, err)
+
+	_, err = blocks.ProcessBlockHeader(beaconState, block)
+	want := "signature did not verify"
+	assert.ErrorContains(t, want, err)
+}
+
+func TestProcessBlockHeader_DifferentSlots(t *testing.T) {
+	validators := make([]*ethpb.Validator, params.BeaconConfig().MinGenesisActiveValidatorCount)
+	for i := 0; i < len(validators); i++ {
+		validators[i] = &ethpb.Validator{
+			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
+			Slashed:   true,
+		}
+	}
+
+	state, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Validators:        validators,
+		Slot:              10,
+		LatestBlockHeader: &ethpb.BeaconBlockHeader{Slot: 9},
+		Fork: &pb.Fork{
+			PreviousVersion: []byte{0, 0, 0, 0},
+			CurrentVersion:  []byte{0, 0, 0, 0},
+		},
+		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+	})
+	require.NoError(t, err)
+
+	lbhsr, err := ssz.HashTreeRoot(state.LatestBlockHeader())
+	require.NoError(t, err)
+	currentEpoch := helpers.CurrentEpoch(state)
+
+	priv := bls.RandKey()
+	blockSig, err := helpers.ComputeDomainAndSign(state, currentEpoch, []byte("hello"), params.BeaconConfig().DomainBeaconProposer, priv)
+	require.NoError(t, err)
+	validators[5896].PublicKey = priv.PublicKey().Marshal()
+	block := &ethpb.SignedBeaconBlock{
+		Block: &ethpb.BeaconBlock{
+			Slot: 1,
+			Body: &ethpb.BeaconBlockBody{
+				RandaoReveal: []byte{'A', 'B', 'C'},
+			},
+			ParentRoot: lbhsr[:],
+		},
+		Signature: blockSig,
+	}
+
+	_, err = blocks.ProcessBlockHeader(state, block)
+	want := "is different than block slot"
+	assert.ErrorContains(t, want, err)
+}
+
+func TestProcessBlockHeader_PreviousBlockRootNotSignedRoot(t *testing.T) {
+	validators := make([]*ethpb.Validator, params.BeaconConfig().MinGenesisActiveValidatorCount)
+	for i := 0; i < len(validators); i++ {
+		validators[i] = &ethpb.Validator{
+			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
+			Slashed:   true,
+		}
+	}
+
+	state, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Validators:        validators,
+		Slot:              10,
+		LatestBlockHeader: &ethpb.BeaconBlockHeader{Slot: 9},
+		Fork: &pb.Fork{
+			PreviousVersion: []byte{0, 0, 0, 0},
+			CurrentVersion:  []byte{0, 0, 0, 0},
+		},
+		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+	})
+	require.NoError(t, err)
+
+	currentEpoch := helpers.CurrentEpoch(state)
+	priv := bls.RandKey()
+	blockSig, err := helpers.ComputeDomainAndSign(state, currentEpoch, []byte("hello"), params.BeaconConfig().DomainBeaconProposer, priv)
+	require.NoError(t, err)
+	validators[5896].PublicKey = priv.PublicKey().Marshal()
+	pID, err := helpers.BeaconProposerIndex(state)
+	require.NoError(t, err)
+	block := &ethpb.SignedBeaconBlock{
+		Block: &ethpb.BeaconBlock{
+			ProposerIndex: pID,
+			Slot:          10,
+			Body: &ethpb.BeaconBlockBody{
+				RandaoReveal: []byte{'A', 'B', 'C'},
+			},
+			ParentRoot: []byte{'A'},
+		},
+		Signature: blockSig,
+	}
+
+	_, err = blocks.ProcessBlockHeader(state, block)
+	want := "does not match"
+	assert.ErrorContains(t, want, err)
+}
+
+func TestProcessBlockHeader_SlashedProposer(t *testing.T) {
+	validators := make([]*ethpb.Validator, params.BeaconConfig().MinGenesisActiveValidatorCount)
+	for i := 0; i < len(validators); i++ {
+		validators[i] = &ethpb.Validator{
+			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
+			Slashed:   true,
+		}
+	}
+
+	state, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Validators:        validators,
+		Slot:              10,
+		LatestBlockHeader: &ethpb.BeaconBlockHeader{Slot: 9},
+		Fork: &pb.Fork{
+			PreviousVersion: []byte{0, 0, 0, 0},
+			CurrentVersion:  []byte{0, 0, 0, 0},
+		},
+		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+	})
+	require.NoError(t, err)
+
+	parentRoot, err := stateutil.BlockHeaderRoot(state.LatestBlockHeader())
+	require.NoError(t, err)
+	currentEpoch := helpers.CurrentEpoch(state)
+	priv := bls.RandKey()
+	blockSig, err := helpers.ComputeDomainAndSign(state, currentEpoch, []byte("hello"), params.BeaconConfig().DomainBeaconProposer, priv)
+	require.NoError(t, err)
+
+	validators[12683].PublicKey = priv.PublicKey().Marshal()
+	pID, err := helpers.BeaconProposerIndex(state)
+	require.NoError(t, err)
+	block := &ethpb.SignedBeaconBlock{
+		Block: &ethpb.BeaconBlock{
+			ProposerIndex: pID,
+			Slot:          10,
+			Body: &ethpb.BeaconBlockBody{
+				RandaoReveal: []byte{'A', 'B', 'C'},
+			},
+			ParentRoot: parentRoot[:],
+		},
+		Signature: blockSig,
+	}
+
+	_, err = blocks.ProcessBlockHeader(state, block)
+	want := "was previously slashed"
+	assert.ErrorContains(t, want, err)
+}
+
+func TestProcessBlockHeader_OK(t *testing.T) {
+	validators := make([]*ethpb.Validator, params.BeaconConfig().MinGenesisActiveValidatorCount)
+	for i := 0; i < len(validators); i++ {
+		validators[i] = &ethpb.Validator{
+			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
+			Slashed:   true,
+		}
+	}
+
+	state, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Validators:        validators,
+		Slot:              10,
+		LatestBlockHeader: &ethpb.BeaconBlockHeader{Slot: 9},
+		Fork: &pb.Fork{
+			PreviousVersion: []byte{0, 0, 0, 0},
+			CurrentVersion:  []byte{0, 0, 0, 0},
+		},
+		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+	})
+	require.NoError(t, err)
+
+	latestBlockSignedRoot, err := stateutil.BlockHeaderRoot(state.LatestBlockHeader())
+	require.NoError(t, err)
+
+	currentEpoch := helpers.CurrentEpoch(state)
+	priv := bls.RandKey()
+	pID, err := helpers.BeaconProposerIndex(state)
+	require.NoError(t, err)
+	block := &ethpb.SignedBeaconBlock{
+		Block: &ethpb.BeaconBlock{
+			ProposerIndex: pID,
+			Slot:          10,
+			Body: &ethpb.BeaconBlockBody{
+				RandaoReveal: []byte{'A', 'B', 'C'},
+			},
+			ParentRoot: latestBlockSignedRoot[:],
+		},
+	}
+	block.Signature, err = helpers.ComputeDomainAndSign(state, currentEpoch, block.Block, params.BeaconConfig().DomainBeaconProposer, priv)
+	require.NoError(t, err)
+	bodyRoot, err := stateutil.BlockBodyRoot(block.Block.Body)
+	require.NoError(t, err, "Failed to hash block bytes got")
+
+	proposerIdx, err := helpers.BeaconProposerIndex(state)
+	require.NoError(t, err)
+	validators[proposerIdx].Slashed = false
+	validators[proposerIdx].PublicKey = priv.PublicKey().Marshal()
+	err = state.UpdateValidatorAtIndex(proposerIdx, validators[proposerIdx])
+	require.NoError(t, err)
+
+	newState, err := blocks.ProcessBlockHeader(state, block)
+	require.NoError(t, err, "Failed to process block header got")
+	var zeroHash [32]byte
+	nsh := newState.LatestBlockHeader()
+	expected := &ethpb.BeaconBlockHeader{
+		ProposerIndex: pID,
+		Slot:          block.Block.Slot,
+		ParentRoot:    latestBlockSignedRoot[:],
+		BodyRoot:      bodyRoot[:],
+		StateRoot:     zeroHash[:],
+	}
+	assert.Equal(t, true, proto.Equal(nsh, expected), "Expected %v, received %v", expected, nsh)
+}
+
+func TestBlockSignatureSet_OK(t *testing.T) {
+	validators := make([]*ethpb.Validator, params.BeaconConfig().MinGenesisActiveValidatorCount)
+	for i := 0; i < len(validators); i++ {
+		validators[i] = &ethpb.Validator{
+			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
+			Slashed:   true,
+		}
+	}
+
+	state, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Validators:        validators,
+		Slot:              10,
+		LatestBlockHeader: &ethpb.BeaconBlockHeader{Slot: 9},
+		Fork: &pb.Fork{
+			PreviousVersion: []byte{0, 0, 0, 0},
+			CurrentVersion:  []byte{0, 0, 0, 0},
+		},
+		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+	})
+	require.NoError(t, err)
+
+	latestBlockSignedRoot, err := stateutil.BlockHeaderRoot(state.LatestBlockHeader())
+	require.NoError(t, err)
+
+	currentEpoch := helpers.CurrentEpoch(state)
+	priv := bls.RandKey()
+	pID, err := helpers.BeaconProposerIndex(state)
+	require.NoError(t, err)
+	block := &ethpb.SignedBeaconBlock{
+		Block: &ethpb.BeaconBlock{
+			ProposerIndex: pID,
+			Slot:          10,
+			Body: &ethpb.BeaconBlockBody{
+				RandaoReveal: []byte{'A', 'B', 'C'},
+			},
+			ParentRoot: latestBlockSignedRoot[:],
+		},
+	}
+	block.Signature, err = helpers.ComputeDomainAndSign(state, currentEpoch, block.Block, params.BeaconConfig().DomainBeaconProposer, priv)
+	require.NoError(t, err)
+	proposerIdx, err := helpers.BeaconProposerIndex(state)
+	require.NoError(t, err)
+	validators[proposerIdx].Slashed = false
+	validators[proposerIdx].PublicKey = priv.PublicKey().Marshal()
+	err = state.UpdateValidatorAtIndex(proposerIdx, validators[proposerIdx])
+	require.NoError(t, err)
+	set, err := blocks.BlockSignatureSet(state, block)
+	require.NoError(t, err)
+
+	verified, err := set.Verify()
+	require.NoError(t, err)
+	assert.Equal(t, true, verified, "Block signature set returned a set which was unable to be verified")
+}

beacon-chain/core/blocks/log.go

@@ -0,0 +1,5 @@
+package blocks
+
+import "github.com/sirupsen/logrus"
+
+var log = logrus.WithField("prefix", "blocks")

beacon-chain/core/blocks/proposer_slashing.go

@@ -0,0 +1,96 @@
+package blocks
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	v "github.com/prysmaticlabs/prysm/beacon-chain/core/validators"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/params"
+)
+
+// ProcessProposerSlashings is one of the operations performed
+// on each processed beacon block to slash proposers based on
+// slashing conditions if any slashable events occurred.
+//
+// Spec pseudocode definition:
+//   def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSlashing) -> None:
+//    """
+//    Process ``ProposerSlashing`` operation.
+//    """
+//    proposer = state.validator_registry[proposer_slashing.proposer_index]
+//    # Verify slots match
+//    assert proposer_slashing.header_1.slot == proposer_slashing.header_2.slot
+//    # But the headers are different
+//    assert proposer_slashing.header_1 != proposer_slashing.header_2
+//    # Check proposer is slashable
+//    assert is_slashable_validator(proposer, get_current_epoch(state))
+//    # Signatures are valid
+//    for header in (proposer_slashing.header_1, proposer_slashing.header_2):
+//        domain = get_domain(state, DOMAIN_BEACON_PROPOSER, slot_to_epoch(header.slot))
+//        assert bls_verify(proposer.pubkey, signing_root(header), header.signature, domain)
+//
+//    slash_validator(state, proposer_slashing.proposer_index)
+func ProcessProposerSlashings(
+	ctx context.Context,
+	beaconState *stateTrie.BeaconState,
+	body *ethpb.BeaconBlockBody,
+) (*stateTrie.BeaconState, error) {
+	var err error
+	for idx, slashing := range body.ProposerSlashings {
+		if slashing == nil {
+			return nil, errors.New("nil proposer slashings in block body")
+		}
+		if err = VerifyProposerSlashing(beaconState, slashing); err != nil {
+			return nil, errors.Wrapf(err, "could not verify proposer slashing %d", idx)
+		}
+		beaconState, err = v.SlashValidator(
+			beaconState, slashing.Header_1.Header.ProposerIndex,
+		)
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not slash proposer index %d", slashing.Header_1.Header.ProposerIndex)
+		}
+	}
+	return beaconState, nil
+}
+
+// VerifyProposerSlashing verifies that the data provided from slashing is valid.
+func VerifyProposerSlashing(
+	beaconState *stateTrie.BeaconState,
+	slashing *ethpb.ProposerSlashing,
+) error {
+	if slashing.Header_1 == nil || slashing.Header_1.Header == nil || slashing.Header_2 == nil || slashing.Header_2.Header == nil {
+		return errors.New("nil header cannot be verified")
+	}
+	hSlot := slashing.Header_1.Header.Slot
+	if hSlot != slashing.Header_2.Header.Slot {
+		return fmt.Errorf("mismatched header slots, received %d == %d", slashing.Header_1.Header.Slot, slashing.Header_2.Header.Slot)
+	}
+	pIdx := slashing.Header_1.Header.ProposerIndex
+	if pIdx != slashing.Header_2.Header.ProposerIndex {
+		return fmt.Errorf("mismatched indices, received %d == %d", slashing.Header_1.Header.ProposerIndex, slashing.Header_2.Header.ProposerIndex)
+	}
+	if proto.Equal(slashing.Header_1, slashing.Header_2) {
+		return errors.New("expected slashing headers to differ")
+	}
+	proposer, err := beaconState.ValidatorAtIndexReadOnly(slashing.Header_1.Header.ProposerIndex)
+	if err != nil {
+		return err
+	}
+	if !helpers.IsSlashableValidatorUsingTrie(proposer, helpers.SlotToEpoch(beaconState.Slot())) {
+		return fmt.Errorf("validator with key %#x is not slashable", proposer.PublicKey())
+	}
+	headers := []*ethpb.SignedBeaconBlockHeader{slashing.Header_1, slashing.Header_2}
+	for _, header := range headers {
+		if err := helpers.ComputeDomainVerifySigningRoot(beaconState, pIdx, helpers.SlotToEpoch(hSlot),
+			header.Header, params.BeaconConfig().DomainBeaconProposer, header.Signature); err != nil {
+			return errors.Wrap(err, "could not verify beacon block header")
+		}
+	}
+
+	return nil
+}