Forkchoice: update read only lock to regular lock (#7633)

* fix snappy errors

* gaz

* Update beacon-chain/p2p/pubsub.go

Co-authored-by: Preston Van Loon <preston@prysmaticlabs.com>

.bazelrc

@@ -5,6 +5,9 @@ test --test_verbose_timeout_warnings
 test --build_tests_only
 test --test_output=errors
 
+# Clearly indicate that coverage is enabled to disable certain nogo checks.
+coverage --define=coverage_enabled=1
+
 # Fix for rules_docker. See: https://github.com/bazelbuild/rules_docker/issues/842
 build --host_force_python=PY2
 test --host_force_python=PY2
@@ -15,10 +18,219 @@ run --host_force_python=PY2
 # Network sandboxing only works on linux.
 --experimental_sandbox_default_allow_network=false
 
-# Use minimal protobufs at runtime
-run --define ssz=minimal
+# Use mainnet protobufs at runtime
+run --define ssz=mainnet
+test --define ssz=mainnet
+build --define ssz=mainnet
 
 # Prevent PATH changes from rebuilding when switching from IDE to command line.
 build --incompatible_strict_action_env
 test --incompatible_strict_action_env
 run --incompatible_strict_action_env
+
+# Disable kafka by default, it takes a long time to build...
+build --define kafka_enabled=false
+test --define kafka_enabled=false
+run --define kafka_enabled=false
+
+# Enable blst by default, we use a config so that our fuzzer stops complaining.
+build --config=blst_enabled
+test --config=blst_enabled
+run --config=blst_enabled
+
+build:kafka_enabled --define kafka_enabled=true
+build:kafka_enabled --define gotags=kafka_enabled
+
+build:blst_enabled --define blst_enabled=true
+build:blst_enabled --define gotags=blst_enabled
+
+# Release flags
+build:release --workspace_status_command=./scripts/workspace_status.sh
+build:release --stamp
+build:release --compilation_mode=opt
+build:release --config=llvm
+
+# LLVM compiler for building C/C++ dependencies.
+build:llvm --crosstool_top=@llvm_toolchain//:toolchain
+build:llvm --define compiler=llvm
+build:llvm --copt -fno-sanitize=vptr,function
+build:llvm --linkopt -fno-sanitize=vptr,function
+
+build:asan --copt -fsanitize=address,undefined
+build:asan --copt -fno-omit-frame-pointer
+build:asan --linkopt -fsanitize=address,undefined
+build:asan --copt -fno-sanitize=vptr,function
+build:asan --linkopt -fno-sanitize=vptr,function
+build:asan --copt -DADDRESS_SANITIZER=1
+build:asan --copt -D__SANITIZE_ADDRESS__
+build:asan --linkopt -ldl
+
+build:llvm-asan --config=llvm
+build:llvm-asan --config=asan
+build:llvm-asan --linkopt -fuse-ld=ld.lld
+
+build:fuzz --define=gotags=libfuzzer
+build:fuzz --config=llvm-asan
+build:fuzz --copt=-fsanitize=fuzzer-no-link
+build:fuzz --linkopt=-fsanitize=fuzzer
+build:fuzz --copt=-fno-omit-frame-pointer
+build:fuzz --define=FUZZING_ENGINE=libfuzzer
+build:fuzz --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+build:fuzz --linkopt -Wl,--no-as-needed
+build:fuzz --define=gc_goopts=-d=libfuzzer,checkptr
+build:fuzz --run_under=//tools:fuzz_wrapper
+build:fuzz --compilation_mode=opt
+build:fuzz --define=blst_enabled=false
+
+test:fuzz --local_test_jobs="HOST_CPUS*.5"
+
+# Build binary with cgo symbolizer for debugging / profiling.
+build:cgo_symbolizer --config=llvm
+build:cgo_symbolizer --copt=-g
+build:cgo_symbolizer --define=USE_CGO_SYMBOLIZER=true
+build:cgo_symbolizer -c dbg
+build:cgo_symbolizer --define=gotags=cgosymbolizer_enabled
+
+# multi-arch cross-compiling toolchain configs:
+-----------------------------------------------
+build:cross --crosstool_top=@prysm_toolchains//:multiarch_toolchain
+build:cross --host_platform=@io_bazel_rules_go//go/toolchain:linux_amd64
+build:cross --host_crosstool_top=@prysm_toolchains//:hostonly_toolchain
+
+# linux_amd64 config for cross compiler toolchain, not strictly necessary since host/exec env is amd64
+build:linux_amd64 --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64_cgo
+
+# osx_amd64 config for cross compiler toolchain
+build:osx_amd64 --config=cross
+build:osx_amd64 --platforms=@io_bazel_rules_go//go/toolchain:darwin_amd64_cgo
+build:osx_amd64 --compiler=osxcross
+
+# windows
+build:windows_amd64 --config=cross
+build:windows_amd64  --platforms=@io_bazel_rules_go//go/toolchain:windows_amd64_cgo
+build:windows_amd64 --compiler=mingw-w64
+
+# linux_arm64 conifg for cross compiler toolchain
+build:linux_arm64 --config=cross
+build:linux_arm64 --platforms=@io_bazel_rules_go//go/toolchain:linux_arm64_cgo
+build:linux_arm64 --copt=-funsafe-math-optimizations
+build:linux_arm64 --copt=-ftree-vectorize
+build:linux_arm64 --copt=-fomit-frame-pointer
+build:linux_arm64 --cpu=aarch64
+build:linux_arm64 --compiler=clang
+build:linux_arm64 --copt=-march=armv8-a
+
+
+# toolchain build debug configs
+#------------------------------
+build:debug --sandbox_debug
+build:debug --toolchain_resolution_debug
+build:debug --verbose_failures
+build:debug -s
+
+# windows debug
+build:windows_amd64_debug --config=windows_amd64
+build:windows_amd64_debug --config=debug
+
+# osx_amd64 debug config
+build:osx_amd64_debug --config=debug
+build:osx_amd64_debug --config=osx_amd64
+
+# linux_arm64_debug
+build:linux_arm64_debug --config=linux_arm64
+build:linux_arm64_debug --config=debug
+
+# linux_amd64_debug
+build:linux_amd64_debug --config=linux_amd64
+build:linux_amd64_debug --config=debug
+
+
+# Docker Sandbox Configs
+#-----------------------
+# Note all docker sandbox configs must run from a linux x86_64 host
+# build:docker-sandbox --experimental_docker_image=gcr.io/prysmaticlabs/rbe-worker:latest
+build:docker-sandbox --spawn_strategy=docker --strategy=Javac=docker --genrule_strategy=docker
+build:docker-sandbox --define=EXECUTOR=remote
+build:docker-sandbox --experimental_docker_verbose
+build:docker-sandbox --experimental_enable_docker_sandbox
+build:docker-sandbox --crosstool_top=@rbe_ubuntu_clang//cc:toolchain 
+build:docker-sandbox --host_javabase=@rbe_ubuntu_clang//java:jdk 
+build:docker-sandbox --javabase=@rbe_ubuntu_clang//java:jdk 
+build:docker-sandbox --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 
+build:docker-sandbox --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 
+build:docker-sandbox --extra_execution_platforms=@rbe_ubuntu_clang//config:platform 
+build:docker-sandbox --host_platform=@rbe_ubuntu_clang//config:platform 
+build:docker-sandbox --platforms=@rbe_ubuntu_clang//config:platform
+build:docker-sandbox --extra_toolchains=@prysm_toolchains//:cc-toolchain-multiarch
+
+# windows_amd64 docker sandbox build config
+build:windows_amd64_docker --config=docker-sandbox --config=windows_amd64
+build:windows_amd64_docker_debug --config=windows_amd64_docker --config=debug
+
+# osx_amd64 docker sandbox build config
+build:osx_amd64_docker --config=docker-sandbox --config=osx_amd64
+build:osx_amd64_docker_debug --config=osx_amd64_docker --config=debug
+
+# linux_arm64 docker sandbox build config
+build:linux_arm64_docker --config=docker-sandbox --config=linux_arm64
+build:linux_arm64_docker_debug --config=linux_arm64_docker --config=debug
+
+# linux_amd64 docker sandbox build config
+build:linux_amd64_docker --config=docker-sandbox --config=linux_amd64
+build:linux_amd64_docker_debug --config=linux_amd64_docker --config=debug
+
+
+# Remote Build Execution
+#-----------------------
+# Originally from https://github.com/bazelbuild/bazel-toolchains/blob/master/bazelrc/bazel-2.0.0.bazelrc
+#
+# Depending on how many machines are in the remote execution instance, setting
+# this higher can make builds faster by allowing more jobs to run in parallel.
+# Setting it too high can result in jobs that timeout, however, while waiting
+# for a remote machine to execute them.
+build:remote --jobs=50
+
+# Set several flags related to specifying the platform, toolchain and java
+# properties.
+# These flags should only be used as is for the rbe-ubuntu16-04 container
+# and need to be adapted to work with other toolchain containers.
+build:remote --host_javabase=@rbe_ubuntu_clang//java:jdk
+build:remote --javabase=@rbe_ubuntu_clang//java:jdk
+build:remote --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
+build:remote --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
+build:remote --crosstool_top=@rbe_ubuntu_clang//cc:toolchain
+build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
+# Platform flags:
+# The toolchain container used for execution is defined in the target indicated
+# by "extra_execution_platforms", "host_platform" and "platforms".
+# More about platforms: https://docs.bazel.build/versions/master/platforms.html
+build:remote --extra_toolchains=@rbe_ubuntu_clang//config:cc-toolchain
+build:remote --extra_execution_platforms=@rbe_ubuntu_clang//config:platform
+build:remote --host_platform=@rbe_ubuntu_clang//config:platform
+build:remote --platforms=@rbe_ubuntu_clang//config:platform
+
+# Starting with Bazel 0.27.0 strategies do not need to be explicitly
+# defined. See https://github.com/bazelbuild/bazel/issues/7480
+build:remote --define=EXECUTOR=remote
+
+# Enable remote execution so actions are performed on the remote systems.
+# build:remote --remote_executor=grpcs://remotebuildexecution.googleapis.com
+
+# Enforce stricter environment rules, which eliminates some non-hermetic
+# behavior and therefore improves both the remote cache hit rate and the
+# correctness and repeatability of the build.
+build:remote --incompatible_strict_action_env=true
+
+# Set a higher timeout value, just in case.
+build:remote --remote_timeout=3600
+
+# Enable authentication. This will pick up application default credentials by
+# default. You can use --google_credentials=some_file.json to use a service
+# account credential instead.
+# build:remote --google_default_credentials=true
+
+# Enable build without the bytes
+# See: https://github.com/bazelbuild/bazel/issues/6862
+build:remote --experimental_remote_download_outputs=toplevel --experimental_inmemory_jdeps_files --experimental_inmemory_dotd_files
+
+build:remote --remote_local_fallback

.bazelversion

@@ -0,0 +1 @@
+3.2.0

.buildkite-bazelrc

@@ -2,33 +2,24 @@
 # across machines, developers, and workspaces.
 # 
 # This config is loaded from https://github.com/bazelbuild/bazel-toolchains/blob/master/bazelrc/latest.bazelrc
-build:remote-cache --remote_cache=grpcs://remotebuildexecution.googleapis.com
 build:remote-cache --remote_timeout=3600
-build:remote-cache --auth_enabled=true
-build:remote-cache --spawn_strategy=standalone
-build:remote-cache --strategy=Javac=standalone
-build:remote-cache --strategy=Closure=standalone
-build:remote-cache --strategy=Genrule=standalone
-
-# Build results backend.                                                        
-build:remote-cache --bes_results_url="https://source.cloud.google.com/results/invocations/"
-build:remote-cache --bes_backend=buildeventservice.googleapis.com               
-build:remote-cache --bes_timeout=60s                                            
-build:remote-cache --project_id=prysmaticlabs
+#build:remote-cache --spawn_strategy=standalone
+#build:remote-cache --strategy=Javac=standalone
+#build:remote-cache --strategy=Closure=standalone
+#build:remote-cache --strategy=Genrule=standalone
 
 # Prysm specific remote-cache properties.
-build:remote-cache --disk_cache=
-build:remote-cache --host_platform_remote_properties_override='properties:{name:\"cache-silo-key\" value:\"prysm\"}' 
-build:remote-cache --remote_instance_name=projects/prysmaticlabs/instances/default_instance
-
-build:remote-cache --experimental_remote_download_outputs=minimal
-build:remote-cache --experimental_inmemory_jdeps_files
-build:remote-cache --experimental_inmemory_dotd_files
+#build:remote-cache --disk_cache=
+build:remote-cache --remote_download_minimal
 
 # Import workspace options.
 import %workspace%/.bazelrc
 
 startup --host_jvm_args=-Xmx1000m --host_jvm_args=-Xms1000m
+query --repository_cache=/tmp/repositorycache
+query --experimental_repository_cache_hardlinks
+build --repository_cache=/tmp/repositorycache
+build --experimental_repository_cache_hardlinks
 build --experimental_strict_action_env
 build --disk_cache=/tmp/bazelbuilds
 build --experimental_multi_threaded_digest
@@ -36,12 +27,12 @@ build --sandbox_tmpfs_path=/tmp
 build --verbose_failures
 build --announce_rc
 build --show_progress_rate_limit=5
-build --curses=yes --color=yes
+build --curses=yes --color=no
 build --keep_going
 build --test_output=errors
 build --flaky_test_attempts=5
-build --jobs=50
-build --stamp
-test --local_test_jobs=2
 # Disabled race detection due to unstable test results under constrained environment build kite
 # build --features=race
+
+# Disable flaky test detection for fuzzing.
+test:fuzz --flaky_test_attempts=1

.deepsource.toml

@@ -0,0 +1,12 @@
+version = 1
+
+[[analyzers]]
+name = "go"
+enabled = true
+
+  [analyzers.meta]
+  import_paths = ["github.com/prysmaticlabs/prysm"]
+
+[[analyzers]]
+name = "test-coverage"
+enabled = true

.github/CODEOWNERS

@@ -0,0 +1,8 @@
+# Automatically require code review from core-team.
+* @prysmaticlabs/core-team
+
+# Starlark code owners
+*.bzl @prestonvanloon
+
+# Anyone on prylabs team can approve dependency updates.
+deps.bzl @prysmaticlabs/core-team

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,59 @@
+---
+name: "\U0001F41EBug report"
+about: Report a bug or problem with running Prysm
+---
+<!--💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎
+
+Hellooo! 😄 
+
+To help us tend to your issue faster, please search our currently open issues before submitting a new one.
+Existing issues often contain information about workarounds, resolution, or progress updates.
+
+💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎-->
+
+# 🐞 Bug Report
+
+### Description
+
+<!-- ✍️--> A clear and concise description of the problem...
+
+### Has this worked before in a previous version?
+
+<!-- Did this behavior use to work in the previous version? -->
+<!-- ✍️--> Yes, the previous version in which this bug was not present was: ....
+
+## 🔬 Minimal Reproduction
+
+<!--
+Please let us know how we can reproduce this issue. Include the exact method you used to run Prysm along with any flags used in your beacon chain and/or validator. Make sure you don't upload any confidential files or private keys.
+-->
+
+## 🔥 Error
+
+<pre><code>
+<!-- If the issue is accompanied by an error, please share the error logs with us below. If you have a lot of logs, place make a paste bin with your logs and share the link with us here: -->
+<!-- ✍️-->
+
+</code></pre>
+
+
+## 🌍  Your Environment
+
+**Operating System:**
+
+<pre>
+  <code>
+
+  </code>
+</pre>
+
+**What version of Prysm are you running? (Which release)**
+
+<pre>
+  <code>
+
+  </code>
+</pre>
+
+**Anything else relevant (validator index / public key)?**
+

.github/ISSUE_TEMPLATE/discussion.md

@@ -0,0 +1,21 @@
+---
+name: "\U0001F48EGeneral issue / discussion"
+about: Any other type of general issue or discussion
+
+---
+<!--💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎
+
+Hellooo! 😄 
+
+To help us tend to your issue faster, please search our currently open issues before submitting a new one.
+Existing issues often contain information about workarounds, resolution, or progress updates.
+
+💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎-->
+
+# 💎 Issue
+
+### Background
+
+<!-- ✍️--> Context and background information on the discussion...
+
+### Description

.github/ISSUE_TEMPLATE/feature_flag.md

@@ -0,0 +1,38 @@
+---
+name: "\U0001F984Feature Flag Tracking"
+about: Track a new feature, in development, in Prysm. This issue template should only be used by developers or contributors!
+labels: Tracking
+---
+<!--💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎
+
+Hellooo! 😄 
+
+Thanks for taking the time to file a tracking issue for your new feature. These issues really help
+us track progress of features as they work their way through development. Be sure to review 
+shared/featureconfig/README.md for the latest documentation around feature flags.
+
+💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎-->
+
+# 🦄 Feature Tracking
+
+**Current status** (opt-in or opt-out): 
+
+**Opt-in feature flag name**:
+
+**Opt-out feature flag name**: 
+
+### Feature Description
+
+<!-- ✍️--> A clear and concise description of the new feature. Provide links to your technical
+design document or other supporting information.
+
+### Tasks
+
+- [ ] Feature flag created as opt-in
+- [ ] Feature implemented / code complete
+- [ ] Feature tested in production for adequate amount of time
+- [ ] Feature flag is inverted to be opt-out and the opt-in flag is deprecated
+- [ ] Feature has made it to a tagged production release as an opt-out flag
+- [ ] Opt-out feature flag is deprecated and old code paths are cleaned up
+
+This issue should be closed after all of the above tasks are complete.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,27 @@
+---
+name: "\U0001F680Feature request"
+about: Suggest a feature for Prysm
+
+---
+<!--💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎
+
+Hellooo! 😄 
+
+To help us tend to your issue faster, please search our currently open issues before submitting a new one.
+Existing issues often contain information about workarounds, resolution, or progress updates.
+
+💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎-->
+
+# 🚀 Feature Request
+
+### Description
+
+<!-- ✍️--> A clear and concise description of the problem or missing capability...
+
+### Describe the solution you'd like
+
+<!-- ✍️--> If you have a solution in mind, please describe it.
+
+### Describe alternatives you've considered
+
+<!-- ✍️--> Have you considered any alternative solutions or workarounds?

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,30 @@
+<!-- Thanks for sending a PR! Before submitting:
+
+1. If this is your first PR, please read CONTRIBUTING.md and sign the CLA
+   first. We cannot review code without a signed CLA.
+2. Please file an issue *first*. All features and most bug fixes should have
+   an associated issue with a design discussed and decided upon. Small bug
+   fixes and documentation improvements don't need issues.
+3. New features and bug fixes must have tests. Documentation may need to
+   be updated. If you're unsure what to update, send the PR, and we'll discuss
+   in review.
+4. Note that PRs updating dependencies and new Go versions are not accepted.
+   Please file an issue instead.
+-->
+
+**What type of PR is this?**
+
+> Uncomment one line below and remove others.
+>
+> Bug fix
+> Feature
+> Documentation
+> Other
+
+**What does this PR do? Why is it needed?**
+
+**Which issues(s) does this PR fix?**
+
+Fixes #
+
+**Other notes for review**

.github/actions/gofmt/Dockerfile

@@ -0,0 +1,5 @@
+FROM cytopia/gofmt
+
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

.github/actions/gofmt/action.yml

@@ -0,0 +1,12 @@
+name: 'Gofmt checker'
+description: 'Checks that all project files have been properly formatted.'
+inputs:
+  path:
+    description: 'Path to check'
+    required: true
+    default: './'
+runs:
+  using: 'docker'
+  image: 'Dockerfile'
+  args:
+    - ${{ inputs.path }}

.github/actions/gofmt/entrypoint.sh

@@ -0,0 +1,15 @@
+#!/bin/sh -l
+set -e
+
+cd $GITHUB_WORKSPACE
+
+# Check if any files are not formatted.
+nonformatted="$(gofmt -l $1 2>&1)"
+
+# Return if `go fmt` passes.
+[ -z "$nonformatted" ] && exit 0
+
+# Notify of issues with formatting.
+echo "Following files need to be properly formatted:"
+echo "$nonformatted"
+exit 1

.github/actions/gomodtidy/Dockerfile

@@ -0,0 +1,5 @@
+FROM golang:alpine
+
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

.github/actions/gomodtidy/action.yml

@@ -0,0 +1,5 @@
+name: 'Go mod tidy checker'
+description: 'Checks that `go mod tidy` has been applied.'
+runs:
+  using: 'docker'
+  image: 'Dockerfile'

.github/actions/gomodtidy/entrypoint.sh

@@ -0,0 +1,34 @@
+#!/bin/sh -l
+set -e
+export PATH=$PATH:/usr/local/go/bin
+
+cd $GITHUB_WORKSPACE
+
+cp go.mod go.mod.orig
+cp go.sum go.sum.orig
+
+go mod tidy
+
+echo "Checking go.mod and go.sum:"
+checks=0
+if [ "$(diff -s go.mod.orig go.mod | grep -c 'Files go.mod.orig and go.mod are identical')" = 1 ]; then
+  echo "- go.mod is up to date."
+  checks=$((checks + 1))
+else
+  echo "- go.mod is NOT up to date."
+fi
+
+if [ "$(diff -s go.sum.orig go.sum | grep -c 'Files go.sum.orig and go.sum are identical')" = 1 ]; then
+  echo "- go.sum is up to date."
+  checks=$((checks + 1))
+else
+  echo "- go.sum is NOT up to date."
+fi
+
+if [ $checks -eq 2 ]; then
+  exit 0
+fi
+
+# Notify of any issues.
+echo "Run 'go mod tidy' to update."
+exit 1

.github/no-response.yml

@@ -0,0 +1,13 @@
+# Configuration for probot-no-response - https://github.com/probot/no-response
+
+# Number of days of inactivity before an Issue is closed for lack of response
+daysUntilClose: 14
+# Label requiring a response
+responseRequiredLabel: Need-Info
+# Comment to post when closing an Issue for lack of response. Set to `false` to disable
+closeComment: >
+  This issue has been automatically closed because there has been no response
+  to our request for more information from the original author. With only the
+  information that is currently in the issue, we don't have enough information
+  to take action. Please reach out if you have or find the answers we need so
+  that we can investigate further.

.github/workflows/dappnode-release-trigger.yml

@@ -0,0 +1,41 @@
+name: Update DAppNodePackages
+
+on:
+  push:
+    tags:
+    - '*'
+
+jobs:
+  dappnode-update-beacon-chain:
+    name: Trigger a beacon-chain release
+    runs-on: ubuntu-latest
+    steps:
+    - name: Get latest tag
+      id: get_tag
+      run: echo ::set-output name=TAG::${GITHUB_REF/refs\/tags\//}
+    - name: Send dispatch event to DAppNodePackage-prysm-beacon-chain
+      env: 
+        DISPATCH_REPO: dappnode/DAppNodePackage-prysm-beacon-chain
+      run: |
+        curl -v -X POST -u "${{ secrets.PAT_GITHUB }}" \
+        -H "Accept: application/vnd.github.everest-preview+json" \
+        -H "Content-Type: application/json" \
+        --data '{"event_type":"new_release", "client_payload": { "tag":"${{ steps.get_tag.outputs.TAG }}"}}' \
+        https://api.github.com/repos/$DISPATCH_REPO/dispatches
+        
+  dappnode-update-validator:
+    name: Trigger a validator release
+    runs-on: ubuntu-latest
+    steps:
+    - name: Get latest tag
+      id: get_tag
+      run: echo ::set-output name=TAG::${GITHUB_REF/refs\/tags\//}
+    - name: Send dispatch event to DAppNodePackage validator repository
+      env: 
+        DISPATCH_REPO: dappnode/DAppNodePackage-prysm-validator
+      run: |
+        curl -v -X POST -u "${{ secrets.PAT_GITHUB }}" \
+        -H "Accept: application/vnd.github.everest-preview+json" \
+        -H "Content-Type: application/json" \
+        --data '{"event_type":"new_release", "client_payload": { "tag":"${{ steps.get_tag.outputs.TAG }}"}}' \
+        https://api.github.com/repos/$DISPATCH_REPO/dispatches

.github/workflows/go.yml

@@ -0,0 +1,51 @@
+name: Go
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ '*' ]
+
+jobs:
+
+  check:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+
+      - name: Go mod tidy checker
+        id: gomodtidy
+        uses: ./.github/actions/gomodtidy
+
+      - name: Gofmt checker
+        id: gofmt
+        uses: ./.github/actions/gofmt
+        with:
+          path: ./
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.14
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Get dependencies
+        run: |
+          go get -v -t -d ./...
+
+      - name: Build
+        # Use blst tag to allow go and bazel builds for blst.
+        run: go build -v --tags=blst_enabled ./...
+
+# Tests run via Bazel for now...
+#      - name: Test
+#        run: go test -v ./...

.gitignore

@@ -4,6 +4,9 @@ bazel-*
 .DS_Store
 .swp
 
+# Ignore VI/Vim swapfiles
+.*.sw?
+
 # IntelliJ
 .idea
 .ijwb
@@ -14,6 +17,7 @@ bazel-*
 # Coverage outputs
 coverage.txt
 profile.out
+profile.grind
 
 # Nodejs
 node_modules
@@ -23,6 +27,9 @@ yarn-error.log
 # Ignore password file
 password.txt
 
-# go dependancy
-/go.mod
-/go.sum
+# Dist files
+dist
+
+# libfuzzer
+oom-*
+crash-*

.policy.yml

@@ -0,0 +1,91 @@
+policy:
+  approval:
+    - or:
+        - only test files are changed
+        - only proto files are changed
+        - touches consensus critical code
+        - touches sync/blockchain/p2p code
+        - large line count
+        - is critical priority
+approval_rules:
+  - name: only test files are changed
+    if:
+      only_changed_files:
+        paths:
+          - "*_test.go"
+          - "*.bazel"
+    options:
+      ignore_commits_by:
+        users: ["bulldozer[bot]"]
+    requires:
+      count: 1
+      teams:
+        - "prysmaticlabs/core-team"
+  - name: only proto files are changed
+    if:
+      only_changed_files:
+        paths:
+          - "*pb.go"
+          - "*pb.gw.go"
+          - "*.bazel"
+    options:
+      ignore_commits_by:
+        users: ["bulldozer[bot]"]
+    requires:
+      count: 1
+      teams:
+        - "prysmaticlabs/core-team"
+  - name: touches consensus critical code
+    if:
+      only_changed_files:
+        paths:
+          - "beacon-chain/core/*"
+          - "beacon-chain/state/*"
+    options:
+      ignore_commits_by:
+        users: ["bulldozer[bot]"]
+    requires:
+      count: 2
+      teams:
+        - "prysmaticlabs/core-team"
+  - name: touches sync/blockchain/p2p code
+    if:
+      only_changed_files:
+        paths:
+          - "beacon-chain/blockchain/*"
+          - "beacon-chain/sync/*"
+          - "beacon-chain/p2p/*"
+    options:
+      ignore_commits_by:
+        users: ["bulldozer[bot]"]
+    requires:
+      count: 2
+      teams:
+        - "prysmaticlabs/core-team"
+  - name: large line count
+    if:
+      modified_lines:
+        total: "> 1000"
+      changed_files:
+        ignore:
+          - "*pb.go"
+          - "*pb.gw.go"
+          - "*.bazel"
+    options:
+      ignore_commits_by:
+        users: ["bulldozer[bot]"]
+    requires:
+      count: 2
+      teams:
+        - "prysmaticlabs/core-team"
+  - name: is critical priority
+    if:
+      has_labels:
+        - "Priority: Critical"
+    options:
+      ignore_commits_by:
+        users: ["bulldozer[bot]"]
+    requires:
+      count: 3
+      teams:
+        - "prysmaticlabs/core-team"

.well-known/security.txt

@@ -4,23 +4,23 @@ Hash: SHA512
 Contact: mailto:security@prysmaticlabs.com
 Encryption: openpgp4fpr:0AE0051D647BA3C1A917AF4072E33E4DF1A5036E
 Encryption: openpgp4fpr:341396BAFACC28C5082327F889725027FC8EC0D4
-Encryption: openpgp4fpr:8B7814F1B221A8E8AA465FC7BDBF744ADE1A0033
+Encryption: openpgp4fpr:FEE44615A19049DF0CA0C2735E2B7E5734DFADCB
 Preferred-Languages: en
 Canonical: https://github.com/prysmaticlabs/prysm/tree/master/.well-known/security.txt
 
 -----BEGIN PGP SIGNATURE-----
 
-iQIzBAEBCgAdFiEECuAFHWR7o8GpF69AcuM+TfGlA24FAlzi0WgACgkQcuM+TfGl
-A241pw/+Ks3Hxx8eGbjRIeuncuK811FkCiofNJS+MY2p4W2/tIrk48DtLRx8/k5L
-Dh1QyypZsqUgofrK7PbGVdEin6oEb2jYbTWUarAVTbhlsUdM4YcxwpgmGVslW7+C
-Hm8wMasQZhCkFfakzhfKX5hIQoFaFI/OvtVKIQsodP8dAieCDaGmtfq1Bs1LgFqi
-KrpeEdC2XbBQs33ADheC5SdGT1mnatP3VX8cOhLsfoPksYgTSpwK0clkoWs1eZOQ
-l1ImfW/FJCpSndBWgBR503ZgaU3Ic+5qxmAIuUP4chl0DFRMlPFEM5OWC6JkkCOd
-5kKrXGRmrhgtQg+pA3zqJnFItRj7gxPBA/ypxCkKPrLEkRvbdpdZEl5vAlYkeBL6
-iKSLHnMswGKldiYxy7ofam5bM3myhYYNFb25boV5pRptrnoUmWOACHioBGQHwWNt
-B0XktD0j7+pCCiJyyYxmOnElsk/Y/u4Tv5pYWvfFuxTF2XOg+P/EH64AIFLWgB1U
-VnITxhakxqejCBxZkuVCFNSzt+TXG0NS9EIj/UOYBY+wxrBZ62ITjdA16RS/3n3z
-DuIDtxOOwUumbOO32+a5zIb+ARmnocYJviI7FuENb01/U6qb+nm9hQI6oIpSCNsv
-Pb4O/ZlOx70U/7mt4Xn/dTKH9bnKOOVhOw00KJWFfAce73AVnLA=
-=Uhqg
+iQIzBAEBCgAdFiEECuAFHWR7o8GpF69AcuM+TfGlA24FAl6HrcwACgkQcuM+TfGl
+A26voQ/8DFB5wUHP0uyY8k7FGbxhLzSeImxomnUHJaUGfdczbCdYPMEHc9zI1iZP
+6LRiy9wS6qhqj/GSKVwvDPr+ZymXuV3L22GOP2lRhl7Z9Mm21ZJNOcoQBFOZnyHu
+DAy9HeTmeuJxYkf8weqZYXyzEoKJBDmfuWmEFjrtMcFXUfT3aJn1E2A/AQdcVQIC
+9L+iGWwFwjsPhcfaMuwcB7QMheDO6KSB7XPPCbrZ036Np8UTZ4qbZ5y73tlfkcOc
+tYTrMSPtS4eNutiDOP5Np36cLzRtNpm/BziAK+7ZKiYY0HI5h9IkCTLO4x2UmAMX
+sPoeaAB5z2QLIwmU9J2NhJrwiNMGTpJ+0bowy8U4cgzAX20CXVjRqGhy+cir8Ewg
+DjEGjWINUw6W0yzJp0mKSKzuOhdTTmzIYBeMBsyce+pgN1KGFCxeIwxGxyJzADdw
+mYQdljRXn4yEYP/KEpu/F2o8L4ptRO2jZWKvTvdzSSGGSyKyF4HsIRJ7m98DaB6S
+0oGq1KpbKKTbQi5g8UShGV2gPeMCs5ZIIqK2b/cRzUet18aUuofLmR4lkKZa9yEG
+rbzuJq/gB2vgQwExUEgVQ3/DfVc+y80e3YZ5s+rzV0vbLxl4Gh4yExpLo7hRf9iY
+EFvMzH+BEEb5VfCwByZyV1BmesZVIosr7K6UmVtPe0bZGvv3uIg=
+=5qpD
 -----END PGP SIGNATURE-----

BUILD.bazel

@@ -4,7 +4,7 @@ load("@com_github_atlassian_bazel_tools//goimports:def.bzl", "goimports")
 load("@io_kubernetes_build//defs:run_in_workspace.bzl", "workspace_binary")
 load("@io_bazel_rules_go//go:def.bzl", "nogo")
 load("@graknlabs_bazel_distribution//common:rules.bzl", "assemble_targz", "assemble_versioned")
-load("//tools:binary_targets.bzl", "binary_targets", "determine_targets")
+load("@bazel_skylib//rules:common_settings.bzl", "string_setting")
 
 prefix = "github.com/prysmaticlabs/prysm"
 
@@ -13,6 +13,8 @@ exports_files([
 ])
 
 # gazelle:prefix github.com/prysmaticlabs/prysm
+# gazelle:map_kind go_library go_library @prysm//tools/go:def.bzl
+# gazelle:map_kind go_repository go_repository @prysm//tools/go:def.bzl
 gazelle(
     name = "gazelle",
     prefix = prefix,
@@ -31,27 +33,21 @@ alias(
 alias(
     name = "grpc_proto_compiler",
     actual = "@io_bazel_rules_go//proto:gogofast_grpc",
-    visibility = [
-        "//proto:__subpackages__",
-    ],
+    visibility = ["//visibility:public"],
 )
 
 # Protobuf gRPC compiler without gogoproto. Required for gRPC gateway.
 alias(
     name = "grpc_nogogo_proto_compiler",
     actual = "@io_bazel_rules_go//proto:go_grpc",
-    visibility = [
-        "//proto:__subpackages__",
-    ],
+    visibility = ["//visibility:public"],
 )
 
 # Protobuf gRPC gateway compiler
 alias(
     name = "grpc_gateway_proto_compiler",
-    actual = "@grpc_ecosystem_grpc_gateway//protoc-gen-grpc-gateway:go_gen_grpc_gateway",
-    visibility = [
-        "//proto:__subpackages__",
-    ],
+    actual = "@com_github_grpc_ecosystem_grpc_gateway//protoc-gen-grpc-gateway:go_gen_grpc_gateway",
+    visibility = ["//visibility:public"],
 )
 
 gometalinter(
@@ -92,15 +88,12 @@ nogo(
         "@org_golang_x_tools//go/analysis/passes/pkgfact:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/nilness:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/nilfunc:go_tool_library",
-        # lost cancel ignore doesn't seem to work when running with coverage
-        #"@org_golang_x_tools//go/analysis/passes/lostcancel:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/loopclosure:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/httpresponse:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/findcall:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/deepequalerrors:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/ctrlflow:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/copylock:go_tool_library",
-        "@org_golang_x_tools//go/analysis/passes/composite:go_tool_library",
         #  "@org_golang_x_tools//go/analysis/passes/cgocall:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/buildtag:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/buildssa:go_tool_library",
@@ -110,20 +103,28 @@ nogo(
         "@org_golang_x_tools//go/analysis/passes/assign:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/inspect:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/asmdecl:go_tool_library",
-    ],
+        "//tools/analyzers/maligned:go_tool_library",
+        "//tools/analyzers/cryptorand:go_tool_library",
+        "//tools/analyzers/errcheck:go_tool_library",
+        "//tools/analyzers/featureconfig:go_tool_library",
+        "//tools/analyzers/comparesame:go_tool_library",
+        "//tools/analyzers/shadowpredecl:go_tool_library",
+        "//tools/analyzers/nop:go_tool_library",
+        "//tools/analyzers/slicedirect:go_tool_library",
+        "//tools/analyzers/ineffassign:go_tool_library",
+    ] + select({
+        # nogo checks that fail with coverage enabled.
+        ":coverage_enabled": [],
+        "//conditions:default": [
+            "@org_golang_x_tools//go/analysis/passes/lostcancel:go_tool_library",
+            "@org_golang_x_tools//go/analysis/passes/composite:go_tool_library",
+        ],
+    }),
 )
 
-assemble_versioned(
-    name = "assemble-versioned-all",
-    tags = ["manual"],
-    targets = [
-        ":assemble-{}-{}-targz".format(
-            pair[0],
-            pair[1],
-        )
-        for pair in binary_targets
-    ],
-    version_file = "//:VERSION",
+config_setting(
+    name = "coverage_enabled",
+    values = {"define": "coverage_enabled=1"},
 )
 
 common_files = {
@@ -131,15 +132,17 @@ common_files = {
     "//:README.md": "README.md",
 }
 
-[assemble_targz(
-    name = "assemble-{}-{}-targz".format(
-        pair[0],
-        pair[1],
-    ),
-    additional_files = determine_targets(pair, common_files),
-    output_filename = "prysm-{}-{}".format(
-        pair[0],
-        pair[1],
-    ),
-    tags = ["manual"],
-) for pair in binary_targets]
+toolchain(
+    name = "built_cmake_toolchain",
+    toolchain = "@rules_foreign_cc//tools/build_defs/native_tools:built_cmake",
+    toolchain_type = "@rules_foreign_cc//tools/build_defs:cmake_toolchain",
+)
+
+string_setting(
+    name = "gotags",
+    build_setting_default = "",
+    values = [
+        "",
+        "libfuzzer",
+    ],
+)

CONTRIBUTING.md

@@ -1,6 +1,8 @@
 # Contribution Guidelines
 
-Excited by our work and want to get involved in building out our sharding releases? Or maybe you haven't learned as much about the Ethereum protocol but are a savvy developer? Our [READINGS.md](https://github.com/prysmaticlabs/prysm/blob/master/docs/READINGS.md) doc includes comprehensive information on Ethereum and sharding for both part-time and core contributors to the project.
+Note: The latest and most up to date documenation can be found on our [docs portal](https://docs.prylabs.network/docs/contribute/contribution-guidelines).
+
+Excited by our work and want to get involved in building out our sharding releases? Or maybe you haven't learned as much about the Ethereum protocol but are a savvy developer? 
 
 You can explore our [Open Issues](https://github.com/prysmaticlabs/prysm/issues) in-the works for our different releases. Feel free to fork our repo and start creating PR’s after assigning yourself to an issue of interest. We are always chatting on [Discord](https://discord.gg/che9auJ) or [Gitter](https://gitter.im/prysmaticlabs/geth-sharding) drop us a line there if you want to get more involved or have any questions on our implementation!
 

DEPENDENCIES.md

@@ -0,0 +1,69 @@
+# Dependency Managagement in Prysm
+
+Prysm is go project with many complicated dependencies, including some c++ based libraries. There
+are two parts to Prysm's dependency management. Go modules and bazel managed dependencies. Be sure 
+to read [Why Bazel?](https://github.com/prysmaticlabs/documentation/issues/138) to fully
+understand the reasoning behind an additional layer of build tooling via Bazel rather than a pure
+"go build" project.
+
+## Go Module support
+
+The Prysm project officially supports go modules with some caveats. 
+
+### Caveat 1: Some c++ libraries are precompiled archives
+
+Given some of Prysm's c++ dependencies have very complicated project structures which make building
+difficult or impossible with "go build" alone. Additionally, building c++ dependencies with certain
+compilers, like clang / LLVM, offer a significant performance improvement. To get around this 
+issue, c++ dependencies have been precompiled as linkable archives. While there isn't necessarily
+anything bad about precompiled archives, these files are a "blackbox" which a 3rd party author
+could have compiled anything for this archive and detecting undesired behavior would be nearly
+impossible. If your risk tolerance is low, always compile everything from source yourself, 
+including complicated c++ dependencies.
+
+*Recommendation: Use go build only for local development and use bazel build for production.*
+
+### Caveat 2: Generated gRPC protobuf libraries
+
+One key advantage of Bazel over vanilla `go build` is that Bazel automatically (re)builds generated
+pb.go files at build time when file changes are present in any protobuf definition file or after
+any updates to the protobuf compiler or other relevant dependencies. Vanilla go users should run
+the following scripts often to ensure their generated files are up to date. Further more, Prysm
+generates SSZ marshal related code based on defined data structures. These generated files must
+also be updated and checked in as frequently.
+
+```bash
+./scripts/update-go-pbs.sh
+./scripts/update-go-ssz.sh
+```
+
+*Recommendation: Use go build only for local development and use bazel build for production.*
+
+### Caveat 3: Compile-time optimizations 
+
+When Prysmatic Labs builds production binaries, they use the "release" configuration of bazel to
+compile with several compiler optimizations and recommended production build configurations.
+Additionally, the release build properly stamps the built binaries to include helpful metadata
+about how and when the binary was built. 
+
+*Recommendation: Use go build only for local development and use bazel build for production.*
+
+```bash
+bazel build //beacon-chain --config=release
+```
+ 
+## Adding / updating dependencies
+
+1. Add your dependency as you would with go modules. I.e. `go get ...`
+1. Run `gazelle update-repos -from_file=go.mod` to update the bazel managed dependencies.
+
+Example:
+
+```bash
+go get github.com/prysmaticlabs/example@v1.2.3
+bazel run //:gazelle -- update-repos -from_file=go.mod -to_macro=deps.bzl%prysm_deps
+```
+
+The deps.bzl file should have been updated with the dependency and any transitive dependencies. 
+
+Do NOT add new `go_repository` to the WORKSPACE file. All dependencies should live in deps.bzl.

INTEROP.md

@@ -45,10 +45,9 @@ Open up two terminal windows, run:
 
 ```
 bazel run //beacon-chain -- \
---no-genesis-delay \
 --bootstrap-node= \
---deposit-contract 0xD775140349E6A5D12524C6ccc3d6A1d4519D4029 \
---clear-db \
+--deposit-contract $(curl -s https://prylabs.net/contract) \
+--force-clear-db \
 --interop-num-validators 64 \
 --interop-eth1data-votes
 ```
@@ -58,11 +57,10 @@ the system with 64 validators and the genesis time set to the current unix times
 Wait a bit until your beacon chain starts, and in the other window:
 
 ```
-bazel run //validator -- --interop-num-validators 64
+bazel run //validator -- --keymanager=interop --keymanageropts='{"keys":64}'
 ```
 
 This will launch and kickstart the system with your 64 validators performing their duties accordingly.
-specify which keys 
 
 ### Launching from `genesis.ssz`
 
@@ -70,10 +68,9 @@ Assuming you generated a `genesis.ssz` file with 64 validators, open up two term
 
 ```
  bazel run //beacon-chain -- \
---no-genesis-delay \
 --bootstrap-node= \
---deposit-contract 0xD775140349E6A5D12524C6ccc3d6A1d4519D4029 \
---clear-db \
+--deposit-contract $(curl -s https://prylabs.net/contract) \
+--force-clear-db \
 --interop-genesis-state /path/to/genesis.ssz \
 --interop-eth1data-votes
 ```
@@ -81,13 +78,7 @@ Assuming you generated a `genesis.ssz` file with 64 validators, open up two term
 Wait a bit until your beacon chain starts, and in the other window:
 
 ```
-bazel run //validator -- --interop-num-validators 64
+bazel run //validator -- --keymanager=interop --keymanageropts='{"keys":64}'
 ```
 
 This will launch and kickstart the system with your 64 validators performing their duties accordingly.
-
-
-
-
-
-

PULL_REQUEST_TEMPLATE.md

@@ -1,13 +0,0 @@
-TODO(you): choose "part of" or "resolves" and the associated github issue #.
-
-[Part of|Resolves] #531 
-
----
-
-# Description
-
-**Write why you are making the changes in this pull request**
-
-**Write a summary of the changes you are making**
-
-**Link anything that would be helpful or relevant to the reviewers**

README.md

@@ -1,245 +1,20 @@
 # Prysm: An Ethereum 2.0 Client Written in Go
 
 [![Build status](https://badge.buildkite.com/b555891daf3614bae4284dcf365b2340cefc0089839526f096.svg?branch=master)](https://buildkite.com/prysmatic-labs/prysm)
-[![ETH2.0_Spec_Version 0.8.1](https://img.shields.io/badge/ETH2.0%20Spec%20Version-v0.8.1-blue.svg)](https://github.com/ethereum/eth2.0-specs/commit/452ecf8e27c7852c7854597f2b1bb4a62b80c7ec)
+[![ETH2.0_Spec_Version 0.12.3](https://img.shields.io/badge/ETH2.0%20Spec%20Version-v0.12.3-blue.svg)](https://github.com/ethereum/eth2.0-specs/tree/v0.12.3)
 [![Discord](https://user-images.githubusercontent.com/7288322/34471967-1df7808a-efbb-11e7-9088-ed0b04151291.png)](https://discord.gg/KSA7rPr)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/prysmaticlabs/geth-sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 
 This is the core repository for Prysm, a [Golang](https://golang.org/) implementation of the Ethereum 2.0 client specifications developed by [Prysmatic Labs](https://prysmaticlabs.com).
 
-### Need assistance?
-A more detailed set of installation and usage instructions as well as breakdowns of each individual component are available in the [official documentation portal](https://prysmaticlabs.gitbook.io/prysm/). If you still have questions, feel free to stop by either our [Discord](https://discord.gg/KSA7rPr) or [Gitter](https://gitter.im/prysmaticlabs/geth-sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) and a member of the team or our community will be happy to assist you.
+### Getting Started
+A detailed set of installation and usage instructions as well as breakdowns of each individual component are available in the [official documentation portal](https://docs.prylabs.network). If you still have questions, feel free to stop by either our [Discord](https://discord.gg/KSA7rPr) or [Gitter](https://gitter.im/prysmaticlabs/geth-sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) and a member of the team or our community will be happy to assist you.
 
 ### Come join the testnet!
-Participation is now open to the public for our Ethereum 2.0 phase 0 testnet release. Visit [prylabs.net](https://prylabs.net) for more information on the project or to sign up as a validator on the network.
-
-# Table of Contents
-
-- [Dependencies](#dependencies)
-- [Installation](#installation)
-    - [Build via Docker](#build-via-docker)
-    - [Build via Bazel](#build-via-bazel)
-- [Connecting to the public testnet: running a beacon node](#connecting-to-the-testnet-running-a-beacon-node)
-    - [Running via Docker](#build-via-docker)
-    - [Running via Bazel](#build-via-bazel)
-- [Staking ETH: running a validator client](#staking-eth-running-a-validator-client)
-    - [Activating your validator: depositing 3.2 Goerli ETH](#activating-your-validator-depositing-32-goerli-eth)
-    - [Starting the validator with Bazel](#starting-the-validator-with-bazel)
-- [Setting up a local ETH2 development chain](#setting-up-a-local-eth2-development-chain)
-    - [Installation and dependencies](#installation-and-dependencies) 
-    - [Running a local beacon node and validator client](#running-a-local-beacon-node-and-validator-client)   
--   [Testing Prysm](#testing-prysm)
--   [Contributing](#contributing)
--   [License](#license)
-
-## Dependencies
-Prysm can be installed either with Docker **(recommended method)** or using our build tool, Bazel. The below instructions include sections for performing both.
-
-**For Docker installations:**
-  - The latest release of [Docker](https://docs.docker.com/install/)
-
-**For Bazel installations:**
-  - The latest release of [Bazel](https://docs.bazel.build/versions/master/install.html)
-  - A modern UNIX operating system (MacOS included)
-
-## Installation
-
-### Build via Docker
-1. Ensure you are running the most recent version of Docker by issuing the command:
-```
-docker -v
-```
-2.  To pull the Prysm images from the server, issue the following commands:
-```
-docker pull gcr.io/prysmaticlabs/prysm/validator:latest
-docker pull gcr.io/prysmaticlabs/prysm/beacon-chain:latest
-```
-This process will also install any related dependencies.
-
-### Build via Bazel
-
-1. Open a terminal window. Ensure you are running the most recent version of Bazel by issuing the command:
-```
-bazel version
-```
-2. Clone this repository and enter the directory:
-```
-git clone https://github.com/prysmaticlabs/prysm
-cd prysm
-```
-3. Build both the beacon chain node implementation and the validator client:
-```
-bazel build //beacon-chain:beacon-chain
-bazel build //validator:validator
-```
-Bazel will automatically pull and install any dependencies as well, including Go and necessary compilers.
-
-4. Build the configuration for the Prysm testnet by issuing the commands:
-
-```
-bazel build --define ssz=minimal //beacon-chain:beacon-chain
-bazel build --define ssz=minimal //validator:validator
-```
-
-The binaries will be built in an architecture-dependent subdirectory of `bazel-bin`, and are supplied as part of Bazel's build process.  To fetch the location, issue the command:
-
-```
-$ bazel build --define ssz=minimal //beacon-chain:beacon-chain
-...
-Target //beacon-chain:beacon-chain up-to-date:
-  bazel-bin/beacon-chain/linux_amd64_stripped/beacon-chain
-...
-```
-
-In the example above, the beacon chain binary has been created in `bazel-bin/beacon-chain/linux_amd64_stripped/beacon-chain`.
-
-## Connecting to the testnet: running a beacon node
-
-This section contains instructions for initialising a beacon node and connecting to the public testnet. To further understand the role that both the beacon node and validator play in Prysm, see [this section of our documentation](https://prysmaticlabs.gitbook.io/prysm/how-prysm-works/overview-technical).
-
-### Running via Docker
-
-#### Docker on Linux/Mac
-
-To start your beacon node, issue the following command:
-
-```
-docker run -v $HOME/prysm-data:/data -p 4000:4000 \
-  --name beacon-node \
-  gcr.io/prysmaticlabs/prysm/beacon-chain:latest \
-  --no-genesis-delay \
-  --datadir=/data
-```
-
-(Optional) If you want to enable gRPC, then run this command instead of the one above:
-
-```
-docker run -v $HOME/prysm-data:/data -p 4000:4000 -p 7000:7000 \
-  --name beacon-node \
-  gcr.io/prysmaticlabs/prysm/beacon-chain:latest \
-  --datadir=/data \
-  --no-genesis-delay \
-  --grpc-gateway-port=7000
-```
-
-You can halt the beacon node using `Ctrl+c` or with the following command:
-
-```
-docker stop beacon-node
-```
-
-To restart the beacon node, issue the command:
-
-```
-docker start -ai beacon-node
-```
-
-To delete a corrupted container, issue the command:
-
-```
-docker rm beacon-node
-```
-
-To recreate a deleted container and refresh the chain database, issue the start command with an additional `--force-clear-db` parameter:
-
-```
-docker run -it -v $HOME/prysm-data:/data -p 4000:4000 --name beacon-node \
-  gcr.io/prysmaticlabs/prysm/beacon-chain:latest \
-  --datadir=/data \
-  --force-clear-db
-```
-
-#### Docker on Windows
-
-1) You will need to share the local drive you wish to mount to to container (e.g. C:).
-    1. Enter Docker settings (right click the tray icon)
-    2. Click 'Shared Drives'
-    3. Select a drive to share
-    4. Click 'Apply'
-
-2) You will next need to create a directory named ```/tmp/prysm-data/``` within your selected shared Drive. This folder will be used as a local data directory for Beacon Node chain data as well as account and keystore information required by the validator. Docker will **not** create this directory if it does not exist already. For the purposes of these instructions, it is assumed that ```C:``` is your prior-selected shared Drive.
-
-4) To run the beacon node, issue the command:
-```
-docker run -it -v c:/tmp/prysm-data:/data -p 4000:4000 gcr.io/prysmaticlabs/prysm/beacon-chain:latest --datadir=/data
-```
-
-### Running via Bazel
-
-1) To start your Beacon Node with Bazel, issue the command:
-```
-bazel run //beacon-chain -- --datadir=/tmp/prysm-data
-```
-This will sync up the Beacon Node with the latest head block in the network. Note that the beacon node must be **completely synced** before attempting to initialise a validator client, otherwise the validator will not be able to complete the deposit and funds will be lost.
-
-
-## Staking ETH: running a validator client
-
-Once your beacon node is up, the chain will be waiting for you to deposit 3.2 Goerli ETH into the Validator Deposit Contract to activate your validator (discussed in the section below). First though, you will need to create a validator client to connect to this node in order to stake and participate. Each validator represents 3.2 Goerli ETH being staked in the system, and it is possible to spin up as many as you desire in order to have more stake in the network.
-
-For more information on the functionality of validator clients, see [this section](https://prysmaticlabs.gitbook.io/prysm/how-prysm-works/validator-clients) of our official documentation.
-
-### Activating your validator: depositing 3.2 Goerli ETH
-
-Using your validator deposit data from the previous step, follow the instructions found on https://prylabs.net/participate to make a deposit.
-
-It will take a while for the nodes in the network to process your deposit, but once your node is active, the validator will begin doing its responsibility. In your validator client, you will be able to frequently see your validator balance as it goes up over time. Note that, should your node ever go offline for a long period, you'll start gradually losing your deposit until you are removed from the system.
-
-### Starting the validator with Bazel
-
-1. Open another terminal window. Enter your Prysm directory and run the validator by issuing the following command:
-```
-cd prysm
-bazel run //validator
-```
-**Congratulations, you are now running Ethereum 2.0 Phase 0!**
-
-## Setting up a local ETH2 development chain
-
-This section outlines the process of setting up Prysm for local interop testing with other Ethereum 2.0 client implementations. See the [INTEROP.md](https://github.com/prysmaticlabs/prysm/blob/master/INTEROP.md) file for advanced configuration options. For more background information on interoperability development, see [this blog post](https://blog.ethereum.org/2019/09/19/eth2-interop-in-review/).
-
-### Installation and dependencies
-
-To begin setting up a local ETH2 development chain, follow the **Bazel** instructions found in the [dependencies](#dependencies) and [installation](#installation) sections respectively. 
-
-### Running a local beacon node and validator client
-
-The example below will deterministically generate a beacon genesis state, initiate Prysm with 64 validators and set the genesis time to your local machines current UNIX time.
-
-1. Open up two terminal windows. In the first, issue the command:
-
-```
-bazel run //beacon-chain -- \
---no-genesis-delay \
---bootstrap-node= \
---deposit-contract 0xD775140349E6A5D12524C6ccc3d6A1d4519D4029 \
---clear-db \
---interop-num-validators 64 \
---interop-eth1data-votes
-```
-
-2. Wait a moment for the beacon chain to start. In the other terminal, issue the command:
-
-```
-bazel run //validator -- --interop-num-validators 64
-```
-
-This command will kickstart the system with your 64 validators performing their duties accordingly.
-
-## Testing Prysm
-
-To run the unit tests of our system, issue the command:
-```
-bazel test //...
-```
-
-To run the linter, ensure you have [golangci-lint](https://github.com/golangci/golangci-lint) installed, then issue the command:
-```
-golangci-lint run
-```
+Participation is now open to the public for our Ethereum 2.0 phase 0 testnet release. Visit [prylabs.net](https://prylabs.net) for more information on the project or to sign up as a validator on the network. You can visualize the nodes in the network on [eth2stats.io](https://eth2stats.io), explore validator rewards/penalties via Bitfly's block explorer: [beaconcha.in](https://beaconcha.in), and follow the latest blocks added to the chain on [Etherscan](https://beacon.etherscan.io).
 
 ## Contributing
-Want to get involved? Check out our [Contribution Guide](https://prysmaticlabs.gitbook.io/prysm/getting-involved/contribution-guidelines) to learn more!
+Want to get involved? Check out our [Contribution Guide](https://docs.prylabs.network/docs/contribute/contribution-guidelines/) to learn more!
 
 ## License
 [GNU General Public License v3.0](https://www.gnu.org/licenses/gpl-3.0.en.html)

TERMS_OF_SERVICE.md

@@ -0,0 +1,45 @@
+## Terms of Use
+
+Effective as of Oct 14, 2020
+
+By downloading, accessing or using the Prysm implementation (“Prysm”), you (referenced herein as “you” or the “user”) certify that you have read and agreed to the terms and conditions below (the “Terms”) which form a binding contract between you and Prysmatic Labs (referenced herein as “we” or “us”). If you do not agree to the Terms, do not download or use Prysm.
+
+### About Prysm
+Prysm is a client implementation for Ethereum 2.0 Phase 0 protocol for a proof-of-stake blockchain. To participate in the network, a user must send ETH from the Eth1.0 chain into a validator deposit contract, which will queue in the user as a validator in the system. Validators participate in proposing and voting on blocks in the protocol, and the network applies rewards/penalties based on their behavior. A detailed set of installation and usage instructions as well as breakdowns of each individual component are available in the official documentation portal, however, we do not warrant the accuracy, completeness or usefulness of this documentation. Any reliance you place on such information is strictly at your own risk.
+
+### Licensing Terms
+Prysm is a fully open-source software program licensed pursuant to the GNU General Public License v3.0.
+
+The Prysmatic Labs name, the term “Prysm” and all related names, logos, product and service names, designs and slogans are trademarks of Prysmatic Labs or its affiliates and/or licensors. You must not use such marks without our prior written permission.
+
+### Risks of Operating Prysm
+The use of Prysm and acting as a validator on the Ethereum 2.0 network can lead to loss of money. Ethereum is still an experimental system and ETH remains a risky investment. You alone are responsible for your actions on Prysm including the security of your ETH and meeting any applicable minimum system requirements.
+
+Use of Prysm and the ability to receive rewards or penalties may be affected at any time by mistakes made by the user or other users, software problems such as bugs, errors, incorrectly constructed transactions, unsafe cryptographic libraries or malware affecting the network, technical failures in the hardware of a user, security problems experienced by a user and/or actions or inactions of third parties and/or events experienced by third parties, among other risks. We cannot and do not guarantee that any user of Prysm will make money, that the Prysm network will operate in accordance with the documentation or that transactions will be effective or secure.
+
+We make no claims that Prysm is appropriate or permitted for use in any specific jurisdiction. Access to Prysm may not be legal by certain persons or in certain jurisdictions or countries. If you access Prysm, you do so on your own initiative and are responsible for compliance with local laws.
+
+Some Internet plans will charge an additional amount for any excess upload bandwidth used that isn’t included in the plan and may terminate your connection without warning because of overuse. We advise that you check whether your Internet connection is subjected to such limitations and monitor your bandwidth use so that you can stop Prysm before you reach your upload limit.
+
+### Warranty Disclaimer
+PRYSM IS PROVIDED ON AN “AS-IS” BASIS AND MAY INCLUDE ERRORS, OMISSIONS, OR OTHER INACCURACIES. PRYSMATIC LABS AND ITS CONTRIBUTORS MAKE NO REPRESENTATIONS OR WARRANTIES ABOUT PRYSM FOR ANY PURPOSE, AND HEREBY EXPRESSLY DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY WARRANTY OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT OR ANY OTHER IMPLIED WARRANTY UNDER THE UNIFORM COMPUTER INFORMATION TRANSACTIONS ACT AS ENACTED BY ANY STATE. WE ALSO MAKE NO REPRESENTATIONS OR WARRANTIES THAT PRYSM WILL OPERATE ERROR-FREE, UNINTERRUPTED, OR IN A MANNER THAT WILL MEET YOUR REQUIREMENTS AND/OR NEEDS. THEREFORE, YOU ASSUME THE ENTIRE RISK REGARDING THE QUALITY AND/OR PERFORMANCE OF PRYSM AND ANY TRANSACTIONS ENTERED INTO THEREON.
+
+### Limitation of Liability
+In no event will Prysmatic Labs or any of its contributors be liable, whether in contract, warranty, tort (including negligence, whether active, passive or imputed), product liability, strict liability or other theory, breach of statutory duty or otherwise arising out of, or in connection with, your use of Prysm, for any direct, indirect, incidental, special or consequential damages (including any loss of profits or data, business interruption or other pecuniary loss, or damage, loss or other compromise of data, in each case whether direct, indirect, incidental, special or consequential) arising out of use Prysm, even if we or other users have been advised of the possibility of such damages. The foregoing limitations and disclaimers shall apply to the maximum extent permitted by applicable law, even if any remedy fails of its essential purpose. You acknowledge and agree that the limitations of liability afforded us hereunder constitute a material and actual inducement and condition to entering into these Terms, and are reasonable, fair and equitable in scope to protect our legitimate interests in light of the fact that we are not receiving consideration from you for providing Prysm.
+
+### Indemnification
+To the maximum extent permitted by law, you will defend, indemnify and hold Prysmatic Labs and its contributors harmless from and against any and all claims, actions, suits, investigations, or proceedings by any third party (including any party or purported party to or beneficiary or purported beneficiary of any transaction on Prysm), as well as any and all losses, liabilities,
+damages, costs, and expenses (including reasonable attorneys’ fees) arising out of, accruing from, or in any way related to (i) your breach of the terms of this Agreement, (ii) any transaction, or the failure to occur of any transaction on Prysm, and (iii) your negligence, fraud, or willful misconduct.
+
+### Compliance with Laws and Tax Obligations
+Your use of Prysm is subject to all applicable laws of any governmental authority, including, without limitation, federal, state and foreign securities laws, tax laws, tariff and trade laws, ordinances, judgments, decrees, injunctions, writs and orders or like actions of any governmental authority and rules, regulations, orders, interpretations, licenses, and permits of any federal,
+regional, state, county, municipal or other governmental authority and you agree to comply with all such laws in your use of Prysm. The users of Prysm are solely responsible to determinate what, if any, taxes apply to their ETH transactions. The owners of, or contributors to, Prysm are not responsible for determining the taxes that apply to ETH transactions.
+
+### Miscellaneous
+These Terms will be construed and enforced in accordance with the laws of the state of Illinois as applied to agreements entered into and completely performed in Illinois. You agree to the personal jurisdiction by and venue in Illinois and waive any objection to such jurisdiction or venue.
+
+We reserve the right to revise these Terms, and your rights and obligations are at all times subject to the then-current Terms provided on Prysm. Your continued use of Prysm constitutes acceptance of such revised Terms.
+
+These Terms constitute the entire agreement between you and Prysmatic Labs regarding use of Prysm and will supersede all prior agreements whether, written or oral. No usage of trade or other regular practice or method of dealing between the parties will be used to modify, interpret, supplement, or alter the terms of these Terms.
+
+If any portion of these Terms is held invalid or unenforceable, such invalidity or enforceability will not affect the other provisions of these Terms, which will remain in full force and effect, and the invalid or unenforceable portion will be given effect to the greatest extent possible. The failure of a party to require performance of any provision will not affect that party’s right to require performance at any time thereafter, nor will a waiver of any breach or default of these Terms or any provision of these Terms constitute a waiver of any subsequent breach or default or a waiver of the provision itself.

bazel.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# This script serves as a wrapper around bazel to limit the scope of environment variables that
+# may change the action output. Using this script should result in a higher cache hit ratio for
+# cached actions with a more heremtic build.
+
+env -i \
+ PATH=/usr/bin:/bin \
+ HOME=$HOME \
+ GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS \
+ bazel "$@"

beacon-chain/BUILD.bazel

@@ -1,7 +1,8 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_test")
 load("@io_bazel_rules_docker//go:image.bzl", "go_image")
 load("@io_bazel_rules_docker//container:container.bzl", "container_bundle")
-load("//tools:binary_targets.bzl", "binary_targets")
+load("//tools:go_image.bzl", "go_image_alpine", "go_image_debug")
 load("@io_bazel_rules_docker//contrib:push-all.bzl", "docker_push")
 
 go_library(
@@ -18,15 +19,17 @@ go_library(
         "//shared/cmd:go_default_library",
         "//shared/debug:go_default_library",
         "//shared/featureconfig:go_default_library",
+        "//shared/journald:go_default_library",
         "//shared/logutil:go_default_library",
+        "//shared/maxprocs:go_default_library",
+        "//shared/tos:go_default_library",
         "//shared/version:go_default_library",
-        "@com_github_ipfs_go_log//:go_default_library",
+        "@com_github_ethereum_go_ethereum//log:go_default_library",
+        "@com_github_ipfs_go_log_v2//:go_default_library",
         "@com_github_joonix_log//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_urfave_cli//:go_default_library",
-        "@com_github_whyrusleeping_go_logging//:go_default_library",
+        "@com_github_urfave_cli_v2//:go_default_library",
         "@com_github_x_cray_logrus_prefixed_formatter//:go_default_library",
-        "@org_uber_go_automaxprocs//:go_default_library",
     ],
 )
 
@@ -36,6 +39,11 @@ go_image(
         "main.go",
         "usage.go",
     ],
+    base = select({
+        "//tools:base_image_alpine": "//tools:alpine_cc_image",
+        "//tools:base_image_cc": "//tools:cc_image",
+        "//conditions:default": "//tools:cc_image",
+    }),
     goarch = "amd64",
     goos = "linux",
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain",
@@ -46,18 +54,20 @@ go_image(
     deps = [
         "//beacon-chain/flags:go_default_library",
         "//beacon-chain/node:go_default_library",
+        "//shared/tos:go_default_library",
         "//shared/cmd:go_default_library",
         "//shared/debug:go_default_library",
         "//shared/featureconfig:go_default_library",
+        "//shared/journald:go_default_library",
         "//shared/logutil:go_default_library",
+        "//shared/maxprocs:go_default_library",
         "//shared/version:go_default_library",
-        "@com_github_ipfs_go_log//:go_default_library",
+        "@com_github_ethereum_go_ethereum//log:go_default_library",
+        "@com_github_ipfs_go_log_v2//:go_default_library",
         "@com_github_joonix_log//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_urfave_cli//:go_default_library",
-        "@com_github_whyrusleeping_go_logging//:go_default_library",
+        "@com_github_urfave_cli_v2//:go_default_library",
         "@com_github_x_cray_logrus_prefixed_formatter//:go_default_library",
-        "@org_uber_go_automaxprocs//:go_default_library",
     ],
 )
 
@@ -66,6 +76,42 @@ container_bundle(
     images = {
         "gcr.io/prysmaticlabs/prysm/beacon-chain:latest": ":image",
         "gcr.io/prysmaticlabs/prysm/beacon-chain:{DOCKER_TAG}": ":image",
+        "index.docker.io/prysmaticlabs/prysm-beacon-chain:latest": ":image",
+        "index.docker.io/prysmaticlabs/prysm-beacon-chain:{DOCKER_TAG}": ":image",
+    },
+    tags = ["manual"],
+)
+
+go_image_debug(
+    name = "image_debug",
+    image = ":image",
+    tags = ["manual"],
+)
+
+container_bundle(
+    name = "image_bundle_debug",
+    images = {
+        "gcr.io/prysmaticlabs/prysm/beacon-chain:latest-debug": ":image_debug",
+        "gcr.io/prysmaticlabs/prysm/beacon-chain:{DOCKER_TAG}-debug": ":image_debug",
+        "index.docker.io/prysmaticlabs/prysm-beacon-chain:latest-debug": ":image_debug",
+        "index.docker.io/prysmaticlabs/prysm-beacon-chain:{DOCKER_TAG}-debug": ":image_debug",
+    },
+    tags = ["manual"],
+)
+
+go_image_alpine(
+    name = "image_alpine",
+    image = ":image",
+    tags = ["manual"],
+)
+
+container_bundle(
+    name = "image_bundle_alpine",
+    images = {
+        "gcr.io/prysmaticlabs/prysm/beacon-chain:latest-alpine": ":image_alpine",
+        "gcr.io/prysmaticlabs/prysm/beacon-chain:{DOCKER_TAG}-alpine": ":image_alpine",
+        "index.docker.io/prysmaticlabs/prysm-beacon-chain:latest-alpine": ":image_alpine",
+        "index.docker.io/prysmaticlabs/prysm-beacon-chain:{DOCKER_TAG}-alpine": ":image_alpine",
     },
     tags = ["manual"],
 )
@@ -76,6 +122,18 @@ docker_push(
     tags = ["manual"],
 )
 
+docker_push(
+    name = "push_images_debug",
+    bundle = ":image_bundle_debug",
+    tags = ["manual"],
+)
+
+docker_push(
+    name = "push_images_alpine",
+    bundle = ":image_bundle_alpine",
+    tags = ["manual"],
+)
+
 go_binary(
     name = "beacon-chain",
     embed = [":go_default_library"],
@@ -90,17 +148,8 @@ go_test(
     size = "small",
     srcs = ["usage_test.go"],
     embed = [":go_default_library"],
-    deps = ["@com_github_urfave_cli//:go_default_library"],
+    deps = [
+        "//shared/featureconfig:go_default_library",
+        "@com_github_urfave_cli_v2//:go_default_library",
+    ],
 )
-
-[go_binary(
-    name = "beacon-chain-{}-{}".format(
-        pair[0],
-        pair[1],
-    ),
-    embed = [":go_default_library"],
-    goarch = pair[1],
-    goos = pair[0],
-    tags = ["manual"],
-    visibility = ["//visibility:public"],
-) for pair in binary_targets]

beacon-chain/README.md

@@ -5,6 +5,6 @@ This is the main project folder for the beacon chain implementation of Ethereum
 [![Discord](https://user-images.githubusercontent.com/7288322/34471967-1df7808a-efbb-11e7-9088-ed0b04151291.png)](https://discord.gg/KSA7rPr)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/prysmaticlabs/prysm?badge&utm_medium=badge&utm_campaign=pr-badge)
 
-Also, read the latest beacon chain [design spec](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md), this design spec serves as a source of truth for the beacon chain implementation we follow at prysmatic labs.
+Also, read the latest beacon chain [design spec](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/phase0/beacon-chain.md), this design spec serves as a source of truth for the beacon chain implementation we follow at prysmatic labs.
 Check out the [FAQs](https://notes.ethereum.org/9MMuzWeFTTSg-3Tz_YeiBA?view). Refer this page on [why](http://email.mg2.substack.com/c/eJwlj9GOhCAMRb9G3jRQQPGBh5mM8xsbhKrsDGIAM9m_X9xN2qZtbpt7rCm4xvSjj5gLOTOmL-809CMbKXFaOKakIl4DZYr2AGyQIGjHOnWH22OiYnoIxmDijaBhhS6fcy7GvjobA9m0mSXOcnZq5GBqLkilXBZhBsus5ZK89VbKkRt-a-BZI6DzZ7iur1lQ953KJ9bemnxgahuQU9XJu6pFPdu8meT8vragzEjpMCwMGLlgLo6h5z1JumQTu4IJd4v15xqMf_8ZLP_Y1bSLdbnrD-LL71i2Kj7DLxaWWF4)
 we are combining sharding and casper together.

beacon-chain/archiver/BUILD.bazel

@@ -1,42 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
-
-go_library(
-    name = "go_default_library",
-    srcs = ["service.go"],
-    importpath = "github.com/prysmaticlabs/prysm/beacon-chain/archiver",
-    visibility = ["//beacon-chain:__subpackages__"],
-    deps = [
-        "//beacon-chain/blockchain:go_default_library",
-        "//beacon-chain/core/epoch:go_default_library",
-        "//beacon-chain/core/helpers:go_default_library",
-        "//beacon-chain/core/statefeed:go_default_library",
-        "//beacon-chain/core/validators:go_default_library",
-        "//beacon-chain/db:go_default_library",
-        "//proto/beacon/p2p/v1:go_default_library",
-        "//proto/eth/v1alpha1:go_default_library",
-        "//shared/params:go_default_library",
-        "@com_github_pkg_errors//:go_default_library",
-        "@com_github_sirupsen_logrus//:go_default_library",
-    ],
-)
-
-go_test(
-    name = "go_default_test",
-    srcs = ["service_test.go"],
-    embed = [":go_default_library"],
-    deps = [
-        "//beacon-chain/blockchain/testing:go_default_library",
-        "//beacon-chain/core/helpers:go_default_library",
-        "//beacon-chain/core/statefeed:go_default_library",
-        "//beacon-chain/db:go_default_library",
-        "//beacon-chain/db/testing:go_default_library",
-        "//proto/beacon/p2p/v1:go_default_library",
-        "//proto/eth/v1alpha1:go_default_library",
-        "//shared/params:go_default_library",
-        "//shared/testutil:go_default_library",
-        "@com_github_gogo_protobuf//proto:go_default_library",
-        "@com_github_prysmaticlabs_go_bitfield//:go_default_library",
-        "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_sirupsen_logrus//hooks/test:go_default_library",
-    ],
-)

beacon-chain/archiver/service.go

@@ -1,187 +0,0 @@
-package archiver
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/beacon-chain/blockchain"
-	epochProcessing "github.com/prysmaticlabs/prysm/beacon-chain/core/epoch"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/statefeed"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/validators"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/sirupsen/logrus"
-)
-
-var log = logrus.WithField("prefix", "archiver")
-
-// Service defining archiver functionality for persisting checkpointed
-// beacon chain information to a database backend for historical purposes.
-type Service struct {
-	ctx               context.Context
-	cancel            context.CancelFunc
-	beaconDB          db.Database
-	headFetcher       blockchain.HeadFetcher
-	stateNotifier     statefeed.Notifier
-	lastArchivedEpoch uint64
-}
-
-// Config options for the archiver service.
-type Config struct {
-	BeaconDB      db.Database
-	HeadFetcher   blockchain.HeadFetcher
-	StateNotifier statefeed.Notifier
-}
-
-// NewArchiverService initializes the service from configuration options.
-func NewArchiverService(ctx context.Context, cfg *Config) *Service {
-	ctx, cancel := context.WithCancel(ctx)
-	return &Service{
-		ctx:           ctx,
-		cancel:        cancel,
-		beaconDB:      cfg.BeaconDB,
-		headFetcher:   cfg.HeadFetcher,
-		stateNotifier: cfg.StateNotifier,
-	}
-}
-
-// Start the archiver service event loop.
-func (s *Service) Start() {
-	go s.run(s.ctx)
-}
-
-// Stop the archiver service event loop.
-func (s *Service) Stop() error {
-	defer s.cancel()
-	return nil
-}
-
-// Status reports the healthy status of the archiver. Returning nil means service
-// is correctly running without error.
-func (s *Service) Status() error {
-	return nil
-}
-
-// We archive committee information pertaining to the head state's epoch.
-func (s *Service) archiveCommitteeInfo(ctx context.Context, headState *pb.BeaconState, epoch uint64) error {
-	proposerSeed, err := helpers.Seed(headState, epoch, params.BeaconConfig().DomainBeaconProposer)
-	if err != nil {
-		return errors.Wrap(err, "could not generate seed")
-	}
-	attesterSeed, err := helpers.Seed(headState, epoch, params.BeaconConfig().DomainBeaconAttester)
-	if err != nil {
-		return errors.Wrap(err, "could not generate seed")
-	}
-
-	info := &ethpb.ArchivedCommitteeInfo{
-		ProposerSeed: proposerSeed[:],
-		AttesterSeed: attesterSeed[:],
-	}
-	if err := s.beaconDB.SaveArchivedCommitteeInfo(ctx, epoch, info); err != nil {
-		return errors.Wrap(err, "could not archive committee info")
-	}
-	return nil
-}
-
-// We archive active validator set changes that happened during the previous epoch.
-func (s *Service) archiveActiveSetChanges(ctx context.Context, headState *pb.BeaconState, epoch uint64) error {
-	prevEpoch := epoch - 1
-	activations := validators.ActivatedValidatorIndices(prevEpoch, headState.Validators)
-	slashings := validators.SlashedValidatorIndices(prevEpoch, headState.Validators)
-	activeValidatorCount, err := helpers.ActiveValidatorCount(headState, prevEpoch)
-	if err != nil {
-		return errors.Wrap(err, "could not get active validator count")
-	}
-	exited, err := validators.ExitedValidatorIndices(headState.Validators, activeValidatorCount)
-	if err != nil {
-		return errors.Wrap(err, "could not determine exited validator indices")
-	}
-	activeSetChanges := &ethpb.ArchivedActiveSetChanges{
-		Activated: activations,
-		Exited:    exited,
-		Slashed:   slashings,
-	}
-	if err := s.beaconDB.SaveArchivedActiveValidatorChanges(ctx, prevEpoch, activeSetChanges); err != nil {
-		return errors.Wrap(err, "could not archive active validator set changes")
-	}
-	return nil
-}
-
-// We compute participation metrics by first retrieving the head state and
-// matching validator attestations during the epoch.
-func (s *Service) archiveParticipation(ctx context.Context, headState *pb.BeaconState, epoch uint64) error {
-	participation, err := epochProcessing.ComputeValidatorParticipation(headState, epoch)
-	if err != nil {
-		return errors.Wrap(err, "could not compute participation")
-	}
-	return s.beaconDB.SaveArchivedValidatorParticipation(ctx, epoch, participation)
-}
-
-// We archive validator balances and active indices.
-func (s *Service) archiveBalances(ctx context.Context, headState *pb.BeaconState, epoch uint64) error {
-	balances := headState.Balances
-	if err := s.beaconDB.SaveArchivedBalances(ctx, epoch, balances); err != nil {
-		return errors.Wrap(err, "could not archive balances")
-	}
-	return nil
-}
-
-func (s *Service) run(ctx context.Context) {
-	subChannel := make(chan *statefeed.Event, 1)
-	sub := s.stateNotifier.StateFeed().Subscribe(subChannel)
-	defer sub.Unsubscribe()
-	for {
-		select {
-		case event := <-subChannel:
-			switch event.Type {
-			case statefeed.BlockProcessed:
-				data := event.Data.(*statefeed.BlockProcessedData)
-				log.WithField("headRoot", fmt.Sprintf("%#x", data.BlockRoot)).Debug("Received block processed event")
-				headState, err := s.headFetcher.HeadState(ctx)
-				if err != nil {
-					log.WithError(err).Error("Head state is not available")
-					continue
-				}
-				currentEpoch := helpers.CurrentEpoch(headState)
-				if !helpers.IsEpochEnd(headState.Slot) && currentEpoch <= s.lastArchivedEpoch {
-					continue
-				}
-				epochToArchive := currentEpoch
-				if !helpers.IsEpochEnd(headState.Slot) {
-					epochToArchive--
-				}
-				if err := s.archiveCommitteeInfo(ctx, headState, epochToArchive); err != nil {
-					log.WithError(err).Error("Could not archive committee info")
-					continue
-				}
-				if err := s.archiveActiveSetChanges(ctx, headState, epochToArchive); err != nil {
-					log.WithError(err).Error("Could not archive active validator set changes")
-					continue
-				}
-				if err := s.archiveParticipation(ctx, headState, epochToArchive); err != nil {
-					log.WithError(err).Error("Could not archive validator participation")
-					continue
-				}
-				if err := s.archiveBalances(ctx, headState, epochToArchive); err != nil {
-					log.WithError(err).Error("Could not archive validator balances and active indices")
-					continue
-				}
-				log.WithField(
-					"epoch",
-					epochToArchive,
-				).Debug("Successfully archived beacon chain data during epoch")
-				s.lastArchivedEpoch = epochToArchive
-			}
-		case <-s.ctx.Done():
-			log.Debug("Context closed, exiting goroutine")
-			return
-		case err := <-sub.Err():
-			log.WithError(err).Error("Subscription to new chain head notifier failed")
-			return
-		}
-	}
-}

beacon-chain/archiver/service_test.go

@@ -1,407 +0,0 @@
-package archiver
-
-import (
-	"context"
-	"fmt"
-	"io/ioutil"
-	"reflect"
-	"testing"
-
-	"github.com/gogo/protobuf/proto"
-	"github.com/prysmaticlabs/go-bitfield"
-	mock "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/statefeed"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db"
-	dbutil "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/prysmaticlabs/prysm/shared/testutil"
-	"github.com/sirupsen/logrus"
-	logTest "github.com/sirupsen/logrus/hooks/test"
-)
-
-func init() {
-	logrus.SetLevel(logrus.DebugLevel)
-	logrus.SetOutput(ioutil.Discard)
-	params.OverrideBeaconConfig(params.MinimalSpecConfig())
-}
-
-func TestArchiverService_ReceivesBlockProcessedEvent(t *testing.T) {
-	hook := logTest.NewGlobal()
-	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
-	svc.headFetcher = &mock.ChainService{
-		State: &pb.BeaconState{Slot: 1},
-	}
-	event := &statefeed.Event{
-		Type: statefeed.BlockProcessed,
-		Data: &statefeed.BlockProcessedData{
-			BlockRoot: [32]byte{1, 2, 3},
-			Verified:  true,
-		},
-	}
-	triggerStateEvent(t, svc, event)
-	testutil.AssertLogsContain(t, hook, fmt.Sprintf("%#x", event.Data.(*statefeed.BlockProcessedData).BlockRoot))
-	testutil.AssertLogsContain(t, hook, "Received block processed event")
-}
-
-func TestArchiverService_OnlyArchiveAtEpochEnd(t *testing.T) {
-	hook := logTest.NewGlobal()
-	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
-	// The head state is NOT an epoch end.
-	svc.headFetcher = &mock.ChainService{
-		State: &pb.BeaconState{Slot: params.BeaconConfig().SlotsPerEpoch - 2},
-	}
-	event := &statefeed.Event{
-		Type: statefeed.BlockProcessed,
-		Data: &statefeed.BlockProcessedData{
-			BlockRoot: [32]byte{1, 2, 3},
-			Verified:  true,
-		},
-	}
-	triggerStateEvent(t, svc, event)
-
-	// The context should have been canceled.
-	if svc.ctx.Err() != context.Canceled {
-		t.Error("context was not canceled")
-	}
-	testutil.AssertLogsContain(t, hook, "Received block processed event")
-	// The service should ONLY log any archival logs if we receive a
-	// head slot that is an epoch end.
-	testutil.AssertLogsDoNotContain(t, hook, "Successfully archived")
-}
-
-func TestArchiverService_ArchivesEvenThroughSkipSlot(t *testing.T) {
-	hook := logTest.NewGlobal()
-	svc, beaconDB := setupService(t)
-	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
-	defer dbutil.TeardownDB(t, beaconDB)
-	event := &statefeed.Event{
-		Type: statefeed.BlockProcessed,
-		Data: &statefeed.BlockProcessedData{
-			BlockRoot: [32]byte{1, 2, 3},
-			Verified:  true,
-		},
-	}
-
-	exitRoutine := make(chan bool)
-	go func() {
-		svc.run(svc.ctx)
-		<-exitRoutine
-	}()
-
-	// Send out an event every slot, skipping the end slot of the epoch.
-	for i := uint64(0); i < params.BeaconConfig().SlotsPerEpoch+1; i++ {
-		headState.Slot = i
-		svc.headFetcher = &mock.ChainService{
-			State: headState,
-		}
-		if helpers.IsEpochEnd(i) {
-			continue
-		}
-		// Send in a loop to ensure it is delivered (busy wait for the service to subscribe to the state feed).
-		for sent := 0; sent == 0; {
-			sent = svc.stateNotifier.StateFeed().Send(event)
-		}
-	}
-	if err := svc.Stop(); err != nil {
-		t.Fatal(err)
-	}
-	exitRoutine <- true
-
-	// The context should have been canceled.
-	if svc.ctx.Err() != context.Canceled {
-		t.Error("context was not canceled")
-	}
-
-	testutil.AssertLogsContain(t, hook, "Received block processed event")
-	// Even though there was a skip slot, we should still be able to archive
-	// upon the next block event afterwards.
-	testutil.AssertLogsContain(t, hook, "Successfully archived")
-}
-
-func TestArchiverService_ComputesAndSavesParticipation(t *testing.T) {
-	hook := logTest.NewGlobal()
-	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
-	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
-	svc.headFetcher = &mock.ChainService{
-		State: headState,
-	}
-	event := &statefeed.Event{
-		Type: statefeed.BlockProcessed,
-		Data: &statefeed.BlockProcessedData{
-			BlockRoot: [32]byte{1, 2, 3},
-			Verified:  true,
-		},
-	}
-	triggerStateEvent(t, svc, event)
-
-	attestedBalance := uint64(1)
-	currentEpoch := helpers.CurrentEpoch(headState)
-	wanted := &ethpb.ValidatorParticipation{
-		VotedEther:              attestedBalance,
-		EligibleEther:           validatorCount * params.BeaconConfig().MaxEffectiveBalance,
-		GlobalParticipationRate: float32(attestedBalance) / float32(validatorCount*params.BeaconConfig().MaxEffectiveBalance),
-	}
-
-	retrieved, err := svc.beaconDB.ArchivedValidatorParticipation(svc.ctx, currentEpoch)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if !proto.Equal(wanted, retrieved) {
-		t.Errorf("Wanted participation for epoch %d %v, retrieved %v", currentEpoch-1, wanted, retrieved)
-	}
-	testutil.AssertLogsContain(t, hook, "Successfully archived")
-}
-
-func TestArchiverService_SavesIndicesAndBalances(t *testing.T) {
-	hook := logTest.NewGlobal()
-	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
-	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
-	svc.headFetcher = &mock.ChainService{
-		State: headState,
-	}
-	event := &statefeed.Event{
-		Type: statefeed.BlockProcessed,
-		Data: &statefeed.BlockProcessedData{
-			BlockRoot: [32]byte{1, 2, 3},
-			Verified:  true,
-		},
-	}
-	triggerStateEvent(t, svc, event)
-
-	retrieved, err := svc.beaconDB.ArchivedBalances(svc.ctx, helpers.CurrentEpoch(headState))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(headState.Balances, retrieved) {
-		t.Errorf(
-			"Wanted balances for epoch %d %v, retrieved %v",
-			helpers.CurrentEpoch(headState),
-			headState.Balances,
-			retrieved,
-		)
-	}
-	testutil.AssertLogsContain(t, hook, "Successfully archived")
-}
-
-func TestArchiverService_SavesCommitteeInfo(t *testing.T) {
-	hook := logTest.NewGlobal()
-	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
-	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
-	svc.headFetcher = &mock.ChainService{
-		State: headState,
-	}
-	event := &statefeed.Event{
-		Type: statefeed.BlockProcessed,
-		Data: &statefeed.BlockProcessedData{
-			BlockRoot: [32]byte{1, 2, 3},
-			Verified:  true,
-		},
-	}
-	triggerStateEvent(t, svc, event)
-
-	currentEpoch := helpers.CurrentEpoch(headState)
-	proposerSeed, err := helpers.Seed(headState, currentEpoch, params.BeaconConfig().DomainBeaconProposer)
-	if err != nil {
-		t.Fatal(err)
-	}
-	attesterSeed, err := helpers.Seed(headState, currentEpoch, params.BeaconConfig().DomainBeaconAttester)
-	if err != nil {
-		t.Fatal(err)
-	}
-	wanted := &ethpb.ArchivedCommitteeInfo{
-		ProposerSeed: proposerSeed[:],
-		AttesterSeed: attesterSeed[:],
-	}
-
-	retrieved, err := svc.beaconDB.ArchivedCommitteeInfo(svc.ctx, helpers.CurrentEpoch(headState))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !proto.Equal(wanted, retrieved) {
-		t.Errorf(
-			"Wanted committee info for epoch %d %v, retrieved %v",
-			helpers.CurrentEpoch(headState),
-			wanted,
-			retrieved,
-		)
-	}
-	testutil.AssertLogsContain(t, hook, "Successfully archived")
-}
-
-func TestArchiverService_SavesActivatedValidatorChanges(t *testing.T) {
-	hook := logTest.NewGlobal()
-	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
-	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
-	svc.headFetcher = &mock.ChainService{
-		State: headState,
-	}
-	prevEpoch := helpers.PrevEpoch(headState)
-	delayedActEpoch := helpers.DelayedActivationExitEpoch(prevEpoch)
-	headState.Validators[4].ActivationEpoch = delayedActEpoch
-	headState.Validators[5].ActivationEpoch = delayedActEpoch
-	event := &statefeed.Event{
-		Type: statefeed.BlockProcessed,
-		Data: &statefeed.BlockProcessedData{
-			BlockRoot: [32]byte{1, 2, 3},
-			Verified:  true,
-		},
-	}
-	triggerStateEvent(t, svc, event)
-
-	retrieved, err := beaconDB.ArchivedActiveValidatorChanges(svc.ctx, prevEpoch)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if retrieved == nil {
-		t.Fatal("Retrieved indices are nil")
-	}
-	if !reflect.DeepEqual(retrieved.Activated, []uint64{4, 5}) {
-		t.Errorf("Wanted indices 4 5 activated, received %v", retrieved.Activated)
-	}
-	testutil.AssertLogsContain(t, hook, "Successfully archived")
-}
-
-func TestArchiverService_SavesSlashedValidatorChanges(t *testing.T) {
-	hook := logTest.NewGlobal()
-	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
-	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
-	svc.headFetcher = &mock.ChainService{
-		State: headState,
-	}
-	prevEpoch := helpers.PrevEpoch(headState)
-	headState.Validators[95].Slashed = true
-	headState.Validators[96].Slashed = true
-	event := &statefeed.Event{
-		Type: statefeed.BlockProcessed,
-		Data: &statefeed.BlockProcessedData{
-			BlockRoot: [32]byte{1, 2, 3},
-			Verified:  true,
-		},
-	}
-	triggerStateEvent(t, svc, event)
-
-	retrieved, err := beaconDB.ArchivedActiveValidatorChanges(svc.ctx, prevEpoch)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if retrieved == nil {
-		t.Fatal("Retrieved indices are nil")
-	}
-	if !reflect.DeepEqual(retrieved.Slashed, []uint64{95, 96}) {
-		t.Errorf("Wanted indices 95, 96 slashed, received %v", retrieved.Slashed)
-	}
-	testutil.AssertLogsContain(t, hook, "Successfully archived")
-}
-
-func TestArchiverService_SavesExitedValidatorChanges(t *testing.T) {
-	hook := logTest.NewGlobal()
-	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
-	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
-	svc.headFetcher = &mock.ChainService{
-		State: headState,
-	}
-	prevEpoch := helpers.PrevEpoch(headState)
-	headState.Validators[95].ExitEpoch = prevEpoch + 1
-	headState.Validators[95].WithdrawableEpoch = prevEpoch + 1 + params.BeaconConfig().MinValidatorWithdrawabilityDelay
-	event := &statefeed.Event{
-		Type: statefeed.BlockProcessed,
-		Data: &statefeed.BlockProcessedData{
-			BlockRoot: [32]byte{1, 2, 3},
-			Verified:  true,
-		},
-	}
-	triggerStateEvent(t, svc, event)
-	testutil.AssertLogsContain(t, hook, "Successfully archived")
-	retrieved, err := beaconDB.ArchivedActiveValidatorChanges(svc.ctx, prevEpoch)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if retrieved == nil {
-		t.Fatal("Retrieved indices are nil")
-	}
-	if !reflect.DeepEqual(retrieved.Exited, []uint64{95}) {
-		t.Errorf("Wanted indices 95 exited, received %v", retrieved.Exited)
-	}
-}
-
-func setupState(t *testing.T, validatorCount uint64) *pb.BeaconState {
-	validators := make([]*ethpb.Validator, validatorCount)
-	balances := make([]uint64, validatorCount)
-	for i := 0; i < len(validators); i++ {
-		validators[i] = &ethpb.Validator{
-			ExitEpoch:         params.BeaconConfig().FarFutureEpoch,
-			WithdrawableEpoch: params.BeaconConfig().FarFutureEpoch,
-			EffectiveBalance:  params.BeaconConfig().MaxEffectiveBalance,
-		}
-		balances[i] = params.BeaconConfig().MaxEffectiveBalance
-	}
-
-	atts := []*pb.PendingAttestation{{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{}}}}
-
-	// We initialize a head state that has attestations from participated
-	// validators in a simulated fashion.
-	return &pb.BeaconState{
-		Slot:                       (2 * params.BeaconConfig().SlotsPerEpoch) - 1,
-		Validators:                 validators,
-		Balances:                   balances,
-		BlockRoots:                 make([][]byte, 128),
-		Slashings:                  []uint64{0, 1e9, 1e9},
-		RandaoMixes:                make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
-		CurrentEpochAttestations:   atts,
-		FinalizedCheckpoint:        &ethpb.Checkpoint{},
-		JustificationBits:          bitfield.Bitvector4{0x00},
-		CurrentJustifiedCheckpoint: &ethpb.Checkpoint{},
-	}
-}
-
-func setupService(t *testing.T) (*Service, db.Database) {
-	beaconDB := dbutil.SetupDB(t)
-	ctx, cancel := context.WithCancel(context.Background())
-	return &Service{
-		beaconDB:      beaconDB,
-		ctx:           ctx,
-		cancel:        cancel,
-		stateNotifier: &mock.ChainService{},
-	}, beaconDB
-}
-
-func triggerStateEvent(t *testing.T, svc *Service, event *statefeed.Event) {
-	exitRoutine := make(chan bool)
-	go func() {
-		svc.run(svc.ctx)
-		<-exitRoutine
-	}()
-
-	// Send in a loop to ensure it is delivered (busy wait for the service to subscribe to the state feed)
-	for sent := 0; sent == 0; {
-		sent = svc.stateNotifier.StateFeed().Send(event)
-	}
-	if err := svc.Stop(); err != nil {
-		t.Fatal(err)
-	}
-	exitRoutine <- true
-
-	// The context should have been canceled.
-	if svc.ctx.Err() != context.Canceled {
-		t.Error("context was not canceled")
-	}
-}

beacon-chain/blockchain/BUILD.bazel

@@ -1,41 +1,67 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_library(
     name = "go_default_library",
     srcs = [
         "chain_info.go",
+        "checkpoint_info_cache.go",
+        "head.go",
         "info.go",
+        "init_sync_process_block.go",
         "log.go",
         "metrics.go",
+        "process_attestation.go",
+        "process_attestation_helpers.go",
+        "process_block.go",
+        "process_block_helpers.go",
         "receive_attestation.go",
         "receive_block.go",
         "service.go",
+        "weak_subjectivity_checks.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/blockchain",
-    visibility = ["//beacon-chain:__subpackages__"],
+    visibility = [
+        "//beacon-chain:__subpackages__",
+        "//fuzz:__pkg__",
+    ],
     deps = [
-        "//beacon-chain/blockchain/forkchoice:go_default_library",
+        "//beacon-chain/cache:go_default_library",
         "//beacon-chain/cache/depositcache:go_default_library",
         "//beacon-chain/core/blocks:go_default_library",
+        "//beacon-chain/core/epoch/precompute:go_default_library",
+        "//beacon-chain/core/feed:go_default_library",
+        "//beacon-chain/core/feed/state:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/state:go_default_library",
-        "//beacon-chain/core/statefeed:go_default_library",
         "//beacon-chain/db:go_default_library",
-        "//beacon-chain/operations:go_default_library",
+        "//beacon-chain/db/filters:go_default_library",
+        "//beacon-chain/flags:go_default_library",
+        "//beacon-chain/forkchoice:go_default_library",
+        "//beacon-chain/forkchoice/protoarray:go_default_library",
+        "//beacon-chain/operations/attestations:go_default_library",
+        "//beacon-chain/operations/slashings:go_default_library",
+        "//beacon-chain/operations/voluntaryexits:go_default_library",
         "//beacon-chain/p2p:go_default_library",
         "//beacon-chain/powchain:go_default_library",
+        "//beacon-chain/state:go_default_library",
+        "//beacon-chain/state/stategen:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
-        "//proto/eth/v1alpha1:go_default_library",
+        "//shared/attestationutil:go_default_library",
+        "//shared/bls:go_default_library",
         "//shared/bytesutil:go_default_library",
-        "//shared/event:go_default_library",
         "//shared/featureconfig:go_default_library",
+        "//shared/hashutil:go_default_library",
         "//shared/params:go_default_library",
+        "//shared/slotutil:go_default_library",
+        "//shared/timeutils:go_default_library",
         "//shared/traceutil:go_default_library",
-        "@com_github_gogo_protobuf//proto:go_default_library",
+        "@com_github_emicklei_dot//:go_default_library",
+        "@com_github_hashicorp_golang_lru//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
-        "@com_github_prysmaticlabs_go_ssz//:go_default_library",
+        "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@io_opencensus_go//trace:go_default_library",
     ],
@@ -54,12 +80,20 @@ go_test(
     size = "medium",
     srcs = [
         "chain_info_test.go",
+        "checkpoint_info_cache_test.go",
+        "head_test.go",
+        "info_test.go",
+        "metrics_test.go",
+        "process_attestation_test.go",
+        "process_block_test.go",
         "receive_attestation_test.go",
         "receive_block_test.go",
         "service_test.go",
+        "weak_subjectivity_checks_test.go",
     ],
     embed = [":go_default_library"],
     deps = [
+        "//beacon-chain/blockchain/testing:go_default_library",
         "//beacon-chain/cache/depositcache:go_default_library",
         "//beacon-chain/core/blocks:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
@@ -68,19 +102,24 @@ go_test(
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/p2p:go_default_library",
         "//beacon-chain/powchain:go_default_library",
+        "//beacon-chain/state/stateutil:go_default_library",
+        "//proto/beacon/db:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
-        "//proto/eth/v1alpha1:go_default_library",
         "//shared/bytesutil:go_default_library",
         "//shared/event:go_default_library",
         "//shared/params:go_default_library",
         "//shared/testutil:go_default_library",
+        "//shared/testutil/assert:go_default_library",
+        "//shared/testutil/require:go_default_library",
         "@com_github_ethereum_go_ethereum//:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_ethereum_go_ethereum//core/types:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
+        "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
         "@com_github_prysmaticlabs_go_ssz//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",
+        "@in_gopkg_d4l3k_messagediff_v1//:go_default_library",
         "@org_golang_x_net//context:go_default_library",
     ],
 )
@@ -89,12 +128,21 @@ go_test(
     name = "go_raceon_test",
     srcs = [
         "chain_info_norace_test.go",
+        "receive_block_test.go",
         "service_norace_test.go",
     ],
     embed = [":go_default_library"],
+    gc_goopts = [
+        # Go 1.14 enables checkptr by default when building with -race or -msan. There is a pointer
+        # issue in boltdb, so must disable checkptr at compile time. This flag can be removed once
+        # the project is migrated to etcd's version of boltdb and the issue has been fixed.
+        # See: https://github.com/etcd-io/bbolt/issues/187.
+        "-d=checkptr=0",
+    ],
     race = "on",
     tags = ["race_on"],
     deps = [
+        "//beacon-chain/blockchain/testing:go_default_library",
         "//beacon-chain/cache/depositcache:go_default_library",
         "//beacon-chain/core/blocks:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
@@ -104,15 +152,17 @@ go_test(
         "//beacon-chain/p2p:go_default_library",
         "//beacon-chain/powchain:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
-        "//proto/eth/v1alpha1:go_default_library",
         "//shared/bytesutil:go_default_library",
         "//shared/event:go_default_library",
         "//shared/params:go_default_library",
         "//shared/testutil:go_default_library",
+        "//shared/testutil/assert:go_default_library",
+        "//shared/testutil/require:go_default_library",
         "@com_github_ethereum_go_ethereum//:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_ethereum_go_ethereum//core/types:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
+        "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
         "@com_github_prysmaticlabs_go_ssz//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",

beacon-chain/blockchain/chain_info.go

@@ -2,12 +2,16 @@ package blockchain
 
 import (
 	"context"
-	"errors"
 	"time"
 
-	"github.com/gogo/protobuf/proto"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"go.opencensus.io/trace"
+
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
@@ -15,28 +19,34 @@ import (
 // directly retrieves chain info related data.
 type ChainInfoFetcher interface {
 	HeadFetcher
-	CanonicalRootFetcher
 	FinalizationFetcher
+	GenesisFetcher
+	CanonicalFetcher
 }
 
-// GenesisTimeFetcher retrieves the Eth2 genesis timestamp.
-type GenesisTimeFetcher interface {
+// TimeFetcher retrieves the Eth2 data that's related to time.
+type TimeFetcher interface {
 	GenesisTime() time.Time
+	CurrentSlot() uint64
+}
+
+// GenesisFetcher retrieves the eth2 data related to its genesis.
+type GenesisFetcher interface {
+	GenesisValidatorRoot() [32]byte
 }
 
 // HeadFetcher defines a common interface for methods in blockchain service which
 // directly retrieves head related data.
 type HeadFetcher interface {
 	HeadSlot() uint64
-	HeadRoot() []byte
-	HeadBlock() *ethpb.BeaconBlock
-	HeadState(ctx context.Context) (*pb.BeaconState, error)
-}
-
-// CanonicalRootFetcher defines a common interface for methods in blockchain service which
-// directly retrieves canonical roots related data.
-type CanonicalRootFetcher interface {
-	CanonicalRoot(slot uint64) []byte
+	HeadRoot(ctx context.Context) ([]byte, error)
+	HeadBlock(ctx context.Context) (*ethpb.SignedBeaconBlock, error)
+	HeadState(ctx context.Context) (*state.BeaconState, error)
+	HeadValidatorsIndices(ctx context.Context, epoch uint64) ([]uint64, error)
+	HeadSeed(ctx context.Context, epoch uint64) ([32]byte, error)
+	HeadGenesisValidatorRoot() [32]byte
+	HeadETH1Data() *ethpb.Eth1Data
+	ProtoArrayStore() *protoarray.Store
 }
 
 // ForkFetcher retrieves the current fork information of the Ethereum beacon chain.
@@ -44,20 +54,45 @@ type ForkFetcher interface {
 	CurrentFork() *pb.Fork
 }
 
-// FinalizationFetcher defines a common interface for methods in blockchain service which
-// directly retrieves finalization related data.
-type FinalizationFetcher interface {
-	FinalizedCheckpt() *ethpb.Checkpoint
+// CanonicalFetcher retrieves the current chain's canonical information.
+type CanonicalFetcher interface {
+	IsCanonical(ctx context.Context, blockRoot [32]byte) (bool, error)
+	VerifyBlkDescendant(ctx context.Context, blockRoot [32]byte) error
 }
 
-// FinalizedCheckpt returns the latest finalized checkpoint tracked in fork choice service.
+// FinalizationFetcher defines a common interface for methods in blockchain service which
+// directly retrieves finalization and justification related data.
+type FinalizationFetcher interface {
+	FinalizedCheckpt() *ethpb.Checkpoint
+	CurrentJustifiedCheckpt() *ethpb.Checkpoint
+	PreviousJustifiedCheckpt() *ethpb.Checkpoint
+}
+
+// FinalizedCheckpt returns the latest finalized checkpoint from head state.
 func (s *Service) FinalizedCheckpt() *ethpb.Checkpoint {
-	cp := s.forkChoiceStore.FinalizedCheckpt()
-	if cp != nil {
-		return cp
+	if s.finalizedCheckpt == nil {
+		return &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
 	}
 
-	return &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	return state.CopyCheckpoint(s.finalizedCheckpt)
+}
+
+// CurrentJustifiedCheckpt returns the current justified checkpoint from head state.
+func (s *Service) CurrentJustifiedCheckpt() *ethpb.Checkpoint {
+	if s.justifiedCheckpt == nil {
+		return &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	}
+
+	return state.CopyCheckpoint(s.justifiedCheckpt)
+}
+
+// PreviousJustifiedCheckpt returns the previous justified checkpoint from head state.
+func (s *Service) PreviousJustifiedCheckpt() *ethpb.Checkpoint {
+	if s.prevJustifiedCheckpt == nil {
+		return &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	}
+
+	return state.CopyCheckpoint(s.prevJustifiedCheckpt)
 }
 
 // HeadSlot returns the slot of the head of the chain.
@@ -65,57 +100,121 @@ func (s *Service) HeadSlot() uint64 {
 	s.headLock.RLock()
 	defer s.headLock.RUnlock()
 
-	return s.headSlot
+	if !s.hasHeadState() {
+		return 0
+	}
+
+	return s.headSlot()
 }
 
 // HeadRoot returns the root of the head of the chain.
-func (s *Service) HeadRoot() []byte {
+func (s *Service) HeadRoot(ctx context.Context) ([]byte, error) {
 	s.headLock.RLock()
 	defer s.headLock.RUnlock()
 
-	root := s.canonicalRoots[s.headSlot]
-	if len(root) != 0 {
-		return root
+	if s.headRoot() != params.BeaconConfig().ZeroHash {
+		r := s.headRoot()
+		return r[:], nil
 	}
 
-	return params.BeaconConfig().ZeroHash[:]
+	b, err := s.beaconDB.HeadBlock(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if b == nil {
+		return params.BeaconConfig().ZeroHash[:], nil
+	}
+
+	r, err := b.Block.HashTreeRoot()
+	if err != nil {
+		return nil, err
+	}
+
+	return r[:], nil
 }
 
 // HeadBlock returns the head block of the chain.
-func (s *Service) HeadBlock() *ethpb.BeaconBlock {
+// If the head is nil from service struct,
+// it will attempt to get the head block from DB.
+func (s *Service) HeadBlock(ctx context.Context) (*ethpb.SignedBeaconBlock, error) {
 	s.headLock.RLock()
 	defer s.headLock.RUnlock()
 
-	return proto.Clone(s.headBlock).(*ethpb.BeaconBlock)
+	if s.hasHeadState() {
+		return s.headBlock(), nil
+	}
+
+	return s.beaconDB.HeadBlock(ctx)
 }
 
 // HeadState returns the head state of the chain.
-// If the head state is nil from service struct,
-// it will attempt to get from DB and error if nil again.
-func (s *Service) HeadState(ctx context.Context) (*pb.BeaconState, error) {
+// If the head is nil from service struct,
+// it will attempt to get the head state from DB.
+func (s *Service) HeadState(ctx context.Context) (*state.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "blockChain.HeadState")
+	defer span.End()
 	s.headLock.RLock()
 	defer s.headLock.RUnlock()
 
-	if s.headState == nil {
-		h, err := s.beaconDB.HeadState(ctx)
-		if err != nil {
-			return nil, err
-		}
-		if h == nil {
-			return nil, errors.New("head state does not exist")
-		}
-		return h, nil
+	ok := s.hasHeadState()
+	span.AddAttributes(trace.BoolAttribute("cache_hit", ok))
+
+	if ok {
+		return s.headState(ctx), nil
 	}
 
-	return proto.Clone(s.headState).(*pb.BeaconState), nil
+	return s.beaconDB.HeadState(ctx)
 }
 
-// CanonicalRoot returns the canonical root of a given slot.
-func (s *Service) CanonicalRoot(slot uint64) []byte {
+// HeadValidatorsIndices returns a list of active validator indices from the head view of a given epoch.
+func (s *Service) HeadValidatorsIndices(ctx context.Context, epoch uint64) ([]uint64, error) {
 	s.headLock.RLock()
 	defer s.headLock.RUnlock()
 
-	return s.canonicalRoots[slot]
+	if !s.hasHeadState() {
+		return []uint64{}, nil
+	}
+	return helpers.ActiveValidatorIndices(s.headState(ctx), epoch)
+}
+
+// HeadSeed returns the seed from the head view of a given epoch.
+func (s *Service) HeadSeed(ctx context.Context, epoch uint64) ([32]byte, error) {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	if !s.hasHeadState() {
+		return [32]byte{}, nil
+	}
+
+	return helpers.Seed(s.headState(ctx), epoch, params.BeaconConfig().DomainBeaconAttester)
+}
+
+// HeadGenesisValidatorRoot returns genesis validator root of the head state.
+func (s *Service) HeadGenesisValidatorRoot() [32]byte {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	if !s.hasHeadState() {
+		return [32]byte{}
+	}
+
+	return s.headGenesisValidatorRoot()
+}
+
+// HeadETH1Data returns the eth1data of the current head state.
+func (s *Service) HeadETH1Data() *ethpb.Eth1Data {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	if !s.hasHeadState() {
+		return &ethpb.Eth1Data{}
+	}
+	return s.head.state.Eth1Data()
+}
+
+// ProtoArrayStore returns the proto array store object.
+func (s *Service) ProtoArrayStore() *protoarray.Store {
+	return s.forkChoiceStore.Store()
 }
 
 // GenesisTime returns the genesis time of beacon chain.
@@ -123,13 +222,39 @@ func (s *Service) GenesisTime() time.Time {
 	return s.genesisTime
 }
 
+// GenesisValidatorRoot returns the genesis validator
+// root of the chain.
+func (s *Service) GenesisValidatorRoot() [32]byte {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	if !s.hasHeadState() {
+		return [32]byte{}
+	}
+	return bytesutil.ToBytes32(s.head.state.GenesisValidatorRoot())
+}
+
 // CurrentFork retrieves the latest fork information of the beacon chain.
 func (s *Service) CurrentFork() *pb.Fork {
-	if s.headState == nil {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	if !s.hasHeadState() {
 		return &pb.Fork{
 			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
 			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
 		}
 	}
-	return proto.Clone(s.headState.Fork).(*pb.Fork)
+	return s.head.state.Fork()
+}
+
+// IsCanonical returns true if the input block root is part of the canonical chain.
+func (s *Service) IsCanonical(ctx context.Context, blockRoot [32]byte) (bool, error) {
+	// If the block has been finalized, the block will always be part of the canonical chain.
+	if s.beaconDB.IsFinalizedBlock(ctx, blockRoot) {
+		return true, nil
+	}
+
+	// If the block has not been finalized, check fork choice store to see if the block is canonical
+	return s.forkChoiceStore.IsCanonical(blockRoot), nil
 }

beacon-chain/blockchain/chain_info_norace_test.go

@@ -4,74 +4,60 @@ import (
 	"context"
 	"testing"
 
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 )
 
 func TestHeadSlot_DataRace(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 	s := &Service{
-		beaconDB:       db,
-		canonicalRoots: make(map[uint64][]byte),
+		beaconDB: db,
 	}
 	go func() {
-		s.saveHead(
-			context.Background(),
-			&ethpb.BeaconBlock{Slot: 777},
-			[32]byte{},
-		)
+		require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
 	}()
 	s.HeadSlot()
 }
 
 func TestHeadRoot_DataRace(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 	s := &Service{
-		beaconDB:       db,
-		canonicalRoots: make(map[uint64][]byte),
+		beaconDB: db,
+		head:     &head{root: [32]byte{'A'}},
+		stateGen: stategen.New(db, sc),
 	}
 	go func() {
-		s.saveHead(
-			context.Background(),
-			&ethpb.BeaconBlock{Slot: 777},
-			[32]byte{},
-		)
+		require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
 	}()
-	s.HeadRoot()
+	_, err := s.HeadRoot(context.Background())
+	require.NoError(t, err)
 }
 
 func TestHeadBlock_DataRace(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 	s := &Service{
-		beaconDB:       db,
-		canonicalRoots: make(map[uint64][]byte),
+		beaconDB: db,
+		head:     &head{block: &ethpb.SignedBeaconBlock{}},
+		stateGen: stategen.New(db, sc),
 	}
 	go func() {
-		s.saveHead(
-			context.Background(),
-			&ethpb.BeaconBlock{Slot: 777},
-			[32]byte{},
-		)
+		require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
 	}()
-	s.HeadBlock()
+	_, err := s.HeadBlock(context.Background())
+	require.NoError(t, err)
 }
 
 func TestHeadState_DataRace(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 	s := &Service{
-		beaconDB:       db,
-		canonicalRoots: make(map[uint64][]byte),
+		beaconDB: db,
+		stateGen: stategen.New(db, sc),
 	}
 	go func() {
-		s.saveHead(
-			context.Background(),
-			&ethpb.BeaconBlock{Slot: 777},
-			[32]byte{},
-		)
+		require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
 	}()
-	s.HeadState(context.Background())
+	_, err := s.HeadState(context.Background())
+	require.NoError(t, err)
 }

beacon-chain/blockchain/chain_info_test.go

@@ -1,113 +1,203 @@
 package blockchain
 
 import (
-	"bytes"
 	"context"
-	"reflect"
 	"testing"
 	"time"
 
+	"github.com/gogo/protobuf/proto"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 )
 
 // Ensure Service implements chain info interface.
-var _ = ChainInfoFetcher(&Service{})
-var _ = GenesisTimeFetcher(&Service{})
-var _ = ForkFetcher(&Service{})
+var _ ChainInfoFetcher = (*Service)(nil)
+var _ TimeFetcher = (*Service)(nil)
+var _ ForkFetcher = (*Service)(nil)
 
 func TestFinalizedCheckpt_Nil(t *testing.T) {
-	c := setupBeaconChain(t, nil)
-	if !bytes.Equal(c.FinalizedCheckpt().Root, params.BeaconConfig().ZeroHash[:]) {
-		t.Error("Incorrect pre chain start value")
-	}
+	db, sc := testDB.SetupDB(t)
+	c := setupBeaconChain(t, db, sc)
+	assert.DeepEqual(t, params.BeaconConfig().ZeroHash[:], c.FinalizedCheckpt().Root, "Incorrect pre chain start value")
 }
 
 func TestHeadRoot_Nil(t *testing.T) {
-	c := setupBeaconChain(t, nil)
-	if !bytes.Equal(c.HeadRoot(), params.BeaconConfig().ZeroHash[:]) {
-		t.Error("Incorrect pre chain start value")
-	}
+	db, sc := testDB.SetupDB(t)
+	c := setupBeaconChain(t, db, sc)
+	headRoot, err := c.HeadRoot(context.Background())
+	require.NoError(t, err)
+	assert.DeepEqual(t, params.BeaconConfig().ZeroHash[:], headRoot, "Incorrect pre chain start value")
 }
 
 func TestFinalizedCheckpt_CanRetrieve(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	ctx := context.Background()
+	db, sc := testDB.SetupDB(t)
 
-	c := setupBeaconChain(t, db)
+	cp := &ethpb.Checkpoint{Epoch: 5, Root: bytesutil.PadTo([]byte("foo"), 32)}
+	c := setupBeaconChain(t, db, sc)
+	c.finalizedCheckpt = cp
 
-	if err := c.forkChoiceStore.GenesisStore(ctx, &ethpb.Checkpoint{}, &ethpb.Checkpoint{}); err != nil {
-		t.Fatal(err)
-	}
+	assert.Equal(t, cp.Epoch, c.FinalizedCheckpt().Epoch, "Unexpected finalized epoch")
+}
 
-	if c.FinalizedCheckpt().Epoch != 0 {
-		t.Errorf("Finalized epoch at genesis should be 0, got: %d", c.FinalizedCheckpt().Epoch)
-	}
+func TestFinalizedCheckpt_GenesisRootOk(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+
+	genesisRoot := [32]byte{'A'}
+	cp := &ethpb.Checkpoint{Root: genesisRoot[:]}
+	c := setupBeaconChain(t, db, sc)
+	c.finalizedCheckpt = cp
+	c.genesisRoot = genesisRoot
+	assert.DeepEqual(t, c.genesisRoot[:], c.FinalizedCheckpt().Root)
+}
+
+func TestCurrentJustifiedCheckpt_CanRetrieve(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+
+	cp := &ethpb.Checkpoint{Epoch: 6, Root: bytesutil.PadTo([]byte("foo"), 32)}
+	c := setupBeaconChain(t, db, sc)
+	c.justifiedCheckpt = cp
+
+	assert.Equal(t, cp.Epoch, c.CurrentJustifiedCheckpt().Epoch, "Unexpected justified epoch")
+}
+
+func TestJustifiedCheckpt_GenesisRootOk(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+
+	genesisRoot := [32]byte{'B'}
+	cp := &ethpb.Checkpoint{Root: genesisRoot[:]}
+	c := setupBeaconChain(t, db, sc)
+	c.justifiedCheckpt = cp
+	c.genesisRoot = genesisRoot
+	assert.DeepEqual(t, c.genesisRoot[:], c.CurrentJustifiedCheckpt().Root)
+}
+
+func TestPreviousJustifiedCheckpt_CanRetrieve(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+
+	cp := &ethpb.Checkpoint{Epoch: 7, Root: bytesutil.PadTo([]byte("foo"), 32)}
+	c := setupBeaconChain(t, db, sc)
+	c.prevJustifiedCheckpt = cp
+	assert.Equal(t, cp.Epoch, c.PreviousJustifiedCheckpt().Epoch, "Unexpected previous justified epoch")
+}
+
+func TestPrevJustifiedCheckpt_GenesisRootOk(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+
+	genesisRoot := [32]byte{'C'}
+	cp := &ethpb.Checkpoint{Root: genesisRoot[:]}
+	c := setupBeaconChain(t, db, sc)
+	c.prevJustifiedCheckpt = cp
+	c.genesisRoot = genesisRoot
+	assert.DeepEqual(t, c.genesisRoot[:], c.PreviousJustifiedCheckpt().Root)
 }
 
 func TestHeadSlot_CanRetrieve(t *testing.T) {
 	c := &Service{}
-	c.headSlot = 100
-	if c.HeadSlot() != 100 {
-		t.Errorf("Wanted head slot: %d, got: %d", 100, c.HeadSlot())
-	}
+	s, err := state.InitializeFromProto(&pb.BeaconState{})
+	require.NoError(t, err)
+	c.head = &head{slot: 100, state: s}
+	assert.Equal(t, uint64(100), c.HeadSlot())
 }
 
 func TestHeadRoot_CanRetrieve(t *testing.T) {
-	c := &Service{canonicalRoots: make(map[uint64][]byte)}
-	c.headSlot = 100
-	c.canonicalRoots[c.headSlot] = []byte{'A'}
-	if !bytes.Equal([]byte{'A'}, c.HeadRoot()) {
-		t.Errorf("Wanted head root: %v, got: %d", []byte{'A'}, c.HeadRoot())
-	}
+	c := &Service{}
+	c.head = &head{root: [32]byte{'A'}}
+	r, err := c.HeadRoot(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, [32]byte{'A'}, bytesutil.ToBytes32(r))
 }
 
 func TestHeadBlock_CanRetrieve(t *testing.T) {
-	b := &ethpb.BeaconBlock{Slot: 1}
-	c := &Service{headBlock: b}
-	if !reflect.DeepEqual(b, c.HeadBlock()) {
-		t.Error("incorrect head block received")
-	}
+	b := testutil.NewBeaconBlock()
+	b.Block.Slot = 1
+	s, err := state.InitializeFromProto(&pb.BeaconState{})
+	require.NoError(t, err)
+	c := &Service{}
+	c.head = &head{block: b, state: s}
+
+	recevied, err := c.HeadBlock(context.Background())
+	require.NoError(t, err)
+	assert.DeepEqual(t, b, recevied, "Incorrect head block received")
 }
 
 func TestHeadState_CanRetrieve(t *testing.T) {
-	s := &pb.BeaconState{Slot: 2}
-	c := &Service{headState: s}
+	s, err := state.InitializeFromProto(&pb.BeaconState{Slot: 2, GenesisValidatorsRoot: params.BeaconConfig().ZeroHash[:]})
+	require.NoError(t, err)
+	c := &Service{}
+	c.head = &head{state: s}
 	headState, err := c.HeadState(context.Background())
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(s, headState) {
-		t.Error("incorrect head state received")
-	}
+	require.NoError(t, err)
+	assert.DeepEqual(t, headState.InnerStateUnsafe(), s.InnerStateUnsafe(), "Incorrect head state received")
 }
 
 func TestGenesisTime_CanRetrieve(t *testing.T) {
 	c := &Service{genesisTime: time.Unix(999, 0)}
 	wanted := time.Unix(999, 0)
-	if c.GenesisTime() != wanted {
-		t.Error("Did not get wanted genesis time")
-	}
+	assert.Equal(t, wanted, c.GenesisTime(), "Did not get wanted genesis time")
 }
 
 func TestCurrentFork_CanRetrieve(t *testing.T) {
 	f := &pb.Fork{Epoch: 999}
-	s := &pb.BeaconState{Fork: f}
-	c := &Service{headState: s}
-	if !reflect.DeepEqual(c.CurrentFork(), f) {
-		t.Error("Recieved incorrect fork version")
+	s, err := state.InitializeFromProto(&pb.BeaconState{Fork: f})
+	require.NoError(t, err)
+	c := &Service{}
+	c.head = &head{state: s}
+	if !proto.Equal(c.CurrentFork(), f) {
+		t.Error("Received incorrect fork version")
 	}
 }
 
-func TestCanonicalRoot_CanRetrieve(t *testing.T) {
-	c := &Service{canonicalRoots: make(map[uint64][]byte)}
-	slot := uint64(123)
-	r := []byte{'B'}
-	c.canonicalRoots[slot] = r
-	if !bytes.Equal(r, c.CanonicalRoot(slot)) {
-		t.Errorf("Wanted head root: %v, got: %d", []byte{'A'}, c.CanonicalRoot(slot))
+func TestGenesisValidatorRoot_CanRetrieve(t *testing.T) {
+	// Should not panic if head state is nil.
+	c := &Service{}
+	assert.Equal(t, [32]byte{}, c.GenesisValidatorRoot(), "Did not get correct genesis validator root")
+
+	s, err := state.InitializeFromProto(&pb.BeaconState{GenesisValidatorsRoot: []byte{'a'}})
+	require.NoError(t, err)
+	c.head = &head{state: s}
+	assert.Equal(t, [32]byte{'a'}, c.GenesisValidatorRoot(), "Did not get correct genesis validator root")
+}
+
+func TestHeadETH1Data_Nil(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+	c := setupBeaconChain(t, db, sc)
+	assert.DeepEqual(t, &ethpb.Eth1Data{}, c.HeadETH1Data(), "Incorrect pre chain start value")
+}
+
+func TestHeadETH1Data_CanRetrieve(t *testing.T) {
+	d := &ethpb.Eth1Data{DepositCount: 999}
+	s, err := state.InitializeFromProto(&pb.BeaconState{Eth1Data: d})
+	require.NoError(t, err)
+	c := &Service{}
+	c.head = &head{state: s}
+	if !proto.Equal(c.HeadETH1Data(), d) {
+		t.Error("Received incorrect eth1 data")
 	}
 }
+
+func TestIsCanonical_Ok(t *testing.T) {
+	ctx := context.Background()
+	db, sc := testDB.SetupDB(t)
+	c := setupBeaconChain(t, db, sc)
+
+	blk := testutil.NewBeaconBlock()
+	blk.Block.Slot = 0
+	root, err := blk.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, db.SaveBlock(ctx, blk))
+	require.NoError(t, db.SaveGenesisBlockRoot(ctx, root))
+	can, err := c.IsCanonical(ctx, root)
+	require.NoError(t, err)
+	assert.Equal(t, true, can)
+
+	can, err = c.IsCanonical(ctx, [32]byte{'a'})
+	require.NoError(t, err)
+	assert.Equal(t, false, can)
+}

beacon-chain/blockchain/checkpoint_info_cache.go

@@ -0,0 +1,83 @@
+package blockchain
+
+import (
+	"sync"
+
+	lru "github.com/hashicorp/golang-lru"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/hashutil"
+)
+
+var (
+	// This defines the max number of checkpoint info this cache can store.
+	// Each cache is calculated at 3MB(30K validators), the total cache size is around 100MB.
+	// Due to reorgs and long finality, it's good to keep the old cache around for quickly switch over.
+	maxInfoSize = 32
+
+	// This tracks the number of check point info requests that aren't present in the cache.
+	infoMiss = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "check_point_info_cache_miss",
+		Help: "The number of check point info requests that aren't present in the cache.",
+	})
+	// This tracks the number of check point info requests that are in the cache.
+	infoHit = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "check_point_info_cache_hit",
+		Help: "The number of check point info requests that are present in the cache.",
+	})
+)
+
+// checkPtInfoCache is a struct with 1 LRU cache for looking up check point info by checkpoint.
+type checkPtInfoCache struct {
+	cache *lru.Cache
+	lock  sync.RWMutex
+}
+
+// newCheckPointInfoCache creates a new checkpoint info cache for storing/accessing processed check point info object.
+func newCheckPointInfoCache() *checkPtInfoCache {
+	cache, err := lru.New(maxInfoSize)
+	if err != nil {
+		panic(err)
+	}
+	return &checkPtInfoCache{
+		cache: cache,
+	}
+}
+
+// get fetches check point info by check point. Returns the reference of the CheckPtInfo, nil if doesn't exist.
+func (c *checkPtInfoCache) get(cp *ethpb.Checkpoint) (*pb.CheckPtInfo, error) {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	h, err := hashutil.HashProto(cp)
+	if err != nil {
+		return nil, err
+	}
+
+	item, exists := c.cache.Get(h)
+
+	if exists && item != nil {
+		infoHit.Inc()
+		// Copy here is unnecessary since the returned check point info object
+		// will only be used to verify attestation signature.
+		return item.(*pb.CheckPtInfo), nil
+	}
+
+	infoMiss.Inc()
+	return nil, nil
+}
+
+// put adds CheckPtInfo object to the cache. This method also trims the least
+// recently added CheckPtInfo object if the cache size has ready the max cache size limit.
+func (c *checkPtInfoCache) put(cp *ethpb.Checkpoint, info *pb.CheckPtInfo) error {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	h, err := hashutil.HashProto(cp)
+	if err != nil {
+		return err
+	}
+
+	c.cache.Add(h, info)
+	return nil
+}

beacon-chain/blockchain/checkpoint_info_cache_test.go

@@ -0,0 +1,41 @@
+package blockchain
+
+import (
+	"testing"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+)
+
+func TestHotStateCache_RoundTrip(t *testing.T) {
+	c := newCheckPointInfoCache()
+	cp := &ethpb.Checkpoint{
+		Epoch: 1,
+		Root:  bytesutil.PadTo([]byte{'a'}, 32),
+	}
+	info, err := c.get(cp)
+	require.NoError(t, err)
+	require.Equal(t, (*pb.CheckPtInfo)(nil), info)
+
+	i := &pb.CheckPtInfo{
+		Seed:          bytesutil.PadTo([]byte{'c'}, 32),
+		GenesisRoot:   bytesutil.PadTo([]byte{'d'}, 32),
+		ActiveIndices: []uint64{0, 1, 2, 3},
+	}
+
+	require.NoError(t, c.put(cp, i))
+	info, err = c.get(cp)
+	require.NoError(t, err)
+	require.DeepEqual(t, info, i)
+}
+
+func TestHotStateCache_CanPrune(t *testing.T) {
+	c := newCheckPointInfoCache()
+	for i := 0; i < maxInfoSize+1; i++ {
+		cp := &ethpb.Checkpoint{Epoch: uint64(i), Root: make([]byte, 32)}
+		require.NoError(t, c.put(cp, &pb.CheckPtInfo{}))
+	}
+	require.Equal(t, len(c.cache.Keys()), maxInfoSize)
+}

beacon-chain/blockchain/forkchoice/benchmark_test.go

@@ -1,178 +0,0 @@
-package forkchoice
-
-import (
-	"context"
-	"testing"
-
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-)
-
-func BenchmarkForkChoiceTree1(b *testing.B) {
-	ctx := context.Background()
-	db := testDB.SetupDB(b)
-	defer testDB.TeardownDB(b, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db)
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	// Benchmark fork choice with 1024 validators
-	validators := make([]*ethpb.Validator, 1024)
-	for i := 0; i < len(validators); i++ {
-		validators[i] = &ethpb.Validator{ExitEpoch: 2, EffectiveBalance: 1e9}
-	}
-	s := &pb.BeaconState{Validators: validators}
-
-	if err := store.GenesisStore(ctx, &ethpb.Checkpoint{}, &ethpb.Checkpoint{}); err != nil {
-		b.Fatal(err)
-	}
-
-	store.justifiedCheckpt.Root = roots[0]
-	if err := store.db.SaveState(ctx, s, bytesutil.ToBytes32(roots[0])); err != nil {
-		b.Fatal(err)
-	}
-
-	if err := store.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: store.justifiedCheckpt,
-		State:      s,
-	}); err != nil {
-		b.Fatal(err)
-	}
-
-	// Spread out the votes evenly for all 3 leaf nodes
-	for i := 0; i < len(validators); i++ {
-		switch {
-		case i < 256:
-			if err := store.db.SaveValidatorLatestVote(ctx, uint64(i), &pb.ValidatorLatestVote{Root: roots[1]}); err != nil {
-				b.Fatal(err)
-			}
-		case i > 768:
-			if err := store.db.SaveValidatorLatestVote(ctx, uint64(i), &pb.ValidatorLatestVote{Root: roots[7]}); err != nil {
-				b.Fatal(err)
-			}
-		default:
-			if err := store.db.SaveValidatorLatestVote(ctx, uint64(i), &pb.ValidatorLatestVote{Root: roots[8]}); err != nil {
-				b.Fatal(err)
-			}
-		}
-	}
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := store.Head(ctx)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func BenchmarkForkChoiceTree2(b *testing.B) {
-	ctx := context.Background()
-	db := testDB.SetupDB(b)
-	defer testDB.TeardownDB(b, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree2(db)
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	// Benchmark fork choice with 1024 validators
-	validators := make([]*ethpb.Validator, 1024)
-	for i := 0; i < len(validators); i++ {
-		validators[i] = &ethpb.Validator{ExitEpoch: 2, EffectiveBalance: 1e9}
-	}
-	s := &pb.BeaconState{Validators: validators}
-	if err := store.GenesisStore(ctx, &ethpb.Checkpoint{}, &ethpb.Checkpoint{}); err != nil {
-		b.Fatal(err)
-	}
-
-	store.justifiedCheckpt.Root = roots[0]
-	if err := store.db.SaveState(ctx, s, bytesutil.ToBytes32(roots[0])); err != nil {
-		b.Fatal(err)
-	}
-
-	if err := store.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: store.justifiedCheckpt,
-		State:      s,
-	}); err != nil {
-		b.Fatal(err)
-	}
-
-	// Spread out the votes evenly for all the leaf nodes. 8 to 15
-	nodeIndex := 8
-	for i := 0; i < len(validators); i++ {
-		if err := store.db.SaveValidatorLatestVote(ctx, uint64(i), &pb.ValidatorLatestVote{Root: roots[nodeIndex]}); err != nil {
-			b.Fatal(err)
-		}
-		if i%155 == 0 {
-			nodeIndex++
-		}
-	}
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := store.Head(ctx)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func BenchmarkForkChoiceTree3(b *testing.B) {
-	ctx := context.Background()
-	db := testDB.SetupDB(b)
-	defer testDB.TeardownDB(b, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree3(db)
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	// Benchmark fork choice with 1024 validators
-	validators := make([]*ethpb.Validator, 1024)
-	for i := 0; i < len(validators); i++ {
-		validators[i] = &ethpb.Validator{ExitEpoch: 2, EffectiveBalance: 1e9}
-	}
-	s := &pb.BeaconState{Validators: validators}
-	if err := store.GenesisStore(ctx, &ethpb.Checkpoint{}, &ethpb.Checkpoint{}); err != nil {
-		b.Fatal(err)
-	}
-
-	store.justifiedCheckpt.Root = roots[0]
-	if err := store.db.SaveState(ctx, s, bytesutil.ToBytes32(roots[0])); err != nil {
-		b.Fatal(err)
-	}
-
-	if err := store.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: store.justifiedCheckpt,
-		State:      s,
-	}); err != nil {
-		b.Fatal(err)
-	}
-
-	// All validators vote on the same head
-	for i := 0; i < len(validators); i++ {
-		if err := store.db.SaveValidatorLatestVote(ctx, uint64(i), &pb.ValidatorLatestVote{Root: roots[len(roots)-1]}); err != nil {
-			b.Fatal(err)
-		}
-	}
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := store.Head(ctx)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}

beacon-chain/blockchain/forkchoice/doc.go

@@ -1,9 +0,0 @@
-/*
-Package forkchoice implements the Latest Message Driven GHOST (Greediest Heaviest Observed
-Sub-Tree) algorithm as the Ethereum Serenity beacon chain fork choice rule. This algorithm is designed to
-properly detect the canonical chain based on validator votes even in the presence of high network
-latency, network partitions, and many conflicting blocks. To read more about fork choice, read the
-official accompanying document:
-https://github.com/ethereum/eth2.0-specs/blob/v0.8.3/specs/core/0_fork-choice.md
-*/
-package forkchoice

beacon-chain/blockchain/forkchoice/lmd_ghost_test.yaml

@@ -1,59 +0,0 @@
-test_cases:
-  # GHOST chooses b3 with the heaviest weight
-  - blocks:
-      - id: 'b0'
-        parent: 'b0'
-      - id: 'b1'
-        parent: 'b0'
-      - id: 'b2'
-        parent: 'b1'
-      - id: 'b3'
-        parent: 'b1'
-    weights:
-      b0: 0
-      b1: 0
-      b2: 5
-      b3: 10
-    head: 'b3'
-  # GHOST chooses b1 with the heaviest weight
-  - blocks:
-      - id: 'b0'
-        parent: 'b0'
-      - id: 'b1'
-        parent: 'b0'
-      - id: 'b2'
-        parent: 'b0'
-      - id: 'b3'
-        parent: 'b0'
-    weights:
-      b1: 5
-      b2: 4
-      b3: 3
-    head: 'b1'
-  # Equal weights children, GHOST chooses b2 because it is higher lexicographically than b3
-  - blocks:
-      - id: 'b0'
-        parent: 'b0'
-      - id: 'b1'
-        parent: 'b0'
-      - id: 'b2'
-        parent: 'b0'
-      - id: 'b3'
-        parent: 'b0'
-    weights:
-      b1: 5
-      b2: 6
-      b3: 6
-    head: 'b3'
-  # Equal weights children, GHOST chooses b2 because it is higher lexicographically than b1
-  - blocks:
-      - id: 'b0'
-        parent: 'b0'
-      - id: 'b1'
-        parent: 'b0'
-      - id: 'b2'
-        parent: 'b0'
-    weights:
-      b1: 0
-      b2: 0
-    head: 'b2'

beacon-chain/blockchain/forkchoice/lmd_ghost_yaml_test.go

@@ -1,140 +0,0 @@
-package forkchoice
-
-import (
-	"bytes"
-	"context"
-	"io/ioutil"
-	"path/filepath"
-	"strconv"
-	"testing"
-
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"gopkg.in/yaml.v2"
-)
-
-type Config struct {
-	TestCases []struct {
-		Blocks []struct {
-			ID     string `yaml:"id"`
-			Parent string `yaml:"parent"`
-		} `yaml:"blocks"`
-		Weights map[string]int `yaml:"weights"`
-		Head    string         `yaml:"head"`
-	} `yaml:"test_cases"`
-}
-
-func TestGetHeadFromYaml(t *testing.T) {
-	ctx := context.Background()
-	filename, _ := filepath.Abs("./lmd_ghost_test.yaml")
-	yamlFile, err := ioutil.ReadFile(filename)
-	if err != nil {
-		t.Fatal(err)
-	}
-	var c *Config
-	err = yaml.Unmarshal(yamlFile, &c)
-
-	for _, test := range c.TestCases {
-		db := testDB.SetupDB(t)
-		defer testDB.TeardownDB(t, db)
-
-		blksRoot := make(map[int][]byte)
-		// Construct block tree from yaml.
-		for _, blk := range test.Blocks {
-			// genesis block condition
-			if blk.ID == blk.Parent {
-				b := &ethpb.BeaconBlock{Slot: 0, ParentRoot: []byte{'g'}}
-				if err := db.SaveBlock(ctx, b); err != nil {
-					t.Fatal(err)
-				}
-				root, err := ssz.SigningRoot(b)
-				if err != nil {
-					t.Fatal(err)
-				}
-				blksRoot[0] = root[:]
-			} else {
-				slot, err := strconv.Atoi(blk.ID[1:])
-				if err != nil {
-					t.Fatal(err)
-				}
-				parentSlot, err := strconv.Atoi(blk.Parent[1:])
-				if err != nil {
-					t.Fatal(err)
-				}
-				b := &ethpb.BeaconBlock{Slot: uint64(slot), ParentRoot: blksRoot[parentSlot]}
-				if err := db.SaveBlock(ctx, b); err != nil {
-					t.Fatal(err)
-				}
-				root, err := ssz.SigningRoot(b)
-				if err != nil {
-					t.Fatal(err)
-				}
-				blksRoot[slot] = root[:]
-			}
-		}
-
-		// Assign validator votes to the blocks as weights.
-		count := 0
-		for blk, votes := range test.Weights {
-			slot, err := strconv.Atoi(blk[1:])
-			if err != nil {
-				t.Fatal(err)
-			}
-			max := count + votes
-			for i := count; i < max; i++ {
-				if err := db.SaveValidatorLatestVote(ctx, uint64(i), &pb.ValidatorLatestVote{Root: blksRoot[slot]}); err != nil {
-					t.Fatal(err)
-				}
-				count++
-			}
-		}
-
-		store := NewForkChoiceService(ctx, db)
-		validators := make([]*ethpb.Validator, count)
-		for i := 0; i < len(validators); i++ {
-			validators[i] = &ethpb.Validator{ExitEpoch: 2, EffectiveBalance: 1e9}
-		}
-
-		s := &pb.BeaconState{Validators: validators}
-
-		if err := store.GenesisStore(ctx, &ethpb.Checkpoint{}, &ethpb.Checkpoint{}); err != nil {
-			t.Fatal(err)
-		}
-
-		store.justifiedCheckpt.Root = blksRoot[0]
-		if err := store.db.SaveState(ctx, s, bytesutil.ToBytes32(blksRoot[0])); err != nil {
-			t.Fatal(err)
-		}
-
-		if err := store.checkpointState.AddCheckpointState(&cache.CheckpointState{
-			Checkpoint: store.justifiedCheckpt,
-			State:      s,
-		}); err != nil {
-			t.Fatal(err)
-		}
-
-		head, err := store.Head(ctx)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		headSlot, err := strconv.Atoi(test.Head[1:])
-		if err != nil {
-			t.Fatal(err)
-		}
-		wantedHead := blksRoot[headSlot]
-
-		if !bytes.Equal(head, wantedHead) {
-			t.Errorf("wanted root %#x, got root %#x", wantedHead, head)
-		}
-
-		helpers.ClearAllCaches()
-		testDB.TeardownDB(t, db)
-
-	}
-}

beacon-chain/blockchain/forkchoice/log.go

@@ -1,40 +0,0 @@
-package forkchoice
-
-import (
-	"fmt"
-
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/sirupsen/logrus"
-)
-
-var log = logrus.WithField("prefix", "forkchoice")
-
-// logs epoch related data during epoch boundary.
-func logEpochData(beaconState *pb.BeaconState) {
-	log.WithFields(logrus.Fields{
-		"epoch":                  helpers.CurrentEpoch(beaconState),
-		"finalizedEpoch":         beaconState.FinalizedCheckpoint.Epoch,
-		"justifiedEpoch":         beaconState.CurrentJustifiedCheckpoint.Epoch,
-		"previousJustifiedEpoch": beaconState.PreviousJustifiedCheckpoint.Epoch,
-	}).Info("Starting next epoch")
-	activeVals, err := helpers.ActiveValidatorIndices(beaconState, helpers.CurrentEpoch(beaconState))
-	if err != nil {
-		log.WithError(err).Error("Could not get active validator indices")
-		return
-	}
-	log.WithFields(logrus.Fields{
-		"totalValidators":  len(beaconState.Validators),
-		"activeValidators": len(activeVals),
-		"averageBalance":   fmt.Sprintf("%.5f ETH", averageBalance(beaconState.Balances)),
-	}).Info("Validator registry information")
-}
-
-func averageBalance(balances []uint64) float64 {
-	total := uint64(0)
-	for i := 0; i < len(balances); i++ {
-		total += balances[i]
-	}
-	return float64(total) / float64(len(balances)) / float64(params.BeaconConfig().GweiPerEth)
-}

beacon-chain/blockchain/forkchoice/metrics.go

@@ -1,92 +0,0 @@
-package forkchoice
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/params"
-)
-
-var (
-	beaconFinalizedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_finalized_epoch",
-		Help: "Last finalized epoch of the processed state",
-	})
-	beaconFinalizedRoot = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_finalized_root",
-		Help: "Last finalized root of the processed state",
-	})
-	beaconCurrentJustifiedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_current_justified_epoch",
-		Help: "Current justified epoch of the processed state",
-	})
-	beaconCurrentJustifiedRoot = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_current_justified_root",
-		Help: "Current justified root of the processed state",
-	})
-	beaconPrevJustifiedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_previous_justified_epoch",
-		Help: "Previous justified epoch of the processed state",
-	})
-	beaconPrevJustifiedRoot = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_previous_justified_root",
-		Help: "Previous justified root of the processed state",
-	})
-	activeValidatorsGauge = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "state_active_validators",
-		Help: "Total number of active validators",
-	})
-	slashedValidatorsGauge = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "state_slashed_validators",
-		Help: "Total slashed validators",
-	})
-	withdrawnValidatorsGauge = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "state_withdrawn_validators",
-		Help: "Total withdrawn validators",
-	})
-	totalValidatorsGauge = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_current_validators",
-		Help: "Number of status=pending|active|exited|withdrawable validators in current epoch",
-	})
-)
-
-func reportEpochMetrics(state *pb.BeaconState) {
-	currentEpoch := state.Slot / params.BeaconConfig().SlotsPerEpoch
-
-	// Validator counts
-	var active float64
-	var slashed float64
-	var withdrawn float64
-	for _, v := range state.Validators {
-		if v.ActivationEpoch <= currentEpoch && currentEpoch < v.ExitEpoch {
-			active++
-		}
-		if v.Slashed {
-			slashed++
-		}
-		if currentEpoch >= v.ExitEpoch {
-			withdrawn++
-		}
-	}
-	activeValidatorsGauge.Set(active)
-	slashedValidatorsGauge.Set(slashed)
-	withdrawnValidatorsGauge.Set(withdrawn)
-	totalValidatorsGauge.Set(float64(len(state.Validators)))
-
-	// Last justified slot
-	if state.CurrentJustifiedCheckpoint != nil {
-		beaconCurrentJustifiedEpoch.Set(float64(state.CurrentJustifiedCheckpoint.Epoch))
-		beaconCurrentJustifiedRoot.Set(float64(bytesutil.ToLowInt64(state.CurrentJustifiedCheckpoint.Root)))
-	}
-	// Last previous justified slot
-	if state.PreviousJustifiedCheckpoint != nil {
-		beaconPrevJustifiedEpoch.Set(float64(state.PreviousJustifiedCheckpoint.Epoch))
-		beaconPrevJustifiedRoot.Set(float64(bytesutil.ToLowInt64(state.PreviousJustifiedCheckpoint.Root)))
-	}
-	// Last finalized slot
-	if state.FinalizedCheckpoint != nil {
-		beaconFinalizedEpoch.Set(float64(state.FinalizedCheckpoint.Epoch))
-		beaconFinalizedRoot.Set(float64(bytesutil.ToLowInt64(state.FinalizedCheckpoint.Root)))
-	}
-}

beacon-chain/blockchain/forkchoice/process_attestation.go

@@ -1,236 +0,0 @@
-package forkchoice
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/gogo/protobuf/proto"
-	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/hashutil"
-	"github.com/sirupsen/logrus"
-	"go.opencensus.io/trace"
-)
-
-// OnAttestation is called whenever an attestation is received, it updates validators latest vote,
-// as well as the fork choice store struct.
-//
-// Spec pseudocode definition:
-//   def on_attestation(store: Store, attestation: Attestation) -> None:
-//    target = attestation.data.target
-//
-//    # Cannot calculate the current shuffling if have not seen the target
-//    assert target.root in store.blocks
-//
-//    # Attestations cannot be from future epochs. If they are, delay consideration until the epoch arrives
-//    base_state = store.block_states[target.root].copy()
-//    assert store.time >= base_state.genesis_time + compute_start_slot_of_epoch(target.epoch) * SECONDS_PER_SLOT
-//
-//    # Store target checkpoint state if not yet seen
-//    if target not in store.checkpoint_states:
-//        process_slots(base_state, compute_start_slot_of_epoch(target.epoch))
-//        store.checkpoint_states[target] = base_state
-//    target_state = store.checkpoint_states[target]
-//
-//    # Attestations can only affect the fork choice of subsequent slots.
-//    # Delay consideration in the fork choice until their slot is in the past.
-//    attestation_slot = get_attestation_data_slot(target_state, attestation.data)
-//    assert store.time >= (attestation_slot + 1) * SECONDS_PER_SLOT
-//
-//    # Get state at the `target` to validate attestation and calculate the committees
-//    indexed_attestation = get_indexed_attestation(target_state, attestation)
-//    assert is_valid_indexed_attestation(target_state, indexed_attestation)
-//
-//    # Update latest messages
-//    for i in indexed_attestation.custody_bit_0_indices + indexed_attestation.custody_bit_1_indices:
-//        if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch:
-//            store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=attestation.data.beacon_block_root)
-func (s *Store) OnAttestation(ctx context.Context, a *ethpb.Attestation) error {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.onAttestation")
-	defer span.End()
-
-	tgt := proto.Clone(a.Data.Target).(*ethpb.Checkpoint)
-	tgtSlot := helpers.StartSlot(tgt.Epoch)
-
-	// Verify beacon node has seen the target block before.
-	if !s.db.HasBlock(ctx, bytesutil.ToBytes32(tgt.Root)) {
-		return fmt.Errorf("target root %#x does not exist in db", bytesutil.Trunc(tgt.Root))
-	}
-
-	// Verify attestation target has had a valid pre state produced by the target block.
-	baseState, err := s.verifyAttPreState(ctx, tgt)
-	if err != nil {
-		return err
-	}
-
-	// Verify Attestations cannot be from future epochs.
-	if err := helpers.VerifySlotTime(baseState.GenesisTime, tgtSlot); err != nil {
-		return errors.Wrap(err, "could not verify attestation target slot")
-	}
-
-	// Store target checkpoint state if not yet seen.
-	baseState, err = s.saveCheckpointState(ctx, baseState, tgt)
-	if err != nil {
-		return err
-	}
-
-	// Verify attestations can only affect the fork choice of subsequent slots.
-	if err := helpers.VerifySlotTime(baseState.GenesisTime, a.Data.Slot+1); err != nil {
-		return err
-	}
-
-	// Use the target state to to validate attestation and calculate the committees.
-	indexedAtt, err := s.verifyAttestation(ctx, baseState, a)
-	if err != nil {
-		return err
-	}
-
-	// Update every validator's latest vote.
-	if err := s.updateAttVotes(ctx, indexedAtt, tgt.Root, tgt.Epoch); err != nil {
-		return err
-	}
-
-	// Mark attestation as seen we don't update votes when it appears in block.
-	if err := s.setSeenAtt(a); err != nil {
-		return err
-	}
-
-	if err := s.db.SaveAttestation(ctx, a); err != nil {
-		return err
-	}
-
-	log := log.WithFields(logrus.Fields{
-		"Slot":               a.Data.Slot,
-		"Index":              a.Data.Index,
-		"AggregatedBitfield": fmt.Sprintf("%08b", a.AggregationBits),
-		"BeaconBlockRoot":    fmt.Sprintf("%#x", bytesutil.Trunc(a.Data.BeaconBlockRoot)),
-	})
-	log.Debug("Updated latest votes")
-
-	return nil
-}
-
-// verifyAttPreState validates input attested check point has a valid pre-state.
-func (s *Store) verifyAttPreState(ctx context.Context, c *ethpb.Checkpoint) (*pb.BeaconState, error) {
-	baseState, err := s.db.State(ctx, bytesutil.ToBytes32(c.Root))
-	if err != nil {
-		return nil, errors.Wrapf(err, "could not get pre state for slot %d", helpers.StartSlot(c.Epoch))
-	}
-	if baseState == nil {
-		return nil, fmt.Errorf("pre state of target block %d does not exist", helpers.StartSlot(c.Epoch))
-	}
-	return baseState, nil
-}
-
-// saveCheckpointState saves and returns the processed state with the associated check point.
-func (s *Store) saveCheckpointState(ctx context.Context, baseState *pb.BeaconState, c *ethpb.Checkpoint) (*pb.BeaconState, error) {
-	s.checkpointStateLock.Lock()
-	defer s.checkpointStateLock.Unlock()
-	cachedState, err := s.checkpointState.StateByCheckpoint(c)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not get cached checkpoint state")
-	}
-	if cachedState != nil {
-		return cachedState, nil
-	}
-
-	// Advance slots only when it's higher than current state slot.
-	if helpers.StartSlot(c.Epoch) > baseState.Slot {
-		stateCopy := proto.Clone(baseState).(*pb.BeaconState)
-		baseState, err = state.ProcessSlots(ctx, stateCopy, helpers.StartSlot(c.Epoch))
-		if err != nil {
-			return nil, errors.Wrapf(err, "could not process slots up to %d", helpers.StartSlot(c.Epoch))
-		}
-	}
-
-	if err := s.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: c,
-		State:      baseState,
-	}); err != nil {
-		return nil, errors.Wrap(err, "could not saved checkpoint state to cache")
-	}
-
-	return baseState, nil
-}
-
-// verifyAttestation validates input attestation is valid.
-func (s *Store) verifyAttestation(ctx context.Context, baseState *pb.BeaconState, a *ethpb.Attestation) (*ethpb.IndexedAttestation, error) {
-	indexedAtt, err := blocks.ConvertToIndexed(ctx, baseState, a)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not convert attestation to indexed attestation")
-	}
-	if err := blocks.VerifyIndexedAttestation(ctx, baseState, indexedAtt); err != nil {
-		return nil, errors.Wrap(err, "could not verify indexed attestation")
-	}
-	return indexedAtt, nil
-}
-
-// updateAttVotes updates validator's latest votes based on the incoming attestation.
-func (s *Store) updateAttVotes(
-	ctx context.Context,
-	indexedAtt *ethpb.IndexedAttestation,
-	tgtRoot []byte,
-	tgtEpoch uint64) error {
-
-	indices := append(indexedAtt.CustodyBit_0Indices, indexedAtt.CustodyBit_1Indices...)
-	newVoteIndices := make([]uint64, 0, len(indices))
-	newVotes := make([]*pb.ValidatorLatestVote, 0, len(indices))
-	for _, i := range indices {
-		vote, err := s.db.ValidatorLatestVote(ctx, i)
-		if err != nil {
-			return errors.Wrapf(err, "could not get latest vote for validator %d", i)
-		}
-		if vote == nil || tgtEpoch > vote.Epoch {
-			newVotes = append(newVotes, &pb.ValidatorLatestVote{
-				Epoch: tgtEpoch,
-				Root:  tgtRoot,
-			})
-			newVoteIndices = append(newVoteIndices, i)
-		}
-	}
-	return s.db.SaveValidatorLatestVotes(ctx, newVoteIndices, newVotes)
-}
-
-// setSeenAtt sets the attestation hash in seen attestation map to true.
-func (s *Store) setSeenAtt(a *ethpb.Attestation) error {
-	s.seenAttsLock.Lock()
-	defer s.seenAttsLock.Unlock()
-
-	r, err := hashutil.HashProto(a)
-	if err != nil {
-		return err
-	}
-	s.seenAtts[r] = true
-
-	return nil
-}
-
-// aggregatedAttestation returns the aggregated attestation after checking saved one in db.
-func (s *Store) aggregatedAttestations(ctx context.Context, att *ethpb.Attestation) ([]*ethpb.Attestation, error) {
-	r, err := ssz.HashTreeRoot(att.Data)
-	if err != nil {
-		return nil, err
-	}
-	saved, err := s.db.AttestationsByDataRoot(ctx, r)
-	if err != nil {
-		return nil, err
-	}
-
-	if saved == nil {
-		return []*ethpb.Attestation{att}, nil
-	}
-
-	aggregated, err := helpers.AggregateAttestations(append(saved, att))
-	if err != nil {
-		return nil, err
-	}
-
-	return aggregated, nil
-}

beacon-chain/blockchain/forkchoice/process_attestation_test.go

@@ -1,212 +0,0 @@
-package forkchoice
-
-import (
-	"context"
-	"reflect"
-	"strings"
-	"testing"
-
-	"github.com/prysmaticlabs/go-bitfield"
-	"github.com/prysmaticlabs/go-ssz"
-	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/shared/params"
-)
-
-func TestStore_OnAttestation(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	_, err := blockTree1(db)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	BlkWithOutState := &ethpb.BeaconBlock{Slot: 0}
-	if err := db.SaveBlock(ctx, BlkWithOutState); err != nil {
-		t.Fatal(err)
-	}
-	BlkWithOutStateRoot, _ := ssz.SigningRoot(BlkWithOutState)
-
-	BlkWithStateBadAtt := &ethpb.BeaconBlock{Slot: 1}
-	if err := db.SaveBlock(ctx, BlkWithStateBadAtt); err != nil {
-		t.Fatal(err)
-	}
-	BlkWithStateBadAttRoot, _ := ssz.SigningRoot(BlkWithStateBadAtt)
-	if err := store.db.SaveState(ctx, &pb.BeaconState{}, BlkWithStateBadAttRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	BlkWithValidState := &ethpb.BeaconBlock{Slot: 2}
-	if err := db.SaveBlock(ctx, BlkWithValidState); err != nil {
-		t.Fatal(err)
-	}
-	BlkWithValidStateRoot, _ := ssz.SigningRoot(BlkWithValidState)
-	if err := store.db.SaveState(ctx, &pb.BeaconState{
-		Fork: &pb.Fork{
-			Epoch:           0,
-			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
-			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
-		},
-		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
-	}, BlkWithValidStateRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	tests := []struct {
-		name          string
-		a             *ethpb.Attestation
-		s             *pb.BeaconState
-		wantErr       bool
-		wantErrString string
-	}{
-		{
-			name:          "attestation's target root not in db",
-			a:             &ethpb.Attestation{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{Root: []byte{'A'}}}},
-			s:             &pb.BeaconState{},
-			wantErr:       true,
-			wantErrString: "target root 0x41 does not exist in db",
-		},
-		{
-			name:          "no pre state for attestations's target block",
-			a:             &ethpb.Attestation{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{Root: BlkWithOutStateRoot[:]}}},
-			s:             &pb.BeaconState{},
-			wantErr:       true,
-			wantErrString: "pre state of target block 0 does not exist",
-		},
-		{
-			name: "process attestation from future epoch",
-			a: &ethpb.Attestation{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{Epoch: params.BeaconConfig().FarFutureEpoch,
-				Root: BlkWithStateBadAttRoot[:]}}},
-			s:             &pb.BeaconState{},
-			wantErr:       true,
-			wantErrString: "could not process slot from the future",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if err := store.GenesisStore(ctx, &ethpb.Checkpoint{}, &ethpb.Checkpoint{}); err != nil {
-				t.Fatal(err)
-			}
-
-			err := store.OnAttestation(ctx, tt.a)
-			if tt.wantErr {
-				if !strings.Contains(err.Error(), tt.wantErrString) {
-					t.Errorf("Store.OnAttestation() error = %v, wantErr = %v", err, tt.wantErrString)
-				}
-			} else {
-				t.Error(err)
-			}
-		})
-	}
-}
-
-func TestStore_SaveCheckpointState(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	params.UseDemoBeaconConfig()
-
-	store := NewForkChoiceService(ctx, db)
-
-	s := &pb.BeaconState{
-		Fork: &pb.Fork{
-			Epoch:           0,
-			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
-			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
-		},
-		RandaoMixes:         make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
-		StateRoots:          make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
-		BlockRoots:          make([][]byte, params.BeaconConfig().SlotsPerHistoricalRoot),
-		LatestBlockHeader:   &ethpb.BeaconBlockHeader{},
-		JustificationBits:   []byte{0},
-		Slashings:           make([]uint64, params.BeaconConfig().EpochsPerSlashingsVector),
-		FinalizedCheckpoint: &ethpb.Checkpoint{},
-	}
-	if err := store.GenesisStore(ctx, &ethpb.Checkpoint{}, &ethpb.Checkpoint{}); err != nil {
-		t.Fatal(err)
-	}
-
-	cp1 := &ethpb.Checkpoint{Epoch: 1, Root: []byte{'A'}}
-	s1, err := store.saveCheckpointState(ctx, s, cp1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if s1.Slot != 1*params.BeaconConfig().SlotsPerEpoch {
-		t.Errorf("Wanted state slot: %d, got: %d", 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot)
-	}
-
-	cp2 := &ethpb.Checkpoint{Epoch: 2, Root: []byte{'B'}}
-	s2, err := store.saveCheckpointState(ctx, s, cp2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if s2.Slot != 2*params.BeaconConfig().SlotsPerEpoch {
-		t.Errorf("Wanted state slot: %d, got: %d", 2*params.BeaconConfig().SlotsPerEpoch, s2.Slot)
-	}
-
-	s1, err = store.saveCheckpointState(ctx, nil, cp1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if s1.Slot != 1*params.BeaconConfig().SlotsPerEpoch {
-		t.Errorf("Wanted state slot: %d, got: %d", 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot)
-	}
-
-	s1, err = store.checkpointState.StateByCheckpoint(cp1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if s1.Slot != 1*params.BeaconConfig().SlotsPerEpoch {
-		t.Errorf("Wanted state slot: %d, got: %d", 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot)
-	}
-
-	s2, err = store.checkpointState.StateByCheckpoint(cp2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if s2.Slot != 2*params.BeaconConfig().SlotsPerEpoch {
-		t.Errorf("Wanted state slot: %d, got: %d", 2*params.BeaconConfig().SlotsPerEpoch, s2.Slot)
-	}
-
-	s.Slot = params.BeaconConfig().SlotsPerEpoch + 1
-	if err := store.GenesisStore(ctx, &ethpb.Checkpoint{}, &ethpb.Checkpoint{}); err != nil {
-		t.Fatal(err)
-	}
-	cp3 := &ethpb.Checkpoint{Epoch: 1, Root: []byte{'C'}}
-	s3, err := store.saveCheckpointState(ctx, s, cp3)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if s3.Slot != s.Slot {
-		t.Errorf("Wanted state slot: %d, got: %d", s.Slot, s3.Slot)
-	}
-}
-
-func TestStore_ReturnAggregatedAttestation(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	a1 := &ethpb.Attestation{Data: &ethpb.AttestationData{}, AggregationBits: bitfield.Bitlist{0x02}}
-	err := store.db.SaveAttestation(ctx, a1)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	a2 := &ethpb.Attestation{Data: &ethpb.AttestationData{}, AggregationBits: bitfield.Bitlist{0x03}}
-	saved, err := store.aggregatedAttestations(ctx, a2)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if !reflect.DeepEqual([]*ethpb.Attestation{a2}, saved) {
-		t.Error("did not retrieve saved attestation")
-	}
-}

beacon-chain/blockchain/forkchoice/process_block.go

@@ -1,445 +0,0 @@
-package forkchoice
-
-import (
-	"bytes"
-	"context"
-	"encoding/hex"
-	"fmt"
-
-	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"github.com/prysmaticlabs/prysm/shared/hashutil"
-	"github.com/prysmaticlabs/prysm/shared/traceutil"
-	"github.com/sirupsen/logrus"
-	"go.opencensus.io/trace"
-)
-
-// OnBlock is called when a gossip block is received. It runs regular state transition on the block and
-// update fork choice store.
-//
-// Spec pseudocode definition:
-//   def on_block(store: Store, block: BeaconBlock) -> None:
-//    # Make a copy of the state to avoid mutability issues
-//    assert block.parent_root in store.block_states
-//    pre_state = store.block_states[block.parent_root].copy()
-//    # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past.
-//    assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT
-//    # Add new block to the store
-//    store.blocks[signing_root(block)] = block
-//    # Check block is a descendant of the finalized block
-//    assert (
-//        get_ancestor(store, signing_root(block), store.blocks[store.finalized_checkpoint.root].slot) ==
-//        store.finalized_checkpoint.root
-//    )
-//    # Check that block is later than the finalized epoch slot
-//    assert block.slot > compute_start_slot_of_epoch(store.finalized_checkpoint.epoch)
-//    # Check the block is valid and compute the post-state
-//    state = state_transition(pre_state, block)
-//    # Add new state for this block to the store
-//    store.block_states[signing_root(block)] = state
-//
-//    # Update justified checkpoint
-//    if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch:
-//        store.justified_checkpoint = state.current_justified_checkpoint
-//
-//    # Update finalized checkpoint
-//    if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch:
-//        store.finalized_checkpoint = state.finalized_checkpoint
-func (s *Store) OnBlock(ctx context.Context, b *ethpb.BeaconBlock) error {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.onBlock")
-	defer span.End()
-
-	// Retrieve incoming block's pre state.
-	preState, err := s.getBlockPreState(ctx, b)
-	if err != nil {
-		return err
-	}
-	preStateValidatorCount := len(preState.Validators)
-
-	root, err := ssz.SigningRoot(b)
-	if err != nil {
-		return errors.Wrapf(err, "could not get signing root of block %d", b.Slot)
-	}
-	log.WithFields(logrus.Fields{
-		"slot": b.Slot,
-		"root": fmt.Sprintf("0x%s...", hex.EncodeToString(root[:])[:8]),
-	}).Info("Executing state transition on block")
-	postState, err := state.ExecuteStateTransition(ctx, preState, b)
-	if err != nil {
-		return errors.Wrap(err, "could not execute state transition")
-	}
-	if err := s.updateBlockAttestationsVotes(ctx, b.Body.Attestations); err != nil {
-		return errors.Wrap(err, "could not update votes for attestations in block")
-	}
-
-	if err := s.db.SaveBlock(ctx, b); err != nil {
-		return errors.Wrapf(err, "could not save block from slot %d", b.Slot)
-	}
-	if err := s.db.SaveState(ctx, postState, root); err != nil {
-		return errors.Wrap(err, "could not save state")
-	}
-
-	// Update justified check point.
-	if postState.CurrentJustifiedCheckpoint.Epoch > s.JustifiedCheckpt().Epoch {
-		s.justifiedCheckpt = postState.CurrentJustifiedCheckpoint
-		if err := s.db.SaveJustifiedCheckpoint(ctx, postState.CurrentJustifiedCheckpoint); err != nil {
-			return errors.Wrap(err, "could not save justified checkpoint")
-		}
-	}
-
-	// Update finalized check point.
-	// Prune the block cache and helper caches on every new finalized epoch.
-	if postState.FinalizedCheckpoint.Epoch > s.finalizedCheckpt.Epoch {
-		s.clearSeenAtts()
-		helpers.ClearAllCaches()
-		if err := s.db.SaveFinalizedCheckpoint(ctx, postState.FinalizedCheckpoint); err != nil {
-			return errors.Wrap(err, "could not save finalized checkpoint")
-		}
-
-		startSlot := helpers.StartSlot(s.prevFinalizedCheckpt.Epoch) + 1
-		endSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
-		if endSlot > startSlot {
-			if err := s.rmStatesOlderThanLastFinalized(ctx, startSlot, endSlot); err != nil {
-				return errors.Wrapf(err, "could not delete states prior to finalized check point, range: %d, %d",
-					startSlot, endSlot)
-			}
-		}
-
-		s.prevFinalizedCheckpt = s.finalizedCheckpt
-		s.finalizedCheckpt = postState.FinalizedCheckpoint
-	}
-
-	// Update validator indices in database as needed.
-	if err := s.saveNewValidators(ctx, preStateValidatorCount, postState); err != nil {
-		return errors.Wrap(err, "could not save finalized checkpoint")
-	}
-	// Save the unseen attestations from block to db.
-	if err := s.saveNewBlockAttestations(ctx, b.Body.Attestations); err != nil {
-		return errors.Wrap(err, "could not save attestations")
-	}
-
-	// Epoch boundary bookkeeping such as logging epoch summaries.
-	if helpers.IsEpochStart(postState.Slot) {
-		logEpochData(postState)
-		reportEpochMetrics(postState)
-
-		// Update committee shuffled indices at the end of every epoch
-		if featureconfig.Get().EnableNewCache {
-			if err := helpers.UpdateCommitteeCache(postState); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-// OnBlockNoVerifyStateTransition is called when an initial sync block is received.
-// It runs state transition on the block and without any BLS verification. The BLS verification
-// includes proposer signature, randao and attestation's aggregated signature.
-func (s *Store) OnBlockNoVerifyStateTransition(ctx context.Context, b *ethpb.BeaconBlock) error {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.onBlock")
-	defer span.End()
-
-	// Retrieve incoming block's pre state.
-	preState, err := s.getBlockPreState(ctx, b)
-	if err != nil {
-		return err
-	}
-	preStateValidatorCount := len(preState.Validators)
-
-	log.WithField("slot", b.Slot).Debug("Executing state transition on block")
-
-	postState, err := state.ExecuteStateTransitionNoVerify(ctx, preState, b)
-	if err != nil {
-		return errors.Wrap(err, "could not execute state transition")
-	}
-
-	if err := s.db.SaveBlock(ctx, b); err != nil {
-		return errors.Wrapf(err, "could not save block from slot %d", b.Slot)
-	}
-	root, err := ssz.SigningRoot(b)
-	if err != nil {
-		return errors.Wrapf(err, "could not get signing root of block %d", b.Slot)
-	}
-	if err := s.db.SaveState(ctx, postState, root); err != nil {
-		return errors.Wrap(err, "could not save state")
-	}
-
-	// Update justified check point.
-	if postState.CurrentJustifiedCheckpoint.Epoch > s.JustifiedCheckpt().Epoch {
-		s.justifiedCheckpt = postState.CurrentJustifiedCheckpoint
-		if err := s.db.SaveJustifiedCheckpoint(ctx, postState.CurrentJustifiedCheckpoint); err != nil {
-			return errors.Wrap(err, "could not save justified checkpoint")
-		}
-	}
-
-	// Update finalized check point.
-	// Prune the block cache and helper caches on every new finalized epoch.
-	if postState.FinalizedCheckpoint.Epoch > s.finalizedCheckpt.Epoch {
-		s.clearSeenAtts()
-		helpers.ClearAllCaches()
-
-		startSlot := helpers.StartSlot(s.prevFinalizedCheckpt.Epoch) + 1
-		endSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
-		if endSlot > startSlot {
-			if err := s.rmStatesOlderThanLastFinalized(ctx, startSlot, endSlot); err != nil {
-				return errors.Wrapf(err, "could not delete states prior to finalized check point, range: %d, %d",
-					startSlot, endSlot)
-			}
-		}
-
-		if err := s.db.SaveFinalizedCheckpoint(ctx, postState.FinalizedCheckpoint); err != nil {
-			return errors.Wrap(err, "could not save finalized checkpoint")
-		}
-
-		s.prevFinalizedCheckpt = s.finalizedCheckpt
-		s.finalizedCheckpt = postState.FinalizedCheckpoint
-	}
-
-	// Update validator indices in database as needed.
-	if err := s.saveNewValidators(ctx, preStateValidatorCount, postState); err != nil {
-		return errors.Wrap(err, "could not save finalized checkpoint")
-	}
-	// Save the unseen attestations from block to db.
-	if err := s.saveNewBlockAttestations(ctx, b.Body.Attestations); err != nil {
-		return errors.Wrap(err, "could not save attestations")
-	}
-
-	// Epoch boundary bookkeeping such as logging epoch summaries.
-	if helpers.IsEpochStart(postState.Slot) {
-		reportEpochMetrics(postState)
-
-		// Update committee shuffled indices at the end of every epoch
-		if featureconfig.Get().EnableNewCache {
-			if err := helpers.UpdateCommitteeCache(postState); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-// getBlockPreState returns the pre state of an incoming block. It uses the parent root of the block
-// to retrieve the state in DB. It verifies the pre state's validity and the incoming block
-// is in the correct time window.
-func (s *Store) getBlockPreState(ctx context.Context, b *ethpb.BeaconBlock) (*pb.BeaconState, error) {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.getBlockPreState")
-	defer span.End()
-
-	// Verify incoming block has a valid pre state.
-	preState, err := s.verifyBlkPreState(ctx, b)
-	if err != nil {
-		return nil, err
-	}
-
-	// Verify block slot time is not from the feature.
-	if err := helpers.VerifySlotTime(preState.GenesisTime, b.Slot); err != nil {
-		return nil, err
-	}
-
-	// Verify block is a descendent of a finalized block.
-	if err := s.verifyBlkDescendant(ctx, bytesutil.ToBytes32(b.ParentRoot), b.Slot); err != nil {
-		return nil, err
-	}
-
-	// Verify block is later than the finalized epoch slot.
-	if err := s.verifyBlkFinalizedSlot(b); err != nil {
-		return nil, err
-	}
-
-	return preState, nil
-}
-
-// updateBlockAttestationsVotes checks the attestations in block and filter out the seen ones,
-// the unseen ones get passed to updateBlockAttestationVote for updating fork choice votes.
-func (s *Store) updateBlockAttestationsVotes(ctx context.Context, atts []*ethpb.Attestation) error {
-	s.seenAttsLock.Lock()
-	defer s.seenAttsLock.Unlock()
-
-	for _, att := range atts {
-		// If we have not seen the attestation yet
-		r, err := hashutil.HashProto(att)
-		if err != nil {
-			return err
-		}
-		if s.seenAtts[r] {
-			continue
-		}
-		if err := s.updateBlockAttestationVote(ctx, att); err != nil {
-			log.WithError(err).Warn("Attestation failed to update vote")
-		}
-		s.seenAtts[r] = true
-	}
-	return nil
-}
-
-// updateBlockAttestationVotes checks the attestation to update validator's latest votes.
-func (s *Store) updateBlockAttestationVote(ctx context.Context, att *ethpb.Attestation) error {
-	tgt := att.Data.Target
-	baseState, err := s.db.State(ctx, bytesutil.ToBytes32(tgt.Root))
-	if err != nil {
-		return errors.Wrap(err, "could not get state for attestation tgt root")
-	}
-	if baseState == nil {
-		return errors.New("no state found in db with attestation tgt root")
-	}
-	indexedAtt, err := blocks.ConvertToIndexed(ctx, baseState, att)
-	if err != nil {
-		return errors.Wrap(err, "could not convert attestation to indexed attestation")
-	}
-	for _, i := range append(indexedAtt.CustodyBit_0Indices, indexedAtt.CustodyBit_1Indices...) {
-		vote, err := s.db.ValidatorLatestVote(ctx, i)
-		if err != nil {
-			return errors.Wrapf(err, "could not get latest vote for validator %d", i)
-		}
-		if vote == nil || tgt.Epoch > vote.Epoch {
-			if err := s.db.SaveValidatorLatestVote(ctx, i, &pb.ValidatorLatestVote{
-				Epoch: tgt.Epoch,
-				Root:  tgt.Root,
-			}); err != nil {
-				return errors.Wrapf(err, "could not save latest vote for validator %d", i)
-			}
-		}
-	}
-	return nil
-}
-
-// verifyBlkPreState validates input block has a valid pre-state.
-func (s *Store) verifyBlkPreState(ctx context.Context, b *ethpb.BeaconBlock) (*pb.BeaconState, error) {
-	preState, err := s.db.State(ctx, bytesutil.ToBytes32(b.ParentRoot))
-	if err != nil {
-		return nil, errors.Wrapf(err, "could not get pre state for slot %d", b.Slot)
-	}
-	if preState == nil {
-		return nil, fmt.Errorf("pre state of slot %d does not exist", b.Slot)
-	}
-	return preState, nil
-}
-
-// verifyBlkDescendant validates input block root is a descendant of the
-// current finalized block root.
-func (s *Store) verifyBlkDescendant(ctx context.Context, root [32]byte, slot uint64) error {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.verifyBlkDescendant")
-	defer span.End()
-
-	finalizedBlk, err := s.db.Block(ctx, bytesutil.ToBytes32(s.finalizedCheckpt.Root))
-	if err != nil || finalizedBlk == nil {
-		return errors.Wrap(err, "could not get finalized block")
-	}
-
-	bFinalizedRoot, err := s.ancestor(ctx, root[:], finalizedBlk.Slot)
-	if err != nil {
-		return errors.Wrap(err, "could not get finalized block root")
-	}
-	if !bytes.Equal(bFinalizedRoot, s.finalizedCheckpt.Root) {
-		err := fmt.Errorf("block from slot %d is not a descendent of the current finalized block slot %d, %#x != %#x",
-			slot, finalizedBlk.Slot, bytesutil.Trunc(bFinalizedRoot), bytesutil.Trunc(s.finalizedCheckpt.Root))
-		traceutil.AnnotateError(span, err)
-		return err
-	}
-	return nil
-}
-
-// verifyBlkFinalizedSlot validates input block is not less than or equal
-// to current finalized slot.
-func (s *Store) verifyBlkFinalizedSlot(b *ethpb.BeaconBlock) error {
-	finalizedSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
-	if finalizedSlot >= b.Slot {
-		return fmt.Errorf("block is equal or earlier than finalized block, slot %d < slot %d", b.Slot, finalizedSlot)
-	}
-	return nil
-}
-
-// saveNewValidators saves newly added validator index from state to db. Does nothing if validator count has not
-// changed.
-func (s *Store) saveNewValidators(ctx context.Context, preStateValidatorCount int, postState *pb.BeaconState) error {
-	postStateValidatorCount := len(postState.Validators)
-	if preStateValidatorCount != postStateValidatorCount {
-		for i := preStateValidatorCount; i < postStateValidatorCount; i++ {
-			pubKey := postState.Validators[i].PublicKey
-			if err := s.db.SaveValidatorIndex(ctx, bytesutil.ToBytes48(pubKey), uint64(i)); err != nil {
-				return errors.Wrapf(err, "could not save activated validator: %d", i)
-			}
-			log.WithFields(logrus.Fields{
-				"index":               i,
-				"pubKey":              hex.EncodeToString(bytesutil.Trunc(pubKey)),
-				"totalValidatorCount": i + 1,
-			}).Info("New validator index saved in DB")
-		}
-	}
-	return nil
-}
-
-// saveNewBlockAttestations saves the new attestations in block to DB.
-func (s *Store) saveNewBlockAttestations(ctx context.Context, atts []*ethpb.Attestation) error {
-	attestations := make([]*ethpb.Attestation, 0, len(atts))
-	for _, att := range atts {
-		aggregated, err := s.aggregatedAttestations(ctx, att)
-		if err != nil {
-			continue
-		}
-		attestations = append(attestations, aggregated...)
-	}
-	if err := s.db.SaveAttestations(ctx, atts); err != nil {
-		return err
-	}
-	return nil
-}
-
-// clearSeenAtts clears seen attestations map, it gets called upon new finalization.
-func (s *Store) clearSeenAtts() {
-	s.seenAttsLock.Lock()
-	s.seenAttsLock.Unlock()
-	s.seenAtts = make(map[[32]byte]bool)
-}
-
-// rmStatesOlderThanLastFinalized deletes the states in db since last finalized check point.
-func (s *Store) rmStatesOlderThanLastFinalized(ctx context.Context, startSlot uint64, endSlot uint64) error {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.rmStatesBySlots")
-	defer span.End()
-
-	if !featureconfig.Get().PruneFinalizedStates {
-		return nil
-	}
-
-	// Make sure finalized slot is not a skipped slot.
-	for i := endSlot; i > 0; i-- {
-		filter := filters.NewFilter().SetStartSlot(i).SetEndSlot(i)
-		b, err := s.db.Blocks(ctx, filter)
-		if err != nil {
-			return err
-		}
-		if len(b) > 0 {
-			endSlot = i - 1
-			break
-		}
-	}
-
-	// Do not remove genesis state
-	if startSlot == 0 {
-		startSlot++
-	}
-
-	// Do not remove finalized state that's in the middle of slot ranges.
-	filter := filters.NewFilter().SetStartSlot(startSlot).SetEndSlot(endSlot)
-	roots, err := s.db.BlockRoots(ctx, filter)
-	if err != nil {
-		return err
-	}
-
-	if err := s.db.DeleteStates(ctx, roots); err != nil {
-		return err
-	}
-
-	return nil
-}

beacon-chain/blockchain/forkchoice/process_block_test.go

@@ -1,343 +0,0 @@
-package forkchoice
-
-import (
-	"context"
-	"reflect"
-	"strings"
-	"testing"
-
-	"github.com/prysmaticlabs/go-bitfield"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
-	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"github.com/prysmaticlabs/prysm/shared/hashutil"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/prysmaticlabs/prysm/shared/testutil"
-)
-
-func init() {
-	fc := featureconfig.Get()
-	fc.PruneFinalizedStates = true
-	featureconfig.Init(fc)
-}
-
-func TestStore_OnBlock(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	randomParentRoot := []byte{'a'}
-	if err := store.db.SaveState(ctx, &pb.BeaconState{}, bytesutil.ToBytes32(randomParentRoot)); err != nil {
-		t.Fatal(err)
-	}
-	randomParentRoot2 := roots[1]
-	if err := store.db.SaveState(ctx, &pb.BeaconState{}, bytesutil.ToBytes32(randomParentRoot2)); err != nil {
-		t.Fatal(err)
-	}
-	validGenesisRoot := []byte{'g'}
-	if err := store.db.SaveState(ctx, &pb.BeaconState{}, bytesutil.ToBytes32(validGenesisRoot)); err != nil {
-		t.Fatal(err)
-	}
-
-	tests := []struct {
-		name          string
-		blk           *ethpb.BeaconBlock
-		s             *pb.BeaconState
-		time          uint64
-		wantErrString string
-	}{
-		{
-			name:          "parent block root does not have a state",
-			blk:           &ethpb.BeaconBlock{},
-			s:             &pb.BeaconState{},
-			wantErrString: "pre state of slot 0 does not exist",
-		},
-		{
-			name:          "block is from the feature",
-			blk:           &ethpb.BeaconBlock{ParentRoot: randomParentRoot, Slot: params.BeaconConfig().FarFutureEpoch},
-			s:             &pb.BeaconState{},
-			wantErrString: "could not process slot from the future",
-		},
-		{
-			name:          "could not get finalized block",
-			blk:           &ethpb.BeaconBlock{ParentRoot: randomParentRoot},
-			s:             &pb.BeaconState{},
-			wantErrString: "block from slot 0 is not a descendent of the current finalized block",
-		},
-		{
-			name:          "same slot as finalized block",
-			blk:           &ethpb.BeaconBlock{Slot: 0, ParentRoot: randomParentRoot2},
-			s:             &pb.BeaconState{},
-			wantErrString: "block is equal or earlier than finalized block, slot 0 < slot 0",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if err := store.GenesisStore(ctx, &ethpb.Checkpoint{}, &ethpb.Checkpoint{}); err != nil {
-				t.Fatal(err)
-			}
-			store.finalizedCheckpt.Root = roots[0]
-
-			err := store.OnBlock(ctx, tt.blk)
-			if !strings.Contains(err.Error(), tt.wantErrString) {
-				t.Errorf("Store.OnBlock() error = %v, wantErr = %v", err, tt.wantErrString)
-			}
-		})
-	}
-}
-
-func TestStore_SaveNewValidators(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	preCount := 2 // validators 0 and validators 1
-	s := &pb.BeaconState{Validators: []*ethpb.Validator{
-		{PublicKey: []byte{0}}, {PublicKey: []byte{1}},
-		{PublicKey: []byte{2}}, {PublicKey: []byte{3}},
-	}}
-	if err := store.saveNewValidators(ctx, preCount, s); err != nil {
-		t.Fatal(err)
-	}
-
-	if !db.HasValidatorIndex(ctx, bytesutil.ToBytes48([]byte{2})) {
-		t.Error("Wanted validator saved in db")
-	}
-	if !db.HasValidatorIndex(ctx, bytesutil.ToBytes48([]byte{3})) {
-		t.Error("Wanted validator saved in db")
-	}
-	if db.HasValidatorIndex(ctx, bytesutil.ToBytes48([]byte{1})) {
-		t.Error("validator not suppose to be saved in db")
-	}
-}
-
-func TestStore_UpdateBlockAttestationVote(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	params.UseMinimalConfig()
-
-	deposits, _, _ := testutil.SetupInitialDeposits(t, 100)
-	beaconState, err := state.GenesisBeaconState(deposits, uint64(0), &ethpb.Eth1Data{BlockHash: make([]byte, 32)})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	store := NewForkChoiceService(ctx, db)
-	r := [32]byte{'A'}
-	att := &ethpb.Attestation{
-		Data: &ethpb.AttestationData{
-			Source: &ethpb.Checkpoint{Epoch: 0, Root: params.BeaconConfig().ZeroHash[:]},
-			Target: &ethpb.Checkpoint{Epoch: 0, Root: r[:]},
-		},
-		AggregationBits: []byte{255},
-		CustodyBits:     []byte{255},
-	}
-	if err := store.db.SaveState(ctx, beaconState, r); err != nil {
-		t.Fatal(err)
-	}
-
-	indices, err := blocks.ConvertToIndexed(ctx, beaconState, att)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var attestedIndices []uint64
-	for _, k := range append(indices.CustodyBit_0Indices, indices.CustodyBit_1Indices...) {
-		attestedIndices = append(attestedIndices, k)
-	}
-
-	if err := store.updateBlockAttestationVote(ctx, att); err != nil {
-		t.Fatal(err)
-	}
-	for _, i := range attestedIndices {
-		v, err := store.db.ValidatorLatestVote(ctx, i)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if !reflect.DeepEqual(v.Root, r[:]) {
-			t.Error("Attested roots don't match")
-		}
-	}
-}
-
-func TestStore_UpdateBlockAttestationsVote(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	params.UseMinimalConfig()
-
-	deposits, _, _ := testutil.SetupInitialDeposits(t, 100)
-	beaconState, err := state.GenesisBeaconState(deposits, uint64(0), &ethpb.Eth1Data{BlockHash: make([]byte, 32)})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	store := NewForkChoiceService(ctx, db)
-	r := [32]byte{'A'}
-	atts := make([]*ethpb.Attestation, 5)
-	hashes := make([][32]byte, 5)
-	for i := 0; i < len(atts); i++ {
-		atts[i] = &ethpb.Attestation{
-			Data: &ethpb.AttestationData{
-				Source: &ethpb.Checkpoint{Epoch: 0, Root: params.BeaconConfig().ZeroHash[:]},
-				Target: &ethpb.Checkpoint{Epoch: 0, Root: r[:]},
-			},
-			AggregationBits: []byte{255},
-			CustodyBits:     []byte{255},
-		}
-		h, _ := hashutil.HashProto(atts[i])
-		hashes[i] = h
-	}
-
-	if err := store.db.SaveState(ctx, beaconState, r); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := store.updateBlockAttestationsVotes(ctx, atts); err != nil {
-		t.Fatal(err)
-	}
-
-	for _, h := range hashes {
-		if !store.seenAtts[h] {
-			t.Error("Seen attestation did not get recorded")
-		}
-	}
-}
-
-func TestStore_SavesNewBlockAttestations(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	a1 := &ethpb.Attestation{Data: &ethpb.AttestationData{}, AggregationBits: bitfield.Bitlist{0b101}, CustodyBits: bitfield.NewBitlist(2)}
-	a2 := &ethpb.Attestation{Data: &ethpb.AttestationData{BeaconBlockRoot: []byte{'A'}}, AggregationBits: bitfield.Bitlist{0b110}, CustodyBits: bitfield.NewBitlist(2)}
-	r1, _ := ssz.HashTreeRoot(a1.Data)
-	r2, _ := ssz.HashTreeRoot(a2.Data)
-
-	if err := store.saveNewBlockAttestations(ctx, []*ethpb.Attestation{a1, a2}); err != nil {
-		t.Fatal(err)
-	}
-
-	saved, err := store.db.AttestationsByDataRoot(ctx, r1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual([]*ethpb.Attestation{a1}, saved) {
-		t.Error("did not retrieve saved attestation")
-	}
-
-	saved, err = store.db.AttestationsByDataRoot(ctx, r2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual([]*ethpb.Attestation{a2}, saved) {
-		t.Error("did not retrieve saved attestation")
-	}
-
-	a1 = &ethpb.Attestation{Data: &ethpb.AttestationData{}, AggregationBits: bitfield.Bitlist{0b111}, CustodyBits: bitfield.NewBitlist(2)}
-	a2 = &ethpb.Attestation{Data: &ethpb.AttestationData{BeaconBlockRoot: []byte{'A'}}, AggregationBits: bitfield.Bitlist{0b111}, CustodyBits: bitfield.NewBitlist(2)}
-
-	if err := store.saveNewBlockAttestations(ctx, []*ethpb.Attestation{a1, a2}); err != nil {
-		t.Fatal(err)
-	}
-
-	saved, err = store.db.AttestationsByDataRoot(ctx, r1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual([]*ethpb.Attestation{a1}, saved) {
-		t.Error("did not retrieve saved attestation")
-	}
-
-	saved, err = store.db.AttestationsByDataRoot(ctx, r2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual([]*ethpb.Attestation{a2}, saved) {
-		t.Error("did not retrieve saved attestation")
-	}
-}
-
-func TestRemoveStateSinceLastFinalized(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	params.UseMinimalConfig()
-	defer params.UseMainnetConfig()
-
-	store := NewForkChoiceService(ctx, db)
-
-	// Save 100 blocks in DB, each has a state.
-	numBlocks := 100
-	totalBlocks := make([]*ethpb.BeaconBlock, numBlocks)
-	blockRoots := make([][32]byte, 0)
-	for i := 0; i < len(totalBlocks); i++ {
-		totalBlocks[i] = &ethpb.BeaconBlock{
-			Slot: uint64(i),
-		}
-		r, err := ssz.SigningRoot(totalBlocks[i])
-		if err != nil {
-			t.Fatal(err)
-		}
-		if err := store.db.SaveState(ctx, &pb.BeaconState{Slot: uint64(i)}, r); err != nil {
-			t.Fatal(err)
-		}
-		if err := store.db.SaveBlock(ctx, totalBlocks[i]); err != nil {
-			t.Fatal(err)
-		}
-		blockRoots = append(blockRoots, r)
-	}
-
-	// New finalized epoch: 1
-	finalizedEpoch := uint64(1)
-	finalizedSlot := finalizedEpoch * params.BeaconConfig().SlotsPerEpoch
-	endSlot := helpers.StartSlot(finalizedEpoch+1) - 1 // Inclusive
-	if err := store.rmStatesOlderThanLastFinalized(ctx, 0, endSlot); err != nil {
-		t.Fatal(err)
-	}
-	for _, r := range blockRoots {
-		s, err := store.db.State(ctx, r)
-		if err != nil {
-			t.Fatal(err)
-		}
-		// Also verifies genesis state didnt get deleted
-		if s != nil && s.Slot != finalizedSlot && s.Slot != 0 && s.Slot < endSlot {
-			t.Errorf("State with slot %d should not be in DB", s.Slot)
-		}
-	}
-
-	// New finalized epoch: 5
-	newFinalizedEpoch := uint64(5)
-	newFinalizedSlot := newFinalizedEpoch * params.BeaconConfig().SlotsPerEpoch
-	endSlot = helpers.StartSlot(newFinalizedEpoch+1) - 1 // Inclusive
-	if err := store.rmStatesOlderThanLastFinalized(ctx, helpers.StartSlot(finalizedEpoch+1)-1, endSlot); err != nil {
-		t.Fatal(err)
-	}
-	for _, r := range blockRoots {
-		s, err := store.db.State(ctx, r)
-		if err != nil {
-			t.Fatal(err)
-		}
-		// Also verifies genesis state didnt get deleted
-		if s != nil && s.Slot != newFinalizedSlot && s.Slot != finalizedSlot && s.Slot != 0 && s.Slot < endSlot {
-			t.Errorf("State with slot %d should not be in DB", s.Slot)
-		}
-	}
-}

beacon-chain/blockchain/forkchoice/service.go

@@ -1,254 +0,0 @@
-package forkchoice
-
-import (
-	"bytes"
-	"context"
-	"sync"
-
-	"github.com/gogo/protobuf/proto"
-	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"go.opencensus.io/trace"
-)
-
-// ForkChoicer defines a common interface for methods useful for directly applying fork choice
-// to beacon blocks to compute head.
-type ForkChoicer interface {
-	Head(ctx context.Context) ([]byte, error)
-	OnBlock(ctx context.Context, b *ethpb.BeaconBlock) error
-	OnBlockNoVerifyStateTransition(ctx context.Context, b *ethpb.BeaconBlock) error
-	OnAttestation(ctx context.Context, a *ethpb.Attestation) error
-	GenesisStore(ctx context.Context, justifiedCheckpoint *ethpb.Checkpoint, finalizedCheckpoint *ethpb.Checkpoint) error
-	FinalizedCheckpt() *ethpb.Checkpoint
-}
-
-// Store represents a service struct that handles the forkchoice
-// logic of managing the full PoS beacon chain.
-type Store struct {
-	ctx                  context.Context
-	cancel               context.CancelFunc
-	db                   db.Database
-	justifiedCheckpt     *ethpb.Checkpoint
-	finalizedCheckpt     *ethpb.Checkpoint
-	prevFinalizedCheckpt *ethpb.Checkpoint
-	checkpointState      *cache.CheckpointStateCache
-	checkpointStateLock  sync.Mutex
-	seenAtts             map[[32]byte]bool
-	seenAttsLock         sync.Mutex
-}
-
-// NewForkChoiceService instantiates a new service instance that will
-// be registered into a running beacon node.
-func NewForkChoiceService(ctx context.Context, db db.Database) *Store {
-	ctx, cancel := context.WithCancel(ctx)
-	return &Store{
-		ctx:             ctx,
-		cancel:          cancel,
-		db:              db,
-		checkpointState: cache.NewCheckpointStateCache(),
-		seenAtts:        make(map[[32]byte]bool),
-	}
-}
-
-// GenesisStore initializes the store struct before beacon chain
-// starts to advance.
-//
-// Spec pseudocode definition:
-//   def get_genesis_store(genesis_state: BeaconState) -> Store:
-//    genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))
-//    root = signing_root(genesis_block)
-//    justified_checkpoint = Checkpoint(epoch=GENESIS_EPOCH, root=root)
-//    finalized_checkpoint = Checkpoint(epoch=GENESIS_EPOCH, root=root)
-//    return Store(
-//        time=genesis_state.genesis_time,
-//        justified_checkpoint=justified_checkpoint,
-//        finalized_checkpoint=finalized_checkpoint,
-//        blocks={root: genesis_block},
-//        block_states={root: genesis_state.copy()},
-//        checkpoint_states={justified_checkpoint: genesis_state.copy()},
-//    )
-func (s *Store) GenesisStore(
-	ctx context.Context,
-	justifiedCheckpoint *ethpb.Checkpoint,
-	finalizedCheckpoint *ethpb.Checkpoint) error {
-
-	s.justifiedCheckpt = proto.Clone(justifiedCheckpoint).(*ethpb.Checkpoint)
-	s.finalizedCheckpt = proto.Clone(finalizedCheckpoint).(*ethpb.Checkpoint)
-	s.prevFinalizedCheckpt = proto.Clone(finalizedCheckpoint).(*ethpb.Checkpoint)
-
-	justifiedState, err := s.db.State(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root))
-	if err != nil {
-		return errors.Wrap(err, "could not retrieve last justified state")
-	}
-
-	if err := s.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: s.justifiedCheckpt,
-		State:      justifiedState,
-	}); err != nil {
-		return errors.Wrap(err, "could not save genesis state in check point cache")
-	}
-
-	return nil
-}
-
-// ancestor returns the block root of an ancestry block from the input block root.
-//
-// Spec pseudocode definition:
-//   def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash:
-//    block = store.blocks[root]
-//    if block.slot > slot:
-//      return get_ancestor(store, block.parent_root, slot)
-//    elif block.slot == slot:
-//      return root
-//    else:
-//      return Bytes32()  # root is older than queried slot: no results.
-func (s *Store) ancestor(ctx context.Context, root []byte, slot uint64) ([]byte, error) {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.ancestor")
-	defer span.End()
-
-	b, err := s.db.Block(ctx, bytesutil.ToBytes32(root))
-	if err != nil {
-		return nil, errors.Wrap(err, "could not get ancestor block")
-	}
-
-	// If we dont have the ancestor in the DB, simply return nil so rest of fork choice
-	// operation can proceed. This is not an error condition.
-	if b == nil || b.Slot < slot {
-		return nil, nil
-	}
-
-	if b.Slot == slot {
-		return root, nil
-	}
-
-	return s.ancestor(ctx, b.ParentRoot, slot)
-}
-
-// latestAttestingBalance returns the staked balance of a block from the input block root.
-//
-// Spec pseudocode definition:
-//   def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei:
-//    state = store.checkpoint_states[store.justified_checkpoint]
-//    active_indices = get_active_validator_indices(state, get_current_epoch(state))
-//    return Gwei(sum(
-//        state.validators[i].effective_balance for i in active_indices
-//        if (i in store.latest_messages
-//            and get_ancestor(store, store.latest_messages[i].root, store.blocks[root].slot) == root)
-//    ))
-func (s *Store) latestAttestingBalance(ctx context.Context, root []byte) (uint64, error) {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.latestAttestingBalance")
-	defer span.End()
-
-	lastJustifiedState, err := s.checkpointState.StateByCheckpoint(s.JustifiedCheckpt())
-	if err != nil {
-		return 0, errors.Wrap(err, "could not retrieve cached state via last justified check point")
-	}
-	if lastJustifiedState == nil {
-		return 0, errors.Wrapf(err, "could not get justified state at epoch %d", s.JustifiedCheckpt().Epoch)
-	}
-
-	lastJustifiedEpoch := helpers.CurrentEpoch(lastJustifiedState)
-	activeIndices, err := helpers.ActiveValidatorIndices(lastJustifiedState, lastJustifiedEpoch)
-	if err != nil {
-		return 0, errors.Wrap(err, "could not get active indices for last justified checkpoint")
-	}
-
-	wantedBlk, err := s.db.Block(ctx, bytesutil.ToBytes32(root))
-	if err != nil {
-		return 0, errors.Wrap(err, "could not get target block")
-	}
-
-	balances := uint64(0)
-	for _, i := range activeIndices {
-		vote, err := s.db.ValidatorLatestVote(ctx, i)
-		if err != nil {
-			return 0, errors.Wrapf(err, "could not get validator %d's latest vote", i)
-		}
-		if vote == nil {
-			continue
-		}
-
-		wantedRoot, err := s.ancestor(ctx, vote.Root, wantedBlk.Slot)
-		if err != nil {
-			return 0, errors.Wrapf(err, "could not get ancestor root for slot %d", wantedBlk.Slot)
-		}
-		if bytes.Equal(wantedRoot, root) {
-			balances += lastJustifiedState.Validators[i].EffectiveBalance
-		}
-	}
-	return balances, nil
-}
-
-// Head returns the head of the beacon chain.
-//
-// Spec pseudocode definition:
-//   def get_head(store: Store) -> Hash:
-//    # Execute the LMD-GHOST fork choice
-//    head = store.justified_checkpoint.root
-//    justified_slot = compute_start_slot_of_epoch(store.justified_checkpoint.epoch)
-//    while True:
-//        children = [
-//            root for root in store.blocks.keys()
-//            if store.blocks[root].parent_root == head and store.blocks[root].slot > justified_slot
-//        ]
-//        if len(children) == 0:
-//            return head
-//        # Sort by latest attesting balance with ties broken lexicographically
-//        head = max(children, key=lambda root: (get_latest_attesting_balance(store, root), root))
-func (s *Store) Head(ctx context.Context) ([]byte, error) {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.head")
-	defer span.End()
-
-	head := s.JustifiedCheckpt().Root
-
-	for {
-		startSlot := s.JustifiedCheckpt().Epoch * params.BeaconConfig().SlotsPerEpoch
-		filter := filters.NewFilter().SetParentRoot(head).SetStartSlot(startSlot)
-		children, err := s.db.BlockRoots(ctx, filter)
-		if err != nil {
-			return nil, errors.Wrap(err, "could not retrieve children info")
-		}
-
-		if len(children) == 0 {
-			return head, nil
-		}
-
-		// if a block has one child, then we don't have to lookup anything to
-		// know that this child will be the best child.
-		head = children[0][:]
-		if len(children) > 1 {
-			highest, err := s.latestAttestingBalance(ctx, head)
-			if err != nil {
-				return nil, errors.Wrap(err, "could not get latest balance")
-			}
-			for _, child := range children[1:] {
-				balance, err := s.latestAttestingBalance(ctx, child[:])
-				if err != nil {
-					return nil, errors.Wrap(err, "could not get latest balance")
-				}
-				// When there's a tie, it's broken lexicographically to favor the higher one.
-				if balance > highest ||
-					balance == highest && bytes.Compare(child[:], head) > 0 {
-					highest = balance
-					head = child[:]
-				}
-			}
-		}
-	}
-}
-
-// JustifiedCheckpt returns the latest justified check point from fork choice store.
-func (s *Store) JustifiedCheckpt() *ethpb.Checkpoint {
-	return proto.Clone(s.justifiedCheckpt).(*ethpb.Checkpoint)
-}
-
-// FinalizedCheckpt returns the latest finalized check point from fork choice store.
-func (s *Store) FinalizedCheckpt() *ethpb.Checkpoint {
-	return proto.Clone(s.finalizedCheckpt).(*ethpb.Checkpoint)
-}

beacon-chain/blockchain/forkchoice/service_test.go

@@ -1,346 +0,0 @@
-package forkchoice
-
-import (
-	"bytes"
-	"context"
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
-	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-)
-
-func TestStore_GenesisStoreOk(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	genesisTime := time.Unix(9999, 0)
-	genesisState := &pb.BeaconState{GenesisTime: uint64(genesisTime.Unix())}
-	genesisStateRoot, err := ssz.HashTreeRoot(genesisState)
-	if err != nil {
-		t.Fatal(err)
-	}
-	genesisBlk := blocks.NewGenesisBlock(genesisStateRoot[:])
-	genesisBlkRoot, err := ssz.SigningRoot(genesisBlk)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveState(ctx, genesisState, genesisBlkRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	checkPoint := &ethpb.Checkpoint{Root: genesisBlkRoot[:]}
-	if err := store.GenesisStore(ctx, checkPoint, checkPoint); err != nil {
-		t.Fatal(err)
-	}
-
-	if !reflect.DeepEqual(store.justifiedCheckpt, checkPoint) {
-		t.Error("Justified check point from genesis store did not match")
-	}
-	if !reflect.DeepEqual(store.finalizedCheckpt, checkPoint) {
-		t.Error("Finalized check point from genesis store did not match")
-	}
-
-	cachedState, err := store.checkpointState.StateByCheckpoint(checkPoint)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(cachedState, genesisState) {
-		t.Error("Incorrect genesis state cached")
-	}
-}
-
-func TestStore_AncestorOk(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db)
-	if err != nil {
-		t.Fatal(err)
-	}
-	type args struct {
-		root []byte
-		slot uint64
-	}
-
-	//    /- B1
-	// B0           /- B5 - B7
-	//    \- B3 - B4 - B6 - B8
-	tests := []struct {
-		args *args
-		want []byte
-	}{
-		{args: &args{roots[1], 0}, want: roots[0]},
-		{args: &args{roots[8], 0}, want: roots[0]},
-		{args: &args{roots[8], 4}, want: roots[4]},
-		{args: &args{roots[7], 4}, want: roots[4]},
-		{args: &args{roots[7], 0}, want: roots[0]},
-	}
-	for _, tt := range tests {
-		got, err := store.ancestor(ctx, tt.args.root, tt.args.slot)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if !reflect.DeepEqual(got, tt.want) {
-			t.Errorf("Store.ancestor(ctx, ) = %v, want %v", got, tt.want)
-		}
-	}
-}
-
-func TestStore_AncestorNotPartOfTheChain(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	//    /- B1
-	// B0           /- B5 - B7
-	//    \- B3 - B4 - B6 - B8
-	root, err := store.ancestor(ctx, roots[8], 1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if root != nil {
-		t.Error("block at slot 1 is not part of the chain")
-	}
-	root, err = store.ancestor(ctx, roots[8], 2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if root != nil {
-		t.Error("block at slot 2 is not part of the chain")
-	}
-}
-
-func TestStore_LatestAttestingBalance(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	validators := make([]*ethpb.Validator, 100)
-	for i := 0; i < len(validators); i++ {
-		validators[i] = &ethpb.Validator{ExitEpoch: 2, EffectiveBalance: 1e9}
-	}
-
-	s := &pb.BeaconState{Validators: validators}
-	stateRoot, err := ssz.HashTreeRoot(s)
-	if err != nil {
-		t.Fatal(err)
-	}
-	b := blocks.NewGenesisBlock(stateRoot[:])
-	blkRoot, err := ssz.SigningRoot(b)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveState(ctx, s, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	checkPoint := &ethpb.Checkpoint{Root: blkRoot[:]}
-	if err := store.GenesisStore(ctx, checkPoint, checkPoint); err != nil {
-		t.Fatal(err)
-	}
-
-	//    /- B1 (33 votes)
-	// B0           /- B5 - B7 (33 votes)
-	//    \- B3 - B4 - B6 - B8 (34 votes)
-	for i := 0; i < len(validators); i++ {
-		switch {
-		case i < 33:
-			if err := store.db.SaveValidatorLatestVote(ctx, uint64(i), &pb.ValidatorLatestVote{Root: roots[1]}); err != nil {
-				t.Fatal(err)
-			}
-		case i > 66:
-			if err := store.db.SaveValidatorLatestVote(ctx, uint64(i), &pb.ValidatorLatestVote{Root: roots[7]}); err != nil {
-				t.Fatal(err)
-			}
-		default:
-			if err := store.db.SaveValidatorLatestVote(ctx, uint64(i), &pb.ValidatorLatestVote{Root: roots[8]}); err != nil {
-				t.Fatal(err)
-			}
-		}
-	}
-
-	tests := []struct {
-		root []byte
-		want uint64
-	}{
-		{root: roots[0], want: 100 * 1e9},
-		{root: roots[1], want: 33 * 1e9},
-		{root: roots[3], want: 67 * 1e9},
-		{root: roots[4], want: 67 * 1e9},
-		{root: roots[7], want: 33 * 1e9},
-		{root: roots[8], want: 34 * 1e9},
-	}
-	for _, tt := range tests {
-		got, err := store.latestAttestingBalance(ctx, tt.root)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if got != tt.want {
-			t.Errorf("Store.latestAttestingBalance(ctx, ) = %v, want %v", got, tt.want)
-		}
-	}
-}
-
-func TestStore_ChildrenBlocksFromParentRoot(t *testing.T) {
-	helpers.ClearAllCaches()
-
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	filter := filters.NewFilter().SetParentRoot(roots[0]).SetStartSlot(0)
-	children, err := store.db.BlockRoots(ctx, filter)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(children, [][32]byte{bytesutil.ToBytes32(roots[1]), bytesutil.ToBytes32(roots[3])}) {
-		t.Error("Did not receive correct children roots")
-	}
-
-	filter = filters.NewFilter().SetParentRoot(roots[0]).SetStartSlot(2)
-	children, err = store.db.BlockRoots(ctx, filter)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(children, [][32]byte{bytesutil.ToBytes32(roots[3])}) {
-		t.Error("Did not receive correct children roots")
-	}
-}
-
-func TestStore_GetHead(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	validators := make([]*ethpb.Validator, 100)
-	for i := 0; i < len(validators); i++ {
-		validators[i] = &ethpb.Validator{ExitEpoch: 2, EffectiveBalance: 1e9}
-	}
-
-	s := &pb.BeaconState{Validators: validators}
-	stateRoot, err := ssz.HashTreeRoot(s)
-	if err != nil {
-		t.Fatal(err)
-	}
-	b := blocks.NewGenesisBlock(stateRoot[:])
-	blkRoot, err := ssz.SigningRoot(b)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	checkPoint := &ethpb.Checkpoint{Root: blkRoot[:]}
-
-	if err := store.GenesisStore(ctx, checkPoint, checkPoint); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveState(ctx, s, bytesutil.ToBytes32(roots[0])); err != nil {
-		t.Fatal(err)
-	}
-	store.justifiedCheckpt.Root = roots[0]
-	if err := store.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: store.justifiedCheckpt,
-		State:      s,
-	}); err != nil {
-		t.Fatal(err)
-	}
-
-	//    /- B1 (33 votes)
-	// B0           /- B5 - B7 (33 votes)
-	//    \- B3 - B4 - B6 - B8 (34 votes)
-	for i := 0; i < len(validators); i++ {
-		switch {
-		case i < 33:
-			if err := store.db.SaveValidatorLatestVote(ctx, uint64(i), &pb.ValidatorLatestVote{Root: roots[1]}); err != nil {
-				t.Fatal(err)
-			}
-		case i > 66:
-			if err := store.db.SaveValidatorLatestVote(ctx, uint64(i), &pb.ValidatorLatestVote{Root: roots[7]}); err != nil {
-				t.Fatal(err)
-			}
-		default:
-			if err := store.db.SaveValidatorLatestVote(ctx, uint64(i), &pb.ValidatorLatestVote{Root: roots[8]}); err != nil {
-				t.Fatal(err)
-			}
-		}
-	}
-
-	// Default head is B8
-	head, err := store.Head(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !bytes.Equal(head, roots[8]) {
-		t.Error("Incorrect head")
-	}
-
-	// 1 validator switches vote to B7 to gain 34%, enough to switch head
-	if err := store.db.SaveValidatorLatestVote(ctx, 50, &pb.ValidatorLatestVote{Root: roots[7]}); err != nil {
-		t.Fatal(err)
-	}
-	head, err = store.Head(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !bytes.Equal(head, roots[7]) {
-		t.Error("Incorrect head")
-	}
-
-	// 18 validators switches vote to B1 to gain 51%, enough to switch head
-	for i := 0; i < 18; i++ {
-		idx := 50 + uint64(i)
-		if err := store.db.SaveValidatorLatestVote(ctx, idx, &pb.ValidatorLatestVote{Root: roots[1]}); err != nil {
-			t.Fatal(err)
-		}
-	}
-	head, err = store.Head(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !bytes.Equal(head, roots[1]) {
-		t.Log(head)
-		t.Error("Incorrect head")
-	}
-}

beacon-chain/blockchain/forkchoice/tree_test.go

@@ -1,144 +0,0 @@
-package forkchoice
-
-import (
-	"context"
-
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-)
-
-// blockTree1 constructs the following tree:
-//    /- B1
-// B0           /- B5 - B7
-//    \- B3 - B4 - B6 - B8
-// (B1, and B3 are all from the same slots)
-func blockTree1(db db.Database) ([][]byte, error) {
-	b0 := &ethpb.BeaconBlock{Slot: 0, ParentRoot: []byte{'g'}}
-	r0, _ := ssz.SigningRoot(b0)
-	b1 := &ethpb.BeaconBlock{Slot: 1, ParentRoot: r0[:]}
-	r1, _ := ssz.SigningRoot(b1)
-	b3 := &ethpb.BeaconBlock{Slot: 3, ParentRoot: r0[:]}
-	r3, _ := ssz.SigningRoot(b3)
-	b4 := &ethpb.BeaconBlock{Slot: 4, ParentRoot: r3[:]}
-	r4, _ := ssz.SigningRoot(b4)
-	b5 := &ethpb.BeaconBlock{Slot: 5, ParentRoot: r4[:]}
-	r5, _ := ssz.SigningRoot(b5)
-	b6 := &ethpb.BeaconBlock{Slot: 6, ParentRoot: r4[:]}
-	r6, _ := ssz.SigningRoot(b6)
-	b7 := &ethpb.BeaconBlock{Slot: 7, ParentRoot: r5[:]}
-	r7, _ := ssz.SigningRoot(b7)
-	b8 := &ethpb.BeaconBlock{Slot: 8, ParentRoot: r6[:]}
-	r8, _ := ssz.SigningRoot(b8)
-	for _, b := range []*ethpb.BeaconBlock{b0, b1, b3, b4, b5, b6, b7, b8} {
-		if err := db.SaveBlock(context.Background(), b); err != nil {
-			return nil, err
-		}
-		if err := db.SaveState(context.Background(), &pb.BeaconState{}, bytesutil.ToBytes32(b.ParentRoot)); err != nil {
-			return nil, err
-		}
-	}
-	return [][]byte{r0[:], r1[:], nil, r3[:], r4[:], r5[:], r6[:], r7[:], r8[:]}, nil
-}
-
-// blockTree2 constructs the following tree:
-// Scenario graph: shorturl.at/loyP6
-//
-//digraph G {
-//    rankdir=LR;
-//    node [shape="none"];
-//
-//    subgraph blocks {
-//        rankdir=LR;
-//        node [shape="box"];
-//        a->b;
-//        a->c;
-//        b->d;
-//        b->e;
-//        c->f;
-//        c->g;
-//        d->h
-//        d->i
-//        d->j
-//        d->k
-//        h->l
-//        h->m
-//        g->n
-//        g->o
-//        e->p
-//    }
-//}
-func blockTree2(db db.Database) ([][]byte, error) {
-	b0 := &ethpb.BeaconBlock{Slot: 0, ParentRoot: []byte{'g'}}
-	r0, _ := ssz.SigningRoot(b0)
-	b1 := &ethpb.BeaconBlock{Slot: 1, ParentRoot: r0[:]}
-	r1, _ := ssz.SigningRoot(b1)
-	b2 := &ethpb.BeaconBlock{Slot: 2, ParentRoot: r0[:]}
-	r2, _ := ssz.SigningRoot(b2)
-	b3 := &ethpb.BeaconBlock{Slot: 3, ParentRoot: r1[:]}
-	r3, _ := ssz.SigningRoot(b3)
-	b4 := &ethpb.BeaconBlock{Slot: 4, ParentRoot: r1[:]}
-	r4, _ := ssz.SigningRoot(b4)
-	b5 := &ethpb.BeaconBlock{Slot: 5, ParentRoot: r2[:]}
-	r5, _ := ssz.SigningRoot(b5)
-	b6 := &ethpb.BeaconBlock{Slot: 6, ParentRoot: r2[:]}
-	r6, _ := ssz.SigningRoot(b6)
-	b7 := &ethpb.BeaconBlock{Slot: 7, ParentRoot: r3[:]}
-	r7, _ := ssz.SigningRoot(b7)
-	b8 := &ethpb.BeaconBlock{Slot: 8, ParentRoot: r3[:]}
-	r8, _ := ssz.SigningRoot(b8)
-	b9 := &ethpb.BeaconBlock{Slot: 9, ParentRoot: r3[:]}
-	r9, _ := ssz.SigningRoot(b9)
-	b10 := &ethpb.BeaconBlock{Slot: 10, ParentRoot: r3[:]}
-	r10, _ := ssz.SigningRoot(b10)
-	b11 := &ethpb.BeaconBlock{Slot: 11, ParentRoot: r4[:]}
-	r11, _ := ssz.SigningRoot(b11)
-	b12 := &ethpb.BeaconBlock{Slot: 12, ParentRoot: r6[:]}
-	r12, _ := ssz.SigningRoot(b12)
-	b13 := &ethpb.BeaconBlock{Slot: 13, ParentRoot: r6[:]}
-	r13, _ := ssz.SigningRoot(b13)
-	b14 := &ethpb.BeaconBlock{Slot: 14, ParentRoot: r7[:]}
-	r14, _ := ssz.SigningRoot(b14)
-	b15 := &ethpb.BeaconBlock{Slot: 15, ParentRoot: r7[:]}
-	r15, _ := ssz.SigningRoot(b15)
-	for _, b := range []*ethpb.BeaconBlock{b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15} {
-		if err := db.SaveBlock(context.Background(), b); err != nil {
-			return nil, err
-		}
-		if err := db.SaveState(context.Background(), &pb.BeaconState{}, bytesutil.ToBytes32(b.ParentRoot)); err != nil {
-			return nil, err
-		}
-	}
-	return [][]byte{r0[:], r1[:], r2[:], r3[:], r4[:], r5[:], r6[:], r7[:], r8[:], r9[:], r10[:], r11[:], r12[:], r13[:], r14[:], r15[:]}, nil
-}
-
-// blockTree3 constructs a tree that is 512 blocks in a row.
-// B0 - B1 - B2 - B3 - .... - B512
-func blockTree3(db db.Database) ([][]byte, error) {
-	blkCount := 512
-	roots := make([][]byte, 0, blkCount)
-	blks := make([]*ethpb.BeaconBlock, 0, blkCount)
-	b0 := &ethpb.BeaconBlock{Slot: 0, ParentRoot: []byte{'g'}}
-	r0, _ := ssz.SigningRoot(b0)
-	roots = append(roots, r0[:])
-	blks = append(blks, b0)
-
-	for i := 1; i < blkCount; i++ {
-		b := &ethpb.BeaconBlock{Slot: uint64(i), ParentRoot: roots[len(roots)-1]}
-		r, _ := ssz.SigningRoot(b)
-		roots = append(roots, r[:])
-		blks = append(blks, b)
-	}
-
-	for _, b := range blks {
-		if err := db.SaveBlock(context.Background(), b); err != nil {
-			return nil, err
-		}
-		if err := db.SaveState(context.Background(), &pb.BeaconState{}, bytesutil.ToBytes32(b.ParentRoot)); err != nil {
-			return nil, err
-		}
-	}
-	return roots, nil
-}

beacon-chain/blockchain/head.go

@@ -0,0 +1,300 @@
+package blockchain
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
+	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/sirupsen/logrus"
+	"go.opencensus.io/trace"
+)
+
+// This defines the current chain service's view of head.
+type head struct {
+	slot  uint64                   // current head slot.
+	root  [32]byte                 // current head root.
+	block *ethpb.SignedBeaconBlock // current head block.
+	state *stateTrie.BeaconState   // current head state.
+}
+
+// Determined the head from the fork choice service and saves its new data
+// (head root, head block, and head state) to the local service cache.
+func (s *Service) updateHead(ctx context.Context, balances []uint64) error {
+	ctx, span := trace.StartSpan(ctx, "blockChain.updateHead")
+	defer span.End()
+
+	// To get the proper head update, a node first checks its best justified
+	// can become justified. This is designed to prevent bounce attack and
+	// ensure head gets its best justified info.
+	if s.bestJustifiedCheckpt.Epoch > s.justifiedCheckpt.Epoch {
+		s.justifiedCheckpt = s.bestJustifiedCheckpt
+		if err := s.cacheJustifiedStateBalances(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root)); err != nil {
+			return err
+		}
+	}
+
+	// Get head from the fork choice service.
+	f := s.finalizedCheckpt
+	j := s.justifiedCheckpt
+	// To get head before the first justified epoch, the fork choice will start with genesis root
+	// instead of zero hashes.
+	headStartRoot := bytesutil.ToBytes32(j.Root)
+	if headStartRoot == params.BeaconConfig().ZeroHash {
+		headStartRoot = s.genesisRoot
+	}
+
+	// In order to process head, fork choice store requires justified info.
+	// If the fork choice store is missing justified block info, a node should
+	// re-initiate fork choice store using the latest justified info.
+	// This recovers a fatal condition and should not happen in run time.
+	if !s.forkChoiceStore.HasNode(headStartRoot) {
+		jb, err := s.beaconDB.Block(ctx, headStartRoot)
+		if err != nil {
+			return err
+		}
+		s.forkChoiceStore = protoarray.New(j.Epoch, f.Epoch, bytesutil.ToBytes32(f.Root))
+		if err := s.insertBlockToForkChoiceStore(ctx, jb.Block, headStartRoot, f, j); err != nil {
+			return err
+		}
+	}
+
+	headRoot, err := s.forkChoiceStore.Head(ctx, j.Epoch, headStartRoot, balances, f.Epoch)
+	if err != nil {
+		return err
+	}
+
+	// Save head to the local service cache.
+	return s.saveHead(ctx, headRoot)
+}
+
+// This saves head info to the local service cache, it also saves the
+// new head root to the DB.
+func (s *Service) saveHead(ctx context.Context, headRoot [32]byte) error {
+	ctx, span := trace.StartSpan(ctx, "blockChain.saveHead")
+	defer span.End()
+
+	// Do nothing if head hasn't changed.
+	r, err := s.HeadRoot(ctx)
+	if err != nil {
+		return err
+	}
+	if headRoot == bytesutil.ToBytes32(r) {
+		return nil
+	}
+
+	// If the head state is not available, just return nil.
+	// There's nothing to cache
+	if !s.stateGen.StateSummaryExists(ctx, headRoot) {
+		return nil
+	}
+
+	// Get the new head block from DB.
+	newHeadBlock, err := s.beaconDB.Block(ctx, headRoot)
+	if err != nil {
+		return err
+	}
+	if newHeadBlock == nil || newHeadBlock.Block == nil {
+		return errors.New("cannot save nil head block")
+	}
+
+	// Get the new head state from cached state or DB.
+	newHeadState, err := s.stateGen.StateByRoot(ctx, headRoot)
+	if err != nil {
+		return errors.Wrap(err, "could not retrieve head state in DB")
+	}
+	if newHeadState == nil {
+		return errors.New("cannot save nil head state")
+	}
+
+	// A chain re-org occurred, so we fire an event notifying the rest of the services.
+	headSlot := s.HeadSlot()
+	if bytesutil.ToBytes32(newHeadBlock.Block.ParentRoot) != bytesutil.ToBytes32(r) {
+		log.WithFields(logrus.Fields{
+			"newSlot": fmt.Sprintf("%d", newHeadBlock.Block.Slot),
+			"oldSlot": fmt.Sprintf("%d", headSlot),
+		}).Debug("Chain reorg occurred")
+		s.stateNotifier.StateFeed().Send(&feed.Event{
+			Type: statefeed.Reorg,
+			Data: &statefeed.ReorgData{
+				NewSlot: newHeadBlock.Block.Slot,
+				OldSlot: headSlot,
+			},
+		})
+
+		reorgCount.Inc()
+	}
+
+	// Cache the new head info.
+	s.setHead(headRoot, newHeadBlock, newHeadState)
+
+	// Save the new head root to DB.
+	if err := s.beaconDB.SaveHeadBlockRoot(ctx, headRoot); err != nil {
+		return errors.Wrap(err, "could not save head root in DB")
+	}
+
+	return nil
+}
+
+// This gets called to update canonical root mapping. It does not save head block
+// root in DB. With the inception of initial-sync-cache-state flag, it uses finalized
+// check point as anchors to resume sync therefore head is no longer needed to be saved on per slot basis.
+func (s *Service) saveHeadNoDB(ctx context.Context, b *ethpb.SignedBeaconBlock, r [32]byte, hs *stateTrie.BeaconState) error {
+	cachedHeadRoot, err := s.HeadRoot(ctx)
+	if err != nil {
+		return errors.Wrap(err, "could not get head root from cache")
+	}
+	if bytes.Equal(r[:], cachedHeadRoot) {
+		return nil
+	}
+
+	if b == nil || b.Block == nil {
+		return errors.New("cannot save nil head block")
+	}
+
+	s.setHeadInitialSync(r, stateTrie.CopySignedBeaconBlock(b), hs)
+	return nil
+}
+
+// This sets head view object which is used to track the head slot, root, block and state.
+func (s *Service) setHead(root [32]byte, block *ethpb.SignedBeaconBlock, state *stateTrie.BeaconState) {
+	s.headLock.Lock()
+	defer s.headLock.Unlock()
+
+	// This does a full copy of the block and state.
+	s.head = &head{
+		slot:  block.Block.Slot,
+		root:  root,
+		block: stateTrie.CopySignedBeaconBlock(block),
+		state: state.Copy(),
+	}
+}
+
+// This sets head view object which is used to track the head slot, root, block and state. The method
+// assumes that state being passed into the method will not be modified by any other alternate
+// caller which holds the state's reference.
+func (s *Service) setHeadInitialSync(root [32]byte, block *ethpb.SignedBeaconBlock, state *stateTrie.BeaconState) {
+	s.headLock.Lock()
+	defer s.headLock.Unlock()
+
+	// This does a full copy of the block only.
+	s.head = &head{
+		slot:  block.Block.Slot,
+		root:  root,
+		block: stateTrie.CopySignedBeaconBlock(block),
+		state: state,
+	}
+}
+
+// This returns the head slot.
+// This is a lock free version.
+func (s *Service) headSlot() uint64 {
+	return s.head.slot
+}
+
+// This returns the head root.
+// It does a full copy on head root for immutability.
+// This is a lock free version.
+func (s *Service) headRoot() [32]byte {
+	if s.head == nil {
+		return params.BeaconConfig().ZeroHash
+	}
+
+	return s.head.root
+}
+
+// This returns the head block.
+// It does a full copy on head block for immutability.
+// This is a lock free version.
+func (s *Service) headBlock() *ethpb.SignedBeaconBlock {
+	return stateTrie.CopySignedBeaconBlock(s.head.block)
+}
+
+// This returns the head state.
+// It does a full copy on head state for immutability.
+// This is a lock free version.
+func (s *Service) headState(ctx context.Context) *stateTrie.BeaconState {
+	ctx, span := trace.StartSpan(ctx, "blockChain.headState")
+	defer span.End()
+
+	return s.head.state.Copy()
+}
+
+// This returns the genesis validator root of the head state.
+// This is a lock free version.
+func (s *Service) headGenesisValidatorRoot() [32]byte {
+	return bytesutil.ToBytes32(s.head.state.GenesisValidatorRoot())
+}
+
+// HasHeadState returns true if head state exists.
+func (s *Service) HasHeadState() bool {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	return s.hasHeadState()
+}
+
+// Returns true if head state exists.
+// This is the lock free version.
+func (s *Service) hasHeadState() bool {
+	return s.head != nil && s.head.state != nil
+}
+
+// This caches justified state balances to be used for fork choice.
+func (s *Service) cacheJustifiedStateBalances(ctx context.Context, justifiedRoot [32]byte) error {
+	if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+		return err
+	}
+
+	s.clearInitSyncBlocks()
+
+	var justifiedState *stateTrie.BeaconState
+	var err error
+	if justifiedRoot == s.genesisRoot {
+		justifiedState, err = s.beaconDB.GenesisState(ctx)
+		if err != nil {
+			return err
+		}
+	} else {
+		justifiedState, err = s.stateGen.StateByRoot(ctx, justifiedRoot)
+		if err != nil {
+			return err
+		}
+	}
+	if justifiedState == nil {
+		return errors.New("justified state can't be nil")
+	}
+
+	epoch := helpers.CurrentEpoch(justifiedState)
+
+	justifiedBalances := make([]uint64, justifiedState.NumValidators())
+	if err := justifiedState.ReadFromEveryValidator(func(idx int, val *stateTrie.ReadOnlyValidator) error {
+		if helpers.IsActiveValidatorUsingTrie(val, epoch) {
+			justifiedBalances[idx] = val.EffectiveBalance()
+		} else {
+			justifiedBalances[idx] = 0
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	s.justifiedBalancesLock.Lock()
+	defer s.justifiedBalancesLock.Unlock()
+	s.justifiedBalances = justifiedBalances
+	return nil
+}
+
+func (s *Service) getJustifiedBalances() []uint64 {
+	s.justifiedBalancesLock.RLock()
+	defer s.justifiedBalancesLock.RUnlock()
+	return s.justifiedBalances
+}

beacon-chain/blockchain/head_test.go

@@ -0,0 +1,123 @@
+package blockchain
+
+import (
+	"bytes"
+	"context"
+	"testing"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+	logTest "github.com/sirupsen/logrus/hooks/test"
+)
+
+func TestSaveHead_Same(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db, sc)
+
+	r := [32]byte{'A'}
+	service.head = &head{slot: 0, root: r}
+
+	require.NoError(t, service.saveHead(context.Background(), r))
+	assert.Equal(t, uint64(0), service.headSlot(), "Head did not stay the same")
+	assert.Equal(t, r, service.headRoot(), "Head did not stay the same")
+}
+
+func TestSaveHead_Different(t *testing.T) {
+	ctx := context.Background()
+	db, sc := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db, sc)
+
+	oldRoot := [32]byte{'A'}
+	service.head = &head{slot: 0, root: oldRoot}
+
+	newHeadSignedBlock := testutil.NewBeaconBlock()
+	newHeadSignedBlock.Block.Slot = 1
+	newHeadBlock := newHeadSignedBlock.Block
+
+	require.NoError(t, service.beaconDB.SaveBlock(context.Background(), newHeadSignedBlock))
+	newRoot, err := newHeadBlock.HashTreeRoot()
+	require.NoError(t, err)
+	headState := testutil.NewBeaconState()
+	require.NoError(t, headState.SetSlot(1))
+	require.NoError(t, service.beaconDB.SaveStateSummary(context.Background(), &pb.StateSummary{Slot: 1, Root: newRoot[:]}))
+	require.NoError(t, service.beaconDB.SaveState(context.Background(), headState, newRoot))
+	require.NoError(t, service.saveHead(context.Background(), newRoot))
+
+	assert.Equal(t, uint64(1), service.HeadSlot(), "Head did not change")
+
+	cachedRoot, err := service.HeadRoot(context.Background())
+	require.NoError(t, err)
+	if !bytes.Equal(cachedRoot, newRoot[:]) {
+		t.Error("Head did not change")
+	}
+	assert.DeepEqual(t, newHeadSignedBlock, service.headBlock(), "Head did not change")
+	assert.DeepEqual(t, headState.CloneInnerState(), service.headState(ctx).CloneInnerState(), "Head did not change")
+}
+
+func TestSaveHead_Different_Reorg(t *testing.T) {
+	ctx := context.Background()
+	hook := logTest.NewGlobal()
+	db, sc := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db, sc)
+
+	oldRoot := [32]byte{'A'}
+	service.head = &head{slot: 0, root: oldRoot}
+
+	reorgChainParent := [32]byte{'B'}
+	newHeadSignedBlock := testutil.NewBeaconBlock()
+	newHeadSignedBlock.Block.Slot = 1
+	newHeadSignedBlock.Block.ParentRoot = reorgChainParent[:]
+	newHeadBlock := newHeadSignedBlock.Block
+
+	require.NoError(t, service.beaconDB.SaveBlock(context.Background(), newHeadSignedBlock))
+	newRoot, err := newHeadBlock.HashTreeRoot()
+	require.NoError(t, err)
+	headState := testutil.NewBeaconState()
+	require.NoError(t, headState.SetSlot(1))
+	require.NoError(t, service.beaconDB.SaveStateSummary(context.Background(), &pb.StateSummary{Slot: 1, Root: newRoot[:]}))
+	require.NoError(t, service.beaconDB.SaveState(context.Background(), headState, newRoot))
+	require.NoError(t, service.saveHead(context.Background(), newRoot))
+
+	assert.Equal(t, uint64(1), service.HeadSlot(), "Head did not change")
+
+	cachedRoot, err := service.HeadRoot(context.Background())
+	require.NoError(t, err)
+	if !bytes.Equal(cachedRoot, newRoot[:]) {
+		t.Error("Head did not change")
+	}
+	assert.DeepEqual(t, newHeadSignedBlock, service.headBlock(), "Head did not change")
+	assert.DeepEqual(t, headState.CloneInnerState(), service.headState(ctx).CloneInnerState(), "Head did not change")
+	require.LogsContain(t, hook, "Chain reorg occurred")
+}
+
+func TestCacheJustifiedStateBalances_CanCache(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db, sc)
+
+	state, _ := testutil.DeterministicGenesisState(t, 100)
+	r := [32]byte{'a'}
+	require.NoError(t, service.beaconDB.SaveStateSummary(context.Background(), &pb.StateSummary{Root: r[:]}))
+	require.NoError(t, service.beaconDB.SaveState(context.Background(), state, r))
+	require.NoError(t, service.cacheJustifiedStateBalances(context.Background(), r))
+	require.DeepEqual(t, service.getJustifiedBalances(), state.Balances(), "Incorrect justified balances")
+}
+
+func TestUpdateHead_MissingJustifiedRoot(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db, sc)
+
+	b := testutil.NewBeaconBlock()
+	require.NoError(t, service.beaconDB.SaveBlock(context.Background(), b))
+	r, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+
+	service.justifiedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	service.finalizedCheckpt = &ethpb.Checkpoint{}
+	service.bestJustifiedCheckpt = &ethpb.Checkpoint{}
+
+	require.NoError(t, service.updateHead(context.Background(), []uint64{}))
+}

beacon-chain/blockchain/info.go

@@ -1,57 +1,99 @@
 package blockchain
 
 import (
-	"bytes"
 	"encoding/hex"
 	"fmt"
 	"net/http"
-	"sort"
 
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/sirupsen/logrus"
+	"github.com/emicklei/dot"
+	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
-const latestSlotCount = 10
+const template = `<html>
+<head>
+    <script src="//cdnjs.cloudflare.com/ajax/libs/viz.js/2.1.2/viz.js"></script>
+    <script src="//cdnjs.cloudflare.com/ajax/libs/viz.js/2.1.2/full.render.js"></script>
+<body>
+    <script type="application/javascript">
+        var graph = ` + "`%s`;" + `
+        var viz = new Viz();
+        viz.renderSVGElement(graph) // reading the graph.
+            .then(function(element) {
+                document.body.appendChild(element); // appends to document.
+            })
+            .catch(error => {
+                // Create a new Viz instance (@see Caveats page for more info)
+                viz = new Viz();
+                // Possibly display the error
+                console.error(error);
+            });
+    </script>
+</head>
+</body>
+</html>`
 
-// HeadsHandler is a handler to serve /heads page in metrics.
-func (s *Service) HeadsHandler(w http.ResponseWriter, _ *http.Request) {
-	buf := new(bytes.Buffer)
-
-	if _, err := fmt.Fprintf(w, "\n %s\t%s\t", "Head slot", "Head root"); err != nil {
-		logrus.WithError(err).Error("Failed to render chain heads page")
+// TreeHandler is a handler to serve /tree page in metrics.
+func (s *Service) TreeHandler(w http.ResponseWriter, r *http.Request) {
+	headState, err := s.HeadState(r.Context())
+	if err != nil {
+		log.WithError(err).Error("Could not get head state")
 		return
 	}
-
-	if _, err := fmt.Fprintf(w, "\n %s\t%s\t", "---------", "---------"); err != nil {
-		logrus.WithError(err).Error("Failed to render chain heads page")
-		return
+	if headState == nil {
+		if _, err := w.Write([]byte("Unavailable during initial syncing")); err != nil {
+			log.WithError(err).Error("Failed to render p2p info page")
+		}
 	}
 
-	slots := s.latestHeadSlots()
-	for _, slot := range slots {
-		r := hex.EncodeToString(bytesutil.Trunc(s.canonicalRoots[uint64(slot)]))
-		if _, err := fmt.Fprintf(w, "\n %d\t\t%s\t", slot, r); err != nil {
-			logrus.WithError(err).Error("Failed to render chain heads page")
-			return
+	nodes := s.forkChoiceStore.Nodes()
+
+	graph := dot.NewGraph(dot.Directed)
+	graph.Attr("rankdir", "RL")
+	graph.Attr("labeljust", "l")
+
+	dotNodes := make([]*dot.Node, len(nodes))
+	avgBalance := uint64(averageBalance(headState.Balances()))
+
+	for i := len(nodes) - 1; i >= 0; i-- {
+		// Construct label for each node.
+		slot := fmt.Sprintf("%d", nodes[i].Slot())
+		weight := fmt.Sprintf("%d", nodes[i].Weight()/1e9) // Convert unit Gwei to unit ETH.
+		votes := fmt.Sprintf("%d", nodes[i].Weight()/1e9/avgBalance)
+		index := fmt.Sprintf("%d", i)
+		g := nodes[i].Graffiti()
+		graffiti := hex.EncodeToString(g[:8])
+		label := "slot: " + slot + "\n votes: " + votes + "\n weight: " + weight + "\n graffiti: " + graffiti
+		var dotN dot.Node
+		if nodes[i].Parent() != ^uint64(0) {
+			dotN = graph.Node(index).Box().Attr("label", label)
+		}
+
+		if nodes[i].Slot() == s.HeadSlot() &&
+			nodes[i].BestDescendant() == ^uint64(0) &&
+			nodes[i].Parent() != ^uint64(0) {
+			dotN = dotN.Attr("color", "green")
+		}
+
+		dotNodes[i] = &dotN
+	}
+
+	for i := len(nodes) - 1; i >= 0; i-- {
+		if nodes[i].Parent() != ^uint64(0) && nodes[i].Parent() < uint64(len(dotNodes)) {
+			graph.Edge(*dotNodes[i], *dotNodes[nodes[i].Parent()])
 		}
 	}
 
 	w.WriteHeader(http.StatusOK)
-	if _, err := w.Write(buf.Bytes()); err != nil {
-		log.WithError(err).Error("Failed to render chain heads page")
+	w.Header().Set("Content-Type", "text/html")
+	if _, err := fmt.Fprintf(w, template, graph.String()); err != nil {
+		log.WithError(err).Error("Failed to render p2p info page")
 	}
-
 }
 
-// This returns the latest head slots in a slice and up to latestSlotCount
-func (s *Service) latestHeadSlots() []int {
-	slots := make([]int, 0, len(s.canonicalRoots))
-	for k := range s.canonicalRoots {
-		slots = append(slots, int(k))
+func averageBalance(balances []uint64) float64 {
+	total := uint64(0)
+	for i := 0; i < len(balances); i++ {
+		total += balances[i]
 	}
-	sort.Ints(slots)
-	if (len(slots)) > latestSlotCount {
-		return slots[len(slots)-latestSlotCount:]
-	}
-	return slots
+	return float64(total) / float64(len(balances)) / float64(params.BeaconConfig().GweiPerEth)
 }

beacon-chain/blockchain/info_test.go

@@ -0,0 +1,47 @@
+package blockchain
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+)
+
+func TestService_TreeHandler(t *testing.T) {
+	req, err := http.NewRequest("GET", "/tree", nil)
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	db, sCache := testDB.SetupDB(t)
+	headState := testutil.NewBeaconState()
+	require.NoError(t, headState.SetBalances([]uint64{params.BeaconConfig().GweiPerEth}))
+	cfg := &Config{
+		BeaconDB: db,
+		ForkChoiceStore: protoarray.New(
+			0, // justifiedEpoch
+			0, // finalizedEpoch
+			[32]byte{'a'},
+		),
+		StateGen: stategen.New(db, sCache),
+	}
+	s, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+	require.NoError(t, s.forkChoiceStore.ProcessBlock(ctx, 0, [32]byte{'a'}, [32]byte{'g'}, [32]byte{'c'}, 0, 0))
+	require.NoError(t, s.forkChoiceStore.ProcessBlock(ctx, 1, [32]byte{'b'}, [32]byte{'a'}, [32]byte{'c'}, 0, 0))
+	s.setHead([32]byte{'a'}, testutil.NewBeaconBlock(), headState)
+
+	rr := httptest.NewRecorder()
+	handler := http.HandlerFunc(s.TreeHandler)
+
+	handler.ServeHTTP(rr, req)
+
+	assert.Equal(t, http.StatusOK, rr.Code)
+}

beacon-chain/blockchain/init_sync_process_block.go

@@ -0,0 +1,50 @@
+package blockchain
+
+import (
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+)
+
+// This saves a beacon block to the initial sync blocks cache.
+func (s *Service) saveInitSyncBlock(r [32]byte, b *ethpb.SignedBeaconBlock) {
+	s.initSyncBlocksLock.Lock()
+	defer s.initSyncBlocksLock.Unlock()
+	s.initSyncBlocks[r] = b
+}
+
+// This checks if a beacon block exists in the initial sync blocks cache using the root
+// of the block.
+func (s *Service) hasInitSyncBlock(r [32]byte) bool {
+	s.initSyncBlocksLock.RLock()
+	defer s.initSyncBlocksLock.RUnlock()
+	_, ok := s.initSyncBlocks[r]
+	return ok
+}
+
+// This retrieves a beacon block from the initial sync blocks cache using the root of
+// the block.
+func (s *Service) getInitSyncBlock(r [32]byte) *ethpb.SignedBeaconBlock {
+	s.initSyncBlocksLock.RLock()
+	defer s.initSyncBlocksLock.RUnlock()
+	b := s.initSyncBlocks[r]
+	return b
+}
+
+// This retrieves all the beacon blocks from the initial sync blocks cache, the returned
+// blocks are unordered.
+func (s *Service) getInitSyncBlocks() []*ethpb.SignedBeaconBlock {
+	s.initSyncBlocksLock.RLock()
+	defer s.initSyncBlocksLock.RUnlock()
+
+	blks := make([]*ethpb.SignedBeaconBlock, 0, len(s.initSyncBlocks))
+	for _, b := range s.initSyncBlocks {
+		blks = append(blks, b)
+	}
+	return blks
+}
+
+// This clears out the initial sync blocks cache.
+func (s *Service) clearInitSyncBlocks() {
+	s.initSyncBlocksLock.Lock()
+	defer s.initSyncBlocksLock.Unlock()
+	s.initSyncBlocks = make(map[[32]byte]*ethpb.SignedBeaconBlock)
+}

beacon-chain/blockchain/log.go

@@ -1,17 +1,35 @@
 package blockchain
 
 import (
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
+	"encoding/hex"
+	"fmt"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/sirupsen/logrus"
 )
 
 var log = logrus.WithField("prefix", "blockchain")
 
 // logs state transition related data every slot.
-func logStateTransitionData(b *ethpb.BeaconBlock, r []byte) {
+func logStateTransitionData(b *ethpb.BeaconBlock) {
 	log.WithFields(logrus.Fields{
-		"slot":         b.Slot,
-		"attestations": len(b.Body.Attestations),
-		"deposits":     len(b.Body.Deposits),
+		"attestations":      len(b.Body.Attestations),
+		"deposits":          len(b.Body.Deposits),
+		"attesterSlashings": len(b.Body.AttesterSlashings),
+		"proposerSlashings": len(b.Body.ProposerSlashings),
+		"voluntaryExits":    len(b.Body.VoluntaryExits),
 	}).Info("Finished applying state transition")
 }
+
+func logBlockSyncStatus(block *ethpb.BeaconBlock, blockRoot [32]byte, finalized *ethpb.Checkpoint) {
+	log.WithFields(logrus.Fields{
+		"slot":           block.Slot,
+		"slotInEpoch":    block.Slot % params.BeaconConfig().SlotsPerEpoch,
+		"block":          fmt.Sprintf("0x%s...", hex.EncodeToString(blockRoot[:])[:8]),
+		"epoch":          helpers.SlotToEpoch(block.Slot),
+		"finalizedEpoch": finalized.Epoch,
+		"finalizedRoot":  fmt.Sprintf("0x%s...", hex.EncodeToString(finalized.Root)[:8]),
+	}).Info("Synced new block")
+}

beacon-chain/blockchain/metrics.go

@@ -1,9 +1,15 @@
 package blockchain
 
 import (
+	"context"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
 var (
@@ -15,42 +21,209 @@ var (
 		Name: "beacon_head_slot",
 		Help: "Slot of the head block of the beacon chain",
 	})
-	beaconHeadRoot = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_head_root",
-		Help: "Root of the head block of the beacon chain, it returns the lowest 8 bytes interpreted as little endian",
+	clockTimeSlot = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_clock_time_slot",
+		Help: "The current slot based on the genesis time and current clock",
 	})
-	competingAtts = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "competing_attestations",
-		Help: "The # of attestations received and processed from a competing chain",
+
+	headFinalizedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "head_finalized_epoch",
+		Help: "Last finalized epoch of the head state",
 	})
-	competingBlks = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "competing_blocks",
-		Help: "The # of blocks received and processed from a competing chain",
+	headFinalizedRoot = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "head_finalized_root",
+		Help: "Last finalized root of the head state",
 	})
-	processedBlkNoPubsub = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "processed_no_pubsub_block_counter",
-		Help: "The # of processed block without pubsub, this usually means the blocks from sync",
+	beaconFinalizedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_finalized_epoch",
+		Help: "Last finalized epoch of the processed state",
 	})
-	processedBlkNoPubsubForkchoice = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "processed_no_pubsub_forkchoice_block_counter",
-		Help: "The # of processed block without pubsub and forkchoice, this means indicate blocks from initial sync",
+	beaconFinalizedRoot = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_finalized_root",
+		Help: "Last finalized root of the processed state",
 	})
-	processedBlk = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "processed_block_counter",
-		Help: "The # of total processed in block chain service, with fork choice and pubsub",
+	beaconCurrentJustifiedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_current_justified_epoch",
+		Help: "Current justified epoch of the processed state",
 	})
-	processedAttNoPubsub = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "processed_no_pubsub_attestation_counter",
-		Help: "The # of processed attestation without pubsub, this usually means the attestations from sync",
+	beaconCurrentJustifiedRoot = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_current_justified_root",
+		Help: "Current justified root of the processed state",
 	})
-	processedAtt = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "processed_attestation_counter",
-		Help: "The # of processed attestation with pubsub and fork choice, this ususally means attestations from rpc",
+	beaconPrevJustifiedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_previous_justified_epoch",
+		Help: "Previous justified epoch of the processed state",
 	})
+	beaconPrevJustifiedRoot = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_previous_justified_root",
+		Help: "Previous justified root of the processed state",
+	})
+	validatorsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "validator_count",
+		Help: "The total number of validators",
+	}, []string{"state"})
+	validatorsBalance = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "validators_total_balance",
+		Help: "The total balance of validators, in GWei",
+	}, []string{"state"})
+	validatorsEffectiveBalance = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "validators_total_effective_balance",
+		Help: "The total effective balance of validators, in GWei",
+	}, []string{"state"})
+	currentEth1DataDepositCount = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "current_eth1_data_deposit_count",
+		Help: "The current eth1 deposit count in the last processed state eth1data field.",
+	})
+	stateTrieReferences = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "field_references",
+		Help: "The number of states a particular field is shared with.",
+	}, []string{"state"})
+	prevEpochActiveBalances = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_prev_epoch_active_gwei",
+		Help: "The total amount of ether, in gwei, that was active for voting of previous epoch",
+	})
+	prevEpochSourceBalances = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_prev_epoch_source_gwei",
+		Help: "The total amount of ether, in gwei, that has been used in voting attestation source of previous epoch",
+	})
+	prevEpochTargetBalances = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_prev_epoch_target_gwei",
+		Help: "The total amount of ether, in gwei, that has been used in voting attestation target of previous epoch",
+	})
+	prevEpochHeadBalances = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_prev_epoch_head_gwei",
+		Help: "The total amount of ether, in gwei, that has been used in voting attestation head of previous epoch",
+	})
+	reorgCount = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "beacon_reorg_total",
+		Help: "Count the number of times beacon chain has a reorg",
+	})
+	attestationInclusionDelay = promauto.NewHistogram(
+		prometheus.HistogramOpts{
+			Name:    "attestation_inclusion_delay_slots",
+			Help:    "The number of slots between att.Slot and block.Slot",
+			Buckets: []float64{1, 2, 3, 4, 6, 32, 64},
+		},
+	)
 )
 
-func (s *Service) reportSlotMetrics(currentSlot uint64) {
-	beaconSlot.Set(float64(currentSlot))
-	beaconHeadSlot.Set(float64(s.HeadSlot()))
-	beaconHeadRoot.Set(float64(bytesutil.ToLowInt64(s.HeadRoot())))
+// reportSlotMetrics reports slot related metrics.
+func reportSlotMetrics(stateSlot, headSlot, clockSlot uint64, finalizedCheckpoint *ethpb.Checkpoint) {
+	clockTimeSlot.Set(float64(clockSlot))
+	beaconSlot.Set(float64(stateSlot))
+	beaconHeadSlot.Set(float64(headSlot))
+	if finalizedCheckpoint != nil {
+		headFinalizedEpoch.Set(float64(finalizedCheckpoint.Epoch))
+		headFinalizedRoot.Set(float64(bytesutil.ToLowInt64(finalizedCheckpoint.Root)))
+	}
+}
+
+// reportEpochMetrics reports epoch related metrics.
+func reportEpochMetrics(ctx context.Context, postState, headState *stateTrie.BeaconState) error {
+	currentEpoch := postState.Slot() / params.BeaconConfig().SlotsPerEpoch
+
+	// Validator instances
+	pendingInstances := 0
+	activeInstances := 0
+	slashingInstances := 0
+	slashedInstances := 0
+	exitingInstances := 0
+	exitedInstances := 0
+	// Validator balances
+	pendingBalance := uint64(0)
+	activeBalance := uint64(0)
+	activeEffectiveBalance := uint64(0)
+	exitingBalance := uint64(0)
+	exitingEffectiveBalance := uint64(0)
+	slashingBalance := uint64(0)
+	slashingEffectiveBalance := uint64(0)
+
+	for i, validator := range postState.Validators() {
+		bal, err := postState.BalanceAtIndex(uint64(i))
+		if err != nil {
+			log.Errorf("Could not load validator balance: %v", err)
+			continue
+		}
+		if validator.Slashed {
+			if currentEpoch < validator.ExitEpoch {
+				slashingInstances++
+				slashingBalance += bal
+				slashingEffectiveBalance += validator.EffectiveBalance
+			} else {
+				slashedInstances++
+			}
+			continue
+		}
+		if validator.ExitEpoch != params.BeaconConfig().FarFutureEpoch {
+			if currentEpoch < validator.ExitEpoch {
+				exitingInstances++
+				exitingBalance += bal
+				exitingEffectiveBalance += validator.EffectiveBalance
+			} else {
+				exitedInstances++
+			}
+			continue
+		}
+		if currentEpoch < validator.ActivationEpoch {
+			pendingInstances++
+			pendingBalance += bal
+			continue
+		}
+		activeInstances++
+		activeBalance += bal
+		activeEffectiveBalance += validator.EffectiveBalance
+	}
+	validatorsCount.WithLabelValues("Pending").Set(float64(pendingInstances))
+	validatorsCount.WithLabelValues("Active").Set(float64(activeInstances))
+	validatorsCount.WithLabelValues("Exiting").Set(float64(exitingInstances))
+	validatorsCount.WithLabelValues("Exited").Set(float64(exitedInstances))
+	validatorsCount.WithLabelValues("Slashing").Set(float64(slashingInstances))
+	validatorsCount.WithLabelValues("Slashed").Set(float64(slashedInstances))
+	validatorsBalance.WithLabelValues("Pending").Set(float64(pendingBalance))
+	validatorsBalance.WithLabelValues("Active").Set(float64(activeBalance))
+	validatorsBalance.WithLabelValues("Exiting").Set(float64(exitingBalance))
+	validatorsBalance.WithLabelValues("Slashing").Set(float64(slashingBalance))
+	validatorsEffectiveBalance.WithLabelValues("Active").Set(float64(activeEffectiveBalance))
+	validatorsEffectiveBalance.WithLabelValues("Exiting").Set(float64(exitingEffectiveBalance))
+	validatorsEffectiveBalance.WithLabelValues("Slashing").Set(float64(slashingEffectiveBalance))
+
+	// Last justified slot
+	beaconCurrentJustifiedEpoch.Set(float64(postState.CurrentJustifiedCheckpoint().Epoch))
+	beaconCurrentJustifiedRoot.Set(float64(bytesutil.ToLowInt64(postState.CurrentJustifiedCheckpoint().Root)))
+
+	// Last previous justified slot
+	beaconPrevJustifiedEpoch.Set(float64(postState.PreviousJustifiedCheckpoint().Epoch))
+	beaconPrevJustifiedRoot.Set(float64(bytesutil.ToLowInt64(postState.PreviousJustifiedCheckpoint().Root)))
+
+	// Last finalized slot
+	beaconFinalizedEpoch.Set(float64(postState.FinalizedCheckpointEpoch()))
+	beaconFinalizedRoot.Set(float64(bytesutil.ToLowInt64(postState.FinalizedCheckpoint().Root)))
+	currentEth1DataDepositCount.Set(float64(postState.Eth1Data().DepositCount))
+
+	// Validator participation should be viewed on the canonical chain.
+	v, b, err := precompute.New(ctx, headState)
+	if err != nil {
+		return err
+	}
+	_, b, err = precompute.ProcessAttestations(ctx, headState, v, b)
+	if err != nil {
+		return err
+	}
+	prevEpochActiveBalances.Set(float64(b.ActivePrevEpoch))
+	prevEpochSourceBalances.Set(float64(b.PrevEpochAttested))
+	prevEpochTargetBalances.Set(float64(b.PrevEpochTargetAttested))
+	prevEpochHeadBalances.Set(float64(b.PrevEpochHeadAttested))
+
+	refMap := postState.FieldReferencesCount()
+	for name, val := range refMap {
+		stateTrieReferences.WithLabelValues(name).Set(float64(val))
+	}
+
+	return nil
+}
+
+func reportAttestationInclusion(blk *ethpb.BeaconBlock) {
+	for _, att := range blk.Body.Attestations {
+		attestationInclusionDelay.Observe(float64(blk.Slot - att.Data.Slot))
+	}
 }

beacon-chain/blockchain/metrics_test.go

@@ -0,0 +1,26 @@
+package blockchain
+
+import (
+	"context"
+	"testing"
+
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+)
+
+func TestReportEpochMetrics_BadHeadState(t *testing.T) {
+	s := testutil.NewBeaconState()
+	h := testutil.NewBeaconState()
+	require.NoError(t, h.SetValidators(nil))
+	err := reportEpochMetrics(context.Background(), s, h)
+	require.ErrorContains(t, "failed to initialize precompute: nil validators in state", err)
+}
+
+func TestReportEpochMetrics_BadAttestation(t *testing.T) {
+	s := testutil.NewBeaconState()
+	h := testutil.NewBeaconState()
+	require.NoError(t, h.SetCurrentEpochAttestations([]*pb.PendingAttestation{{InclusionDelay: 0}}))
+	err := reportEpochMetrics(context.Background(), s, h)
+	require.ErrorContains(t, "attestation with inclusion delay of 0", err)
+}

beacon-chain/blockchain/process_attestation.go

@@ -0,0 +1,162 @@
+package blockchain
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/attestationutil"
+	"github.com/prysmaticlabs/prysm/shared/bls"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/timeutils"
+	"go.opencensus.io/trace"
+)
+
+// ErrTargetRootNotInDB returns when the target block root of an attestation cannot be found in the
+// beacon database.
+var ErrTargetRootNotInDB = errors.New("target root does not exist in db")
+
+// onAttestation is called whenever an attestation is received, verifies the attestation is valid and saves
+// it to the DB. As a stateless function, this does not hold nor delay attestation based on the spec descriptions.
+// The delay is handled by the caller in `processAttestation`.
+//
+// Spec pseudocode definition:
+//   def on_attestation(store: Store, attestation: Attestation) -> None:
+//    """
+//    Run ``on_attestation`` upon receiving a new ``attestation`` from either within a block or directly on the wire.
+//
+//    An ``attestation`` that is asserted as invalid may be valid at a later time,
+//    consider scheduling it for later processing in such case.
+//    """
+//    validate_on_attestation(store, attestation)
+//    store_target_checkpoint_state(store, attestation.data.target)
+//
+//    # Get state at the `target` to fully validate attestation
+//    target_state = store.checkpoint_states[attestation.data.target]
+//    indexed_attestation = get_indexed_attestation(target_state, attestation)
+//    assert is_valid_indexed_attestation(target_state, indexed_attestation)
+//
+//    # Update latest messages for attesting indices
+//    update_latest_messages(store, indexed_attestation.attesting_indices, attestation)
+// TODO(#6072): This code path is highly untested. Requires comprehensive tests and simpler refactoring.
+func (s *Service) onAttestation(ctx context.Context, a *ethpb.Attestation) ([]uint64, error) {
+	ctx, span := trace.StartSpan(ctx, "blockChain.onAttestation")
+	defer span.End()
+
+	if a == nil {
+		return nil, errors.New("nil attestation")
+	}
+	if a.Data == nil {
+		return nil, errors.New("nil attestation.Data field")
+	}
+	if a.Data.Target == nil {
+		return nil, errors.New("nil attestation.Data.Target field")
+	}
+
+	tgt := stateTrie.CopyCheckpoint(a.Data.Target)
+
+	if helpers.SlotToEpoch(a.Data.Slot) != a.Data.Target.Epoch {
+		return nil, fmt.Errorf("data slot is not in the same epoch as target %d != %d", helpers.SlotToEpoch(a.Data.Slot), a.Data.Target.Epoch)
+	}
+
+	// Verify beacon node has seen the target block before.
+	if !s.hasBlock(ctx, bytesutil.ToBytes32(tgt.Root)) {
+		return nil, ErrTargetRootNotInDB
+	}
+
+	if featureconfig.Get().UseCheckPointInfoCache {
+		c, err := s.AttestationCheckPtInfo(ctx, a)
+		if err != nil {
+			return nil, err
+		}
+		if err := s.verifyAttTargetEpoch(ctx, uint64(s.genesisTime.Unix()), uint64(timeutils.Now().Unix()), tgt); err != nil {
+			return nil, err
+		}
+		if err := s.verifyBeaconBlock(ctx, a.Data); err != nil {
+			return nil, errors.Wrap(err, "could not verify attestation beacon block")
+		}
+		if err := s.verifyLMDFFGConsistent(ctx, a.Data.Target.Epoch, a.Data.Target.Root, a.Data.BeaconBlockRoot); err != nil {
+			return nil, errors.Wrap(err, "could not verify attestation beacon block")
+		}
+		if err := helpers.VerifySlotTime(uint64(s.genesisTime.Unix()), a.Data.Slot+1, params.BeaconNetworkConfig().MaximumGossipClockDisparity); err != nil {
+			return nil, err
+		}
+		committee, err := helpers.BeaconCommittee(c.ActiveIndices, bytesutil.ToBytes32(c.Seed), a.Data.Slot, a.Data.CommitteeIndex)
+		if err != nil {
+			return nil, err
+		}
+		indexedAtt := attestationutil.ConvertToIndexed(ctx, a, committee)
+		if err := attestationutil.IsValidAttestationIndices(ctx, indexedAtt); err != nil {
+			return nil, err
+		}
+		domain, err := helpers.Domain(c.Fork, indexedAtt.Data.Target.Epoch, params.BeaconConfig().DomainBeaconAttester, c.GenesisRoot)
+		if err != nil {
+			return nil, err
+		}
+		indices := indexedAtt.AttestingIndices
+		var pubkeys []bls.PublicKey
+		for i := 0; i < len(indices); i++ {
+			pubkeyAtIdx := c.PubKeys[indices[i]]
+			pk, err := bls.PublicKeyFromBytes(pubkeyAtIdx)
+			if err != nil {
+				return nil, err
+			}
+			pubkeys = append(pubkeys, pk)
+		}
+		if err := attestationutil.VerifyIndexedAttestationSig(ctx, indexedAtt, pubkeys, domain); err != nil {
+			return nil, err
+		}
+		s.forkChoiceStore.ProcessAttestation(ctx, indexedAtt.AttestingIndices, bytesutil.ToBytes32(a.Data.BeaconBlockRoot), a.Data.Target.Epoch)
+
+		return indexedAtt.AttestingIndices, nil
+	}
+
+	// Retrieve attestation's data beacon block pre state. Advance pre state to latest epoch if necessary and
+	// save it to the cache.
+	baseState, err := s.getAttPreState(ctx, tgt)
+	if err != nil {
+		return nil, err
+	}
+
+	genesisTime := baseState.GenesisTime()
+
+	// Verify attestation target is from current epoch or previous epoch.
+	if err := s.verifyAttTargetEpoch(ctx, genesisTime, uint64(timeutils.Now().Unix()), tgt); err != nil {
+		return nil, err
+	}
+
+	// Verify attestation beacon block is known and not from the future.
+	if err := s.verifyBeaconBlock(ctx, a.Data); err != nil {
+		return nil, errors.Wrap(err, "could not verify attestation beacon block")
+	}
+
+	// Verify LMG GHOST and FFG votes are consistent with each other.
+	if err := s.verifyLMDFFGConsistent(ctx, a.Data.Target.Epoch, a.Data.Target.Root, a.Data.BeaconBlockRoot); err != nil {
+		return nil, errors.Wrap(err, "could not verify attestation beacon block")
+	}
+
+	// Verify attestations can only affect the fork choice of subsequent slots.
+	if err := helpers.VerifySlotTime(genesisTime, a.Data.Slot+1, params.BeaconNetworkConfig().MaximumGossipClockDisparity); err != nil {
+		return nil, err
+	}
+
+	// Use the target state to validate attestation and calculate the committees.
+	indexedAtt, err := s.verifyAttestation(ctx, baseState, a)
+	if err != nil {
+		return nil, err
+	}
+
+	if indexedAtt.AttestingIndices == nil {
+		return nil, errors.New("nil attesting indices")
+	}
+
+	// Update forkchoice store with the new attestation for updating weight.
+	s.forkChoiceStore.ProcessAttestation(ctx, indexedAtt.AttestingIndices, bytesutil.ToBytes32(a.Data.BeaconBlockRoot), a.Data.Target.Epoch)
+
+	return indexedAtt.AttestingIndices, nil
+}

beacon-chain/blockchain/process_attestation_helpers.go

@@ -0,0 +1,192 @@
+package blockchain
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/attestationutil"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+)
+
+// getAttPreState retrieves the att pre state by either from the cache or the DB.
+func (s *Service) getAttPreState(ctx context.Context, c *ethpb.Checkpoint) (*stateTrie.BeaconState, error) {
+	s.checkpointStateLock.Lock()
+	defer s.checkpointStateLock.Unlock()
+
+	cachedState, err := s.checkpointState.StateByCheckpoint(c)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get cached checkpoint state")
+	}
+	if cachedState != nil {
+		return cachedState, nil
+	}
+
+	baseState, err := s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(c.Root))
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not get pre state for epoch %d", c.Epoch)
+	}
+
+	epochStartSlot, err := helpers.StartSlot(c.Epoch)
+	if err != nil {
+		return nil, err
+	}
+	if epochStartSlot > baseState.Slot() {
+		baseState = baseState.Copy()
+		baseState, err = state.ProcessSlots(ctx, baseState, epochStartSlot)
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not process slots up to epoch %d", c.Epoch)
+		}
+		if err := s.checkpointState.AddCheckpointState(c, baseState); err != nil {
+			return nil, errors.Wrap(err, "could not saved checkpoint state to cache")
+		}
+		return baseState, nil
+	}
+
+	has, err := s.stateGen.HasState(ctx, bytesutil.ToBytes32(c.Root))
+	if err != nil {
+		return nil, err
+	}
+	if !has {
+		if err := s.checkpointState.AddCheckpointState(c, baseState); err != nil {
+			return nil, errors.Wrap(err, "could not saved checkpoint state to cache")
+		}
+	}
+	return baseState, nil
+
+}
+
+// getAttCheckPtInfo retrieves the check point info given a check point. Check point info enables the node
+// to efficiently verify attestation signature without using beacon state. This function utilizes
+// the checkpoint info cache and will update the check point info cache on miss.
+func (s *Service) getAttCheckPtInfo(ctx context.Context, c *ethpb.Checkpoint, e uint64) (*pb.CheckPtInfo, error) {
+	// Return checkpoint info if exists in cache.
+	info, err := s.checkPtInfoCache.get(c)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get cached checkpoint state")
+	}
+	if info != nil {
+		return info, nil
+	}
+
+	// Retrieve checkpoint state to compute checkpoint info.
+	baseState, err := s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(c.Root))
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not get pre state for epoch %d", c.Epoch)
+	}
+	epochStartSlot, err := helpers.StartSlot(c.Epoch)
+	if err != nil {
+		return nil, err
+	}
+	if epochStartSlot > baseState.Slot() {
+		baseState = baseState.Copy()
+		baseState, err = state.ProcessSlots(ctx, baseState, epochStartSlot)
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not process slots up to epoch %d", c.Epoch)
+		}
+	}
+	f := baseState.Fork()
+	g := bytesutil.ToBytes32(baseState.GenesisValidatorRoot())
+	seed, err := helpers.Seed(baseState, e, params.BeaconConfig().DomainBeaconAttester)
+	if err != nil {
+		return nil, err
+	}
+	indices, err := helpers.ActiveValidatorIndices(baseState, e)
+	if err != nil {
+		return nil, err
+	}
+	validators := baseState.ValidatorsReadOnly()
+	pks := make([][]byte, len(validators))
+	for i := 0; i < len(pks); i++ {
+		pk := validators[i].PublicKey()
+		pks[i] = pk[:]
+	}
+
+	// Cache and return the checkpoint info.
+	info = &pb.CheckPtInfo{
+		Fork:          f,
+		GenesisRoot:   g[:],
+		Seed:          seed[:],
+		ActiveIndices: indices,
+		PubKeys:       pks,
+	}
+	if err := s.checkPtInfoCache.put(c, info); err != nil {
+		return nil, err
+	}
+
+	return info, nil
+}
+
+// verifyAttTargetEpoch validates attestation is from the current or previous epoch.
+func (s *Service) verifyAttTargetEpoch(_ context.Context, genesisTime, nowTime uint64, c *ethpb.Checkpoint) error {
+	currentSlot := (nowTime - genesisTime) / params.BeaconConfig().SecondsPerSlot
+	currentEpoch := helpers.SlotToEpoch(currentSlot)
+	var prevEpoch uint64
+	// Prevents previous epoch under flow
+	if currentEpoch > 1 {
+		prevEpoch = currentEpoch - 1
+	}
+	if c.Epoch != prevEpoch && c.Epoch != currentEpoch {
+		return fmt.Errorf("target epoch %d does not match current epoch %d or prev epoch %d", c.Epoch, currentEpoch, prevEpoch)
+	}
+	return nil
+}
+
+// verifyBeaconBlock verifies beacon head block is known and not from the future.
+func (s *Service) verifyBeaconBlock(ctx context.Context, data *ethpb.AttestationData) error {
+	r := bytesutil.ToBytes32(data.BeaconBlockRoot)
+	b, err := s.beaconDB.Block(ctx, r)
+	if err != nil {
+		return err
+	}
+	// If the block does not exist in db, check again if block exists in initial sync block cache.
+	// This could happen as the node first syncs to head.
+	if b == nil && s.hasInitSyncBlock(r) {
+		b = s.getInitSyncBlock(r)
+	}
+	if b == nil || b.Block == nil {
+		return fmt.Errorf("beacon block %#x does not exist", bytesutil.Trunc(data.BeaconBlockRoot))
+	}
+	if b.Block.Slot > data.Slot {
+		return fmt.Errorf("could not process attestation for future block, block.Slot=%d > attestation.Data.Slot=%d", b.Block.Slot, data.Slot)
+	}
+	return nil
+}
+
+// verifyLMDFFGConsistent verifies LMD GHOST and FFG votes are consistent with each other.
+func (s *Service) verifyLMDFFGConsistent(ctx context.Context, ffgEpoch uint64, ffgRoot, lmdRoot []byte) error {
+	ffgSlot, err := helpers.StartSlot(ffgEpoch)
+	if err != nil {
+		return err
+	}
+	r, err := s.ancestor(ctx, lmdRoot, ffgSlot)
+	if err != nil {
+		return err
+	}
+	if !bytes.Equal(ffgRoot, r) {
+		return errors.New("FFG and LMD votes are not consistent")
+	}
+
+	return nil
+}
+
+// verifyAttestation validates input attestation is valid.
+func (s *Service) verifyAttestation(ctx context.Context, baseState *stateTrie.BeaconState, a *ethpb.Attestation) (*ethpb.IndexedAttestation, error) {
+	committee, err := helpers.BeaconCommitteeFromState(baseState, a.Data.Slot, a.Data.CommitteeIndex)
+	if err != nil {
+		return nil, err
+	}
+	indexedAtt := attestationutil.ConvertToIndexed(ctx, a, committee)
+	if err := blocks.VerifyIndexedAttestation(ctx, baseState, indexedAtt); err != nil {
+		return nil, errors.Wrap(err, "could not verify indexed attestation")
+	}
+	return indexedAtt, nil
+}

beacon-chain/blockchain/process_attestation_test.go

@@ -0,0 +1,620 @@
+package blockchain
+
+import (
+	"context"
+	"testing"
+
+	"github.com/gogo/protobuf/proto"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
+	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+)
+
+func TestStore_OnAttestation(t *testing.T) {
+	ctx := context.Background()
+	db, sc := testDB.SetupDB(t)
+
+	cfg := &Config{
+		BeaconDB:        db,
+		ForkChoiceStore: protoarray.New(0, 0, [32]byte{}),
+		StateGen:        stategen.New(db, sc),
+	}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	_, err = blockTree1(db, []byte{'g'})
+	require.NoError(t, err)
+
+	BlkWithOutState := testutil.NewBeaconBlock()
+	BlkWithOutState.Block.Slot = 0
+	require.NoError(t, db.SaveBlock(ctx, BlkWithOutState))
+	BlkWithOutStateRoot, err := BlkWithOutState.Block.HashTreeRoot()
+	require.NoError(t, err)
+
+	BlkWithStateBadAtt := testutil.NewBeaconBlock()
+	BlkWithStateBadAtt.Block.Slot = 1
+	require.NoError(t, db.SaveBlock(ctx, BlkWithStateBadAtt))
+	BlkWithStateBadAttRoot, err := BlkWithStateBadAtt.Block.HashTreeRoot()
+	require.NoError(t, err)
+
+	s := testutil.NewBeaconState()
+	require.NoError(t, s.SetSlot(100*params.BeaconConfig().SlotsPerEpoch))
+	require.NoError(t, service.beaconDB.SaveState(ctx, s, BlkWithStateBadAttRoot))
+
+	BlkWithValidState := testutil.NewBeaconBlock()
+	BlkWithValidState.Block.Slot = 2
+	require.NoError(t, db.SaveBlock(ctx, BlkWithValidState))
+
+	BlkWithValidStateRoot, err := BlkWithValidState.Block.HashTreeRoot()
+	require.NoError(t, err)
+	s = testutil.NewBeaconState()
+	err = s.SetFork(&pb.Fork{
+		Epoch:           0,
+		CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
+		PreviousVersion: params.BeaconConfig().GenesisForkVersion,
+	})
+	require.NoError(t, err)
+	require.NoError(t, service.beaconDB.SaveState(ctx, s, BlkWithValidStateRoot))
+
+	tests := []struct {
+		name      string
+		a         *ethpb.Attestation
+		wantedErr string
+	}{
+		{
+			name:      "attestation's data slot not aligned with target vote",
+			a:         &ethpb.Attestation{Data: &ethpb.AttestationData{Slot: params.BeaconConfig().SlotsPerEpoch, Target: &ethpb.Checkpoint{Root: make([]byte, 32)}}},
+			wantedErr: "data slot is not in the same epoch as target 1 != 0",
+		},
+		{
+			name:      "attestation's target root not in db",
+			a:         &ethpb.Attestation{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{Root: bytesutil.PadTo([]byte{'A'}, 32)}}},
+			wantedErr: "target root does not exist in db",
+		},
+		{
+			name:      "no pre state for attestations's target block",
+			a:         &ethpb.Attestation{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{Root: BlkWithOutStateRoot[:]}}},
+			wantedErr: "could not get pre state for epoch 0",
+		},
+		{
+			name: "process attestation doesn't match current epoch",
+			a: &ethpb.Attestation{Data: &ethpb.AttestationData{Slot: 100 * params.BeaconConfig().SlotsPerEpoch, Target: &ethpb.Checkpoint{Epoch: 100,
+				Root: BlkWithStateBadAttRoot[:]}}},
+			wantedErr: "target epoch 100 does not match current epoch",
+		},
+		{
+			name:      "process nil attestation",
+			a:         nil,
+			wantedErr: "nil attestation",
+		},
+		{
+			name:      "process nil field (a.Data) in attestation",
+			a:         &ethpb.Attestation{},
+			wantedErr: "nil attestation.Data field",
+		},
+		{
+			name: "process nil field (a.Target) in attestation",
+			a: &ethpb.Attestation{
+				Data: &ethpb.AttestationData{
+					BeaconBlockRoot: make([]byte, 32),
+					Target:          nil,
+					Source:          &ethpb.Checkpoint{Root: make([]byte, 32)},
+				},
+				AggregationBits: make([]byte, 1),
+				Signature:       make([]byte, 96),
+			},
+			wantedErr: "nil attestation.Data.Target field",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, err := service.onAttestation(ctx, tt.a)
+			if tt.wantedErr != "" {
+				assert.ErrorContains(t, tt.wantedErr, err)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestStore_OnAttestationUsingCheckptCache(t *testing.T) {
+	resetCfg := featureconfig.InitWithReset(&featureconfig.Flags{UseCheckPointInfoCache: true})
+	defer resetCfg()
+
+	ctx := context.Background()
+	db, sc := testDB.SetupDB(t)
+
+	cfg := &Config{
+		BeaconDB:        db,
+		ForkChoiceStore: protoarray.New(0, 0, [32]byte{}),
+		StateGen:        stategen.New(db, sc),
+	}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	_, err = blockTree1(db, []byte{'g'})
+	require.NoError(t, err)
+
+	BlkWithOutState := testutil.NewBeaconBlock()
+	BlkWithOutState.Block.Slot = 0
+	require.NoError(t, db.SaveBlock(ctx, BlkWithOutState))
+	BlkWithOutStateRoot, err := BlkWithOutState.Block.HashTreeRoot()
+	require.NoError(t, err)
+
+	BlkWithStateBadAtt := testutil.NewBeaconBlock()
+	BlkWithStateBadAtt.Block.Slot = 1
+	require.NoError(t, db.SaveBlock(ctx, BlkWithStateBadAtt))
+	BlkWithStateBadAttRoot, err := BlkWithStateBadAtt.Block.HashTreeRoot()
+	require.NoError(t, err)
+
+	s := testutil.NewBeaconState()
+	require.NoError(t, s.SetSlot(100*params.BeaconConfig().SlotsPerEpoch))
+	require.NoError(t, service.beaconDB.SaveState(ctx, s, BlkWithStateBadAttRoot))
+
+	BlkWithValidState := testutil.NewBeaconBlock()
+	BlkWithValidState.Block.Slot = 2
+	require.NoError(t, db.SaveBlock(ctx, BlkWithValidState))
+
+	BlkWithValidStateRoot, err := BlkWithValidState.Block.HashTreeRoot()
+	require.NoError(t, err)
+	s = testutil.NewBeaconState()
+	err = s.SetFork(&pb.Fork{
+		Epoch:           0,
+		CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
+		PreviousVersion: params.BeaconConfig().GenesisForkVersion,
+	})
+	require.NoError(t, err)
+	require.NoError(t, service.beaconDB.SaveState(ctx, s, BlkWithValidStateRoot))
+
+	tests := []struct {
+		name      string
+		a         *ethpb.Attestation
+		wantedErr string
+	}{
+		{
+			name:      "attestation's data slot not aligned with target vote",
+			a:         &ethpb.Attestation{Data: &ethpb.AttestationData{Slot: params.BeaconConfig().SlotsPerEpoch, Target: &ethpb.Checkpoint{Root: make([]byte, 32)}}},
+			wantedErr: "data slot is not in the same epoch as target 1 != 0",
+		},
+		{
+			name:      "attestation's target root not in db",
+			a:         &ethpb.Attestation{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{Root: bytesutil.PadTo([]byte{'A'}, 32)}}},
+			wantedErr: "target root does not exist in db",
+		},
+		{
+			name:      "no pre state for attestations's target block",
+			a:         &ethpb.Attestation{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{Root: BlkWithOutStateRoot[:]}}},
+			wantedErr: "could not get pre state for epoch 0",
+		},
+		{
+			name: "process attestation doesn't match current epoch",
+			a: &ethpb.Attestation{Data: &ethpb.AttestationData{Slot: 100 * params.BeaconConfig().SlotsPerEpoch, Target: &ethpb.Checkpoint{Epoch: 100,
+				Root: BlkWithStateBadAttRoot[:]}}},
+			wantedErr: "target epoch 100 does not match current epoch",
+		},
+		{
+			name:      "process nil attestation",
+			a:         nil,
+			wantedErr: "nil attestation",
+		},
+		{
+			name:      "process nil field (a.Data) in attestation",
+			a:         &ethpb.Attestation{},
+			wantedErr: "nil attestation.Data field",
+		},
+		{
+			name: "process nil field (a.Target) in attestation",
+			a: &ethpb.Attestation{
+				Data: &ethpb.AttestationData{
+					BeaconBlockRoot: make([]byte, 32),
+					Target:          nil,
+					Source:          &ethpb.Checkpoint{Root: make([]byte, 32)},
+				},
+				AggregationBits: make([]byte, 1),
+				Signature:       make([]byte, 96),
+			},
+			wantedErr: "nil attestation.Data.Target field",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, err := service.onAttestation(ctx, tt.a)
+			if tt.wantedErr != "" {
+				assert.ErrorContains(t, tt.wantedErr, err)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestStore_SaveCheckpointState(t *testing.T) {
+	ctx := context.Background()
+	db, sc := testDB.SetupDB(t)
+
+	cfg := &Config{
+		BeaconDB: db,
+		StateGen: stategen.New(db, sc),
+	}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	s := testutil.NewBeaconState()
+	err = s.SetFinalizedCheckpoint(&ethpb.Checkpoint{Root: bytesutil.PadTo([]byte{'A'}, 32)})
+	require.NoError(t, err)
+	val := &ethpb.Validator{
+		PublicKey:             bytesutil.PadTo([]byte("foo"), 48),
+		WithdrawalCredentials: bytesutil.PadTo([]byte("bar"), 32),
+	}
+	err = s.SetValidators([]*ethpb.Validator{val})
+	require.NoError(t, err)
+	err = s.SetBalances([]uint64{0})
+	require.NoError(t, err)
+	r := [32]byte{'g'}
+	require.NoError(t, service.beaconDB.SaveState(ctx, s, r))
+
+	service.justifiedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	service.bestJustifiedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	service.finalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	service.prevFinalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+
+	r = bytesutil.ToBytes32([]byte{'A'})
+	cp1 := &ethpb.Checkpoint{Epoch: 1, Root: bytesutil.PadTo([]byte{'A'}, 32)}
+	require.NoError(t, service.beaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'A'})))
+	require.NoError(t, service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Root: bytesutil.PadTo([]byte{'A'}, 32)}))
+
+	s1, err := service.getAttPreState(ctx, cp1)
+	require.NoError(t, err)
+	assert.Equal(t, 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot(), "Unexpected state slot")
+
+	cp2 := &ethpb.Checkpoint{Epoch: 2, Root: bytesutil.PadTo([]byte{'B'}, 32)}
+	require.NoError(t, service.beaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'B'})))
+	require.NoError(t, service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Root: bytesutil.PadTo([]byte{'B'}, 32)}))
+	s2, err := service.getAttPreState(ctx, cp2)
+	require.NoError(t, err)
+	assert.Equal(t, 2*params.BeaconConfig().SlotsPerEpoch, s2.Slot(), "Unexpected state slot")
+
+	s1, err = service.getAttPreState(ctx, cp1)
+	require.NoError(t, err)
+	assert.Equal(t, 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot(), "Unexpected state slot")
+
+	s1, err = service.checkpointState.StateByCheckpoint(cp1)
+	require.NoError(t, err)
+	assert.Equal(t, 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot(), "Unexpected state slot")
+
+	s2, err = service.checkpointState.StateByCheckpoint(cp2)
+	require.NoError(t, err)
+	assert.Equal(t, 2*params.BeaconConfig().SlotsPerEpoch, s2.Slot(), "Unexpected state slot")
+
+	require.NoError(t, s.SetSlot(params.BeaconConfig().SlotsPerEpoch+1))
+	service.justifiedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	service.bestJustifiedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	service.finalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	service.prevFinalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	cp3 := &ethpb.Checkpoint{Epoch: 1, Root: bytesutil.PadTo([]byte{'C'}, 32)}
+	require.NoError(t, service.beaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'C'})))
+	require.NoError(t, service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Root: bytesutil.PadTo([]byte{'C'}, 32)}))
+	s3, err := service.getAttPreState(ctx, cp3)
+	require.NoError(t, err)
+	assert.Equal(t, s.Slot(), s3.Slot(), "Unexpected state slot")
+}
+
+func TestStore_UpdateCheckpointState(t *testing.T) {
+	ctx := context.Background()
+	db, sc := testDB.SetupDB(t)
+
+	cfg := &Config{
+		BeaconDB: db,
+		StateGen: stategen.New(db, sc),
+	}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	epoch := uint64(1)
+	baseState, _ := testutil.DeterministicGenesisState(t, 1)
+	checkpoint := &ethpb.Checkpoint{Epoch: epoch, Root: bytesutil.PadTo([]byte("hi"), 32)}
+	require.NoError(t, service.beaconDB.SaveState(ctx, baseState, bytesutil.ToBytes32(checkpoint.Root)))
+	returned, err := service.getAttPreState(ctx, checkpoint)
+	require.NoError(t, err)
+	assert.Equal(t, returned.Slot(), checkpoint.Epoch*params.BeaconConfig().SlotsPerEpoch, "Incorrectly returned base state")
+
+	cached, err := service.checkpointState.StateByCheckpoint(checkpoint)
+	require.NoError(t, err)
+	assert.Equal(t, returned.Slot(), cached.Slot(), "State should have been cached")
+
+	epoch = uint64(2)
+	newCheckpoint := &ethpb.Checkpoint{Epoch: epoch, Root: bytesutil.PadTo([]byte("bye"), 32)}
+	require.NoError(t, service.beaconDB.SaveState(ctx, baseState, bytesutil.ToBytes32(newCheckpoint.Root)))
+	returned, err = service.getAttPreState(ctx, newCheckpoint)
+	require.NoError(t, err)
+	s, err := helpers.StartSlot(newCheckpoint.Epoch)
+	require.NoError(t, err)
+	baseState, err = state.ProcessSlots(ctx, baseState, s)
+	require.NoError(t, err)
+	assert.Equal(t, returned.Slot(), baseState.Slot(), "Incorrectly returned base state")
+
+	cached, err = service.checkpointState.StateByCheckpoint(newCheckpoint)
+	require.NoError(t, err)
+	if !proto.Equal(returned.InnerStateUnsafe(), cached.InnerStateUnsafe()) {
+		t.Error("Incorrectly cached base state")
+	}
+}
+
+func TestAttEpoch_MatchPrevEpoch(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	nowTime := params.BeaconConfig().SlotsPerEpoch * params.BeaconConfig().SecondsPerSlot
+	require.NoError(t, service.verifyAttTargetEpoch(ctx, 0, nowTime, &ethpb.Checkpoint{Root: make([]byte, 32)}))
+}
+
+func TestAttEpoch_MatchCurrentEpoch(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	nowTime := params.BeaconConfig().SlotsPerEpoch * params.BeaconConfig().SecondsPerSlot
+	require.NoError(t, service.verifyAttTargetEpoch(ctx, 0, nowTime, &ethpb.Checkpoint{Epoch: 1}))
+}
+
+func TestAttEpoch_NotMatch(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	nowTime := 2 * params.BeaconConfig().SlotsPerEpoch * params.BeaconConfig().SecondsPerSlot
+	err = service.verifyAttTargetEpoch(ctx, 0, nowTime, &ethpb.Checkpoint{Root: make([]byte, 32)})
+	assert.ErrorContains(t, "target epoch 0 does not match current epoch 2 or prev epoch 1", err)
+}
+
+func TestVerifyBeaconBlock_NoBlock(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	d := &ethpb.AttestationData{
+		BeaconBlockRoot: make([]byte, 32),
+		Target:          &ethpb.Checkpoint{Root: make([]byte, 32)},
+		Source:          &ethpb.Checkpoint{Root: make([]byte, 32)},
+	}
+	assert.ErrorContains(t, "beacon block 0x000000000000 does not exist", service.verifyBeaconBlock(ctx, d))
+}
+
+func TestVerifyBeaconBlock_futureBlock(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	b := testutil.NewBeaconBlock()
+	b.Block.Slot = 2
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, b))
+	r, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+	d := &ethpb.AttestationData{Slot: 1, BeaconBlockRoot: r[:]}
+
+	assert.ErrorContains(t, "could not process attestation for future block", service.verifyBeaconBlock(ctx, d))
+}
+
+func TestVerifyBeaconBlock_OK(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	b := testutil.NewBeaconBlock()
+	b.Block.Slot = 2
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, b))
+	r, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+	d := &ethpb.AttestationData{Slot: 2, BeaconBlockRoot: r[:]}
+
+	assert.NoError(t, service.verifyBeaconBlock(ctx, d), "Did not receive the wanted error")
+}
+
+func TestVerifyLMDFFGConsistent_NotOK(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db, ForkChoiceStore: protoarray.New(0, 0, [32]byte{})}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	b32 := testutil.NewBeaconBlock()
+	b32.Block.Slot = 32
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, b32))
+	r32, err := b32.Block.HashTreeRoot()
+	require.NoError(t, err)
+	b33 := testutil.NewBeaconBlock()
+	b33.Block.Slot = 33
+	b33.Block.ParentRoot = r32[:]
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, b33))
+	r33, err := b33.Block.HashTreeRoot()
+	require.NoError(t, err)
+
+	wanted := "FFG and LMD votes are not consistent"
+	assert.ErrorContains(t, wanted, service.verifyLMDFFGConsistent(context.Background(), 1, []byte{'a'}, r33[:]))
+}
+
+func TestVerifyLMDFFGConsistent_OK(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db, ForkChoiceStore: protoarray.New(0, 0, [32]byte{})}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	b32 := testutil.NewBeaconBlock()
+	b32.Block.Slot = 32
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, b32))
+	r32, err := b32.Block.HashTreeRoot()
+	require.NoError(t, err)
+	b33 := testutil.NewBeaconBlock()
+	b33.Block.Slot = 33
+	b33.Block.ParentRoot = r32[:]
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, b33))
+	r33, err := b33.Block.HashTreeRoot()
+	require.NoError(t, err)
+
+	err = service.verifyLMDFFGConsistent(context.Background(), 1, r32[:], r33[:])
+	assert.NoError(t, err, "Could not verify LMD and FFG votes to be consistent")
+}
+
+func TestVerifyFinalizedConsistency_InconsistentRoot(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db, ForkChoiceStore: protoarray.New(0, 0, [32]byte{})}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	b32 := testutil.NewBeaconBlock()
+	b32.Block.Slot = 32
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, b32))
+	r32, err := b32.Block.HashTreeRoot()
+	require.NoError(t, err)
+
+	service.finalizedCheckpt = &ethpb.Checkpoint{Epoch: 1}
+
+	b33 := testutil.NewBeaconBlock()
+	b33.Block.Slot = 33
+	b33.Block.ParentRoot = r32[:]
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, b33))
+	r33, err := b33.Block.HashTreeRoot()
+	require.NoError(t, err)
+
+	err = service.VerifyFinalizedConsistency(context.Background(), r33[:])
+	require.ErrorContains(t, "Root and finalized store are not consistent", err)
+}
+
+func TestVerifyFinalizedConsistency_OK(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db, ForkChoiceStore: protoarray.New(0, 0, [32]byte{})}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	b32 := testutil.NewBeaconBlock()
+	b32.Block.Slot = 32
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, b32))
+	r32, err := b32.Block.HashTreeRoot()
+	require.NoError(t, err)
+
+	service.finalizedCheckpt = &ethpb.Checkpoint{Epoch: 1, Root: r32[:]}
+
+	b33 := testutil.NewBeaconBlock()
+	b33.Block.Slot = 33
+	b33.Block.ParentRoot = r32[:]
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, b33))
+	r33, err := b33.Block.HashTreeRoot()
+	require.NoError(t, err)
+
+	err = service.VerifyFinalizedConsistency(context.Background(), r33[:])
+	require.NoError(t, err)
+}
+
+func TestVerifyFinalizedConsistency_IsCanonical(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db, ForkChoiceStore: protoarray.New(0, 0, [32]byte{})}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	b32 := testutil.NewBeaconBlock()
+	b32.Block.Slot = 32
+	r32, err := b32.Block.HashTreeRoot()
+	require.NoError(t, err)
+
+	service.finalizedCheckpt = &ethpb.Checkpoint{Epoch: 1, Root: r32[:]}
+
+	b33 := testutil.NewBeaconBlock()
+	b33.Block.Slot = 33
+	b33.Block.ParentRoot = r32[:]
+	r33, err := b33.Block.HashTreeRoot()
+	require.NoError(t, err)
+
+	require.NoError(t, service.forkChoiceStore.ProcessBlock(ctx, b32.Block.Slot, r32, [32]byte{}, [32]byte{}, 0, 0))
+	require.NoError(t, service.forkChoiceStore.ProcessBlock(ctx, b33.Block.Slot, r33, r32, [32]byte{}, 0, 0))
+
+	_, err = service.forkChoiceStore.Head(ctx, 0, r32, []uint64{}, 0)
+	require.NoError(t, err)
+	err = service.VerifyFinalizedConsistency(context.Background(), r33[:])
+	require.NoError(t, err)
+}
+
+func TestGetAttCheckptInfo(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+	cfg := &Config{BeaconDB: db, StateGen: stategen.New(db, cache.NewStateSummaryCache())}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	baseState, _ := testutil.DeterministicGenesisState(t, 128)
+	b := testutil.NewBeaconBlock()
+	r, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, service.beaconDB.SaveState(ctx, baseState, r))
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, b))
+	require.NoError(t, service.beaconDB.SaveGenesisBlockRoot(ctx, r))
+	checkpoint := &ethpb.Checkpoint{Root: r[:]}
+
+	returned, err := service.getAttCheckPtInfo(ctx, checkpoint, 0)
+	require.NoError(t, err)
+
+	seed, err := helpers.Seed(baseState, 0, params.BeaconConfig().DomainBeaconAttester)
+	require.NoError(t, err)
+	indices, err := helpers.ActiveValidatorIndices(baseState, 0)
+	require.NoError(t, err)
+	validators := baseState.ValidatorsReadOnly()
+	pks := make([][]byte, len(validators))
+	for i := 0; i < len(pks); i++ {
+		pk := validators[i].PublicKey()
+		pks[i] = pk[:]
+	}
+
+	wanted := &pb.CheckPtInfo{
+		Fork:          baseState.Fork(),
+		GenesisRoot:   baseState.GenesisValidatorRoot(),
+		Seed:          seed[:],
+		ActiveIndices: indices,
+		PubKeys:       pks,
+	}
+	require.DeepEqual(t, wanted, returned)
+
+	cached, err := service.checkPtInfoCache.get(checkpoint)
+	require.NoError(t, err)
+	require.DeepEqual(t, wanted, cached)
+}

beacon-chain/blockchain/process_block.go

@@ -0,0 +1,399 @@
+package blockchain
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/attestationutil"
+	"github.com/prysmaticlabs/prysm/shared/bls"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"go.opencensus.io/trace"
+)
+
+// This defines size of the upper bound for initial sync block cache.
+var initialSyncBlockCacheSize = 2 * params.BeaconConfig().SlotsPerEpoch
+
+// onBlock is called when a gossip block is received. It runs regular state transition on the block.
+// The block's signing root should be computed before calling this method to avoid redundant
+// computation in this method and methods it calls into.
+//
+// Spec pseudocode definition:
+//   def on_block(store: Store, signed_block: SignedBeaconBlock) -> None:
+//    block = signed_block.message
+//    # Parent block must be known
+//    assert block.parent_root in store.block_states
+//    # Make a copy of the state to avoid mutability issues
+//    pre_state = copy(store.block_states[block.parent_root])
+//    # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past.
+//    assert get_current_slot(store) >= block.slot
+//
+//    # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor)
+//    finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)
+//    assert block.slot > finalized_slot
+//    # Check block is a descendant of the finalized block at the checkpoint finalized slot
+//    assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root
+//
+//    # Check the block is valid and compute the post-state
+//    state = state_transition(pre_state, signed_block, True)
+//    # Add new block to the store
+//    store.blocks[hash_tree_root(block)] = block
+//    # Add new state for this block to the store
+//    store.block_states[hash_tree_root(block)] = state
+//
+//    # Update justified checkpoint
+//    if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch:
+//        if state.current_justified_checkpoint.epoch > store.best_justified_checkpoint.epoch:
+//            store.best_justified_checkpoint = state.current_justified_checkpoint
+//        if should_update_justified_checkpoint(store, state.current_justified_checkpoint):
+//            store.justified_checkpoint = state.current_justified_checkpoint
+//
+//    # Update finalized checkpoint
+//    if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch:
+//        store.finalized_checkpoint = state.finalized_checkpoint
+//
+//        # Potentially update justified if different from store
+//        if store.justified_checkpoint != state.current_justified_checkpoint:
+//            # Update justified if new justified is later than store justified
+//            if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch:
+//                store.justified_checkpoint = state.current_justified_checkpoint
+//                return
+//
+//            # Update justified if store justified is not in chain with finalized checkpoint
+//            finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)
+//            ancestor_at_finalized_slot = get_ancestor(store, store.justified_checkpoint.root, finalized_slot)
+//            if ancestor_at_finalized_slot != store.finalized_checkpoint.root:
+//                store.justified_checkpoint = state.current_justified_checkpoint
+func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
+	ctx, span := trace.StartSpan(ctx, "blockChain.onBlock")
+	defer span.End()
+
+	if signed == nil || signed.Block == nil {
+		return errors.New("nil block")
+	}
+	b := signed.Block
+
+	preState, err := s.getBlockPreState(ctx, b)
+	if err != nil {
+		return err
+	}
+
+	postState, err := state.ExecuteStateTransition(ctx, preState, signed)
+	if err != nil {
+		return errors.Wrap(err, "could not execute state transition")
+	}
+
+	if err := s.savePostStateInfo(ctx, blockRoot, signed, postState, false /* reg sync */); err != nil {
+		return err
+	}
+
+	// Update justified check point.
+	if postState.CurrentJustifiedCheckpoint().Epoch > s.justifiedCheckpt.Epoch {
+		if err := s.updateJustified(ctx, postState); err != nil {
+			return err
+		}
+	}
+
+	// Update finalized check point.
+	if postState.FinalizedCheckpointEpoch() > s.finalizedCheckpt.Epoch {
+		if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+			return err
+		}
+		s.clearInitSyncBlocks()
+
+		if err := s.updateFinalized(ctx, postState.FinalizedCheckpoint()); err != nil {
+			return err
+		}
+
+		fRoot := bytesutil.ToBytes32(postState.FinalizedCheckpoint().Root)
+		if err := s.forkChoiceStore.Prune(ctx, fRoot); err != nil {
+			return errors.Wrap(err, "could not prune proto array fork choice nodes")
+		}
+
+		if err := s.finalizedImpliesNewJustified(ctx, postState); err != nil {
+			return errors.Wrap(err, "could not save new justified")
+		}
+
+		// Update deposit cache.
+		finalizedState, err := s.stateGen.StateByRoot(ctx, fRoot)
+		if err != nil {
+			return errors.Wrap(err, "could not fetch finalized state")
+		}
+		// We update the cache up to the last deposit index in the finalized block's state.
+		// We can be confident that these deposits will be included in some block
+		// because the Eth1 follow distance makes such long-range reorgs extremely unlikely.
+		eth1DepositIndex := int64(finalizedState.Eth1Data().DepositCount - 1)
+		s.depositCache.InsertFinalizedDeposits(ctx, eth1DepositIndex)
+		if featureconfig.Get().EnablePruningDepositProofs {
+			// Deposit proofs are only used during state transition and can be safely removed to save space.
+			if err = s.depositCache.PruneProofs(ctx, eth1DepositIndex); err != nil {
+				return errors.Wrap(err, "could not prune deposit proofs")
+			}
+		}
+	}
+
+	defer reportAttestationInclusion(b)
+
+	return s.handleEpochBoundary(ctx, postState)
+}
+
+// onBlockInitialSyncStateTransition is called when an initial sync block is received.
+// It runs state transition on the block and without fork choice and post operation pool processes.
+// The block's signing root should be computed before calling this method to avoid redundant
+// computation in this method and methods it calls into.
+func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
+	ctx, span := trace.StartSpan(ctx, "blockChain.onBlockInitialSyncStateTransition")
+	defer span.End()
+
+	if signed == nil || signed.Block == nil {
+		return errors.New("nil block")
+	}
+
+	b := signed.Block
+
+	// Retrieve incoming block's pre state.
+	if err := s.verifyBlkPreState(ctx, b); err != nil {
+		return err
+	}
+
+	preState, err := s.stateGen.StateByRootInitialSync(ctx, bytesutil.ToBytes32(signed.Block.ParentRoot))
+	if err != nil {
+		return err
+	}
+	if preState == nil {
+		return fmt.Errorf("nil pre state for slot %d", b.Slot)
+	}
+
+	// Exit early if the pre state slot is higher than incoming block's slot.
+	if preState.Slot() >= signed.Block.Slot {
+		return nil
+	}
+
+	postState, err := state.ExecuteStateTransition(ctx, preState, signed)
+	if err != nil {
+		return errors.Wrap(err, "could not execute state transition")
+	}
+
+	if err := s.savePostStateInfo(ctx, blockRoot, signed, postState, true /* init sync */); err != nil {
+		return err
+	}
+	// Save the latest block as head in cache.
+	if err := s.saveHeadNoDB(ctx, signed, blockRoot, postState); err != nil {
+		return err
+	}
+
+	// Rate limit how many blocks (2 epochs worth of blocks) a node keeps in the memory.
+	if uint64(len(s.getInitSyncBlocks())) > initialSyncBlockCacheSize {
+		if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+			return err
+		}
+		s.clearInitSyncBlocks()
+	}
+
+	if postState.CurrentJustifiedCheckpoint().Epoch > s.justifiedCheckpt.Epoch {
+		if err := s.updateJustifiedInitSync(ctx, postState.CurrentJustifiedCheckpoint()); err != nil {
+			return err
+		}
+	}
+
+	// Update finalized check point. Prune the block cache and helper caches on every new finalized epoch.
+	if postState.FinalizedCheckpointEpoch() > s.finalizedCheckpt.Epoch {
+		if err := s.updateFinalized(ctx, postState.FinalizedCheckpoint()); err != nil {
+			return err
+		}
+	}
+
+	return s.handleEpochBoundary(ctx, postState)
+}
+
+func (s *Service) onBlockBatch(ctx context.Context, blks []*ethpb.SignedBeaconBlock,
+	blockRoots [][32]byte) ([]*ethpb.Checkpoint, []*ethpb.Checkpoint, error) {
+	ctx, span := trace.StartSpan(ctx, "blockChain.onBlockBatch")
+	defer span.End()
+
+	if len(blks) == 0 || len(blockRoots) == 0 {
+		return nil, nil, errors.New("no blocks provided")
+	}
+	if blks[0] == nil || blks[0].Block == nil {
+		return nil, nil, errors.New("nil block")
+	}
+	b := blks[0].Block
+
+	// Retrieve incoming block's pre state.
+	if err := s.verifyBlkPreState(ctx, b); err != nil {
+		return nil, nil, err
+	}
+	preState, err := s.stateGen.StateByRootInitialSync(ctx, bytesutil.ToBytes32(b.ParentRoot))
+	if err != nil {
+		return nil, nil, err
+	}
+	if preState == nil {
+		return nil, nil, fmt.Errorf("nil pre state for slot %d", b.Slot)
+	}
+
+	jCheckpoints := make([]*ethpb.Checkpoint, len(blks))
+	fCheckpoints := make([]*ethpb.Checkpoint, len(blks))
+	sigSet := &bls.SignatureSet{
+		Signatures: [][]byte{},
+		PublicKeys: []bls.PublicKey{},
+		Messages:   [][32]byte{},
+	}
+	var set *bls.SignatureSet
+	boundaries := make(map[[32]byte]*stateTrie.BeaconState)
+	for i, b := range blks {
+		set, preState, err = state.ExecuteStateTransitionNoVerifyAnySig(ctx, preState, b)
+		if err != nil {
+			return nil, nil, err
+		}
+		// Save potential boundary states.
+		if helpers.IsEpochStart(preState.Slot()) {
+			boundaries[blockRoots[i]] = preState.Copy()
+			if err := s.handleEpochBoundary(ctx, preState); err != nil {
+				return nil, nil, errors.Wrap(err, "could not handle epoch boundary state")
+			}
+		}
+		jCheckpoints[i] = preState.CurrentJustifiedCheckpoint()
+		fCheckpoints[i] = preState.FinalizedCheckpoint()
+		sigSet.Join(set)
+	}
+	verify, err := sigSet.Verify()
+	if err != nil {
+		return nil, nil, err
+	}
+	if !verify {
+		return nil, nil, errors.New("batch block signature verification failed")
+	}
+	for r, st := range boundaries {
+		if err := s.stateGen.SaveState(ctx, r, st); err != nil {
+			return nil, nil, err
+		}
+	}
+	// Also saves the last post state which to be used as pre state for the next batch.
+	lastB := blks[len(blks)-1]
+	lastBR := blockRoots[len(blockRoots)-1]
+	if err := s.stateGen.SaveState(ctx, lastBR, preState); err != nil {
+		return nil, nil, err
+	}
+	if err := s.saveHeadNoDB(ctx, lastB, lastBR, preState); err != nil {
+		return nil, nil, err
+	}
+	return fCheckpoints, jCheckpoints, nil
+}
+
+// handles a block after the block's batch has been verified, where we can save blocks
+// their state summaries and split them off to relative hot/cold storage.
+func (s *Service) handleBlockAfterBatchVerify(ctx context.Context, signed *ethpb.SignedBeaconBlock,
+	blockRoot [32]byte, fCheckpoint, jCheckpoint *ethpb.Checkpoint) error {
+	b := signed.Block
+
+	s.saveInitSyncBlock(blockRoot, signed)
+	if err := s.insertBlockToForkChoiceStore(ctx, b, blockRoot, fCheckpoint, jCheckpoint); err != nil {
+		return err
+	}
+	s.stateGen.SaveStateSummary(ctx, signed, blockRoot)
+
+	// Rate limit how many blocks (2 epochs worth of blocks) a node keeps in the memory.
+	if uint64(len(s.getInitSyncBlocks())) > initialSyncBlockCacheSize {
+		if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+			return err
+		}
+		s.clearInitSyncBlocks()
+	}
+
+	if jCheckpoint.Epoch > s.justifiedCheckpt.Epoch {
+		if err := s.updateJustifiedInitSync(ctx, jCheckpoint); err != nil {
+			return err
+		}
+	}
+
+	// Update finalized check point. Prune the block cache and helper caches on every new finalized epoch.
+	if fCheckpoint.Epoch > s.finalizedCheckpt.Epoch {
+		if err := s.updateFinalized(ctx, fCheckpoint); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Epoch boundary bookkeeping such as logging epoch summaries.
+func (s *Service) handleEpochBoundary(ctx context.Context, postState *stateTrie.BeaconState) error {
+	if postState.Slot() >= s.nextEpochBoundarySlot {
+		if err := reportEpochMetrics(ctx, postState, s.head.state); err != nil {
+			return err
+		}
+		var err error
+		s.nextEpochBoundarySlot, err = helpers.StartSlot(helpers.NextEpoch(postState))
+		if err != nil {
+			return err
+		}
+		// Update committees cache at epoch boundary slot.
+		if err := helpers.UpdateCommitteeCache(postState, helpers.CurrentEpoch(postState)); err != nil {
+			return err
+		}
+		if err := helpers.UpdateProposerIndicesInCache(postState, helpers.CurrentEpoch(postState)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// This feeds in the block and block's attestations to fork choice store. It's allows fork choice store
+// to gain information on the most current chain.
+func (s *Service) insertBlockAndAttestationsToForkChoiceStore(ctx context.Context, blk *ethpb.BeaconBlock, root [32]byte,
+	state *stateTrie.BeaconState) error {
+	fCheckpoint := state.FinalizedCheckpoint()
+	jCheckpoint := state.CurrentJustifiedCheckpoint()
+	if err := s.insertBlockToForkChoiceStore(ctx, blk, root, fCheckpoint, jCheckpoint); err != nil {
+		return err
+	}
+	// Feed in block's attestations to fork choice store.
+	for _, a := range blk.Body.Attestations {
+		committee, err := helpers.BeaconCommitteeFromState(state, a.Data.Slot, a.Data.CommitteeIndex)
+		if err != nil {
+			return err
+		}
+		indices := attestationutil.AttestingIndices(a.AggregationBits, committee)
+		s.forkChoiceStore.ProcessAttestation(ctx, indices, bytesutil.ToBytes32(a.Data.BeaconBlockRoot), a.Data.Target.Epoch)
+	}
+	return nil
+}
+
+func (s *Service) insertBlockToForkChoiceStore(ctx context.Context, blk *ethpb.BeaconBlock,
+	root [32]byte, fCheckpoint, jCheckpoint *ethpb.Checkpoint) error {
+	if err := s.fillInForkChoiceMissingBlocks(ctx, blk, fCheckpoint, jCheckpoint); err != nil {
+		return err
+	}
+	// Feed in block to fork choice store.
+	if err := s.forkChoiceStore.ProcessBlock(ctx,
+		blk.Slot, root, bytesutil.ToBytes32(blk.ParentRoot), bytesutil.ToBytes32(blk.Body.Graffiti),
+		jCheckpoint.Epoch,
+		fCheckpoint.Epoch); err != nil {
+		return errors.Wrap(err, "could not process block for proto array fork choice")
+	}
+	return nil
+}
+
+// This saves post state info to DB or cache. This also saves post state info to fork choice store.
+// Post state info consists of processed block and state. Do not call this method unless the block and state are verified.
+func (s *Service) savePostStateInfo(ctx context.Context, r [32]byte, b *ethpb.SignedBeaconBlock, state *stateTrie.BeaconState, initSync bool) error {
+	ctx, span := trace.StartSpan(ctx, "blockChain.savePostStateInfo")
+	defer span.End()
+	if initSync {
+		s.saveInitSyncBlock(r, b)
+	} else if err := s.beaconDB.SaveBlock(ctx, b); err != nil {
+		return errors.Wrapf(err, "could not save block from slot %d", b.Block.Slot)
+	}
+	if err := s.stateGen.SaveState(ctx, r, state); err != nil {
+		return errors.Wrap(err, "could not save state")
+	}
+	if err := s.insertBlockAndAttestationsToForkChoiceStore(ctx, b.Block, r, state); err != nil {
+		return errors.Wrapf(err, "could not insert block %d to fork choice store", b.Block.Slot)
+	}
+	return nil
+}

beacon-chain/blockchain/process_block_helpers.go

@@ -0,0 +1,432 @@
+package blockchain
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/attestationutil"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/traceutil"
+	"go.opencensus.io/trace"
+)
+
+// CurrentSlot returns the current slot based on time.
+func (s *Service) CurrentSlot() uint64 {
+	return helpers.CurrentSlot(uint64(s.genesisTime.Unix()))
+}
+
+// getBlockPreState returns the pre state of an incoming block. It uses the parent root of the block
+// to retrieve the state in DB. It verifies the pre state's validity and the incoming block
+// is in the correct time window.
+func (s *Service) getBlockPreState(ctx context.Context, b *ethpb.BeaconBlock) (*stateTrie.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "blockChain.getBlockPreState")
+	defer span.End()
+
+	// Verify incoming block has a valid pre state.
+	if err := s.verifyBlkPreState(ctx, b); err != nil {
+		return nil, err
+	}
+
+	preState, err := s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(b.ParentRoot))
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not get pre state for slot %d", b.Slot)
+	}
+	if preState == nil {
+		return nil, errors.Wrapf(err, "nil pre state for slot %d", b.Slot)
+	}
+
+	// Verify block slot time is not from the future.
+	if err := helpers.VerifySlotTime(preState.GenesisTime(), b.Slot, params.BeaconNetworkConfig().MaximumGossipClockDisparity); err != nil {
+		return nil, err
+	}
+
+	// Verify block is later than the finalized epoch slot.
+	if err := s.verifyBlkFinalizedSlot(b); err != nil {
+		return nil, err
+	}
+
+	return preState, nil
+}
+
+// verifyBlkPreState validates input block has a valid pre-state.
+func (s *Service) verifyBlkPreState(ctx context.Context, b *ethpb.BeaconBlock) error {
+	ctx, span := trace.StartSpan(ctx, "blockChain.verifyBlkPreState")
+	defer span.End()
+
+	parentRoot := bytesutil.ToBytes32(b.ParentRoot)
+	// Loosen the check to HasBlock because state summary gets saved in batches
+	// during initial syncing. There's no risk given a state summary object is just a
+	// a subset of the block object.
+	if !s.stateGen.StateSummaryExists(ctx, parentRoot) && !s.beaconDB.HasBlock(ctx, parentRoot) {
+		return errors.New("could not reconstruct parent state")
+	}
+
+	if err := s.VerifyBlkDescendant(ctx, bytesutil.ToBytes32(b.ParentRoot)); err != nil {
+		return err
+	}
+
+	has, err := s.stateGen.HasState(ctx, parentRoot)
+	if err != nil {
+		return err
+	}
+	if !has {
+		if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+			return errors.Wrap(err, "could not save initial sync blocks")
+		}
+		s.clearInitSyncBlocks()
+	}
+	return nil
+}
+
+// VerifyBlkDescendant validates input block root is a descendant of the
+// current finalized block root.
+func (s *Service) VerifyBlkDescendant(ctx context.Context, root [32]byte) error {
+	ctx, span := trace.StartSpan(ctx, "blockChain.VerifyBlkDescendant")
+	defer span.End()
+	fRoot := s.ensureRootNotZeros(bytesutil.ToBytes32(s.finalizedCheckpt.Root))
+	finalizedBlkSigned, err := s.beaconDB.Block(ctx, fRoot)
+	if err != nil {
+		return err
+	}
+	if finalizedBlkSigned == nil || finalizedBlkSigned.Block == nil {
+		return errors.New("nil finalized block")
+	}
+	finalizedBlk := finalizedBlkSigned.Block
+	bFinalizedRoot, err := s.ancestor(ctx, root[:], finalizedBlk.Slot)
+	if err != nil {
+		return errors.Wrap(err, "could not get finalized block root")
+	}
+	if bFinalizedRoot == nil {
+		return fmt.Errorf("no finalized block known for block %#x", bytesutil.Trunc(root[:]))
+	}
+
+	if !bytes.Equal(bFinalizedRoot, fRoot[:]) {
+		err := fmt.Errorf("block %#x is not a descendent of the current finalized block slot %d, %#x != %#x",
+			bytesutil.Trunc(root[:]), finalizedBlk.Slot, bytesutil.Trunc(bFinalizedRoot),
+			bytesutil.Trunc(fRoot[:]))
+		traceutil.AnnotateError(span, err)
+		return err
+	}
+	return nil
+}
+
+// verifyBlkFinalizedSlot validates input block is not less than or equal
+// to current finalized slot.
+func (s *Service) verifyBlkFinalizedSlot(b *ethpb.BeaconBlock) error {
+	finalizedSlot, err := helpers.StartSlot(s.finalizedCheckpt.Epoch)
+	if err != nil {
+		return err
+	}
+	if finalizedSlot >= b.Slot {
+		return fmt.Errorf("block is equal or earlier than finalized block, slot %d < slot %d", b.Slot, finalizedSlot)
+	}
+	return nil
+}
+
+// shouldUpdateCurrentJustified prevents bouncing attack, by only update conflicting justified
+// checkpoints in the fork choice if in the early slots of the epoch.
+// Otherwise, delay incorporation of new justified checkpoint until next epoch boundary.
+// See https://ethresear.ch/t/prevention-of-bouncing-attack-on-ffg/6114 for more detailed analysis and discussion.
+func (s *Service) shouldUpdateCurrentJustified(ctx context.Context, newJustifiedCheckpt *ethpb.Checkpoint) (bool, error) {
+	if helpers.SlotsSinceEpochStarts(s.CurrentSlot()) < params.BeaconConfig().SafeSlotsToUpdateJustified {
+		return true, nil
+	}
+	var newJustifiedBlockSigned *ethpb.SignedBeaconBlock
+	justifiedRoot := s.ensureRootNotZeros(bytesutil.ToBytes32(newJustifiedCheckpt.Root))
+	var err error
+	if s.hasInitSyncBlock(justifiedRoot) {
+		newJustifiedBlockSigned = s.getInitSyncBlock(justifiedRoot)
+	} else {
+		newJustifiedBlockSigned, err = s.beaconDB.Block(ctx, justifiedRoot)
+		if err != nil {
+			return false, err
+		}
+	}
+	if newJustifiedBlockSigned == nil || newJustifiedBlockSigned.Block == nil {
+		return false, errors.New("nil new justified block")
+	}
+
+	newJustifiedBlock := newJustifiedBlockSigned.Block
+	jSlot, err := helpers.StartSlot(s.justifiedCheckpt.Epoch)
+	if err != nil {
+		return false, err
+	}
+	if newJustifiedBlock.Slot <= jSlot {
+		return false, nil
+	}
+	var justifiedBlockSigned *ethpb.SignedBeaconBlock
+	cachedJustifiedRoot := s.ensureRootNotZeros(bytesutil.ToBytes32(s.justifiedCheckpt.Root))
+	if s.hasInitSyncBlock(cachedJustifiedRoot) {
+		justifiedBlockSigned = s.getInitSyncBlock(cachedJustifiedRoot)
+	} else {
+		justifiedBlockSigned, err = s.beaconDB.Block(ctx, cachedJustifiedRoot)
+		if err != nil {
+			return false, err
+		}
+	}
+
+	if justifiedBlockSigned == nil || justifiedBlockSigned.Block == nil {
+		return false, errors.New("nil justified block")
+	}
+	justifiedBlock := justifiedBlockSigned.Block
+	b, err := s.ancestor(ctx, justifiedRoot[:], justifiedBlock.Slot)
+	if err != nil {
+		return false, err
+	}
+	if !bytes.Equal(b, s.justifiedCheckpt.Root) {
+		return false, nil
+	}
+	return true, nil
+}
+
+func (s *Service) updateJustified(ctx context.Context, state *stateTrie.BeaconState) error {
+	cpt := state.CurrentJustifiedCheckpoint()
+	if cpt.Epoch > s.bestJustifiedCheckpt.Epoch {
+		s.bestJustifiedCheckpt = cpt
+	}
+	canUpdate, err := s.shouldUpdateCurrentJustified(ctx, cpt)
+	if err != nil {
+		return err
+	}
+
+	if canUpdate {
+		s.prevJustifiedCheckpt = s.justifiedCheckpt
+		s.justifiedCheckpt = cpt
+		if err := s.cacheJustifiedStateBalances(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root)); err != nil {
+			return err
+		}
+	}
+
+	return s.beaconDB.SaveJustifiedCheckpoint(ctx, cpt)
+}
+
+// This caches input checkpoint as justified for the service struct. It rotates current justified to previous justified,
+// caches justified checkpoint balances for fork choice and save justified checkpoint in DB.
+// This method does not have defense against fork choice bouncing attack, which is why it's only recommend to be used during initial syncing.
+func (s *Service) updateJustifiedInitSync(ctx context.Context, cp *ethpb.Checkpoint) error {
+	s.prevJustifiedCheckpt = s.justifiedCheckpt
+	s.justifiedCheckpt = cp
+	if err := s.cacheJustifiedStateBalances(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root)); err != nil {
+		return err
+	}
+
+	return s.beaconDB.SaveJustifiedCheckpoint(ctx, cp)
+}
+
+func (s *Service) updateFinalized(ctx context.Context, cp *ethpb.Checkpoint) error {
+	// Blocks need to be saved so that we can retrieve finalized block from
+	// DB when migrating states.
+	if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+		return err
+	}
+	s.clearInitSyncBlocks()
+
+	if err := s.beaconDB.SaveFinalizedCheckpoint(ctx, cp); err != nil {
+		return err
+	}
+
+	s.prevFinalizedCheckpt = s.finalizedCheckpt
+	s.finalizedCheckpt = cp
+
+	fRoot := bytesutil.ToBytes32(cp.Root)
+	if err := s.stateGen.MigrateToCold(ctx, fRoot); err != nil {
+		return errors.Wrap(err, "could not migrate to cold")
+	}
+
+	return nil
+}
+
+// ancestor returns the block root of an ancestry block from the input block root.
+//
+// Spec pseudocode definition:
+//   def get_ancestor(store: Store, root: Root, slot: Slot) -> Root:
+//    block = store.blocks[root]
+//    if block.slot > slot:
+//        return get_ancestor(store, block.parent_root, slot)
+//    elif block.slot == slot:
+//        return root
+//    else:
+//        # root is older than queried slot, thus a skip slot. Return most recent root prior to slot
+//        return root
+func (s *Service) ancestor(ctx context.Context, root []byte, slot uint64) ([]byte, error) {
+	ctx, span := trace.StartSpan(ctx, "blockChain.ancestor")
+	defer span.End()
+
+	r := bytesutil.ToBytes32(root)
+	// Get ancestor root from fork choice store instead of recursively looking up blocks in DB.
+	// This is most optimal outcome.
+	ar, err := s.ancestorByForkChoiceStore(ctx, r, slot)
+	if err != nil {
+		// Try getting ancestor root from DB when failed to retrieve from fork choice store.
+		// This is the second line of defense for retrieving ancestor root.
+		ar, err = s.ancestorByDB(ctx, r, slot)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return ar, nil
+}
+
+// This retrieves an ancestor root using fork choice store. The look up is looping through the a flat array structure.
+func (s *Service) ancestorByForkChoiceStore(ctx context.Context, r [32]byte, slot uint64) ([]byte, error) {
+	ctx, span := trace.StartSpan(ctx, "blockChain.ancestorByForkChoiceStore")
+	defer span.End()
+
+	if !s.forkChoiceStore.HasParent(r) {
+		return nil, errors.New("could not find root in fork choice store")
+	}
+	return s.forkChoiceStore.AncestorRoot(ctx, r, slot)
+}
+
+// This retrieves an ancestor root using DB. The look up is recursively looking up DB. Slower than `ancestorByForkChoiceStore`.
+func (s *Service) ancestorByDB(ctx context.Context, r [32]byte, slot uint64) ([]byte, error) {
+	ctx, span := trace.StartSpan(ctx, "blockChain.ancestorByDB")
+	defer span.End()
+
+	// Stop recursive ancestry lookup if context is cancelled.
+	if ctx.Err() != nil {
+		return nil, ctx.Err()
+	}
+
+	signed, err := s.beaconDB.Block(ctx, r)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get ancestor block")
+	}
+
+	if s.hasInitSyncBlock(r) {
+		signed = s.getInitSyncBlock(r)
+	}
+
+	if signed == nil || signed.Block == nil {
+		return nil, errors.New("nil block")
+	}
+	b := signed.Block
+	if b.Slot == slot || b.Slot < slot {
+		return r[:], nil
+	}
+
+	return s.ancestorByDB(ctx, bytesutil.ToBytes32(b.ParentRoot), slot)
+}
+
+// This updates justified check point in store, if the new justified is later than stored justified or
+// the store's justified is not in chain with finalized check point.
+//
+// Spec definition:
+//   # Potentially update justified if different from store
+//        if store.justified_checkpoint != state.current_justified_checkpoint:
+//            # Update justified if new justified is later than store justified
+//            if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch:
+//                store.justified_checkpoint = state.current_justified_checkpoint
+//                return
+//            # Update justified if store justified is not in chain with finalized checkpoint
+//            finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)
+//            ancestor_at_finalized_slot = get_ancestor(store, store.justified_checkpoint.root, finalized_slot)
+//            if ancestor_at_finalized_slot != store.finalized_checkpoint.root:
+//                store.justified_checkpoint = state.current_justified_checkpoint
+func (s *Service) finalizedImpliesNewJustified(ctx context.Context, state *stateTrie.BeaconState) error {
+	// Update justified if it's different than the one cached in the store.
+	if !attestationutil.CheckPointIsEqual(s.justifiedCheckpt, state.CurrentJustifiedCheckpoint()) {
+		if state.CurrentJustifiedCheckpoint().Epoch > s.justifiedCheckpt.Epoch {
+			s.justifiedCheckpt = state.CurrentJustifiedCheckpoint()
+			return s.cacheJustifiedStateBalances(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root))
+		}
+
+		// Update justified if store justified is not in chain with finalized check point.
+		finalizedSlot, err := helpers.StartSlot(s.finalizedCheckpt.Epoch)
+		if err != nil {
+			return err
+		}
+		justifiedRoot := s.ensureRootNotZeros(bytesutil.ToBytes32(s.justifiedCheckpt.Root))
+		anc, err := s.ancestor(ctx, justifiedRoot[:], finalizedSlot)
+		if err != nil {
+			return err
+		}
+		if !bytes.Equal(anc, s.finalizedCheckpt.Root) {
+			s.justifiedCheckpt = state.CurrentJustifiedCheckpoint()
+			if err := s.cacheJustifiedStateBalances(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root)); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// This retrieves missing blocks from DB (ie. the blocks that couldn't be received over sync) and inserts them to fork choice store.
+// This is useful for block tree visualizer and additional vote accounting.
+func (s *Service) fillInForkChoiceMissingBlocks(ctx context.Context, blk *ethpb.BeaconBlock,
+	fCheckpoint, jCheckpoint *ethpb.Checkpoint) error {
+	pendingNodes := make([]*ethpb.BeaconBlock, 0)
+
+	parentRoot := bytesutil.ToBytes32(blk.ParentRoot)
+	slot := blk.Slot
+	// Fork choice only matters from last finalized slot.
+	fSlot, err := helpers.StartSlot(s.finalizedCheckpt.Epoch)
+	if err != nil {
+		return err
+	}
+	higherThanFinalized := slot > fSlot
+	// As long as parent node is not in fork choice store, and parent node is in DB.
+	for !s.forkChoiceStore.HasNode(parentRoot) && s.beaconDB.HasBlock(ctx, parentRoot) && higherThanFinalized {
+		b, err := s.beaconDB.Block(ctx, parentRoot)
+		if err != nil {
+			return err
+		}
+
+		pendingNodes = append(pendingNodes, b.Block)
+		parentRoot = bytesutil.ToBytes32(b.Block.ParentRoot)
+		slot = b.Block.Slot
+		higherThanFinalized = slot > fSlot
+	}
+
+	// Insert parent nodes to fork choice store in reverse order.
+	// Lower slots should be at the end of the list.
+	for i := len(pendingNodes) - 1; i >= 0; i-- {
+		b := pendingNodes[i]
+		r, err := b.HashTreeRoot()
+		if err != nil {
+			return err
+		}
+
+		if err := s.forkChoiceStore.ProcessBlock(ctx,
+			b.Slot, r, bytesutil.ToBytes32(b.ParentRoot), bytesutil.ToBytes32(b.Body.Graffiti),
+			jCheckpoint.Epoch,
+			fCheckpoint.Epoch); err != nil {
+			return errors.Wrap(err, "could not process block for proto array fork choice")
+		}
+	}
+
+	return nil
+}
+
+// The deletes input attestations from the attestation pool, so proposers don't include them in a block for the future.
+func (s *Service) deletePoolAtts(atts []*ethpb.Attestation) error {
+	for _, att := range atts {
+		if helpers.IsAggregated(att) {
+			if err := s.attPool.DeleteAggregatedAttestation(att); err != nil {
+				return err
+			}
+		} else {
+			if err := s.attPool.DeleteUnaggregatedAttestation(att); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// This ensures that the input root defaults to using genesis root instead of zero hashes. This is needed for handling
+// fork choice justification routine.
+func (s *Service) ensureRootNotZeros(root [32]byte) [32]byte {
+	if root == params.BeaconConfig().ZeroHash {
+		return s.genesisRoot
+	}
+	return root
+}

beacon-chain/blockchain/process_block_test.go

@@ -0,0 +1,851 @@
+package blockchain
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/attestationutil"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+	"github.com/prysmaticlabs/prysm/shared/timeutils"
+)
+
+func TestStore_OnBlock(t *testing.T) {
+	ctx := context.Background()
+	db, sc := testDB.SetupDB(t)
+
+	cfg := &Config{
+		BeaconDB:        db,
+		StateGen:        stategen.New(db, sc),
+		ForkChoiceStore: protoarray.New(0, 0, [32]byte{}),
+	}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+	genesisStateRoot := [32]byte{}
+	genesis := blocks.NewGenesisBlock(genesisStateRoot[:])
+	assert.NoError(t, db.SaveBlock(ctx, genesis))
+	validGenesisRoot, err := genesis.Block.HashTreeRoot()
+	require.NoError(t, err)
+	st := testutil.NewBeaconState()
+	require.NoError(t, service.beaconDB.SaveState(ctx, st.Copy(), validGenesisRoot))
+	roots, err := blockTree1(db, validGenesisRoot[:])
+	require.NoError(t, err)
+	random := testutil.NewBeaconBlock()
+	random.Block.Slot = 1
+	random.Block.ParentRoot = validGenesisRoot[:]
+	assert.NoError(t, db.SaveBlock(ctx, random))
+	randomParentRoot, err := random.Block.HashTreeRoot()
+	assert.NoError(t, err)
+	require.NoError(t, service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Slot: st.Slot(), Root: randomParentRoot[:]}))
+	require.NoError(t, service.beaconDB.SaveState(ctx, st.Copy(), randomParentRoot))
+	randomParentRoot2 := roots[1]
+	require.NoError(t, service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Slot: st.Slot(), Root: randomParentRoot2}))
+	require.NoError(t, service.beaconDB.SaveState(ctx, st.Copy(), bytesutil.ToBytes32(randomParentRoot2)))
+
+	tests := []struct {
+		name          string
+		blk           *ethpb.SignedBeaconBlock
+		s             *stateTrie.BeaconState
+		time          uint64
+		wantErrString string
+	}{
+		{
+			name:          "parent block root does not have a state",
+			blk:           testutil.NewBeaconBlock(),
+			s:             st.Copy(),
+			wantErrString: "could not reconstruct parent state",
+		},
+		{
+			name: "block is from the future",
+			blk: func() *ethpb.SignedBeaconBlock {
+				b := testutil.NewBeaconBlock()
+				b.Block.ParentRoot = randomParentRoot2
+				b.Block.Slot = params.BeaconConfig().FarFutureEpoch
+				return b
+			}(),
+			s:             st.Copy(),
+			wantErrString: "is in the far distant future",
+		},
+		{
+			name: "could not get finalized block",
+			blk: func() *ethpb.SignedBeaconBlock {
+				b := testutil.NewBeaconBlock()
+				b.Block.ParentRoot = randomParentRoot[:]
+				return b
+			}(),
+			s:             st.Copy(),
+			wantErrString: "is not a descendent of the current finalized block",
+		},
+		{
+			name: "same slot as finalized block",
+			blk: func() *ethpb.SignedBeaconBlock {
+				b := testutil.NewBeaconBlock()
+				b.Block.Slot = 0
+				b.Block.ParentRoot = randomParentRoot2
+				return b
+			}(),
+			s:             st.Copy(),
+			wantErrString: "block is equal or earlier than finalized block, slot 0 < slot 0",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			service.justifiedCheckpt = &ethpb.Checkpoint{Root: validGenesisRoot[:]}
+			service.bestJustifiedCheckpt = &ethpb.Checkpoint{Root: validGenesisRoot[:]}
+			service.finalizedCheckpt = &ethpb.Checkpoint{Root: validGenesisRoot[:]}
+			service.prevFinalizedCheckpt = &ethpb.Checkpoint{Root: validGenesisRoot[:]}
+			service.finalizedCheckpt.Root = roots[0]
+
+			root, err := tt.blk.Block.HashTreeRoot()
+			assert.NoError(t, err)
+			err = service.onBlock(ctx, tt.blk, root)
+			assert.ErrorContains(t, tt.wantErrString, err)
+		})
+	}
+}
+
+func TestStore_OnBlockBatch(t *testing.T) {
+	ctx := context.Background()
+	db, sc := testDB.SetupDB(t)
+
+	cfg := &Config{
+		BeaconDB: db,
+		StateGen: stategen.New(db, sc),
+	}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	genesisStateRoot := [32]byte{}
+	genesis := blocks.NewGenesisBlock(genesisStateRoot[:])
+	assert.NoError(t, db.SaveBlock(ctx, genesis))
+	gRoot, err := genesis.Block.HashTreeRoot()
+	require.NoError(t, err)
+	service.finalizedCheckpt = &ethpb.Checkpoint{
+		Root: gRoot[:],
+	}
+	service.forkChoiceStore = protoarray.New(0, 0, [32]byte{})
+	service.saveInitSyncBlock(gRoot, genesis)
+
+	st, keys := testutil.DeterministicGenesisState(t, 64)
+
+	bState := st.Copy()
+
+	var blks []*ethpb.SignedBeaconBlock
+	var blkRoots [][32]byte
+	var firstState *stateTrie.BeaconState
+	for i := 1; i < 10; i++ {
+		b, err := testutil.GenerateFullBlock(bState, keys, testutil.DefaultBlockGenConfig(), uint64(i))
+		require.NoError(t, err)
+		bState, err = state.ExecuteStateTransition(ctx, bState, b)
+		require.NoError(t, err)
+		if i == 1 {
+			firstState = bState.Copy()
+		}
+		root, err := b.Block.HashTreeRoot()
+		require.NoError(t, err)
+		service.saveInitSyncBlock(root, b)
+		blks = append(blks, b)
+		blkRoots = append(blkRoots, root)
+	}
+
+	blks[0].Block.ParentRoot = gRoot[:]
+	require.NoError(t, db.SaveBlock(context.Background(), blks[0]))
+	require.NoError(t, service.stateGen.SaveState(ctx, blkRoots[0], firstState))
+	_, _, err = service.onBlockBatch(ctx, blks[1:], blkRoots[1:])
+	require.NoError(t, err)
+}
+
+func TestRemoveStateSinceLastFinalized_EmptyStartSlot(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+	params.UseMinimalConfig()
+	defer params.UseMainnetConfig()
+
+	cfg := &Config{BeaconDB: db, ForkChoiceStore: protoarray.New(0, 0, [32]byte{})}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+	service.genesisTime = time.Now()
+
+	update, err := service.shouldUpdateCurrentJustified(ctx, &ethpb.Checkpoint{Root: make([]byte, 32)})
+	require.NoError(t, err)
+	assert.Equal(t, true, update, "Should be able to update justified")
+	lastJustifiedBlk := testutil.NewBeaconBlock()
+	lastJustifiedBlk.Block.ParentRoot = bytesutil.PadTo([]byte{'G'}, 32)
+	lastJustifiedRoot, err := lastJustifiedBlk.Block.HashTreeRoot()
+	require.NoError(t, err)
+	newJustifiedBlk := testutil.NewBeaconBlock()
+	newJustifiedBlk.Block.Slot = 1
+	newJustifiedBlk.Block.ParentRoot = bytesutil.PadTo(lastJustifiedRoot[:], 32)
+	newJustifiedRoot, err := newJustifiedBlk.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, newJustifiedBlk))
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, lastJustifiedBlk))
+
+	diff := (params.BeaconConfig().SlotsPerEpoch - 1) * params.BeaconConfig().SecondsPerSlot
+	service.genesisTime = time.Unix(time.Now().Unix()-int64(diff), 0)
+	service.justifiedCheckpt = &ethpb.Checkpoint{Root: lastJustifiedRoot[:]}
+	update, err = service.shouldUpdateCurrentJustified(ctx, &ethpb.Checkpoint{Root: newJustifiedRoot[:]})
+	require.NoError(t, err)
+	assert.Equal(t, true, update, "Should be able to update justified")
+}
+
+func TestShouldUpdateJustified_ReturnFalse(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+	params.UseMinimalConfig()
+	defer params.UseMainnetConfig()
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+	lastJustifiedBlk := testutil.NewBeaconBlock()
+	lastJustifiedBlk.Block.ParentRoot = bytesutil.PadTo([]byte{'G'}, 32)
+	lastJustifiedRoot, err := lastJustifiedBlk.Block.HashTreeRoot()
+	require.NoError(t, err)
+	newJustifiedBlk := testutil.NewBeaconBlock()
+	newJustifiedBlk.Block.ParentRoot = bytesutil.PadTo(lastJustifiedRoot[:], 32)
+	newJustifiedRoot, err := newJustifiedBlk.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, newJustifiedBlk))
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, lastJustifiedBlk))
+
+	diff := (params.BeaconConfig().SlotsPerEpoch - 1) * params.BeaconConfig().SecondsPerSlot
+	service.genesisTime = time.Unix(time.Now().Unix()-int64(diff), 0)
+	service.justifiedCheckpt = &ethpb.Checkpoint{Root: lastJustifiedRoot[:]}
+
+	update, err := service.shouldUpdateCurrentJustified(ctx, &ethpb.Checkpoint{Root: newJustifiedRoot[:]})
+	require.NoError(t, err)
+	assert.Equal(t, false, update, "Should not be able to update justified, received true")
+}
+
+func TestCachedPreState_CanGetFromStateSummary(t *testing.T) {
+	ctx := context.Background()
+	db, sc := testDB.SetupDB(t)
+
+	cfg := &Config{
+		BeaconDB: db,
+		StateGen: stategen.New(db, sc),
+	}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	s, err := stateTrie.InitializeFromProto(&pb.BeaconState{Slot: 1, GenesisValidatorsRoot: params.BeaconConfig().ZeroHash[:]})
+	require.NoError(t, err)
+
+	genesisStateRoot := [32]byte{}
+	genesis := blocks.NewGenesisBlock(genesisStateRoot[:])
+	assert.NoError(t, db.SaveBlock(ctx, genesis))
+	gRoot, err := genesis.Block.HashTreeRoot()
+	require.NoError(t, err)
+	service.finalizedCheckpt = &ethpb.Checkpoint{
+		Root: gRoot[:],
+	}
+	service.forkChoiceStore = protoarray.New(0, 0, [32]byte{})
+	service.saveInitSyncBlock(gRoot, genesis)
+
+	b := testutil.NewBeaconBlock()
+	b.Block.Slot = 1
+	b.Block.ParentRoot = gRoot[:]
+	require.NoError(t, service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Slot: 1, Root: gRoot[:]}))
+	require.NoError(t, service.stateGen.SaveState(ctx, gRoot, s))
+	require.NoError(t, service.verifyBlkPreState(ctx, b.Block))
+}
+
+func TestCachedPreState_CanGetFromDB(t *testing.T) {
+	ctx := context.Background()
+	db, sc := testDB.SetupDB(t)
+
+	cfg := &Config{
+		BeaconDB: db,
+		StateGen: stategen.New(db, sc),
+	}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	genesisStateRoot := [32]byte{}
+	genesis := blocks.NewGenesisBlock(genesisStateRoot[:])
+	assert.NoError(t, db.SaveBlock(ctx, genesis))
+	gRoot, err := genesis.Block.HashTreeRoot()
+	require.NoError(t, err)
+	service.finalizedCheckpt = &ethpb.Checkpoint{
+		Root: gRoot[:],
+	}
+	service.forkChoiceStore = protoarray.New(0, 0, [32]byte{})
+	service.saveInitSyncBlock(gRoot, genesis)
+
+	b := testutil.NewBeaconBlock()
+	b.Block.Slot = 1
+	service.finalizedCheckpt = &ethpb.Checkpoint{Root: gRoot[:]}
+	err = service.verifyBlkPreState(ctx, b.Block)
+	wanted := "could not reconstruct parent state"
+	assert.ErrorContains(t, wanted, err)
+
+	b.Block.ParentRoot = gRoot[:]
+	s, err := stateTrie.InitializeFromProto(&pb.BeaconState{Slot: 1})
+	require.NoError(t, err)
+	require.NoError(t, service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Slot: 1, Root: gRoot[:]}))
+	require.NoError(t, service.stateGen.SaveState(ctx, gRoot, s))
+	require.NoError(t, service.verifyBlkPreState(ctx, b.Block))
+}
+
+func TestUpdateJustified_CouldUpdateBest(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db, StateGen: stategen.New(db, cache.NewStateSummaryCache())}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	signedBlock := testutil.NewBeaconBlock()
+	require.NoError(t, db.SaveBlock(ctx, signedBlock))
+	r, err := signedBlock.Block.HashTreeRoot()
+	require.NoError(t, err)
+	service.justifiedCheckpt = &ethpb.Checkpoint{Root: []byte{'A'}}
+	service.bestJustifiedCheckpt = &ethpb.Checkpoint{Root: []byte{'A'}}
+	st := testutil.NewBeaconState()
+	service.initSyncState[r] = st.Copy()
+	require.NoError(t, db.SaveState(ctx, st.Copy(), r))
+
+	// Could update
+	s := testutil.NewBeaconState()
+	require.NoError(t, s.SetCurrentJustifiedCheckpoint(&ethpb.Checkpoint{Epoch: 1, Root: r[:]}))
+	require.NoError(t, service.updateJustified(context.Background(), s))
+
+	assert.Equal(t, s.CurrentJustifiedCheckpoint().Epoch, service.bestJustifiedCheckpt.Epoch, "Incorrect justified epoch in service")
+
+	// Could not update
+	service.bestJustifiedCheckpt.Epoch = 2
+	require.NoError(t, service.updateJustified(context.Background(), s))
+
+	assert.Equal(t, uint64(2), service.bestJustifiedCheckpt.Epoch, "Incorrect justified epoch in service")
+}
+
+func TestFillForkChoiceMissingBlocks_CanSave(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+	service.forkChoiceStore = protoarray.New(0, 0, [32]byte{'A'})
+	service.finalizedCheckpt = &ethpb.Checkpoint{Root: make([]byte, 32)}
+
+	genesisStateRoot := [32]byte{}
+	genesis := blocks.NewGenesisBlock(genesisStateRoot[:])
+	require.NoError(t, db.SaveBlock(ctx, genesis))
+	validGenesisRoot, err := genesis.Block.HashTreeRoot()
+	require.NoError(t, err)
+	st := testutil.NewBeaconState()
+
+	require.NoError(t, service.beaconDB.SaveState(ctx, st.Copy(), validGenesisRoot))
+	roots, err := blockTree1(db, validGenesisRoot[:])
+	require.NoError(t, err)
+
+	beaconState, _ := testutil.DeterministicGenesisState(t, 32)
+	block := testutil.NewBeaconBlock()
+	block.Block.Slot = 9
+	block.Block.ParentRoot = roots[8]
+
+	err = service.fillInForkChoiceMissingBlocks(
+		context.Background(), block.Block, beaconState.FinalizedCheckpoint(), beaconState.CurrentJustifiedCheckpoint())
+	require.NoError(t, err)
+
+	// 5 nodes from the block tree 1. B0 - B3 - B4 - B6 - B8
+	assert.Equal(t, 5, len(service.forkChoiceStore.Nodes()), "Miss match nodes")
+	assert.Equal(t, true, service.forkChoiceStore.HasNode(bytesutil.ToBytes32(roots[4])), "Didn't save node")
+	assert.Equal(t, true, service.forkChoiceStore.HasNode(bytesutil.ToBytes32(roots[6])), "Didn't save node")
+	assert.Equal(t, true, service.forkChoiceStore.HasNode(bytesutil.ToBytes32(roots[8])), "Didn't save node")
+}
+
+func TestFillForkChoiceMissingBlocks_FilterFinalized(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+	service.forkChoiceStore = protoarray.New(0, 0, [32]byte{'A'})
+	// Set finalized epoch to 1.
+	service.finalizedCheckpt = &ethpb.Checkpoint{Epoch: 1}
+
+	genesisStateRoot := [32]byte{}
+	genesis := blocks.NewGenesisBlock(genesisStateRoot[:])
+	assert.NoError(t, db.SaveBlock(ctx, genesis))
+	validGenesisRoot, err := genesis.Block.HashTreeRoot()
+	assert.NoError(t, err)
+	st := testutil.NewBeaconState()
+
+	require.NoError(t, service.beaconDB.SaveState(ctx, st.Copy(), validGenesisRoot))
+
+	// Define a tree branch, slot 63 <- 64 <- 65
+	b63 := testutil.NewBeaconBlock()
+	b63.Block.Slot = 63
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, b63))
+	r63, err := b63.Block.HashTreeRoot()
+	require.NoError(t, err)
+	b64 := testutil.NewBeaconBlock()
+	b64.Block.Slot = 64
+	b64.Block.ParentRoot = r63[:]
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, b64))
+	r64, err := b64.Block.HashTreeRoot()
+	require.NoError(t, err)
+	b65 := testutil.NewBeaconBlock()
+	b65.Block.Slot = 65
+	b65.Block.ParentRoot = r64[:]
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, b65))
+
+	beaconState, _ := testutil.DeterministicGenesisState(t, 32)
+	err = service.fillInForkChoiceMissingBlocks(
+		context.Background(), b65.Block, beaconState.FinalizedCheckpoint(), beaconState.CurrentJustifiedCheckpoint())
+	require.NoError(t, err)
+
+	// There should be 2 nodes, block 65 and block 64.
+	assert.Equal(t, 2, len(service.forkChoiceStore.Nodes()), "Miss match nodes")
+
+	// Block with slot 63 should be in fork choice because it's less than finalized epoch 1.
+	assert.Equal(t, true, service.forkChoiceStore.HasNode(r63), "Didn't save node")
+}
+
+// blockTree1 constructs the following tree:
+//    /- B1
+// B0           /- B5 - B7
+//    \- B3 - B4 - B6 - B8
+// (B1, and B3 are all from the same slots)
+func blockTree1(db db.Database, genesisRoot []byte) ([][]byte, error) {
+	genesisRoot = bytesutil.PadTo(genesisRoot, 32)
+	b0 := testutil.NewBeaconBlock()
+	b0.Block.Slot = 0
+	b0.Block.ParentRoot = genesisRoot
+	r0, err := b0.Block.HashTreeRoot()
+	if err != nil {
+		return nil, err
+	}
+	b1 := testutil.NewBeaconBlock()
+	b1.Block.Slot = 1
+	b1.Block.ParentRoot = r0[:]
+	r1, err := b1.Block.HashTreeRoot()
+	if err != nil {
+		return nil, err
+	}
+	b3 := testutil.NewBeaconBlock()
+	b3.Block.Slot = 3
+	b3.Block.ParentRoot = r0[:]
+	r3, err := b3.Block.HashTreeRoot()
+	if err != nil {
+		return nil, err
+	}
+	b4 := testutil.NewBeaconBlock()
+	b4.Block.Slot = 4
+	b4.Block.ParentRoot = r3[:]
+	r4, err := b4.Block.HashTreeRoot()
+	if err != nil {
+		return nil, err
+	}
+	b5 := testutil.NewBeaconBlock()
+	b5.Block.Slot = 5
+	b5.Block.ParentRoot = r4[:]
+	r5, err := b5.Block.HashTreeRoot()
+	if err != nil {
+		return nil, err
+	}
+	b6 := testutil.NewBeaconBlock()
+	b6.Block.Slot = 6
+	b6.Block.ParentRoot = r4[:]
+	r6, err := b6.Block.HashTreeRoot()
+	if err != nil {
+		return nil, err
+	}
+	b7 := testutil.NewBeaconBlock()
+	b7.Block.Slot = 7
+	b7.Block.ParentRoot = r5[:]
+	r7, err := b7.Block.HashTreeRoot()
+	if err != nil {
+		return nil, err
+	}
+	b8 := testutil.NewBeaconBlock()
+	b8.Block.Slot = 8
+	b8.Block.ParentRoot = r6[:]
+	r8, err := b8.Block.HashTreeRoot()
+	if err != nil {
+		return nil, err
+	}
+	st := testutil.NewBeaconState()
+
+	for _, b := range []*ethpb.SignedBeaconBlock{b0, b1, b3, b4, b5, b6, b7, b8} {
+		beaconBlock := testutil.NewBeaconBlock()
+		beaconBlock.Block.Slot = b.Block.Slot
+		beaconBlock.Block.ParentRoot = bytesutil.PadTo(b.Block.ParentRoot, 32)
+		if err := db.SaveBlock(context.Background(), beaconBlock); err != nil {
+			return nil, err
+		}
+		if err := db.SaveState(context.Background(), st.Copy(), bytesutil.ToBytes32(beaconBlock.Block.ParentRoot)); err != nil {
+			return nil, errors.Wrap(err, "could not save state")
+		}
+	}
+	if err := db.SaveState(context.Background(), st.Copy(), r1); err != nil {
+		return nil, err
+	}
+	if err := db.SaveState(context.Background(), st.Copy(), r7); err != nil {
+		return nil, err
+	}
+	if err := db.SaveState(context.Background(), st.Copy(), r8); err != nil {
+		return nil, err
+	}
+	return [][]byte{r0[:], r1[:], nil, r3[:], r4[:], r5[:], r6[:], r7[:], r8[:]}, nil
+}
+
+func TestCurrentSlot_HandlesOverflow(t *testing.T) {
+	svc := Service{genesisTime: timeutils.Now().Add(1 * time.Hour)}
+
+	slot := svc.CurrentSlot()
+	require.Equal(t, uint64(0), slot, "Unexpected slot")
+}
+func TestAncestorByDB_CtxErr(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	service, err := NewService(ctx, &Config{})
+	require.NoError(t, err)
+
+	cancel()
+	_, err = service.ancestorByDB(ctx, [32]byte{}, 0)
+	require.ErrorContains(t, "context canceled", err)
+}
+
+func TestAncestor_HandleSkipSlot(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db, ForkChoiceStore: protoarray.New(0, 0, [32]byte{})}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	b1 := testutil.NewBeaconBlock()
+	b1.Block.Slot = 1
+	b1.Block.ParentRoot = bytesutil.PadTo([]byte{'a'}, 32)
+	r1, err := b1.Block.HashTreeRoot()
+	require.NoError(t, err)
+	b100 := testutil.NewBeaconBlock()
+	b100.Block.Slot = 100
+	b100.Block.ParentRoot = r1[:]
+	r100, err := b100.Block.HashTreeRoot()
+	require.NoError(t, err)
+	b200 := testutil.NewBeaconBlock()
+	b200.Block.Slot = 200
+	b200.Block.ParentRoot = r100[:]
+	r200, err := b200.Block.HashTreeRoot()
+	require.NoError(t, err)
+	for _, b := range []*ethpb.SignedBeaconBlock{b1, b100, b200} {
+		beaconBlock := testutil.NewBeaconBlock()
+		beaconBlock.Block.Slot = b.Block.Slot
+		beaconBlock.Block.ParentRoot = bytesutil.PadTo(b.Block.ParentRoot, 32)
+		require.NoError(t, db.SaveBlock(context.Background(), beaconBlock))
+	}
+
+	// Slots 100 to 200 are skip slots. Requesting root at 150 will yield root at 100. The last physical block.
+	r, err := service.ancestor(context.Background(), r200[:], 150)
+	require.NoError(t, err)
+	if bytesutil.ToBytes32(r) != r100 {
+		t.Error("Did not get correct root")
+	}
+
+	// Slots 1 to 100 are skip slots. Requesting root at 50 will yield root at 1. The last physical block.
+	r, err = service.ancestor(context.Background(), r200[:], 50)
+	require.NoError(t, err)
+	if bytesutil.ToBytes32(r) != r1 {
+		t.Error("Did not get correct root")
+	}
+}
+
+func TestAncestor_CanUseForkchoice(t *testing.T) {
+	ctx := context.Background()
+	cfg := &Config{ForkChoiceStore: protoarray.New(0, 0, [32]byte{})}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	b1 := testutil.NewBeaconBlock()
+	b1.Block.Slot = 1
+	b1.Block.ParentRoot = bytesutil.PadTo([]byte{'a'}, 32)
+	r1, err := b1.Block.HashTreeRoot()
+	require.NoError(t, err)
+	b100 := testutil.NewBeaconBlock()
+	b100.Block.Slot = 100
+	b100.Block.ParentRoot = r1[:]
+	r100, err := b100.Block.HashTreeRoot()
+	require.NoError(t, err)
+	b200 := testutil.NewBeaconBlock()
+	b200.Block.Slot = 200
+	b200.Block.ParentRoot = r100[:]
+	r200, err := b200.Block.HashTreeRoot()
+	require.NoError(t, err)
+	for _, b := range []*ethpb.SignedBeaconBlock{b1, b100, b200} {
+		beaconBlock := testutil.NewBeaconBlock()
+		beaconBlock.Block.Slot = b.Block.Slot
+		beaconBlock.Block.ParentRoot = bytesutil.PadTo(b.Block.ParentRoot, 32)
+		r, err := b.Block.HashTreeRoot()
+		require.NoError(t, err)
+		require.NoError(t, service.forkChoiceStore.ProcessBlock(context.Background(), b.Block.Slot, r, bytesutil.ToBytes32(b.Block.ParentRoot), [32]byte{}, 0, 0)) // Saves blocks to fork choice store.
+	}
+
+	r, err := service.ancestor(context.Background(), r200[:], 150)
+	require.NoError(t, err)
+	if bytesutil.ToBytes32(r) != r100 {
+		t.Error("Did not get correct root")
+	}
+}
+
+func TestAncestor_CanUseDB(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db, ForkChoiceStore: protoarray.New(0, 0, [32]byte{})}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	b1 := testutil.NewBeaconBlock()
+	b1.Block.Slot = 1
+	b1.Block.ParentRoot = bytesutil.PadTo([]byte{'a'}, 32)
+	r1, err := b1.Block.HashTreeRoot()
+	require.NoError(t, err)
+	b100 := testutil.NewBeaconBlock()
+	b100.Block.Slot = 100
+	b100.Block.ParentRoot = r1[:]
+	r100, err := b100.Block.HashTreeRoot()
+	require.NoError(t, err)
+	b200 := testutil.NewBeaconBlock()
+	b200.Block.Slot = 200
+	b200.Block.ParentRoot = r100[:]
+	r200, err := b200.Block.HashTreeRoot()
+	require.NoError(t, err)
+	for _, b := range []*ethpb.SignedBeaconBlock{b1, b100, b200} {
+		beaconBlock := testutil.NewBeaconBlock()
+		beaconBlock.Block.Slot = b.Block.Slot
+		beaconBlock.Block.ParentRoot = bytesutil.PadTo(b.Block.ParentRoot, 32)
+		require.NoError(t, db.SaveBlock(context.Background(), beaconBlock)) // Saves blocks to DB.
+	}
+
+	require.NoError(t, service.forkChoiceStore.ProcessBlock(context.Background(), 200, r200, r200, [32]byte{}, 0, 0))
+
+	r, err := service.ancestor(context.Background(), r200[:], 150)
+	require.NoError(t, err)
+	if bytesutil.ToBytes32(r) != r100 {
+		t.Error("Did not get correct root")
+	}
+}
+
+func TestEnsureRootNotZeroHashes(t *testing.T) {
+	ctx := context.Background()
+	cfg := &Config{}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+	service.genesisRoot = [32]byte{'a'}
+
+	r := service.ensureRootNotZeros(params.BeaconConfig().ZeroHash)
+	assert.Equal(t, service.genesisRoot, r, "Did not get wanted justified root")
+	root := [32]byte{'b'}
+	r = service.ensureRootNotZeros(root)
+	assert.Equal(t, root, r, "Did not get wanted justified root")
+}
+
+func TestFinalizedImpliesNewJustified(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+	ctx := context.Background()
+	type args struct {
+		cachedCheckPoint        *ethpb.Checkpoint
+		stateCheckPoint         *ethpb.Checkpoint
+		diffFinalizedCheckPoint bool
+	}
+	tests := []struct {
+		name string
+		args args
+		want *ethpb.Checkpoint
+	}{
+		{
+			name: "Same justified, do nothing",
+			args: args{
+				cachedCheckPoint: &ethpb.Checkpoint{Epoch: 1, Root: []byte{'a'}},
+				stateCheckPoint:  &ethpb.Checkpoint{Epoch: 1, Root: []byte{'a'}},
+			},
+			want: &ethpb.Checkpoint{Epoch: 1, Root: []byte{'a'}},
+		},
+		{
+			name: "Different justified, higher epoch, cache new justified",
+			args: args{
+				cachedCheckPoint: &ethpb.Checkpoint{Epoch: 1, Root: []byte{'a'}},
+				stateCheckPoint:  &ethpb.Checkpoint{Epoch: 2, Root: []byte{'b'}},
+			},
+			want: &ethpb.Checkpoint{Epoch: 2, Root: []byte{'b'}},
+		},
+		{
+			name: "finalized has different justified, cache new justified",
+			args: args{
+				cachedCheckPoint:        &ethpb.Checkpoint{Epoch: 1, Root: []byte{'a'}},
+				stateCheckPoint:         &ethpb.Checkpoint{Epoch: 1, Root: []byte{'b'}},
+				diffFinalizedCheckPoint: true,
+			},
+			want: &ethpb.Checkpoint{Epoch: 1, Root: []byte{'b'}},
+		},
+	}
+	for _, test := range tests {
+		beaconState := testutil.NewBeaconState()
+		require.NoError(t, beaconState.SetCurrentJustifiedCheckpoint(test.args.stateCheckPoint))
+		service, err := NewService(ctx, &Config{BeaconDB: db, StateGen: stategen.New(db, sc), ForkChoiceStore: protoarray.New(0, 0, [32]byte{})})
+		require.NoError(t, err)
+		service.justifiedCheckpt = test.args.cachedCheckPoint
+		require.NoError(t, service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Root: bytesutil.PadTo(test.want.Root, 32)}))
+		genesisState := testutil.NewBeaconState()
+		require.NoError(t, service.beaconDB.SaveState(ctx, genesisState, bytesutil.ToBytes32(test.want.Root)))
+
+		if test.args.diffFinalizedCheckPoint {
+			b1 := testutil.NewBeaconBlock()
+			b1.Block.Slot = 1
+			b1.Block.ParentRoot = bytesutil.PadTo([]byte{'a'}, 32)
+			r1, err := b1.Block.HashTreeRoot()
+			require.NoError(t, err)
+			b100 := testutil.NewBeaconBlock()
+			b100.Block.Slot = 100
+			b100.Block.ParentRoot = r1[:]
+			r100, err := b100.Block.HashTreeRoot()
+			require.NoError(t, err)
+			for _, b := range []*ethpb.SignedBeaconBlock{b1, b100} {
+				beaconBlock := testutil.NewBeaconBlock()
+				beaconBlock.Block.Slot = b.Block.Slot
+				beaconBlock.Block.ParentRoot = bytesutil.PadTo(b.Block.ParentRoot, 32)
+				require.NoError(t, service.beaconDB.SaveBlock(context.Background(), beaconBlock))
+			}
+			service.finalizedCheckpt = &ethpb.Checkpoint{Root: []byte{'c'}, Epoch: 1}
+			service.justifiedCheckpt.Root = r100[:]
+		}
+
+		require.NoError(t, service.finalizedImpliesNewJustified(ctx, beaconState))
+		assert.Equal(t, true, attestationutil.CheckPointIsEqual(test.want, service.justifiedCheckpt), "Did not get wanted check point")
+	}
+}
+
+func TestVerifyBlkDescendant(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+	ctx := context.Background()
+
+	b := testutil.NewBeaconBlock()
+	b.Block.Slot = 1
+	r, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, db.SaveBlock(ctx, b))
+
+	b1 := testutil.NewBeaconBlock()
+	b1.Block.Slot = 1
+	b1.Block.Body.Graffiti = bytesutil.PadTo([]byte{'a'}, 32)
+	r1, err := b1.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, db.SaveBlock(ctx, b1))
+
+	type args struct {
+		parentRoot    [32]byte
+		finalizedRoot [32]byte
+		finalizedSlot uint64
+	}
+	tests := []struct {
+		name      string
+		args      args
+		wantedErr string
+	}{
+		{
+			name: "could not get finalized block in block service cache",
+			args: args{
+				finalizedRoot: [32]byte{'a'},
+			},
+			wantedErr: "nil finalized block",
+		},
+		{
+			name: "could not get finalized block root in DB",
+			args: args{
+				finalizedRoot: r,
+				parentRoot:    [32]byte{'a'},
+			},
+			wantedErr: "could not get finalized block root",
+		},
+		{
+			name: "is not descendant",
+			args: args{
+				finalizedRoot: r1,
+				parentRoot:    r,
+			},
+			wantedErr: "is not a descendent of the current finalized block slot",
+		},
+		{
+			name: "is descendant",
+			args: args{
+				finalizedRoot: r,
+				parentRoot:    r,
+			},
+		},
+	}
+	for _, tt := range tests {
+		service, err := NewService(ctx, &Config{BeaconDB: db, StateGen: stategen.New(db, sc), ForkChoiceStore: protoarray.New(0, 0, [32]byte{})})
+		require.NoError(t, err)
+		service.finalizedCheckpt = &ethpb.Checkpoint{
+			Root: tt.args.finalizedRoot[:],
+		}
+		err = service.VerifyBlkDescendant(ctx, tt.args.parentRoot)
+		if tt.wantedErr != "" {
+			assert.ErrorContains(t, tt.wantedErr, err)
+		} else if err != nil {
+			assert.NoError(t, err)
+		}
+	}
+}
+
+func TestUpdateJustifiedInitSync(t *testing.T) {
+	db, _ := testDB.SetupDB(t)
+	ctx := context.Background()
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	gBlk := testutil.NewBeaconBlock()
+	gRoot, err := gBlk.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, service.beaconDB.SaveBlock(ctx, gBlk))
+	require.NoError(t, service.beaconDB.SaveGenesisBlockRoot(ctx, gRoot))
+	require.NoError(t, service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Root: gRoot[:]}))
+	beaconState, _ := testutil.DeterministicGenesisState(t, 32)
+	require.NoError(t, service.beaconDB.SaveState(ctx, beaconState, gRoot))
+	service.genesisRoot = gRoot
+	currentCp := &ethpb.Checkpoint{Epoch: 1}
+	service.justifiedCheckpt = currentCp
+	newCp := &ethpb.Checkpoint{Epoch: 2, Root: gRoot[:]}
+
+	require.NoError(t, service.updateJustifiedInitSync(ctx, newCp))
+
+	assert.DeepEqual(t, currentCp, service.prevJustifiedCheckpt, "Incorrect previous justified checkpoint")
+	assert.DeepEqual(t, newCp, service.CurrentJustifiedCheckpt(), "Incorrect current justified checkpoint in cache")
+	cp, err := service.beaconDB.JustifiedCheckpoint(ctx)
+	require.NoError(t, err)
+	assert.DeepEqual(t, newCp, cp, "Incorrect current justified checkpoint in db")
+}
+
+func TestHandleEpochBoundary_BadMetrics(t *testing.T) {
+	ctx := context.Background()
+	cfg := &Config{}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+
+	s := testutil.NewBeaconState()
+	require.NoError(t, s.SetSlot(1))
+	service.head = &head{}
+	require.ErrorContains(t, "failed to initialize precompute: nil inner state", service.handleEpochBoundary(ctx, s))
+}

beacon-chain/blockchain/receive_attestation.go

@@ -3,21 +3,34 @@ package blockchain
 import (
 	"bytes"
 	"context"
-	"time"
+	"fmt"
 
 	"github.com/pkg/errors"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/slotutil"
+	"github.com/prysmaticlabs/prysm/shared/timeutils"
+	"github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
 )
 
 // AttestationReceiver interface defines the methods of chain service receive and processing new attestations.
 type AttestationReceiver interface {
 	ReceiveAttestationNoPubsub(ctx context.Context, att *ethpb.Attestation) error
+	IsValidAttestation(ctx context.Context, att *ethpb.Attestation) bool
+	AttestationPreState(ctx context.Context, att *ethpb.Attestation) (*state.BeaconState, error)
+	AttestationCheckPtInfo(ctx context.Context, att *ethpb.Attestation) (*pb.CheckPtInfo, error)
+	VerifyLmdFfgConsistency(ctx context.Context, att *ethpb.Attestation) error
+	VerifyFinalizedConsistency(ctx context.Context, root []byte) error
 }
 
-// ReceiveAttestationNoPubsub is a function that defines the operations that are preformed on
+// ReceiveAttestationNoPubsub is a function that defines the operations that are performed on
 // attestation that is received from regular sync. The operations consist of:
 //  1. Validate attestation, update validator's latest vote
 //  2. Apply fork choice to the processed attestation
@@ -26,52 +39,161 @@ func (s *Service) ReceiveAttestationNoPubsub(ctx context.Context, att *ethpb.Att
 	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveAttestationNoPubsub")
 	defer span.End()
 
-	// Update forkchoice store for the new attestation
-	if err := s.forkChoiceStore.OnAttestation(ctx, att); err != nil {
-		return errors.Wrap(err, "could not process attestation from fork choice service")
-	}
-
-	// Run fork choice for head block after updating fork choice store.
-	headRoot, err := s.forkChoiceStore.Head(ctx)
+	_, err := s.onAttestation(ctx, att)
 	if err != nil {
-		return errors.Wrap(err, "could not get head from fork choice service")
-	}
-	// Only save head if it's different than the current head.
-	if !bytes.Equal(headRoot, s.HeadRoot()) {
-		headBlk, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(headRoot))
-		if err != nil {
-			return errors.Wrap(err, "could not compute state from block head")
-		}
-		if err := s.saveHead(ctx, headBlk, bytesutil.ToBytes32(headRoot)); err != nil {
-			return errors.Wrap(err, "could not save head")
-		}
+		return errors.Wrap(err, "could not process attestation")
+	}
+
+	if err := s.updateHead(ctx, s.getJustifiedBalances()); err != nil {
+		log.Warnf("Resolving fork due to new attestation: %v", err)
+		return nil
+	}
+
+	return nil
+}
+
+// IsValidAttestation returns true if the attestation can be verified against its pre-state.
+func (s *Service) IsValidAttestation(ctx context.Context, att *ethpb.Attestation) bool {
+	baseState, err := s.AttestationPreState(ctx, att)
+	if err != nil {
+		log.WithError(err).Error("Failed to get attestation pre state")
+		return false
+	}
+
+	if err := blocks.VerifyAttestationSignature(ctx, baseState, att); err != nil {
+		log.WithError(err).Error("Failed to validate attestation")
+		return false
+	}
+
+	return true
+}
+
+// AttestationPreState returns the pre state of attestation.
+func (s *Service) AttestationPreState(ctx context.Context, att *ethpb.Attestation) (*state.BeaconState, error) {
+	ss, err := helpers.StartSlot(att.Data.Target.Epoch)
+	if err != nil {
+		return nil, err
+	}
+	if err := helpers.ValidateSlotClock(ss, uint64(s.genesisTime.Unix())); err != nil {
+		return nil, err
+	}
+	return s.getAttPreState(ctx, att.Data.Target)
+}
+
+// AttestationCheckPtInfo returns the check point info of attestation that can be used to verify the attestation
+// contents and signatures.
+func (s *Service) AttestationCheckPtInfo(ctx context.Context, att *ethpb.Attestation) (*pb.CheckPtInfo, error) {
+	ss, err := helpers.StartSlot(att.Data.Target.Epoch)
+	if err != nil {
+		return nil, err
+	}
+	if err := helpers.ValidateSlotClock(ss, uint64(s.genesisTime.Unix())); err != nil {
+		return nil, err
+	}
+	return s.getAttCheckPtInfo(ctx, att.Data.Target, helpers.SlotToEpoch(att.Data.Slot))
+}
+
+// VerifyLmdFfgConsistency verifies that attestation's LMD and FFG votes are consistency to each other.
+func (s *Service) VerifyLmdFfgConsistency(ctx context.Context, a *ethpb.Attestation) error {
+	return s.verifyLMDFFGConsistent(ctx, a.Data.Target.Epoch, a.Data.Target.Root, a.Data.BeaconBlockRoot)
+}
+
+// VerifyFinalizedConsistency verifies input root is consistent with finalized store.
+// When the input root is not be consistent with finalized store then we know it is not
+// on the finalized check point that leads to current canonical chain and should be rejected accordingly.
+func (s *Service) VerifyFinalizedConsistency(ctx context.Context, root []byte) error {
+	// A canonical root implies the root to has an ancestor that aligns with finalized check point.
+	// In this case, we could exit early to save on additional computation.
+	if s.forkChoiceStore.IsCanonical(bytesutil.ToBytes32(root)) {
+		return nil
+	}
+
+	f := s.FinalizedCheckpt()
+	ss, err := helpers.StartSlot(f.Epoch)
+	if err != nil {
+		return err
+	}
+	r, err := s.ancestor(ctx, root, ss)
+	if err != nil {
+		return err
+	}
+	if !bytes.Equal(f.Root, r) {
+		return errors.New("Root and finalized store are not consistent")
 	}
 
-	processedAttNoPubsub.Inc()
 	return nil
 }
 
 // This processes attestations from the attestation pool to account for validator votes and fork choice.
-func (s *Service) processAttestation() {
-	period := time.Duration(params.BeaconConfig().SecondsPerSlot) * time.Second
-	ticker := time.NewTicker(period)
-	for {
-		ctx := context.Background()
-		select {
-		case <-ticker.C:
-			atts, err := s.opsPoolService.AttestationPoolForForkchoice(ctx)
-			if err != nil {
-				log.WithError(err).Error("Could not retrieve attestation from pool")
-			}
+func (s *Service) processAttestation(subscribedToStateEvents chan struct{}) {
+	// Wait for state to be initialized.
+	stateChannel := make(chan *feed.Event, 1)
+	stateSub := s.stateNotifier.StateFeed().Subscribe(stateChannel)
+	subscribedToStateEvents <- struct{}{}
+	<-stateChannel
+	stateSub.Unsubscribe()
 
+	st := slotutil.GetSlotTicker(s.genesisTime, params.BeaconConfig().SecondsPerSlot)
+	for {
+		select {
+		case <-s.ctx.Done():
+			return
+		case <-st.C():
+			ctx := s.ctx
+			atts := s.attPool.ForkchoiceAttestations()
 			for _, a := range atts {
+				// Based on the spec, don't process the attestation until the subsequent slot.
+				// This delays consideration in the fork choice until their slot is in the past.
+				// https://github.com/ethereum/eth2.0-specs/blob/dev/specs/phase0/fork-choice.md#validate_on_attestation
+				nextSlot := a.Data.Slot + 1
+				if err := helpers.VerifySlotTime(uint64(s.genesisTime.Unix()), nextSlot, params.BeaconNetworkConfig().MaximumGossipClockDisparity); err != nil {
+					continue
+				}
+
+				hasState := s.stateGen.StateSummaryExists(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
+				hasBlock := s.hasBlock(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
+				if !(hasState && hasBlock) {
+					continue
+				}
+
+				if err := s.attPool.DeleteForkchoiceAttestation(a); err != nil {
+					log.WithError(err).Error("Could not delete fork choice attestation in pool")
+				}
+
+				if !s.verifyCheckpointEpoch(a.Data.Target) {
+					continue
+				}
+
 				if err := s.ReceiveAttestationNoPubsub(ctx, a); err != nil {
-					log.WithError(err).Error("Could not receive attestation in chain service")
+					log.WithFields(logrus.Fields{
+						"slot":             a.Data.Slot,
+						"committeeIndex":   a.Data.CommitteeIndex,
+						"beaconBlockRoot":  fmt.Sprintf("%#x", bytesutil.Trunc(a.Data.BeaconBlockRoot)),
+						"targetRoot":       fmt.Sprintf("%#x", bytesutil.Trunc(a.Data.Target.Root)),
+						"aggregationCount": a.AggregationBits.Count(),
+					}).WithError(err).Warn("Could not receive attestation in chain service")
 				}
 			}
-		case <-s.ctx.Done():
-			log.Debug("Context closed, exiting routine")
-			return
 		}
 	}
 }
+
+// This verifies the epoch of input checkpoint is within current epoch and previous epoch
+// with respect to current time. Returns true if it's within, false if it's not.
+func (s *Service) verifyCheckpointEpoch(c *ethpb.Checkpoint) bool {
+	now := uint64(timeutils.Now().Unix())
+	genesisTime := uint64(s.genesisTime.Unix())
+	currentSlot := (now - genesisTime) / params.BeaconConfig().SecondsPerSlot
+	currentEpoch := helpers.SlotToEpoch(currentSlot)
+
+	var prevEpoch uint64
+	if currentEpoch > 1 {
+		prevEpoch = currentEpoch - 1
+	}
+
+	if c.Epoch != prevEpoch && c.Epoch != currentEpoch {
+		return false
+	}
+
+	return true
+}

beacon-chain/blockchain/receive_attestation_test.go

@@ -1,46 +1,49 @@
 package blockchain
 
 import (
+	"context"
 	"testing"
+	"time"
 
-	"github.com/prysmaticlabs/go-ssz"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/shared/testutil"
-	logTest "github.com/sirupsen/logrus/hooks/test"
-	"golang.org/x/net/context"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 )
 
-func TestReceiveAttestationNoPubsub_ProcessCorrectly(t *testing.T) {
-	hook := logTest.NewGlobal()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	ctx := context.Background()
+func TestVerifyCheckpointEpoch_Ok(t *testing.T) {
+	helpers.ClearCache()
+	db, sc := testDB.SetupDB(t)
 
-	chainService := setupBeaconChain(t, db)
-	r, _ := ssz.SigningRoot(&ethpb.BeaconBlock{})
-	chainService.forkChoiceStore = &store{headRoot: r[:]}
+	chainService := setupBeaconChain(t, db, sc)
+	chainService.genesisTime = time.Now()
 
-	b := &ethpb.BeaconBlock{}
-	if err := chainService.beaconDB.SaveBlock(ctx, b); err != nil {
-		t.Fatal(err)
-	}
-	root, err := ssz.SigningRoot(b)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := chainService.beaconDB.SaveState(ctx, &pb.BeaconState{}, root); err != nil {
-		t.Fatal(err)
-	}
-
-	a := &ethpb.Attestation{Data: &ethpb.AttestationData{
-		Target: &ethpb.Checkpoint{Root: root[:]},
-	}}
-	if err := chainService.ReceiveAttestationNoPubsub(ctx, a); err != nil {
-		t.Fatal(err)
-	}
-
-	testutil.AssertLogsContain(t, hook, "Saved new head info")
-	testutil.AssertLogsDoNotContain(t, hook, "Broadcasting attestation")
+	assert.Equal(t, true, chainService.verifyCheckpointEpoch(&ethpb.Checkpoint{Root: make([]byte, 32)}))
+	assert.Equal(t, false, chainService.verifyCheckpointEpoch(&ethpb.Checkpoint{Epoch: 1}))
+}
+
+func TestAttestationPreState_FarFutureSlot(t *testing.T) {
+	helpers.ClearCache()
+	db, sc := testDB.SetupDB(t)
+
+	chainService := setupBeaconChain(t, db, sc)
+	chainService.genesisTime = time.Now()
+
+	e := helpers.MaxSlotBuffer/params.BeaconConfig().SlotsPerEpoch + 1
+	_, err := chainService.AttestationCheckPtInfo(context.Background(), &ethpb.Attestation{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{Epoch: e}}})
+	require.ErrorContains(t, "exceeds max allowed value relative to the local clock", err)
+}
+
+func TestAttestationCheckPtInfo_FarFutureSlot(t *testing.T) {
+	helpers.ClearCache()
+	db, sc := testDB.SetupDB(t)
+
+	chainService := setupBeaconChain(t, db, sc)
+	chainService.genesisTime = time.Now()
+
+	e := helpers.MaxSlotBuffer/params.BeaconConfig().SlotsPerEpoch + 1
+	_, err := chainService.AttestationPreState(context.Background(), &ethpb.Attestation{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{Epoch: e}}})
+	require.ErrorContains(t, "exceeds max allowed value relative to the local clock", err)
 }

beacon-chain/blockchain/receive_block.go

@@ -1,249 +1,208 @@
 package blockchain
 
 import (
-	"bytes"
 	"context"
-	"encoding/hex"
 
-	"github.com/gogo/protobuf/proto"
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/statefeed"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
+	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/traceutil"
 	"github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
 )
 
+// This defines how many epochs since finality the run time will begin to save hot state on to the DB.
+var epochsSinceFinalitySaveHotStateDB = 100
+
 // BlockReceiver interface defines the methods of chain service receive and processing new blocks.
 type BlockReceiver interface {
-	ReceiveBlock(ctx context.Context, block *ethpb.BeaconBlock) error
-	ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.BeaconBlock) error
-	ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *ethpb.BeaconBlock) error
-	ReceiveBlockNoVerify(ctx context.Context, block *ethpb.BeaconBlock) error
+	ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error
+	ReceiveBlockInitialSync(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error
+	ReceiveBlockBatch(ctx context.Context, blocks []*ethpb.SignedBeaconBlock, blkRoots [][32]byte) error
+	HasInitSyncBlock(root [32]byte) bool
 }
 
-// ReceiveBlock is a function that defines the operations that are preformed on
-// blocks that is received from rpc service. The operations consists of:
-//   1. Gossip block to other peers
-//   2. Validate block, apply state transition and update check points
-//   3. Apply fork choice to the processed block
-//   4. Save latest head info
-func (s *Service) ReceiveBlock(ctx context.Context, block *ethpb.BeaconBlock) error {
-	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlock")
-	defer span.End()
-
-	root, err := ssz.SigningRoot(block)
-	if err != nil {
-		return errors.Wrap(err, "could not get signing root on received block")
-	}
-
-	// Broadcast the new block to the network.
-	if err := s.p2p.Broadcast(ctx, block); err != nil {
-		return errors.Wrap(err, "could not broadcast block")
-	}
-	log.WithFields(logrus.Fields{
-		"blockRoot": hex.EncodeToString(root[:]),
-	}).Debug("Broadcasting block")
-
-	if err := s.ReceiveBlockNoPubsub(ctx, block); err != nil {
-		return err
-	}
-
-	processedBlk.Inc()
-	return nil
-}
-
-// ReceiveBlockNoPubsub is a function that defines the the operations (minus pubsub)
-// that are preformed on blocks that is received from regular sync service. The operations consists of:
+// ReceiveBlock is a function that defines the the operations (minus pubsub)
+// that are performed on blocks that is received from regular sync service. The operations consists of:
 //   1. Validate block, apply state transition and update check points
 //   2. Apply fork choice to the processed block
 //   3. Save latest head info
-func (s *Service) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.BeaconBlock) error {
-	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlockNoPubsub")
+func (s *Service) ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
+	ctx, span := trace.StartSpan(ctx, "blockChain.ReceiveBlock")
 	defer span.End()
-	blockCopy := proto.Clone(block).(*ethpb.BeaconBlock)
+	blockCopy := stateTrie.CopySignedBeaconBlock(block)
 
 	// Apply state transition on the new block.
-	if err := s.forkChoiceStore.OnBlock(ctx, blockCopy); err != nil {
-		err := errors.Wrap(err, "could not process block from fork choice service")
+	if err := s.onBlock(ctx, blockCopy, blockRoot); err != nil {
+		err := errors.Wrap(err, "could not process block")
 		traceutil.AnnotateError(span, err)
 		return err
 	}
-	root, err := ssz.SigningRoot(blockCopy)
-	if err != nil {
-		return errors.Wrap(err, "could not get signing root on received block")
-	}
 
-	// Run fork choice after applying state transition on the new block.
-	headRoot, err := s.forkChoiceStore.Head(ctx)
-	if err != nil {
-		return errors.Wrap(err, "could not get head from fork choice service")
-	}
-	headBlk, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(headRoot))
-	if err != nil {
-		return errors.Wrap(err, "could not compute state from block head")
-	}
-
-	// Only save head if it's different than the current head.
-	if !bytes.Equal(headRoot, s.HeadRoot()) {
-		if err := s.saveHead(ctx, headBlk, bytesutil.ToBytes32(headRoot)); err != nil {
-			return errors.Wrap(err, "could not save head")
-		}
-	}
-
-	// Remove block's contained deposits, attestations, and other operations from persistent storage.
-	if err := s.cleanupBlockOperations(ctx, blockCopy); err != nil {
-		return errors.Wrap(err, "could not clean up block deposits, attestations, and other operations")
+	// Update and save head block after fork choice.
+	if err := s.updateHead(ctx, s.getJustifiedBalances()); err != nil {
+		log.WithError(err).Warn("Could not update head")
 	}
 
 	// Send notification of the processed block to the state feed.
-	s.stateNotifier.StateFeed().Send(&statefeed.Event{
+	s.stateNotifier.StateFeed().Send(&feed.Event{
 		Type: statefeed.BlockProcessed,
 		Data: &statefeed.BlockProcessedData{
-			BlockRoot: root,
+			Slot:      blockCopy.Block.Slot,
+			BlockRoot: blockRoot,
 			Verified:  true,
 		},
 	})
 
-	// Reports on block and fork choice metrics.
-	s.reportSlotMetrics(blockCopy.Slot)
+	// Handle post block operations such as attestations and exits.
+	if err := s.handlePostBlockOperations(blockCopy.Block); err != nil {
+		return err
+	}
 
-	// Log if block is a competing block.
-	isCompetingBlock(root[:], blockCopy.Slot, headRoot, headBlk.Slot)
+	// Have we been finalizing? Should we start saving hot states to db?
+	if err := s.checkSaveHotStateDB(ctx); err != nil {
+		return err
+	}
+
+	// Reports on block and fork choice metrics.
+	reportSlotMetrics(blockCopy.Block.Slot, s.HeadSlot(), s.CurrentSlot(), s.finalizedCheckpt)
+
+	// Log block sync status.
+	logBlockSyncStatus(blockCopy.Block, blockRoot, s.finalizedCheckpt)
 
 	// Log state transition data.
-	logStateTransitionData(blockCopy, root[:])
-
-	processedBlkNoPubsub.Inc()
+	logStateTransitionData(blockCopy.Block)
 
 	return nil
 }
 
-// ReceiveBlockNoPubsubForkchoice is a function that defines the all operations (minus pubsub and forkchoice)
-// that are preformed blocks that is received from initial sync service. The operations consists of:
-//   1. Validate block, apply state transition and update check points
-//   2. Save latest head info
-func (s *Service) ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *ethpb.BeaconBlock) error {
-	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlockNoForkchoice")
+// ReceiveBlockInitialSync processes the input block for the purpose of initial syncing.
+// This method should only be used on blocks during initial syncing phase.
+func (s *Service) ReceiveBlockInitialSync(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
+	ctx, span := trace.StartSpan(ctx, "blockChain.ReceiveBlockNoVerify")
 	defer span.End()
-	blockCopy := proto.Clone(block).(*ethpb.BeaconBlock)
+	blockCopy := stateTrie.CopySignedBeaconBlock(block)
 
-	// Apply state transition on the incoming newly received block.
-	if err := s.forkChoiceStore.OnBlock(ctx, blockCopy); err != nil {
-		err := errors.Wrap(err, "could not process block from fork choice service")
+	// Apply state transition on the new block.
+	if err := s.onBlockInitialSyncStateTransition(ctx, blockCopy, blockRoot); err != nil {
+		err := errors.Wrap(err, "could not process block")
 		traceutil.AnnotateError(span, err)
 		return err
 	}
-	root, err := ssz.SigningRoot(blockCopy)
-	if err != nil {
-		return errors.Wrap(err, "could not get signing root on received block")
-	}
-
-	if !bytes.Equal(root[:], s.HeadRoot()) {
-		if err := s.saveHead(ctx, blockCopy, root); err != nil {
-			return errors.Wrap(err, "could not save head")
-		}
-	}
-
-	// Remove block's contained deposits, attestations, and other operations from persistent storage.
-	if err := s.cleanupBlockOperations(ctx, blockCopy); err != nil {
-		return errors.Wrap(err, "could not clean up block deposits, attestations, and other operations")
-	}
 
 	// Send notification of the processed block to the state feed.
-	s.stateNotifier.StateFeed().Send(&statefeed.Event{
+	s.stateNotifier.StateFeed().Send(&feed.Event{
 		Type: statefeed.BlockProcessed,
 		Data: &statefeed.BlockProcessedData{
-			BlockRoot: root,
+			Slot:      blockCopy.Block.Slot,
+			BlockRoot: blockRoot,
 			Verified:  true,
 		},
 	})
 
-	// Reports on block and fork choice metrics.
-	s.reportSlotMetrics(blockCopy.Slot)
-
-	// Log state transition data.
-	logStateTransitionData(blockCopy, root[:])
-
-	processedBlkNoPubsubForkchoice.Inc()
-	return nil
-}
-
-// ReceiveBlockNoVerify runs state transition on a input block without verifying the block's BLS contents.
-// Depends on the security model, this is the "minimal" work a node can do to sync the chain.
-// It simulates light client behavior and assumes 100% trust with the syncing peer.
-func (s *Service) ReceiveBlockNoVerify(ctx context.Context, block *ethpb.BeaconBlock) error {
-	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlockNoVerify")
-	defer span.End()
-	blockCopy := proto.Clone(block).(*ethpb.BeaconBlock)
-
-	// Apply state transition on the incoming newly received blockCopy without verifying its BLS contents.
-	if err := s.forkChoiceStore.OnBlockNoVerifyStateTransition(ctx, blockCopy); err != nil {
-		return errors.Wrap(err, "could not process blockCopy from fork choice service")
-	}
-	root, err := ssz.SigningRoot(blockCopy)
-	if err != nil {
-		return errors.Wrap(err, "could not get signing root on received blockCopy")
-	}
-
-	if !bytes.Equal(root[:], s.HeadRoot()) {
-		if err := s.saveHead(ctx, blockCopy, root); err != nil {
-			err := errors.Wrap(err, "could not save head")
-			traceutil.AnnotateError(span, err)
-			return err
-		}
-	}
-
-	// Send notification of the processed block to the state feed.
-	s.stateNotifier.StateFeed().Send(&statefeed.Event{
-		Type: statefeed.BlockProcessed,
-		Data: &statefeed.BlockProcessedData{
-			BlockRoot: root,
-			Verified:  false,
-		},
-	})
-
 	// Reports on blockCopy and fork choice metrics.
-	s.reportSlotMetrics(blockCopy.Slot)
+	reportSlotMetrics(blockCopy.Block.Slot, s.HeadSlot(), s.CurrentSlot(), s.finalizedCheckpt)
 
 	// Log state transition data.
 	log.WithFields(logrus.Fields{
-		"slot":         blockCopy.Slot,
-		"attestations": len(blockCopy.Body.Attestations),
-		"deposits":     len(blockCopy.Body.Deposits),
+		"slot":         blockCopy.Block.Slot,
+		"attestations": len(blockCopy.Block.Body.Attestations),
+		"deposits":     len(blockCopy.Block.Body.Deposits),
 	}).Debug("Finished applying state transition")
 
 	return nil
 }
 
-// cleanupBlockOperations processes and cleans up any block operations relevant to the beacon node
-// such as attestations, exits, and deposits. We update the latest seen attestation by validator
-// in the local node's runtime, cleanup and remove pending deposits which have been included in the block
-// from our node's local cache, and process validator exits and more.
-func (s *Service) cleanupBlockOperations(ctx context.Context, block *ethpb.BeaconBlock) error {
-	// Forward processed block to operation pool to remove individual operation from DB.
-	if s.opsPoolService.IncomingProcessedBlockFeed().Send(block) == 0 {
-		log.Error("Sent processed block to no subscribers")
+// ReceiveBlockBatch processes the whole block batch at once, assuming the block batch is linear ,transitioning
+// the state, performing batch verification of all collected signatures and then performing the appropriate
+// actions for a block post-transition.
+func (s *Service) ReceiveBlockBatch(ctx context.Context, blocks []*ethpb.SignedBeaconBlock, blkRoots [][32]byte) error {
+	ctx, span := trace.StartSpan(ctx, "blockChain.ReceiveBlockBatch")
+	defer span.End()
+
+	// Apply state transition on the incoming newly received blockCopy without verifying its BLS contents.
+	fCheckpoints, jCheckpoints, err := s.onBlockBatch(ctx, blocks, blkRoots)
+	if err != nil {
+		err := errors.Wrap(err, "could not process block in batch")
+		traceutil.AnnotateError(span, err)
+		return err
 	}
 
-	// Remove pending deposits from the deposit queue.
-	for _, dep := range block.Body.Deposits {
-		s.depositCache.RemovePendingDeposit(ctx, dep)
+	for i, b := range blocks {
+		blockCopy := stateTrie.CopySignedBeaconBlock(b)
+		if err = s.handleBlockAfterBatchVerify(ctx, blockCopy, blkRoots[i], fCheckpoints[i], jCheckpoints[i]); err != nil {
+			traceutil.AnnotateError(span, err)
+			return err
+		}
+		// Send notification of the processed block to the state feed.
+		s.stateNotifier.StateFeed().Send(&feed.Event{
+			Type: statefeed.BlockProcessed,
+			Data: &statefeed.BlockProcessedData{
+				Slot:      blockCopy.Block.Slot,
+				BlockRoot: blkRoots[i],
+				Verified:  true,
+			},
+		})
+
+		// Reports on blockCopy and fork choice metrics.
+		reportSlotMetrics(blockCopy.Block.Slot, s.HeadSlot(), s.CurrentSlot(), s.finalizedCheckpt)
+	}
+
+	if err := s.VerifyWeakSubjectivityRoot(s.ctx); err != nil {
+		// log.Fatalf will prevent defer from being called
+		span.End()
+		// Exit run time if the node failed to verify weak subjectivity checkpoint.
+		log.Fatalf("Could not verify weak subjectivity checkpoint: %v", err)
+	}
+
+	return nil
+}
+
+// HasInitSyncBlock returns true if the block of the input root exists in initial sync blocks cache.
+func (s *Service) HasInitSyncBlock(root [32]byte) bool {
+	return s.hasInitSyncBlock(root)
+}
+
+func (s *Service) handlePostBlockOperations(b *ethpb.BeaconBlock) error {
+	// Delete the processed block attestations from attestation pool.
+	if err := s.deletePoolAtts(b.Body.Attestations); err != nil {
+		return err
+	}
+
+	// Add block attestations to the fork choice pool to compute head.
+	if err := s.attPool.SaveBlockAttestations(b.Body.Attestations); err != nil {
+		log.Errorf("Could not save block attestations for fork choice: %v", err)
+		return nil
+	}
+	// Mark block exits as seen so we don't include same ones in future blocks.
+	for _, e := range b.Body.VoluntaryExits {
+		s.exitPool.MarkIncluded(e)
+	}
+
+	//  Mark attester slashings as seen so we don't include same ones in future blocks.
+	for _, as := range b.Body.AttesterSlashings {
+		s.slashingPool.MarkIncludedAttesterSlashing(as)
 	}
 	return nil
 }
 
-// This checks if the block is from a competing chain, emits warning and updates metrics.
-func isCompetingBlock(root []byte, slot uint64, headRoot []byte, headSlot uint64) {
-	if !bytes.Equal(root[:], headRoot) {
-		log.WithFields(logrus.Fields{
-			"blkSlot":  slot,
-			"blkRoot":  hex.EncodeToString(root[:]),
-			"headSlot": headSlot,
-			"headRoot": hex.EncodeToString(headRoot),
-		}).Warn("Calculated head diffs from new block")
-		competingBlks.Inc()
+// This checks whether it's time to start saving hot state to DB.
+// It's time when there's `epochsSinceFinalitySaveHotStateDB` epochs of non-finality.
+func (s *Service) checkSaveHotStateDB(ctx context.Context) error {
+	currentEpoch := helpers.SlotToEpoch(s.CurrentSlot())
+	// Prevent `sinceFinality` going underflow.
+	var sinceFinality uint64
+	if currentEpoch > s.finalizedCheckpt.Epoch {
+		sinceFinality = currentEpoch - s.finalizedCheckpt.Epoch
 	}
+
+	if sinceFinality >= uint64(epochsSinceFinalitySaveHotStateDB) {
+		s.stateGen.EnableSaveHotStateToDB(ctx)
+		return nil
+	}
+
+	return s.stateGen.DisableSaveHotStateToDB(ctx)
 }

beacon-chain/blockchain/receive_block_test.go

@@ -1,275 +1,415 @@
 package blockchain
 
 import (
-	"bytes"
 	"context"
-	"reflect"
+	"sync"
 	"testing"
+	"time"
 
-	"github.com/prysmaticlabs/go-ssz"
-	b "github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	blockchainTesting "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	"github.com/prysmaticlabs/prysm/beacon-chain/operations/attestations"
+	"github.com/prysmaticlabs/prysm/beacon-chain/operations/voluntaryexits"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 	logTest "github.com/sirupsen/logrus/hooks/test"
 )
 
-func TestReceiveBlock_ProcessCorrectly(t *testing.T) {
-	hook := logTest.NewGlobal()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+func TestService_ReceiveBlock(t *testing.T) {
 	ctx := context.Background()
 
-	chainService := setupBeaconChain(t, db)
-	deposits, _, privKeys := testutil.SetupInitialDeposits(t, 100)
-	beaconState, err := state.GenesisBeaconState(deposits, 0, &ethpb.Eth1Data{BlockHash: make([]byte, 32)})
-	if err != nil {
-		t.Fatal(err)
-	}
-	beaconState.Eth1Data.BlockHash = nil
-	beaconState.Eth1DepositIndex = 100
-	stateRoot, err := ssz.HashTreeRoot(beaconState)
-	if err != nil {
-		t.Fatal(err)
-	}
-	genesis := b.NewGenesisBlock(stateRoot[:])
-	bodyRoot, err := ssz.HashTreeRoot(genesis.Body)
-	if err != nil {
-		t.Fatal(err)
+	genesis, keys := testutil.DeterministicGenesisState(t, 64)
+	genFullBlock := func(t *testing.T, conf *testutil.BlockGenConfig, slot uint64) *ethpb.SignedBeaconBlock {
+		blk, err := testutil.GenerateFullBlock(genesis, keys, conf, slot)
+		assert.NoError(t, err)
+		return blk
 	}
+	bc := params.BeaconConfig()
+	bc.ShardCommitteePeriod = 0 // Required for voluntary exits test in reasonable time.
+	params.OverrideBeaconConfig(bc)
 
-	genesisBlkRoot, err := ssz.SigningRoot(genesis)
-	if err != nil {
-		t.Fatal(err)
+	type args struct {
+		block *ethpb.SignedBeaconBlock
 	}
-	cp := &ethpb.Checkpoint{Root: genesisBlkRoot[:]}
-	if err := chainService.forkChoiceStore.GenesisStore(ctx, cp, cp); err != nil {
-		t.Fatal(err)
-	}
-
-	beaconState.LatestBlockHeader = &ethpb.BeaconBlockHeader{
-		Slot:       genesis.Slot,
-		ParentRoot: genesis.ParentRoot,
-		BodyRoot:   bodyRoot[:],
-		StateRoot:  genesis.StateRoot,
-	}
-	if err := chainService.beaconDB.SaveBlock(ctx, genesis); err != nil {
-		t.Fatalf("Could not save block to db: %v", err)
-	}
-	parentRoot, err := ssz.SigningRoot(genesis)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err := db.SaveState(ctx, beaconState, parentRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	slot := beaconState.Slot + 1
-	epoch := helpers.SlotToEpoch(slot)
-	beaconState.Slot++
-	randaoReveal, err := testutil.CreateRandaoReveal(beaconState, epoch, privKeys)
-	if err != nil {
-		t.Fatal(err)
-	}
-	beaconState.Slot--
-
-	block := &ethpb.BeaconBlock{
-		Slot:       slot,
-		ParentRoot: parentRoot[:],
-		Body: &ethpb.BeaconBlockBody{
-			Eth1Data: &ethpb.Eth1Data{
-				DepositCount: uint64(len(deposits)),
-				DepositRoot:  []byte("a"),
-				BlockHash:    []byte("b"),
+	tests := []struct {
+		name      string
+		args      args
+		wantedErr string
+		check     func(*testing.T, *Service)
+	}{
+		{
+			name: "applies block with state transition",
+			args: args{
+				block: genFullBlock(t, testutil.DefaultBlockGenConfig(), 2 /*slot*/),
+			},
+			check: func(t *testing.T, s *Service) {
+				if hs := s.head.state.Slot(); hs != 2 {
+					t.Errorf("Unexpected state slot. Got %d but wanted %d", hs, 2)
+				}
+				if bs := s.head.block.Block.Slot; bs != 2 {
+					t.Errorf("Unexpected head block slot. Got %d but wanted %d", bs, 2)
+				}
+			},
+		},
+		{
+			name: "saves attestations to pool",
+			args: args{
+				block: genFullBlock(t,
+					&testutil.BlockGenConfig{
+						NumProposerSlashings: 0,
+						NumAttesterSlashings: 0,
+						NumAttestations:      2,
+						NumDeposits:          0,
+						NumVoluntaryExits:    0,
+					},
+					1, /*slot*/
+				),
+			},
+			check: func(t *testing.T, s *Service) {
+				if baCount := len(s.attPool.BlockAttestations()); baCount != 2 {
+					t.Errorf("Did not get the correct number of block attestations saved to the pool. "+
+						"Got %d but wanted %d", baCount, 2)
+				}
+			},
+		},
+
+		{
+			name: "updates exit pool",
+			args: args{
+				block: genFullBlock(t, &testutil.BlockGenConfig{
+					NumProposerSlashings: 0,
+					NumAttesterSlashings: 0,
+					NumAttestations:      0,
+					NumDeposits:          0,
+					NumVoluntaryExits:    3,
+				},
+					1, /*slot*/
+				),
+			},
+			check: func(t *testing.T, s *Service) {
+				var n int
+				for i := uint64(0); int(i) < genesis.NumValidators(); i++ {
+					if s.exitPool.HasBeenIncluded(i) {
+						n++
+					}
+				}
+				if n != 3 {
+					t.Errorf("Did not mark the correct number of exits. Got %d but wanted %d", n, 3)
+				}
+			},
+		},
+		{
+			name: "notifies block processed on state feed",
+			args: args{
+				block: genFullBlock(t, testutil.DefaultBlockGenConfig(), 1 /*slot*/),
+			},
+			check: func(t *testing.T, s *Service) {
+				if recvd := len(s.stateNotifier.(*blockchainTesting.MockStateNotifier).ReceivedEvents()); recvd < 1 {
+					t.Errorf("Received %d state notifications, expected at least 1", recvd)
+				}
 			},
-			RandaoReveal: randaoReveal[:],
-			Attestations: nil,
 		},
 	}
 
-	stateRootCandidate, err := state.ExecuteStateTransitionNoVerify(context.Background(), beaconState, block)
-	if err != nil {
-		t.Fatal(err)
-	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			db, stateSummaryCache := testDB.SetupDB(t)
+			genesisBlockRoot := bytesutil.ToBytes32(nil)
+			require.NoError(t, db.SaveState(ctx, genesis, genesisBlockRoot))
 
-	stateRoot, err = ssz.HashTreeRoot(stateRootCandidate)
-	if err != nil {
-		t.Fatal(err)
+			cfg := &Config{
+				BeaconDB: db,
+				ForkChoiceStore: protoarray.New(
+					0, // justifiedEpoch
+					0, // finalizedEpoch
+					genesisBlockRoot,
+				),
+				AttPool:       attestations.NewPool(),
+				ExitPool:      voluntaryexits.NewPool(),
+				StateNotifier: &blockchainTesting.MockStateNotifier{RecordEvents: true},
+				StateGen:      stategen.New(db, stateSummaryCache),
+			}
+			s, err := NewService(ctx, cfg)
+			require.NoError(t, err)
+			require.NoError(t, s.saveGenesisData(ctx, genesis))
+			gBlk, err := s.beaconDB.GenesisBlock(ctx)
+			require.NoError(t, err)
+			gRoot, err := gBlk.Block.HashTreeRoot()
+			require.NoError(t, err)
+			s.finalizedCheckpt = &ethpb.Checkpoint{Root: gRoot[:]}
+			root, err := tt.args.block.Block.HashTreeRoot()
+			require.NoError(t, err)
+			err = s.ReceiveBlock(ctx, tt.args.block, root)
+			if tt.wantedErr != "" {
+				assert.ErrorContains(t, tt.wantedErr, err)
+			} else {
+				assert.NoError(t, err)
+				tt.check(t, s)
+			}
+		})
 	}
-	block.StateRoot = stateRoot[:]
-
-	block, err = testutil.SignBlock(beaconState, block, privKeys)
-	if err != nil {
-		t.Error(err)
-	}
-
-	if err := chainService.beaconDB.SaveBlock(ctx, block); err != nil {
-		t.Fatal(err)
-	}
-	if err := chainService.ReceiveBlock(context.Background(), block); err != nil {
-		t.Errorf("Block failed processing: %v", err)
-	}
-	testutil.AssertLogsContain(t, hook, "Finished applying state transition")
 }
 
-func TestReceiveReceiveBlockNoPubsub_CanSaveHeadInfo(t *testing.T) {
-	hook := logTest.NewGlobal()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+func TestService_ReceiveBlockUpdateHead(t *testing.T) {
 	ctx := context.Background()
-
-	chainService := setupBeaconChain(t, db)
-
-	headBlk := &ethpb.BeaconBlock{Slot: 100}
-	if err := db.SaveBlock(ctx, headBlk); err != nil {
-		t.Fatal(err)
+	genesis, keys := testutil.DeterministicGenesisState(t, 64)
+	b, err := testutil.GenerateFullBlock(genesis, keys, testutil.DefaultBlockGenConfig(), 1)
+	assert.NoError(t, err)
+	db, stateSummaryCache := testDB.SetupDB(t)
+	genesisBlockRoot := bytesutil.ToBytes32(nil)
+	require.NoError(t, db.SaveState(ctx, genesis, genesisBlockRoot))
+	cfg := &Config{
+		BeaconDB: db,
+		ForkChoiceStore: protoarray.New(
+			0, // justifiedEpoch
+			0, // finalizedEpoch
+			genesisBlockRoot,
+		),
+		AttPool:       attestations.NewPool(),
+		ExitPool:      voluntaryexits.NewPool(),
+		StateNotifier: &blockchainTesting.MockStateNotifier{RecordEvents: true},
+		StateGen:      stategen.New(db, stateSummaryCache),
 	}
-	r, err := ssz.SigningRoot(headBlk)
-	if err != nil {
-		t.Fatal(err)
+	s, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+	require.NoError(t, s.saveGenesisData(ctx, genesis))
+	gBlk, err := s.beaconDB.GenesisBlock(ctx)
+	require.NoError(t, err)
+	gRoot, err := gBlk.Block.HashTreeRoot()
+	require.NoError(t, err)
+	s.finalizedCheckpt = &ethpb.Checkpoint{Root: gRoot[:]}
+	root, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		require.NoError(t, s.ReceiveBlock(ctx, b, root))
+		wg.Done()
+	}()
+	wg.Wait()
+	if recvd := len(s.stateNotifier.(*blockchainTesting.MockStateNotifier).ReceivedEvents()); recvd < 1 {
+		t.Errorf("Received %d state notifications, expected at least 1", recvd)
 	}
-	chainService.forkChoiceStore = &store{headRoot: r[:]}
-
-	if err := chainService.ReceiveBlockNoPubsub(ctx, &ethpb.BeaconBlock{
-		Slot: 1,
-		Body: &ethpb.BeaconBlockBody{}}); err != nil {
-		t.Fatal(err)
-	}
-
-	if !bytes.Equal(r[:], chainService.HeadRoot()) {
-		t.Error("Incorrect head root saved")
-	}
-
-	if !reflect.DeepEqual(headBlk, chainService.HeadBlock()) {
-		t.Error("Incorrect head block saved")
-	}
-
-	testutil.AssertLogsContain(t, hook, "Saved new head info")
+	// Verify fork choice has processed the block. (Genesis block and the new block)
+	assert.Equal(t, 2, len(s.forkChoiceStore.Nodes()))
 }
 
-func TestReceiveReceiveBlockNoPubsub_SameHead(t *testing.T) {
-	hook := logTest.NewGlobal()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+func TestService_ReceiveBlockInitialSync(t *testing.T) {
 	ctx := context.Background()
 
-	chainService := setupBeaconChain(t, db)
-
-	headBlk := &ethpb.BeaconBlock{}
-	if err := db.SaveBlock(ctx, headBlk); err != nil {
-		t.Fatal(err)
-	}
-	newBlk := &ethpb.BeaconBlock{
-		Slot: 1,
-		Body: &ethpb.BeaconBlockBody{}}
-	newRoot, _ := ssz.SigningRoot(newBlk)
-	if err := db.SaveBlock(ctx, newBlk); err != nil {
-		t.Fatal(err)
+	genesis, keys := testutil.DeterministicGenesisState(t, 64)
+	genFullBlock := func(t *testing.T, conf *testutil.BlockGenConfig, slot uint64) *ethpb.SignedBeaconBlock {
+		blk, err := testutil.GenerateFullBlock(genesis, keys, conf, slot)
+		assert.NoError(t, err)
+		return blk
 	}
 
-	chainService.forkChoiceStore = &store{headRoot: newRoot[:]}
-	chainService.canonicalRoots[0] = newRoot[:]
-
-	if err := chainService.ReceiveBlockNoPubsub(ctx, newBlk); err != nil {
-		t.Fatal(err)
+	type args struct {
+		block *ethpb.SignedBeaconBlock
 	}
-
-	testutil.AssertLogsDoNotContain(t, hook, "Saved new head info")
-}
-
-func TestReceiveBlockNoPubsubForkchoice_ProcessCorrectly(t *testing.T) {
-	hook := logTest.NewGlobal()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	ctx := context.Background()
-
-	chainService := setupBeaconChain(t, db)
-	deposits, _, privKeys := testutil.SetupInitialDeposits(t, 100)
-	beaconState, err := state.GenesisBeaconState(deposits, 0, &ethpb.Eth1Data{BlockHash: make([]byte, 32)})
-	if err != nil {
-		t.Fatal(err)
-	}
-	beaconState.Eth1DepositIndex = 100
-	stateRoot, err := ssz.HashTreeRoot(beaconState)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	genesis := b.NewGenesisBlock(stateRoot[:])
-	bodyRoot, err := ssz.HashTreeRoot(genesis.Body)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := chainService.forkChoiceStore.GenesisStore(ctx, &ethpb.Checkpoint{}, &ethpb.Checkpoint{}); err != nil {
-		t.Fatal(err)
-	}
-
-	beaconState.LatestBlockHeader = &ethpb.BeaconBlockHeader{
-		Slot:       genesis.Slot,
-		ParentRoot: genesis.ParentRoot,
-		BodyRoot:   bodyRoot[:],
-		StateRoot:  genesis.StateRoot,
-	}
-	if err := chainService.beaconDB.SaveBlock(ctx, genesis); err != nil {
-		t.Fatalf("Could not save block to db: %v", err)
-	}
-	parentRoot, err := ssz.SigningRoot(genesis)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err := db.SaveState(ctx, beaconState, parentRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	slot := beaconState.Slot + 1
-	epoch := helpers.SlotToEpoch(slot)
-	beaconState.Slot++
-	randaoReveal, err := testutil.CreateRandaoReveal(beaconState, epoch, privKeys)
-	if err != nil {
-		t.Fatal(err)
-	}
-	beaconState.Slot--
-
-	block := &ethpb.BeaconBlock{
-		Slot:       slot,
-		ParentRoot: parentRoot[:],
-		Body: &ethpb.BeaconBlockBody{
-			Eth1Data: &ethpb.Eth1Data{
-				DepositCount: uint64(len(deposits)),
-				DepositRoot:  []byte("a"),
-				BlockHash:    []byte("b"),
+	tests := []struct {
+		name      string
+		args      args
+		wantedErr string
+		check     func(*testing.T, *Service)
+	}{
+		{
+			name: "applies block with state transition",
+			args: args{
+				block: genFullBlock(t, testutil.DefaultBlockGenConfig(), 2 /*slot*/),
+			},
+			check: func(t *testing.T, s *Service) {
+				assert.Equal(t, uint64(2), s.head.state.Slot(), "Incorrect head state slot")
+				assert.Equal(t, uint64(2), s.head.block.Block.Slot, "Incorrect head block slot")
+			},
+		},
+		{
+			name: "notifies block processed on state feed",
+			args: args{
+				block: genFullBlock(t, testutil.DefaultBlockGenConfig(), 1 /*slot*/),
+			},
+			check: func(t *testing.T, s *Service) {
+				if recvd := len(s.stateNotifier.(*blockchainTesting.MockStateNotifier).ReceivedEvents()); recvd < 1 {
+					t.Errorf("Received %d state notifications, expected at least 1", recvd)
+				}
 			},
-			RandaoReveal: randaoReveal[:],
-			Attestations: nil,
 		},
 	}
 
-	stateRootCandidate, err := state.ExecuteStateTransitionNoVerify(context.Background(), beaconState, block)
-	if err != nil {
-		t.Fatal(err)
-	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			db, stateSummaryCache := testDB.SetupDB(t)
+			genesisBlockRoot := bytesutil.ToBytes32(nil)
 
-	stateRoot, err = ssz.HashTreeRoot(stateRootCandidate)
-	if err != nil {
-		t.Fatal(err)
-	}
-	block.StateRoot = stateRoot[:]
+			cfg := &Config{
+				BeaconDB: db,
+				ForkChoiceStore: protoarray.New(
+					0, // justifiedEpoch
+					0, // finalizedEpoch
+					genesisBlockRoot,
+				),
+				StateNotifier: &blockchainTesting.MockStateNotifier{RecordEvents: true},
+				StateGen:      stategen.New(db, stateSummaryCache),
+			}
+			s, err := NewService(ctx, cfg)
+			require.NoError(t, err)
+			err = s.saveGenesisData(ctx, genesis)
+			require.NoError(t, err)
+			gBlk, err := s.beaconDB.GenesisBlock(ctx)
+			require.NoError(t, err)
 
-	block, err = testutil.SignBlock(beaconState, block, privKeys)
-	if err != nil {
-		t.Error(err)
-	}
+			gRoot, err := gBlk.Block.HashTreeRoot()
+			require.NoError(t, err)
+			s.finalizedCheckpt = &ethpb.Checkpoint{Root: gRoot[:]}
+			root, err := tt.args.block.Block.HashTreeRoot()
+			require.NoError(t, err)
 
-	if err := chainService.beaconDB.SaveBlock(ctx, block); err != nil {
-		t.Fatal(err)
+			err = s.ReceiveBlockInitialSync(ctx, tt.args.block, root)
+			if tt.wantedErr != "" {
+				assert.ErrorContains(t, tt.wantedErr, err)
+			} else {
+				assert.NoError(t, err)
+				tt.check(t, s)
+			}
+		})
 	}
-	if err := chainService.ReceiveBlockNoPubsubForkchoice(context.Background(), block); err != nil {
-		t.Errorf("Block failed processing: %v", err)
-	}
-	testutil.AssertLogsContain(t, hook, "Finished applying state transition")
-	testutil.AssertLogsDoNotContain(t, hook, "Finished fork choice")
+}
+
+func TestService_ReceiveBlockBatch(t *testing.T) {
+	ctx := context.Background()
+
+	genesis, keys := testutil.DeterministicGenesisState(t, 64)
+	genFullBlock := func(t *testing.T, conf *testutil.BlockGenConfig, slot uint64) *ethpb.SignedBeaconBlock {
+		blk, err := testutil.GenerateFullBlock(genesis, keys, conf, slot)
+		assert.NoError(t, err)
+		return blk
+	}
+
+	type args struct {
+		block *ethpb.SignedBeaconBlock
+	}
+	tests := []struct {
+		name      string
+		args      args
+		wantedErr string
+		check     func(*testing.T, *Service)
+	}{
+		{
+			name: "applies block with state transition",
+			args: args{
+				block: genFullBlock(t, testutil.DefaultBlockGenConfig(), 2 /*slot*/),
+			},
+			check: func(t *testing.T, s *Service) {
+				assert.Equal(t, uint64(2), s.head.state.Slot(), "Incorrect head state slot")
+				assert.Equal(t, uint64(2), s.head.block.Block.Slot, "Incorrect head block slot")
+			},
+		},
+		{
+			name: "notifies block processed on state feed",
+			args: args{
+				block: genFullBlock(t, testutil.DefaultBlockGenConfig(), 1 /*slot*/),
+			},
+			check: func(t *testing.T, s *Service) {
+				if recvd := len(s.stateNotifier.(*blockchainTesting.MockStateNotifier).ReceivedEvents()); recvd < 1 {
+					t.Errorf("Received %d state notifications, expected at least 1", recvd)
+				}
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			db, stateSummaryCache := testDB.SetupDB(t)
+			genesisBlockRoot, err := genesis.HashTreeRoot(ctx)
+			require.NoError(t, err)
+			cfg := &Config{
+				BeaconDB: db,
+				ForkChoiceStore: protoarray.New(
+					0, // justifiedEpoch
+					0, // finalizedEpoch
+					genesisBlockRoot,
+				),
+				StateNotifier: &blockchainTesting.MockStateNotifier{RecordEvents: true},
+				StateGen:      stategen.New(db, stateSummaryCache),
+			}
+			s, err := NewService(ctx, cfg)
+			require.NoError(t, err)
+			err = s.saveGenesisData(ctx, genesis)
+			require.NoError(t, err)
+			gBlk, err := s.beaconDB.GenesisBlock(ctx)
+			require.NoError(t, err)
+
+			gRoot, err := gBlk.Block.HashTreeRoot()
+			require.NoError(t, err)
+			s.finalizedCheckpt = &ethpb.Checkpoint{Root: gRoot[:]}
+			root, err := tt.args.block.Block.HashTreeRoot()
+			require.NoError(t, err)
+			blks := []*ethpb.SignedBeaconBlock{tt.args.block}
+			roots := [][32]byte{root}
+			err = s.ReceiveBlockBatch(ctx, blks, roots)
+			if tt.wantedErr != "" {
+				assert.ErrorContains(t, tt.wantedErr, err)
+			} else {
+				assert.NoError(t, err)
+				tt.check(t, s)
+			}
+		})
+	}
+}
+
+func TestService_HasInitSyncBlock(t *testing.T) {
+	s, err := NewService(context.Background(), &Config{StateNotifier: &blockchainTesting.MockStateNotifier{}})
+	require.NoError(t, err)
+	r := [32]byte{'a'}
+	if s.HasInitSyncBlock(r) {
+		t.Error("Should not have block")
+	}
+	s.saveInitSyncBlock(r, testutil.NewBeaconBlock())
+	if !s.HasInitSyncBlock(r) {
+		t.Error("Should have block")
+	}
+}
+
+func TestCheckSaveHotStateDB_Enabling(t *testing.T) {
+	db, stateSummaryCache := testDB.SetupDB(t)
+	hook := logTest.NewGlobal()
+	s, err := NewService(context.Background(), &Config{StateGen: stategen.New(db, stateSummaryCache)})
+	require.NoError(t, err)
+	st := params.BeaconConfig().SlotsPerEpoch * uint64(epochsSinceFinalitySaveHotStateDB)
+	s.genesisTime = time.Now().Add(time.Duration(-1*int64(st)*int64(params.BeaconConfig().SecondsPerSlot)) * time.Second)
+	s.finalizedCheckpt = &ethpb.Checkpoint{}
+
+	require.NoError(t, s.checkSaveHotStateDB(context.Background()))
+	assert.LogsContain(t, hook, "Entering mode to save hot states in DB")
+}
+
+func TestCheckSaveHotStateDB_Disabling(t *testing.T) {
+	db, stateSummaryCache := testDB.SetupDB(t)
+	hook := logTest.NewGlobal()
+	s, err := NewService(context.Background(), &Config{StateGen: stategen.New(db, stateSummaryCache)})
+	require.NoError(t, err)
+	s.finalizedCheckpt = &ethpb.Checkpoint{}
+	require.NoError(t, s.checkSaveHotStateDB(context.Background()))
+	s.genesisTime = time.Now()
+
+	require.NoError(t, s.checkSaveHotStateDB(context.Background()))
+	assert.LogsContain(t, hook, "Exiting mode to save hot states in DB")
+}
+
+func TestCheckSaveHotStateDB_Overflow(t *testing.T) {
+	db, stateSummaryCache := testDB.SetupDB(t)
+	hook := logTest.NewGlobal()
+	s, err := NewService(context.Background(), &Config{StateGen: stategen.New(db, stateSummaryCache)})
+	require.NoError(t, err)
+	s.finalizedCheckpt = &ethpb.Checkpoint{Epoch: 10000000}
+	s.genesisTime = time.Now()
+
+	require.NoError(t, s.checkSaveHotStateDB(context.Background()))
+	assert.LogsDoNotContain(t, hook, "Entering mode to save hot states in DB")
 }

beacon-chain/blockchain/service.go

@@ -1,6 +1,6 @@
-// Package blockchain defines the life-cycle and status of the beacon chain
-// as well as the Ethereum Serenity beacon chain fork-choice rule based on
-// Casper Proof of Stake finality.
+// Package blockchain defines the life-cycle of the blockchain at the core of
+// eth2, including processing of new blocks and attestations using casper
+// proof of stake.
 package blockchain
 
 import (
@@ -11,142 +11,269 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/blockchain/forkchoice"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
+	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/statefeed"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
-	"github.com/prysmaticlabs/prysm/beacon-chain/operations"
+	"github.com/prysmaticlabs/prysm/beacon-chain/flags"
+	f "github.com/prysmaticlabs/prysm/beacon-chain/forkchoice"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	"github.com/prysmaticlabs/prysm/beacon-chain/operations/attestations"
+	"github.com/prysmaticlabs/prysm/beacon-chain/operations/slashings"
+	"github.com/prysmaticlabs/prysm/beacon-chain/operations/voluntaryexits"
 	"github.com/prysmaticlabs/prysm/beacon-chain/p2p"
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/event"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"github.com/sirupsen/logrus"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/slotutil"
 	"go.opencensus.io/trace"
 )
 
-// ChainFeeds interface defines the methods of the Service which provide state related
-// information feeds to consumers.
-type ChainFeeds interface {
-	StateInitializedFeed() *event.Feed
-}
-
 // Service represents a service that handles the internal
 // logic of managing the full PoS beacon chain.
 type Service struct {
-	ctx                  context.Context
-	cancel               context.CancelFunc
-	beaconDB             db.Database
-	depositCache         *depositcache.DepositCache
-	chainStartFetcher    powchain.ChainStartFetcher
-	opsPoolService       operations.OperationFeeds
-	forkChoiceStore      forkchoice.ForkChoicer
-	chainStartChan       chan time.Time
-	genesisTime          time.Time
-	stateInitializedFeed *event.Feed
-	p2p                  p2p.Broadcaster
-	maxRoutines          int64
-	headSlot             uint64
-	headBlock            *ethpb.BeaconBlock
-	headState            *pb.BeaconState
-	canonicalRoots       map[uint64][]byte
-	headLock             sync.RWMutex
-
-	stateNotifier statefeed.Notifier
+	ctx                   context.Context
+	cancel                context.CancelFunc
+	beaconDB              db.HeadAccessDatabase
+	depositCache          *depositcache.DepositCache
+	chainStartFetcher     powchain.ChainStartFetcher
+	attPool               attestations.Pool
+	slashingPool          *slashings.Pool
+	exitPool              *voluntaryexits.Pool
+	genesisTime           time.Time
+	p2p                   p2p.Broadcaster
+	maxRoutines           int
+	head                  *head
+	headLock              sync.RWMutex
+	stateNotifier         statefeed.Notifier
+	genesisRoot           [32]byte
+	forkChoiceStore       f.ForkChoicer
+	justifiedCheckpt      *ethpb.Checkpoint
+	prevJustifiedCheckpt  *ethpb.Checkpoint
+	bestJustifiedCheckpt  *ethpb.Checkpoint
+	finalizedCheckpt      *ethpb.Checkpoint
+	prevFinalizedCheckpt  *ethpb.Checkpoint
+	nextEpochBoundarySlot uint64
+	initSyncState         map[[32]byte]*stateTrie.BeaconState
+	boundaryRoots         [][32]byte
+	checkpointState       *cache.CheckpointStateCache
+	checkpointStateLock   sync.Mutex
+	stateGen              *stategen.State
+	opsService            *attestations.Service
+	initSyncBlocks        map[[32]byte]*ethpb.SignedBeaconBlock
+	initSyncBlocksLock    sync.RWMutex
+	justifiedBalances     []uint64
+	justifiedBalancesLock sync.RWMutex
+	checkPtInfoCache      *checkPtInfoCache
+	wsEpoch               uint64
+	wsRoot                []byte
+	wsVerified            bool
 }
 
 // Config options for the service.
 type Config struct {
 	BeaconBlockBuf    int
 	ChainStartFetcher powchain.ChainStartFetcher
-	BeaconDB          db.Database
+	BeaconDB          db.HeadAccessDatabase
 	DepositCache      *depositcache.DepositCache
-	OpsPoolService    operations.OperationFeeds
+	AttPool           attestations.Pool
+	ExitPool          *voluntaryexits.Pool
+	SlashingPool      *slashings.Pool
 	P2p               p2p.Broadcaster
-	MaxRoutines       int64
+	MaxRoutines       int
 	StateNotifier     statefeed.Notifier
+	ForkChoiceStore   f.ForkChoicer
+	OpsService        *attestations.Service
+	StateGen          *stategen.State
+	WspBlockRoot      []byte
+	WspEpoch          uint64
 }
 
 // NewService instantiates a new block service instance that will
 // be registered into a running beacon node.
 func NewService(ctx context.Context, cfg *Config) (*Service, error) {
 	ctx, cancel := context.WithCancel(ctx)
-	store := forkchoice.NewForkChoiceService(ctx, cfg.BeaconDB)
 	return &Service{
-		ctx:                  ctx,
-		cancel:               cancel,
-		beaconDB:             cfg.BeaconDB,
-		depositCache:         cfg.DepositCache,
-		chainStartFetcher:    cfg.ChainStartFetcher,
-		opsPoolService:       cfg.OpsPoolService,
-		forkChoiceStore:      store,
-		chainStartChan:       make(chan time.Time),
-		stateInitializedFeed: new(event.Feed),
-		p2p:                  cfg.P2p,
-		canonicalRoots:       make(map[uint64][]byte),
-		maxRoutines:          cfg.MaxRoutines,
-		stateNotifier:        cfg.StateNotifier,
+		ctx:               ctx,
+		cancel:            cancel,
+		beaconDB:          cfg.BeaconDB,
+		depositCache:      cfg.DepositCache,
+		chainStartFetcher: cfg.ChainStartFetcher,
+		attPool:           cfg.AttPool,
+		exitPool:          cfg.ExitPool,
+		slashingPool:      cfg.SlashingPool,
+		p2p:               cfg.P2p,
+		maxRoutines:       cfg.MaxRoutines,
+		stateNotifier:     cfg.StateNotifier,
+		forkChoiceStore:   cfg.ForkChoiceStore,
+		initSyncState:     make(map[[32]byte]*stateTrie.BeaconState),
+		boundaryRoots:     [][32]byte{},
+		checkpointState:   cache.NewCheckpointStateCache(),
+		opsService:        cfg.OpsService,
+		stateGen:          cfg.StateGen,
+		initSyncBlocks:    make(map[[32]byte]*ethpb.SignedBeaconBlock),
+		justifiedBalances: make([]uint64, 0),
+		checkPtInfoCache:  newCheckPointInfoCache(),
+		wsEpoch:           cfg.WspEpoch,
+		wsRoot:            cfg.WspBlockRoot,
 	}, nil
 }
 
 // Start a blockchain service's main event loop.
 func (s *Service) Start() {
-	ctx := context.TODO()
-	beaconState, err := s.beaconDB.HeadState(ctx)
+	beaconState, err := s.beaconDB.HeadState(s.ctx)
 	if err != nil {
 		log.Fatalf("Could not fetch beacon state: %v", err)
 	}
+
+	// For running initial sync with state cache, in an event of restart, we use
+	// last finalized check point as start point to sync instead of head
+	// state. This is because we no longer save state every slot during sync.
+	cp, err := s.beaconDB.FinalizedCheckpoint(s.ctx)
+	if err != nil {
+		log.Fatalf("Could not fetch finalized cp: %v", err)
+	}
+
+	if beaconState == nil {
+		r := bytesutil.ToBytes32(cp.Root)
+		// Before the first finalized epoch, in the current epoch,
+		// the finalized root is defined as zero hashes instead of genesis root hash.
+		// We want to use genesis root to retrieve for state.
+		if r == params.BeaconConfig().ZeroHash {
+			genesisBlock, err := s.beaconDB.GenesisBlock(s.ctx)
+			if err != nil {
+				log.Fatalf("Could not fetch finalized cp: %v", err)
+			}
+			if genesisBlock != nil {
+				r, err = genesisBlock.Block.HashTreeRoot()
+				if err != nil {
+					log.Fatalf("Could not tree hash genesis block: %v", err)
+				}
+			}
+		}
+		beaconState, err = s.stateGen.StateByRoot(s.ctx, r)
+		if err != nil {
+			log.Fatalf("Could not fetch beacon state by root: %v", err)
+		}
+	}
+
+	// Make sure that attestation processor is subscribed and ready for state initializing event.
+	attestationProcessorSubscribed := make(chan struct{}, 1)
+
 	// If the chain has already been initialized, simply start the block processing routine.
 	if beaconState != nil {
 		log.Info("Blockchain data already exists in DB, initializing...")
-		s.genesisTime = time.Unix(int64(beaconState.GenesisTime), 0)
-		if err := s.initializeChainInfo(ctx); err != nil {
+		s.genesisTime = time.Unix(int64(beaconState.GenesisTime()), 0)
+		s.opsService.SetGenesisTime(beaconState.GenesisTime())
+		if err := s.initializeChainInfo(s.ctx); err != nil {
 			log.Fatalf("Could not set up chain info: %v", err)
 		}
-		justifiedCheckpoint, err := s.beaconDB.JustifiedCheckpoint(ctx)
+
+		// We start a counter to genesis, if needed.
+		gState, err := s.beaconDB.GenesisState(s.ctx)
+		if err != nil {
+			log.Fatalf("Could not retrieve genesis state: %v", err)
+		}
+		go slotutil.CountdownToGenesis(s.ctx, s.genesisTime, uint64(gState.NumValidators()))
+
+		justifiedCheckpoint, err := s.beaconDB.JustifiedCheckpoint(s.ctx)
 		if err != nil {
 			log.Fatalf("Could not get justified checkpoint: %v", err)
 		}
-		finalizedCheckpoint, err := s.beaconDB.FinalizedCheckpoint(ctx)
+		finalizedCheckpoint, err := s.beaconDB.FinalizedCheckpoint(s.ctx)
 		if err != nil {
 			log.Fatalf("Could not get finalized checkpoint: %v", err)
 		}
-		if err := s.forkChoiceStore.GenesisStore(ctx, justifiedCheckpoint, finalizedCheckpoint); err != nil {
-			log.Fatalf("Could not start fork choice service: %v", err)
+
+		// Resume fork choice.
+		s.justifiedCheckpt = stateTrie.CopyCheckpoint(justifiedCheckpoint)
+		if err := s.cacheJustifiedStateBalances(s.ctx, s.ensureRootNotZeros(bytesutil.ToBytes32(s.justifiedCheckpt.Root))); err != nil {
+			log.Fatalf("Could not cache justified state balances: %v", err)
 		}
-		s.stateInitializedFeed.Send(s.genesisTime)
+		s.prevJustifiedCheckpt = stateTrie.CopyCheckpoint(justifiedCheckpoint)
+		s.bestJustifiedCheckpt = stateTrie.CopyCheckpoint(justifiedCheckpoint)
+		s.finalizedCheckpt = stateTrie.CopyCheckpoint(finalizedCheckpoint)
+		s.prevFinalizedCheckpt = stateTrie.CopyCheckpoint(finalizedCheckpoint)
+		s.resumeForkChoice(justifiedCheckpoint, finalizedCheckpoint)
+
+		if err := s.VerifyWeakSubjectivityRoot(s.ctx); err != nil {
+			// Exit run time if the node failed to verify weak subjectivity checkpoint.
+			log.Fatalf("Could not verify weak subjectivity checkpoint: %v", err)
+		}
+
+		s.stateNotifier.StateFeed().Send(&feed.Event{
+			Type: statefeed.Initialized,
+			Data: &statefeed.InitializedData{
+				StartTime:             s.genesisTime,
+				GenesisValidatorsRoot: beaconState.GenesisValidatorRoot(),
+			},
+		})
 	} else {
 		log.Info("Waiting to reach the validator deposit threshold to start the beacon chain...")
 		if s.chainStartFetcher == nil {
 			log.Fatal("Not configured web3Service for POW chain")
 			return // return need for TestStartUninitializedChainWithoutConfigPOWChain.
 		}
-		subChainStart := s.chainStartFetcher.ChainStartFeed().Subscribe(s.chainStartChan)
 		go func() {
-			genesisTime := <-s.chainStartChan
-			s.processChainStartTime(ctx, genesisTime, subChainStart)
-			return
+			stateChannel := make(chan *feed.Event, 1)
+			stateSub := s.stateNotifier.StateFeed().Subscribe(stateChannel)
+			defer stateSub.Unsubscribe()
+			<-attestationProcessorSubscribed
+			for {
+				select {
+				case event := <-stateChannel:
+					if event.Type == statefeed.ChainStarted {
+						data, ok := event.Data.(*statefeed.ChainStartedData)
+						if !ok {
+							log.Error("event data is not type *statefeed.ChainStartedData")
+							return
+						}
+						log.WithField("starttime", data.StartTime).Debug("Received chain start event")
+						s.processChainStartTime(s.ctx, data.StartTime)
+						return
+					}
+				case <-s.ctx.Done():
+					log.Debug("Context closed, exiting goroutine")
+					return
+				case err := <-stateSub.Err():
+					log.WithError(err).Error("Subscription to state notifier failed")
+					return
+				}
+			}
 		}()
 	}
 
-	go s.processAttestation()
+	go s.processAttestation(attestationProcessorSubscribed)
 }
 
 // processChainStartTime initializes a series of deposits from the ChainStart deposits in the eth1
 // deposit contract, initializes the beacon chain's state, and kicks off the beacon chain.
-func (s *Service) processChainStartTime(ctx context.Context, genesisTime time.Time, chainStartSub event.Subscription) {
-	initialDeposits := s.chainStartFetcher.ChainStartDeposits()
-	if err := s.initializeBeaconChain(ctx, genesisTime, initialDeposits, s.chainStartFetcher.ChainStartEth1Data()); err != nil {
+func (s *Service) processChainStartTime(ctx context.Context, genesisTime time.Time) {
+	preGenesisState := s.chainStartFetcher.PreGenesisState()
+	initializedState, err := s.initializeBeaconChain(ctx, genesisTime, preGenesisState, s.chainStartFetcher.ChainStartEth1Data())
+	if err != nil {
 		log.Fatalf("Could not initialize beacon chain: %v", err)
 	}
-	s.stateInitializedFeed.Send(genesisTime)
-	chainStartSub.Unsubscribe()
+	// We start a counter to genesis, if needed.
+	go slotutil.CountdownToGenesis(ctx, genesisTime, uint64(initializedState.NumValidators()))
+
+	// We send out a state initialized event to the rest of the services
+	// running in the beacon node.
+	s.stateNotifier.StateFeed().Send(&feed.Event{
+		Type: statefeed.Initialized,
+		Data: &statefeed.InitializedData{
+			StartTime:             genesisTime,
+			GenesisValidatorsRoot: initializedState.GenesisValidatorRoot(),
+		},
+	})
 }
 
 // initializes the state and genesis block of the beacon chain to persistent storage
@@ -155,138 +282,168 @@ func (s *Service) processChainStartTime(ctx context.Context, genesisTime time.Ti
 func (s *Service) initializeBeaconChain(
 	ctx context.Context,
 	genesisTime time.Time,
-	deposits []*ethpb.Deposit,
-	eth1data *ethpb.Eth1Data) error {
-	_, span := trace.StartSpan(context.Background(), "beacon-chain.Service.initializeBeaconChain")
+	preGenesisState *stateTrie.BeaconState,
+	eth1data *ethpb.Eth1Data) (*stateTrie.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "beacon-chain.Service.initializeBeaconChain")
 	defer span.End()
-	log.Info("Genesis time reached, starting the beacon chain")
 	s.genesisTime = genesisTime
 	unixTime := uint64(genesisTime.Unix())
 
-	genesisState, err := state.GenesisBeaconState(deposits, unixTime, eth1data)
+	genesisState, err := state.OptimizedGenesisBeaconState(unixTime, preGenesisState, eth1data)
 	if err != nil {
-		return errors.Wrap(err, "could not initialize genesis state")
+		return nil, errors.Wrap(err, "could not initialize genesis state")
 	}
 
 	if err := s.saveGenesisData(ctx, genesisState); err != nil {
-		return errors.Wrap(err, "could not save genesis data")
+		return nil, errors.Wrap(err, "could not save genesis data")
 	}
 
+	log.Info("Initialized beacon chain genesis state")
+
+	// Clear out all pre-genesis data now that the state is initialized.
+	s.chainStartFetcher.ClearPreGenesisData()
+
 	// Update committee shuffled indices for genesis epoch.
-	if featureconfig.Get().EnableNewCache {
-		if err := helpers.UpdateCommitteeCache(genesisState); err != nil {
-			return err
-		}
+	if err := helpers.UpdateCommitteeCache(genesisState, 0 /* genesis epoch */); err != nil {
+		return nil, err
+	}
+	if err := helpers.UpdateProposerIndicesInCache(genesisState, 0 /* genesis epoch */); err != nil {
+		return nil, err
 	}
 
-	return nil
+	s.opsService.SetGenesisTime(genesisState.GenesisTime())
+
+	return genesisState, nil
 }
 
 // Stop the blockchain service's main event loop and associated goroutines.
 func (s *Service) Stop() error {
 	defer s.cancel()
-	return nil
+
+	if s.stateGen != nil && s.head != nil && s.head.state != nil {
+		if err := s.stateGen.ForceCheckpoint(s.ctx, s.head.state.FinalizedCheckpoint().Root); err != nil {
+			return err
+		}
+	}
+
+	// Save initial sync cached blocks to the DB before stop.
+	return s.beaconDB.SaveBlocks(s.ctx, s.getInitSyncBlocks())
 }
 
 // Status always returns nil unless there is an error condition that causes
 // this service to be unhealthy.
 func (s *Service) Status() error {
-	if runtime.NumGoroutine() > int(s.maxRoutines) {
+	if runtime.NumGoroutine() > s.maxRoutines {
 		return fmt.Errorf("too many goroutines %d", runtime.NumGoroutine())
 	}
 	return nil
 }
 
-// StateInitializedFeed returns a feed that is written to
-// when the beacon state is first initialized.
-func (s *Service) StateInitializedFeed() *event.Feed {
-	return s.stateInitializedFeed
+// ClearCachedStates removes all stored caches states. This is done after the node
+// is synced.
+func (s *Service) ClearCachedStates() {
+	s.initSyncState = map[[32]byte]*stateTrie.BeaconState{}
 }
 
-// This gets called to update canonical root mapping.
-func (s *Service) saveHead(ctx context.Context, b *ethpb.BeaconBlock, r [32]byte) error {
-	s.headLock.Lock()
-	defer s.headLock.Unlock()
-
-	s.headSlot = b.Slot
-
-	s.canonicalRoots[b.Slot] = r[:]
-
-	if err := s.beaconDB.SaveHeadBlockRoot(ctx, r); err != nil {
-		return errors.Wrap(err, "could not save head root in DB")
-	}
-	s.headBlock = b
-
-	headState, err := s.beaconDB.State(ctx, r)
+// This gets called when beacon chain is first initialized to save genesis data (state, block, and more) in db.
+func (s *Service) saveGenesisData(ctx context.Context, genesisState *stateTrie.BeaconState) error {
+	stateRoot, err := genesisState.HashTreeRoot(ctx)
 	if err != nil {
-		return errors.Wrap(err, "could not retrieve head state in DB")
-	}
-	s.headState = headState
-
-	log.WithFields(logrus.Fields{
-		"slot":     b.Slot,
-		"headRoot": fmt.Sprintf("%#x", r),
-	}).Debug("Saved new head info")
-	return nil
-}
-
-// This gets called when beacon chain is first initialized to save validator indices and pubkeys in db
-func (s *Service) saveGenesisValidators(ctx context.Context, state *pb.BeaconState) error {
-	for i, v := range state.Validators {
-		if err := s.beaconDB.SaveValidatorIndex(ctx, bytesutil.ToBytes48(v.PublicKey), uint64(i)); err != nil {
-			return errors.Wrapf(err, "could not save validator index: %d", i)
-		}
-	}
-	return nil
-}
-
-// This gets called when beacon chain is first initialized to save genesis data (state, block, and more) in db
-func (s *Service) saveGenesisData(ctx context.Context, genesisState *pb.BeaconState) error {
-	s.headLock.Lock()
-	defer s.headLock.Unlock()
-
-	stateRoot, err := ssz.HashTreeRoot(genesisState)
-	if err != nil {
-		return errors.Wrap(err, "could not tree hash genesis state")
+		return err
 	}
 	genesisBlk := blocks.NewGenesisBlock(stateRoot[:])
-	genesisBlkRoot, err := ssz.SigningRoot(genesisBlk)
+	genesisBlkRoot, err := genesisBlk.Block.HashTreeRoot()
 	if err != nil {
 		return errors.Wrap(err, "could not get genesis block root")
 	}
+	s.genesisRoot = genesisBlkRoot
 
 	if err := s.beaconDB.SaveBlock(ctx, genesisBlk); err != nil {
 		return errors.Wrap(err, "could not save genesis block")
 	}
+	if err := s.beaconDB.SaveState(ctx, genesisState, genesisBlkRoot); err != nil {
+		return errors.Wrap(err, "could not save genesis state")
+	}
+	if err := s.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{
+		Slot: 0,
+		Root: genesisBlkRoot[:],
+	}); err != nil {
+		return err
+	}
+
+	s.stateGen.SaveFinalizedState(0, genesisBlkRoot, genesisState)
+
 	if err := s.beaconDB.SaveHeadBlockRoot(ctx, genesisBlkRoot); err != nil {
 		return errors.Wrap(err, "could not save head block root")
 	}
 	if err := s.beaconDB.SaveGenesisBlockRoot(ctx, genesisBlkRoot); err != nil {
-		return errors.Wrap(err, "could save genesis block root")
-	}
-	if err := s.beaconDB.SaveState(ctx, genesisState, genesisBlkRoot); err != nil {
-		return errors.Wrap(err, "could not save genesis state")
-	}
-	if err := s.saveGenesisValidators(ctx, genesisState); err != nil {
-		return errors.Wrap(err, "could not save genesis validators")
+		return errors.Wrap(err, "could not save genesis block root")
 	}
 
-	genesisCheckpoint := &ethpb.Checkpoint{Root: genesisBlkRoot[:]}
-	if err := s.forkChoiceStore.GenesisStore(ctx, genesisCheckpoint, genesisCheckpoint); err != nil {
-		return errors.Wrap(err, "Could not start fork choice service: %v")
+	// Finalized checkpoint at genesis is a zero hash.
+	genesisCheckpoint := genesisState.FinalizedCheckpoint()
+
+	s.justifiedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
+	if err := s.cacheJustifiedStateBalances(ctx, genesisBlkRoot); err != nil {
+		return err
+	}
+	s.prevJustifiedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
+	s.bestJustifiedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
+	s.finalizedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
+	s.prevFinalizedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
+
+	if err := s.forkChoiceStore.ProcessBlock(ctx,
+		genesisBlk.Block.Slot,
+		genesisBlkRoot,
+		params.BeaconConfig().ZeroHash,
+		[32]byte{},
+		genesisCheckpoint.Epoch,
+		genesisCheckpoint.Epoch); err != nil {
+		log.Fatalf("Could not process genesis block for fork choice: %v", err)
 	}
 
-	s.headBlock = genesisBlk
-	s.headState = genesisState
-	s.canonicalRoots[genesisState.Slot] = genesisBlkRoot[:]
+	s.setHead(genesisBlkRoot, genesisBlk, genesisState)
 
 	return nil
 }
 
 // This gets called to initialize chain info variables using the finalized checkpoint stored in DB
 func (s *Service) initializeChainInfo(ctx context.Context) error {
-	s.headLock.Lock()
-	defer s.headLock.Unlock()
+	genesisBlock, err := s.beaconDB.GenesisBlock(ctx)
+	if err != nil {
+		return errors.Wrap(err, "could not get genesis block from db")
+	}
+	if genesisBlock == nil {
+		return errors.New("no genesis block in db")
+	}
+	genesisBlkRoot, err := genesisBlock.Block.HashTreeRoot()
+	if err != nil {
+		return errors.Wrap(err, "could not get signing root of genesis block")
+	}
+	s.genesisRoot = genesisBlkRoot
+
+	if flags.Get().HeadSync {
+		headBlock, err := s.beaconDB.HeadBlock(ctx)
+		if err != nil {
+			return errors.Wrap(err, "could not retrieve head block")
+		}
+		headRoot, err := headBlock.Block.HashTreeRoot()
+		if err != nil {
+			return errors.Wrap(err, "could not hash head block")
+		}
+		finalizedState, err := s.stateGen.Resume(ctx)
+		if err != nil {
+			return errors.Wrap(err, "could not get finalized state from db")
+		}
+		log.Infof("Regenerating state from the last checkpoint at slot %d to current head slot of %d."+
+			"This process may take a while, please wait.", finalizedState.Slot(), headBlock.Block.Slot)
+		headState, err := s.stateGen.StateByRoot(ctx, headRoot)
+		if err != nil {
+			return errors.Wrap(err, "could not retrieve head state")
+		}
+		s.setHead(headRoot, headBlock, headState)
+		return nil
+	}
 
 	finalized, err := s.beaconDB.FinalizedCheckpoint(ctx)
 	if err != nil {
@@ -297,17 +454,42 @@ func (s *Service) initializeChainInfo(ctx context.Context) error {
 		// would be the genesis state and block.
 		return errors.New("no finalized epoch in the database")
 	}
-	s.headState, err = s.beaconDB.State(ctx, bytesutil.ToBytes32(finalized.Root))
+	finalizedRoot := s.ensureRootNotZeros(bytesutil.ToBytes32(finalized.Root))
+	var finalizedState *stateTrie.BeaconState
+
+	finalizedState, err = s.stateGen.Resume(ctx)
 	if err != nil {
 		return errors.Wrap(err, "could not get finalized state from db")
 	}
-	s.headBlock, err = s.beaconDB.Block(ctx, bytesutil.ToBytes32(finalized.Root))
+
+	finalizedBlock, err := s.beaconDB.Block(ctx, finalizedRoot)
 	if err != nil {
 		return errors.Wrap(err, "could not get finalized block from db")
 	}
 
-	s.headSlot = s.headState.Slot
-	s.canonicalRoots[s.headSlot] = finalized.Root
+	if finalizedState == nil || finalizedBlock == nil {
+		return errors.New("finalized state and block can't be nil")
+	}
+	s.setHead(finalizedRoot, finalizedBlock, finalizedState)
 
 	return nil
 }
+
+// This is called when a client starts from non-genesis slot. This passes last justified and finalized
+// information to fork choice service to initializes fork choice store.
+func (s *Service) resumeForkChoice(justifiedCheckpoint, finalizedCheckpoint *ethpb.Checkpoint) {
+	store := protoarray.New(justifiedCheckpoint.Epoch, finalizedCheckpoint.Epoch, bytesutil.ToBytes32(finalizedCheckpoint.Root))
+	s.forkChoiceStore = store
+}
+
+// This returns true if block has been processed before. Two ways to verify the block has been processed:
+// 1.) Check fork choice store.
+// 2.) Check DB.
+// Checking 1.) is ten times faster than checking 2.)
+func (s *Service) hasBlock(ctx context.Context, root [32]byte) bool {
+	if s.forkChoiceStore.HasNode(root) {
+		return true
+	}
+
+	return s.beaconDB.HasBlock(ctx, root)
+}

beacon-chain/blockchain/service_norace_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 	"github.com/sirupsen/logrus"
 )
 
@@ -16,22 +16,12 @@ func init() {
 }
 
 func TestChainService_SaveHead_DataRace(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 	s := &Service{
-		beaconDB:       db,
-		canonicalRoots: make(map[uint64][]byte),
+		beaconDB: db,
 	}
 	go func() {
-		s.saveHead(
-			context.Background(),
-			&ethpb.BeaconBlock{Slot: 777},
-			[32]byte{},
-		)
+		require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
 	}()
-	s.saveHead(
-		context.Background(),
-		&ethpb.BeaconBlock{Slot: 888},
-		[32]byte{},
-	)
+	require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
 }

beacon-chain/blockchain/service_test.go

@@ -3,76 +3,49 @@ package blockchain
 import (
 	"bytes"
 	"context"
-	"encoding/hex"
-	"errors"
 	"io/ioutil"
-	"math/big"
 	"reflect"
 	"testing"
 	"time"
 
-	ethereum "github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
-	gethTypes "github.com/ethereum/go-ethereum/core/types"
 	"github.com/gogo/protobuf/proto"
-	ssz "github.com/prysmaticlabs/go-ssz"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache"
 	b "github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	"github.com/prysmaticlabs/prysm/beacon-chain/operations/attestations"
 	"github.com/prysmaticlabs/prysm/beacon-chain/p2p"
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
+	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	protodb "github.com/prysmaticlabs/prysm/proto/beacon/db"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/event"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 	"github.com/sirupsen/logrus"
 	logTest "github.com/sirupsen/logrus/hooks/test"
 )
 
-// Ensure Service implements interfaces.
-var _ = ChainFeeds(&Service{})
-
 func init() {
 	logrus.SetLevel(logrus.DebugLevel)
 	logrus.SetOutput(ioutil.Discard)
 }
 
-type store struct {
-	headRoot []byte
-}
-
-func (s *store) OnBlock(ctx context.Context, b *ethpb.BeaconBlock) error {
-	return nil
-}
-
-func (s *store) OnBlockNoVerifyStateTransition(ctx context.Context, b *ethpb.BeaconBlock) error {
-	return nil
-}
-
-func (s *store) OnAttestation(ctx context.Context, a *ethpb.Attestation) error {
-	return nil
-}
-
-func (s *store) GenesisStore(ctx context.Context, justifiedCheckpoint *ethpb.Checkpoint, finalizedCheckpoint *ethpb.Checkpoint) error {
-	return nil
-}
-
-func (s *store) FinalizedCheckpt() *ethpb.Checkpoint {
-	return nil
-}
-
-func (s *store) Head(ctx context.Context) ([]byte, error) {
-	return s.headRoot, nil
-}
-
 type mockBeaconNode struct {
 	stateFeed *event.Feed
 }
 
+// StateFeed mocks the same method in the beacon node.
 func (mbn *mockBeaconNode) StateFeed() *event.Feed {
 	if mbn.stateFeed == nil {
 		mbn.stateFeed = new(event.Feed)
@@ -80,115 +53,6 @@ func (mbn *mockBeaconNode) StateFeed() *event.Feed {
 	return mbn.stateFeed
 }
 
-type mockOperationService struct{}
-
-func (ms *mockOperationService) IncomingProcessedBlockFeed() *event.Feed {
-	return new(event.Feed)
-}
-
-func (ms *mockOperationService) IncomingAttFeed() *event.Feed {
-	return nil
-}
-
-func (ms *mockOperationService) AttestationPool(ctx context.Context, requestedSlot uint64) ([]*ethpb.Attestation, error) {
-	return nil, nil
-}
-
-func (ms *mockOperationService) AttestationPoolNoVerify(ctx context.Context) ([]*ethpb.Attestation, error) {
-	return nil, nil
-}
-
-func (ms *mockOperationService) AttestationPoolForForkchoice(ctx context.Context) ([]*ethpb.Attestation, error) {
-	return nil, nil
-}
-
-type mockClient struct{}
-
-func (m *mockClient) SubscribeNewHead(ctx context.Context, ch chan<- *gethTypes.Header) (ethereum.Subscription, error) {
-	return new(event.Feed).Subscribe(ch), nil
-}
-
-func (m *mockClient) BlockByHash(ctx context.Context, hash common.Hash) (*gethTypes.Block, error) {
-	head := &gethTypes.Header{Number: big.NewInt(0), Difficulty: big.NewInt(100)}
-	return gethTypes.NewBlockWithHeader(head), nil
-}
-
-func (m *mockClient) BlockByNumber(ctx context.Context, number *big.Int) (*gethTypes.Block, error) {
-	head := &gethTypes.Header{Number: big.NewInt(0), Difficulty: big.NewInt(100)}
-	return gethTypes.NewBlockWithHeader(head), nil
-}
-
-func (m *mockClient) HeaderByNumber(ctx context.Context, number *big.Int) (*gethTypes.Header, error) {
-	return &gethTypes.Header{Number: big.NewInt(0), Difficulty: big.NewInt(100)}, nil
-}
-
-func (m *mockClient) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- gethTypes.Log) (ethereum.Subscription, error) {
-	return new(event.Feed).Subscribe(ch), nil
-}
-
-func (m *mockClient) CallContract(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) {
-	return []byte{'t', 'e', 's', 't'}, nil
-}
-
-func (m *mockClient) CodeAt(ctx context.Context, account common.Address, blockNumber *big.Int) ([]byte, error) {
-	return []byte{'t', 'e', 's', 't'}, nil
-}
-
-func (m *mockClient) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]gethTypes.Log, error) {
-	logs := make([]gethTypes.Log, 3)
-	for i := 0; i < len(logs); i++ {
-		logs[i].Address = common.Address{}
-		logs[i].Topics = make([]common.Hash, 5)
-		logs[i].Topics[0] = common.Hash{'a'}
-		logs[i].Topics[1] = common.Hash{'b'}
-		logs[i].Topics[2] = common.Hash{'c'}
-
-	}
-	return logs, nil
-}
-
-func (m *mockClient) LatestBlockHash() common.Hash {
-	return common.BytesToHash([]byte{'A'})
-}
-
-type faultyClient struct{}
-
-func (f *faultyClient) SubscribeNewHead(ctx context.Context, ch chan<- *gethTypes.Header) (ethereum.Subscription, error) {
-	return new(event.Feed).Subscribe(ch), nil
-}
-
-func (f *faultyClient) BlockByHash(ctx context.Context, hash common.Hash) (*gethTypes.Block, error) {
-	return nil, errors.New("failed")
-}
-
-func (f *faultyClient) BlockByNumber(ctx context.Context, number *big.Int) (*gethTypes.Block, error) {
-	return nil, errors.New("failed")
-}
-
-func (f *faultyClient) HeaderByNumber(ctx context.Context, number *big.Int) (*gethTypes.Header, error) {
-	return nil, errors.New("failed")
-}
-
-func (f *faultyClient) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- gethTypes.Log) (ethereum.Subscription, error) {
-	return new(event.Feed).Subscribe(ch), nil
-}
-
-func (f *faultyClient) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]gethTypes.Log, error) {
-	return nil, errors.New("unable to retrieve logs")
-}
-
-func (f *faultyClient) CallContract(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) {
-	return []byte{}, errors.New("unable to retrieve contract code")
-}
-
-func (f *faultyClient) CodeAt(ctx context.Context, account common.Address, blockNumber *big.Int) ([]byte, error) {
-	return []byte{}, errors.New("unable to retrieve contract code")
-}
-
-func (f *faultyClient) LatestBlockHash() common.Hash {
-	return common.BytesToHash([]byte{'A'})
-}
-
 type mockBroadcaster struct {
 	broadcastCalled bool
 }
@@ -198,226 +62,359 @@ func (mb *mockBroadcaster) Broadcast(_ context.Context, _ proto.Message) error {
 	return nil
 }
 
-var _ = p2p.Broadcaster(&mockBroadcaster{})
-
-func setupGenesisBlock(t *testing.T, cs *Service) ([32]byte, *ethpb.BeaconBlock) {
-	genesis := b.NewGenesisBlock([]byte{})
-	if err := cs.beaconDB.SaveBlock(context.Background(), genesis); err != nil {
-		t.Fatalf("could not save block to db: %v", err)
-	}
-	parentHash, err := ssz.SigningRoot(genesis)
-	if err != nil {
-		t.Fatalf("unable to get tree hash root of canonical head: %v", err)
-	}
-	return parentHash, genesis
+func (mb *mockBroadcaster) BroadcastAttestation(_ context.Context, _ uint64, _ *ethpb.Attestation) error {
+	mb.broadcastCalled = true
+	return nil
 }
 
-func setupBeaconChain(t *testing.T, beaconDB db.Database) *Service {
-	endpoint := "ws://127.0.0.1"
+var _ p2p.Broadcaster = (*mockBroadcaster)(nil)
+
+func setupBeaconChain(t *testing.T, beaconDB db.Database, sc *cache.StateSummaryCache) *Service {
+	endpoint := "http://127.0.0.1"
 	ctx := context.Background()
 	var web3Service *powchain.Service
 	var err error
+	bState, _ := testutil.DeterministicGenesisState(t, 10)
+	err = beaconDB.SavePowchainData(ctx, &protodb.ETH1ChainData{
+		BeaconState: bState.InnerStateUnsafe(),
+		Trie:        &protodb.SparseMerkleTrie{},
+		CurrentEth1Data: &protodb.LatestETH1Data{
+			BlockHash: make([]byte, 32),
+		},
+		ChainstartData: &protodb.ChainStartData{
+			Eth1Data: &ethpb.Eth1Data{
+				DepositRoot:  make([]byte, 32),
+				DepositCount: 0,
+				BlockHash:    make([]byte, 32),
+			},
+		},
+		DepositContainers: []*protodb.DepositContainer{},
+	})
+	require.NoError(t, err)
 	web3Service, err = powchain.NewService(ctx, &powchain.Web3ServiceConfig{
-		ETH1Endpoint:    endpoint,
+		BeaconDB:        beaconDB,
+		HTTPEndPoint:    endpoint,
 		DepositContract: common.Address{},
 	})
-	if err != nil {
-		t.Fatalf("unable to set up web3 service: %v", err)
-	}
+	require.NoError(t, err, "Unable to set up web3 service")
+
+	opsService, err := attestations.NewService(ctx, &attestations.Config{Pool: attestations.NewPool()})
+	require.NoError(t, err)
+
+	depositCache, err := depositcache.New()
+	require.NoError(t, err)
 
 	cfg := &Config{
 		BeaconBlockBuf:    0,
 		BeaconDB:          beaconDB,
-		DepositCache:      depositcache.NewDepositCache(),
+		DepositCache:      depositCache,
 		ChainStartFetcher: web3Service,
-		OpsPoolService:    &mockOperationService{},
 		P2p:               &mockBroadcaster{},
 		StateNotifier:     &mockBeaconNode{},
+		AttPool:           attestations.NewPool(),
+		StateGen:          stategen.New(beaconDB, sc),
+		ForkChoiceStore:   protoarray.New(0, 0, params.BeaconConfig().ZeroHash),
+		OpsService:        opsService,
 	}
-	if err != nil {
-		t.Fatalf("could not register blockchain service: %v", err)
-	}
+
+	// Safe a state in stategen to purposes of testing a service stop / shutdown.
+	require.NoError(t, cfg.StateGen.SaveState(ctx, bytesutil.ToBytes32(bState.FinalizedCheckpoint().Root), bState))
+
 	chainService, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatalf("unable to setup chain service: %v", err)
-	}
+	require.NoError(t, err, "Unable to setup chain service")
+	chainService.genesisTime = time.Unix(1, 0) // non-zero time
 
 	return chainService
 }
 
-func TestChainStartStop_Uninitialized(t *testing.T) {
-	helpers.ClearAllCaches()
-	hook := logTest.NewGlobal()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	chainService := setupBeaconChain(t, db)
-
-	// Test the start function.
-	genesisChan := make(chan time.Time, 0)
-	sub := chainService.stateInitializedFeed.Subscribe(genesisChan)
-	defer sub.Unsubscribe()
-	chainService.Start()
-	chainService.chainStartChan <- time.Unix(0, 0)
-	genesisTime := <-genesisChan
-	if genesisTime != time.Unix(0, 0) {
-		t.Errorf(
-			"Expected genesis time to equal chainstart time (%v), received %v",
-			time.Unix(0, 0),
-			genesisTime,
-		)
-	}
-
-	beaconState, err := db.HeadState(context.Background())
-	if err != nil {
-		t.Fatal(err)
-	}
-	if beaconState == nil || beaconState.Slot != 0 {
-		t.Error("Expected canonical state feed to send a state with genesis block")
-	}
-	if err := chainService.Stop(); err != nil {
-		t.Fatalf("Unable to stop chain service: %v", err)
-	}
-	// The context should have been canceled.
-	if chainService.ctx.Err() != context.Canceled {
-		t.Error("Context was not canceled")
-	}
-	testutil.AssertLogsContain(t, hook, "Waiting")
-	testutil.AssertLogsContain(t, hook, "Genesis time reached")
-}
-
 func TestChainStartStop_Initialized(t *testing.T) {
 	hook := logTest.NewGlobal()
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
-	chainService := setupBeaconChain(t, db)
+	chainService := setupBeaconChain(t, db, sc)
 
-	genesisBlk := b.NewGenesisBlock([]byte{})
-	blkRoot, err := ssz.SigningRoot(genesisBlk)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveBlock(ctx, genesisBlk); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveHeadBlockRoot(ctx, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveGenesisBlockRoot(ctx, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveState(ctx, &pb.BeaconState{Slot: 1}, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveJustifiedCheckpoint(ctx, &ethpb.Checkpoint{Root: blkRoot[:]}); err != nil {
-		t.Fatal(err)
-	}
+	genesisBlk := testutil.NewBeaconBlock()
+	blkRoot, err := genesisBlk.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, db.SaveBlock(ctx, genesisBlk))
+	s := testutil.NewBeaconState()
+	require.NoError(t, s.SetSlot(1))
+	require.NoError(t, db.SaveState(ctx, s, blkRoot))
+	require.NoError(t, db.SaveHeadBlockRoot(ctx, blkRoot))
+	require.NoError(t, db.SaveGenesisBlockRoot(ctx, blkRoot))
+	require.NoError(t, db.SaveJustifiedCheckpoint(ctx, &ethpb.Checkpoint{Root: blkRoot[:]}))
+	require.NoError(t, db.SaveFinalizedCheckpoint(ctx, &ethpb.Checkpoint{Root: blkRoot[:]}))
 
 	// Test the start function.
 	chainService.Start()
 
-	if err := chainService.Stop(); err != nil {
-		t.Fatalf("unable to stop chain service: %v", err)
-	}
+	require.NoError(t, chainService.Stop(), "Unable to stop chain service")
 
 	// The context should have been canceled.
-	if chainService.ctx.Err() != context.Canceled {
-		t.Error("context was not canceled")
-	}
-	testutil.AssertLogsContain(t, hook, "data already exists")
+	assert.Equal(t, context.Canceled, chainService.ctx.Err(), "Context was not canceled")
+	require.LogsContain(t, hook, "data already exists")
+}
+
+func TestChainStartStop_GenesisZeroHashes(t *testing.T) {
+	hook := logTest.NewGlobal()
+	ctx := context.Background()
+	db, sc := testDB.SetupDB(t)
+
+	chainService := setupBeaconChain(t, db, sc)
+
+	genesisBlk := testutil.NewBeaconBlock()
+	blkRoot, err := genesisBlk.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, db.SaveBlock(ctx, genesisBlk))
+	s := testutil.NewBeaconState()
+	require.NoError(t, db.SaveState(ctx, s, blkRoot))
+	require.NoError(t, db.SaveGenesisBlockRoot(ctx, blkRoot))
+	require.NoError(t, db.SaveJustifiedCheckpoint(ctx, &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}))
+
+	// Test the start function.
+	chainService.Start()
+
+	require.NoError(t, chainService.Stop(), "Unable to stop chain service")
+
+	// The context should have been canceled.
+	assert.Equal(t, context.Canceled, chainService.ctx.Err(), "Context was not canceled")
+	require.LogsContain(t, hook, "data already exists")
 }
 
 func TestChainService_InitializeBeaconChain(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	helpers.ClearCache()
+	db, sc := testDB.SetupDB(t)
 	ctx := context.Background()
 
-	bc := setupBeaconChain(t, db)
+	bc := setupBeaconChain(t, db, sc)
+	var err error
 
 	// Set up 10 deposits pre chain start for validators to register
 	count := uint64(10)
-	deposits, _, _ := testutil.SetupInitialDeposits(t, count)
-	if err := bc.initializeBeaconChain(ctx, time.Unix(0, 0), deposits, &ethpb.Eth1Data{}); err != nil {
-		t.Fatal(err)
-	}
+	deposits, _, err := testutil.DeterministicDepositsAndKeys(count)
+	require.NoError(t, err)
+	trie, _, err := testutil.DepositTrieFromDeposits(deposits)
+	require.NoError(t, err)
+	hashTreeRoot := trie.HashTreeRoot()
+	genState, err := state.EmptyGenesisState()
+	require.NoError(t, err)
+	err = genState.SetEth1Data(&ethpb.Eth1Data{
+		DepositRoot:  hashTreeRoot[:],
+		DepositCount: uint64(len(deposits)),
+		BlockHash:    make([]byte, 32),
+	})
+	require.NoError(t, err)
+	genState, err = b.ProcessPreGenesisDeposits(ctx, genState, deposits)
+	require.NoError(t, err)
 
-	s, err := bc.beaconDB.State(ctx, bytesutil.ToBytes32(bc.canonicalRoots[0]))
-	if err != nil {
-		t.Fatal(err)
-	}
+	_, err = bc.initializeBeaconChain(ctx, time.Unix(0, 0), genState, &ethpb.Eth1Data{DepositRoot: hashTreeRoot[:], BlockHash: make([]byte, 32)})
+	require.NoError(t, err)
 
-	for _, v := range s.Validators {
-		if !db.HasValidatorIndex(ctx, bytesutil.ToBytes48(v.PublicKey)) {
-			t.Errorf("Validator %s missing from db", hex.EncodeToString(v.PublicKey))
-		}
-	}
-
-	if _, err := bc.HeadState(ctx); err != nil {
-		t.Error(err)
-	}
-	if bc.HeadBlock() == nil {
+	_, err = bc.HeadState(ctx)
+	assert.NoError(t, err)
+	headBlk, err := bc.HeadBlock(ctx)
+	require.NoError(t, err)
+	if headBlk == nil {
 		t.Error("Head state can't be nil after initialize beacon chain")
 	}
-	if bc.CanonicalRoot(0) == nil {
-		t.Error("Canonical root for slot 0 can't be nil after initialize beacon chain")
+	r, err := bc.HeadRoot(ctx)
+	require.NoError(t, err)
+	if bytesutil.ToBytes32(r) == params.BeaconConfig().ZeroHash {
+		t.Error("Canonical root for slot 0 can't be zeros after initialize beacon chain")
 	}
 }
 
+func TestChainService_CorrectGenesisRoots(t *testing.T) {
+	ctx := context.Background()
+	db, sc := testDB.SetupDB(t)
+
+	chainService := setupBeaconChain(t, db, sc)
+
+	genesisBlk := testutil.NewBeaconBlock()
+	blkRoot, err := genesisBlk.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, db.SaveBlock(ctx, genesisBlk))
+	s := testutil.NewBeaconState()
+	require.NoError(t, s.SetSlot(0))
+	require.NoError(t, db.SaveState(ctx, s, blkRoot))
+	require.NoError(t, db.SaveHeadBlockRoot(ctx, blkRoot))
+	require.NoError(t, db.SaveGenesisBlockRoot(ctx, blkRoot))
+	require.NoError(t, db.SaveFinalizedCheckpoint(ctx, &ethpb.Checkpoint{Root: blkRoot[:]}))
+
+	// Test the start function.
+	chainService.Start()
+
+	require.DeepEqual(t, blkRoot[:], chainService.finalizedCheckpt.Root, "Finalize Checkpoint root is incorrect")
+	require.DeepEqual(t, params.BeaconConfig().ZeroHash[:], chainService.justifiedCheckpt.Root, "Justified Checkpoint root is incorrect")
+
+	require.NoError(t, chainService.Stop(), "Unable to stop chain service")
+
+}
+
 func TestChainService_InitializeChainInfo(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 	ctx := context.Background()
 
-	genesis := b.NewGenesisBlock([]byte{})
-	genesisRoot, err := ssz.SigningRoot(genesis)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveGenesisBlockRoot(ctx, genesisRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveBlock(ctx, genesis); err != nil {
-		t.Fatal(err)
-	}
+	genesis := testutil.NewBeaconBlock()
+	genesisRoot, err := genesis.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, db.SaveGenesisBlockRoot(ctx, genesisRoot))
+	require.NoError(t, db.SaveBlock(ctx, genesis))
 
 	finalizedSlot := params.BeaconConfig().SlotsPerEpoch*2 + 1
-	headBlock := &ethpb.BeaconBlock{Slot: finalizedSlot, ParentRoot: genesisRoot[:]}
-	headState := &pb.BeaconState{Slot: finalizedSlot}
-	headRoot, _ := ssz.SigningRoot(headBlock)
-	if err := db.SaveState(ctx, headState, headRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveBlock(ctx, headBlock); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveFinalizedCheckpoint(ctx, &ethpb.Checkpoint{
-		Epoch: helpers.SlotToEpoch(finalizedSlot),
-		Root:  headRoot[:],
-	}); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveBlock(ctx, headBlock); err != nil {
-		t.Fatal(err)
-	}
-	c := &Service{beaconDB: db, canonicalRoots: make(map[uint64][]byte)}
-	if err := c.initializeChainInfo(ctx); err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(c.HeadBlock(), headBlock) {
-		t.Error("head block incorrect")
-	}
+	headBlock := testutil.NewBeaconBlock()
+	headBlock.Block.Slot = finalizedSlot
+	headBlock.Block.ParentRoot = bytesutil.PadTo(genesisRoot[:], 32)
+	headState := testutil.NewBeaconState()
+	require.NoError(t, headState.SetSlot(finalizedSlot))
+	require.NoError(t, headState.SetGenesisValidatorRoot(params.BeaconConfig().ZeroHash[:]))
+	headRoot, err := headBlock.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, db.SaveState(ctx, headState, headRoot))
+	require.NoError(t, db.SaveState(ctx, headState, genesisRoot))
+	require.NoError(t, db.SaveBlock(ctx, headBlock))
+	require.NoError(t, db.SaveFinalizedCheckpoint(ctx, &ethpb.Checkpoint{Epoch: helpers.SlotToEpoch(finalizedSlot), Root: headRoot[:]}))
+	c := &Service{beaconDB: db, stateGen: stategen.New(db, sc)}
+	require.NoError(t, c.initializeChainInfo(ctx))
+	headBlk, err := c.HeadBlock(ctx)
+	require.NoError(t, err)
+	assert.DeepEqual(t, headBlock, headBlk, "Head block incorrect")
 	s, err := c.HeadState(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(s, headState) {
-		t.Error("head state incorrect")
-	}
-	if headBlock.Slot != c.HeadSlot() {
+	require.NoError(t, err)
+	assert.DeepEqual(t, headState.InnerStateUnsafe(), s.InnerStateUnsafe(), "Head state incorrect")
+	assert.Equal(t, c.HeadSlot(), headBlock.Block.Slot, "Head slot incorrect")
+	r, err := c.HeadRoot(context.Background())
+	require.NoError(t, err)
+	if !bytes.Equal(headRoot[:], r) {
 		t.Error("head slot incorrect")
 	}
-	if !bytes.Equal(headRoot[:], c.HeadRoot()) {
-		t.Error("head slot incorrect")
+	assert.Equal(t, genesisRoot, c.genesisRoot, "Genesis block root incorrect")
+}
+
+func TestChainService_InitializeChainInfo_SetHeadAtGenesis(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+	ctx := context.Background()
+
+	genesis := testutil.NewBeaconBlock()
+	genesisRoot, err := genesis.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, db.SaveGenesisBlockRoot(ctx, genesisRoot))
+	require.NoError(t, db.SaveBlock(ctx, genesis))
+
+	finalizedSlot := params.BeaconConfig().SlotsPerEpoch*2 + 1
+	headBlock := testutil.NewBeaconBlock()
+	headBlock.Block.Slot = finalizedSlot
+	headBlock.Block.ParentRoot = bytesutil.PadTo(genesisRoot[:], 32)
+	headState := testutil.NewBeaconState()
+	require.NoError(t, headState.SetSlot(finalizedSlot))
+	require.NoError(t, headState.SetGenesisValidatorRoot(params.BeaconConfig().ZeroHash[:]))
+	headRoot, err := headBlock.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, db.SaveState(ctx, headState, headRoot))
+	require.NoError(t, db.SaveState(ctx, headState, genesisRoot))
+	require.NoError(t, db.SaveBlock(ctx, headBlock))
+	c := &Service{beaconDB: db, stateGen: stategen.New(db, sc)}
+	require.NoError(t, c.initializeChainInfo(ctx))
+	s, err := c.HeadState(ctx)
+	require.NoError(t, err)
+	assert.DeepEqual(t, headState.InnerStateUnsafe(), s.InnerStateUnsafe(), "Head state incorrect")
+	assert.Equal(t, genesisRoot, c.genesisRoot, "Genesis block root incorrect")
+	assert.DeepEqual(t, genesis, c.head.block)
+}
+
+func TestChainService_SaveHeadNoDB(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+	ctx := context.Background()
+	s := &Service{
+		beaconDB: db,
+		stateGen: stategen.New(db, sc),
+	}
+	b := testutil.NewBeaconBlock()
+	b.Block.Slot = 1
+	r, err := b.HashTreeRoot()
+	require.NoError(t, err)
+	newState := testutil.NewBeaconState()
+	require.NoError(t, s.stateGen.SaveState(ctx, r, newState))
+	require.NoError(t, s.saveHeadNoDB(ctx, b, r, newState))
+
+	newB, err := s.beaconDB.HeadBlock(ctx)
+	require.NoError(t, err)
+	if reflect.DeepEqual(newB, b) {
+		t.Error("head block should not be equal")
+	}
+}
+
+func TestHasBlock_ForkChoiceAndDB(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+	s := &Service{
+		forkChoiceStore:  protoarray.New(0, 0, [32]byte{}),
+		finalizedCheckpt: &ethpb.Checkpoint{Root: make([]byte, 32)},
+		beaconDB:         db,
+	}
+	block := testutil.NewBeaconBlock()
+	r, err := block.Block.HashTreeRoot()
+	require.NoError(t, err)
+	state := testutil.NewBeaconState()
+	require.NoError(t, s.insertBlockAndAttestationsToForkChoiceStore(ctx, block.Block, r, state))
+
+	assert.Equal(t, false, s.hasBlock(ctx, [32]byte{}), "Should not have block")
+	assert.Equal(t, true, s.hasBlock(ctx, r), "Should have block")
+}
+
+func TestServiceStop_SaveCachedBlocks(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	db, _ := testDB.SetupDB(t)
+	s := &Service{
+		ctx:            ctx,
+		cancel:         cancel,
+		beaconDB:       db,
+		initSyncBlocks: make(map[[32]byte]*ethpb.SignedBeaconBlock),
+	}
+	b := testutil.NewBeaconBlock()
+	r, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+	s.saveInitSyncBlock(r, b)
+	require.NoError(t, s.Stop())
+	require.Equal(t, true, s.beaconDB.HasBlock(ctx, r))
+}
+
+func BenchmarkHasBlockDB(b *testing.B) {
+	db, _ := testDB.SetupDB(b)
+	ctx := context.Background()
+	s := &Service{
+		beaconDB: db,
+	}
+	block := testutil.NewBeaconBlock()
+	require.NoError(b, s.beaconDB.SaveBlock(ctx, block))
+	r, err := block.Block.HashTreeRoot()
+	require.NoError(b, err)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		require.Equal(b, true, s.beaconDB.HasBlock(ctx, r), "Block is not in DB")
+	}
+}
+
+func BenchmarkHasBlockForkChoiceStore(b *testing.B) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(b)
+	s := &Service{
+		forkChoiceStore:  protoarray.New(0, 0, [32]byte{}),
+		finalizedCheckpt: &ethpb.Checkpoint{Root: make([]byte, 32)},
+		beaconDB:         db,
+	}
+	block := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Body: &ethpb.BeaconBlockBody{}}}
+	r, err := block.Block.HashTreeRoot()
+	require.NoError(b, err)
+	bs := &pb.BeaconState{FinalizedCheckpoint: &ethpb.Checkpoint{Root: make([]byte, 32)}, CurrentJustifiedCheckpoint: &ethpb.Checkpoint{Root: make([]byte, 32)}}
+	state, err := beaconstate.InitializeFromProto(bs)
+	require.NoError(b, err)
+	require.NoError(b, s.insertBlockAndAttestationsToForkChoiceStore(ctx, block.Block, r, state))
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		require.Equal(b, true, s.forkChoiceStore.HasNode(r), "Block is not in fork choice store")
 	}
 }

beacon-chain/blockchain/testing/BUILD.bazel

@@ -1,18 +1,30 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@prysm//tools/go:def.bzl", "go_library")
 
 go_library(
     name = "go_default_library",
     testonly = True,
     srcs = ["mock.go"],
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing",
-    visibility = ["//beacon-chain:__subpackages__"],
+    visibility = [
+        "//beacon-chain:__subpackages__",
+        "//fuzz:__pkg__",
+    ],
     deps = [
+        "//beacon-chain/core/epoch/precompute:go_default_library",
+        "//beacon-chain/core/feed:go_default_library",
+        "//beacon-chain/core/feed/block:go_default_library",
+        "//beacon-chain/core/feed/operation:go_default_library",
+        "//beacon-chain/core/feed/state:go_default_library",
+        "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/db:go_default_library",
+        "//beacon-chain/forkchoice/protoarray:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
-        "//proto/eth/v1alpha1:go_default_library",
+        "//shared/bytesutil:go_default_library",
         "//shared/event:go_default_library",
+        "//shared/params:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
-        "@com_github_prysmaticlabs_go_ssz//:go_default_library",
+        "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
     ],
 )

beacon-chain/blockchain/testing/mock.go

@@ -1,59 +1,163 @@
+// Package testing includes useful mocks for writing unit
+// tests which depend on logic from the blockchain package.
 package testing
 
 import (
 	"bytes"
 	"context"
+	"sync"
 	"time"
 
 	"github.com/pkg/errors"
-	"github.com/prysmaticlabs/go-ssz"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
+	blockfeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/block"
+	opfeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/operation"
+	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/event"
+	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/sirupsen/logrus"
 )
 
 // ChainService defines the mock interface for testing
 type ChainService struct {
-	State                    *pb.BeaconState
-	Root                     []byte
-	Block                    *ethpb.BeaconBlock
-	FinalizedCheckPoint      *ethpb.Checkpoint
-	MockStateInitializedFeed *event.Feed
-	BlocksReceived           []*ethpb.BeaconBlock
-	Genesis                  time.Time
-	Fork                     *pb.Fork
-	DB                       db.Database
-	MockStateFeed            *event.Feed
+	State                       *stateTrie.BeaconState
+	Root                        []byte
+	Block                       *ethpb.SignedBeaconBlock
+	FinalizedCheckPoint         *ethpb.Checkpoint
+	CurrentJustifiedCheckPoint  *ethpb.Checkpoint
+	PreviousJustifiedCheckPoint *ethpb.Checkpoint
+	BlocksReceived              []*ethpb.SignedBeaconBlock
+	Balance                     *precompute.Balance
+	Genesis                     time.Time
+	ValidatorsRoot              [32]byte
+	CanonicalRoots              map[[32]byte]bool
+	Fork                        *pb.Fork
+	ETH1Data                    *ethpb.Eth1Data
+	DB                          db.Database
+	stateNotifier               statefeed.Notifier
+	blockNotifier               blockfeed.Notifier
+	opNotifier                  opfeed.Notifier
+	ValidAttestation            bool
+	ForkChoiceStore             *protoarray.Store
+	VerifyBlkDescendantErr      error
 }
 
-// ReceiveBlock mocks ReceiveBlock method in chain service.
-func (ms *ChainService) ReceiveBlock(ctx context.Context, block *ethpb.BeaconBlock) error {
-	return nil
+// StateNotifier mocks the same method in the chain service.
+func (ms *ChainService) StateNotifier() statefeed.Notifier {
+	if ms.stateNotifier == nil {
+		ms.stateNotifier = &MockStateNotifier{}
+	}
+	return ms.stateNotifier
 }
 
-// ReceiveBlockNoVerify mocks ReceiveBlockNoVerify method in chain service.
-func (ms *ChainService) ReceiveBlockNoVerify(ctx context.Context, block *ethpb.BeaconBlock) error {
-	return nil
+// BlockNotifier mocks the same method in the chain service.
+func (ms *ChainService) BlockNotifier() blockfeed.Notifier {
+	if ms.blockNotifier == nil {
+		ms.blockNotifier = &MockBlockNotifier{}
+	}
+	return ms.blockNotifier
 }
 
-// ReceiveBlockNoPubsub mocks ReceiveBlockNoPubsub method in chain service.
-func (ms *ChainService) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.BeaconBlock) error {
-	return nil
+// MockBlockNotifier mocks the block notifier.
+type MockBlockNotifier struct {
+	feed *event.Feed
 }
 
-// ReceiveBlockNoPubsubForkchoice mocks ReceiveBlockNoPubsubForkchoice method in chain service.
-func (ms *ChainService) ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *ethpb.BeaconBlock) error {
+// BlockFeed returns a block feed.
+func (msn *MockBlockNotifier) BlockFeed() *event.Feed {
+	if msn.feed == nil {
+		msn.feed = new(event.Feed)
+	}
+	return msn.feed
+}
+
+// MockStateNotifier mocks the state notifier.
+type MockStateNotifier struct {
+	feed     *event.Feed
+	feedLock sync.Mutex
+
+	recv     []*feed.Event
+	recvLock sync.Mutex
+	recvCh   chan *feed.Event
+
+	RecordEvents bool
+}
+
+// ReceivedEvents returns the events received by the state feed in this mock.
+func (msn *MockStateNotifier) ReceivedEvents() []*feed.Event {
+	msn.recvLock.Lock()
+	defer msn.recvLock.Unlock()
+	return msn.recv
+}
+
+// StateFeed returns a state feed.
+func (msn *MockStateNotifier) StateFeed() *event.Feed {
+	msn.feedLock.Lock()
+	defer msn.feedLock.Unlock()
+
+	if msn.feed == nil && msn.recvCh == nil {
+		msn.feed = new(event.Feed)
+		if msn.RecordEvents {
+			msn.recvCh = make(chan *feed.Event)
+			sub := msn.feed.Subscribe(msn.recvCh)
+
+			go func() {
+				select {
+				case evt := <-msn.recvCh:
+					msn.recvLock.Lock()
+					msn.recv = append(msn.recv, evt)
+					msn.recvLock.Unlock()
+				case <-sub.Err():
+					sub.Unsubscribe()
+				}
+			}()
+		}
+	}
+	return msn.feed
+}
+
+// OperationNotifier mocks the same method in the chain service.
+func (ms *ChainService) OperationNotifier() opfeed.Notifier {
+	if ms.opNotifier == nil {
+		ms.opNotifier = &MockOperationNotifier{}
+	}
+	return ms.opNotifier
+}
+
+// MockOperationNotifier mocks the operation notifier.
+type MockOperationNotifier struct {
+	feed *event.Feed
+}
+
+// OperationFeed returns an operation feed.
+func (mon *MockOperationNotifier) OperationFeed() *event.Feed {
+	if mon.feed == nil {
+		mon.feed = new(event.Feed)
+	}
+	return mon.feed
+}
+
+// ReceiveBlockInitialSync mocks ReceiveBlockInitialSync method in chain service.
+func (ms *ChainService) ReceiveBlockInitialSync(ctx context.Context, block *ethpb.SignedBeaconBlock, _ [32]byte) error {
 	if ms.State == nil {
-		ms.State = &pb.BeaconState{}
+		ms.State = &stateTrie.BeaconState{}
 	}
-	if !bytes.Equal(ms.Root, block.ParentRoot) {
-		return errors.Errorf("wanted %#x but got %#x", ms.Root, block.ParentRoot)
+	if !bytes.Equal(ms.Root, block.Block.ParentRoot) {
+		return errors.Errorf("wanted %#x but got %#x", ms.Root, block.Block.ParentRoot)
+	}
+	if err := ms.State.SetSlot(block.Block.Slot); err != nil {
+		return err
 	}
-	ms.State.Slot = block.Slot
 	ms.BlocksReceived = append(ms.BlocksReceived, block)
-	signingRoot, err := ssz.SigningRoot(block)
+	signingRoot, err := block.Block.HashTreeRoot()
 	if err != nil {
 		return err
 	}
@@ -61,7 +165,63 @@ func (ms *ChainService) ReceiveBlockNoPubsubForkchoice(ctx context.Context, bloc
 		if err := ms.DB.SaveBlock(ctx, block); err != nil {
 			return err
 		}
-		logrus.Infof("Saved block with root: %#x at slot %d", signingRoot, block.Slot)
+		logrus.Infof("Saved block with root: %#x at slot %d", signingRoot, block.Block.Slot)
+	}
+	ms.Root = signingRoot[:]
+	ms.Block = block
+	return nil
+}
+
+// ReceiveBlockBatch processes blocks in batches from initial-sync.
+func (ms *ChainService) ReceiveBlockBatch(ctx context.Context, blks []*ethpb.SignedBeaconBlock, _ [][32]byte) error {
+	if ms.State == nil {
+		ms.State = &stateTrie.BeaconState{}
+	}
+	for _, block := range blks {
+		if !bytes.Equal(ms.Root, block.Block.ParentRoot) {
+			return errors.Errorf("wanted %#x but got %#x", ms.Root, block.Block.ParentRoot)
+		}
+		if err := ms.State.SetSlot(block.Block.Slot); err != nil {
+			return err
+		}
+		ms.BlocksReceived = append(ms.BlocksReceived, block)
+		signingRoot, err := block.Block.HashTreeRoot()
+		if err != nil {
+			return err
+		}
+		if ms.DB != nil {
+			if err := ms.DB.SaveBlock(ctx, block); err != nil {
+				return err
+			}
+			logrus.Infof("Saved block with root: %#x at slot %d", signingRoot, block.Block.Slot)
+		}
+		ms.Root = signingRoot[:]
+		ms.Block = block
+	}
+	return nil
+}
+
+// ReceiveBlock mocks ReceiveBlock method in chain service.
+func (ms *ChainService) ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock, _ [32]byte) error {
+	if ms.State == nil {
+		ms.State = &stateTrie.BeaconState{}
+	}
+	if !bytes.Equal(ms.Root, block.Block.ParentRoot) {
+		return errors.Errorf("wanted %#x but got %#x", ms.Root, block.Block.ParentRoot)
+	}
+	if err := ms.State.SetSlot(block.Block.Slot); err != nil {
+		return err
+	}
+	ms.BlocksReceived = append(ms.BlocksReceived, block)
+	signingRoot, err := block.Block.HashTreeRoot()
+	if err != nil {
+		return err
+	}
+	if ms.DB != nil {
+		if err := ms.DB.SaveBlock(ctx, block); err != nil {
+			return err
+		}
+		logrus.Infof("Saved block with root: %#x at slot %d", signingRoot, block.Block.Slot)
 	}
 	ms.Root = signingRoot[:]
 	ms.Block = block
@@ -70,23 +230,27 @@ func (ms *ChainService) ReceiveBlockNoPubsubForkchoice(ctx context.Context, bloc
 
 // HeadSlot mocks HeadSlot method in chain service.
 func (ms *ChainService) HeadSlot() uint64 {
-	return ms.State.Slot
-
+	if ms.State == nil {
+		return 0
+	}
+	return ms.State.Slot()
 }
 
 // HeadRoot mocks HeadRoot method in chain service.
-func (ms *ChainService) HeadRoot() []byte {
-	return ms.Root
-
+func (ms *ChainService) HeadRoot(_ context.Context) ([]byte, error) {
+	if len(ms.Root) > 0 {
+		return ms.Root, nil
+	}
+	return make([]byte, 32), nil
 }
 
 // HeadBlock mocks HeadBlock method in chain service.
-func (ms *ChainService) HeadBlock() *ethpb.BeaconBlock {
-	return ms.Block
+func (ms *ChainService) HeadBlock(context.Context) (*ethpb.SignedBeaconBlock, error) {
+	return ms.Block, nil
 }
 
 // HeadState mocks HeadState method in chain service.
-func (ms *ChainService) HeadState(context.Context) (*pb.BeaconState, error) {
+func (ms *ChainService) HeadState(context.Context) (*stateTrie.BeaconState, error) {
 	return ms.State, nil
 }
 
@@ -100,8 +264,18 @@ func (ms *ChainService) FinalizedCheckpt() *ethpb.Checkpoint {
 	return ms.FinalizedCheckPoint
 }
 
+// CurrentJustifiedCheckpt mocks CurrentJustifiedCheckpt method in chain service.
+func (ms *ChainService) CurrentJustifiedCheckpt() *ethpb.Checkpoint {
+	return ms.CurrentJustifiedCheckPoint
+}
+
+// PreviousJustifiedCheckpt mocks PreviousJustifiedCheckpt method in chain service.
+func (ms *ChainService) PreviousJustifiedCheckpt() *ethpb.Checkpoint {
+	return ms.PreviousJustifiedCheckPoint
+}
+
 // ReceiveAttestation mocks ReceiveAttestation method in chain service.
-func (ms *ChainService) ReceiveAttestation(context.Context, *ethpb.Attestation) error {
+func (ms *ChainService) ReceiveAttestation(_ context.Context, _ *ethpb.Attestation) error {
 	return nil
 }
 
@@ -110,24 +284,129 @@ func (ms *ChainService) ReceiveAttestationNoPubsub(context.Context, *ethpb.Attes
 	return nil
 }
 
-// StateInitializedFeed mocks the same method in the chain service.
-func (ms *ChainService) StateInitializedFeed() *event.Feed {
-	if ms.MockStateInitializedFeed != nil {
-		return ms.MockStateInitializedFeed
-	}
-	ms.MockStateInitializedFeed = new(event.Feed)
-	return ms.MockStateInitializedFeed
+// AttestationPreState mocks AttestationPreState method in chain service.
+func (ms *ChainService) AttestationPreState(_ context.Context, _ *ethpb.Attestation) (*stateTrie.BeaconState, error) {
+	return ms.State, nil
 }
 
-// StateFeed mocks the same method in the chain service.
-func (ms *ChainService) StateFeed() *event.Feed {
-	if ms.MockStateFeed == nil {
-		ms.MockStateFeed = new(event.Feed)
+// HeadValidatorsIndices mocks the same method in the chain service.
+func (ms *ChainService) HeadValidatorsIndices(_ context.Context, epoch uint64) ([]uint64, error) {
+	if ms.State == nil {
+		return []uint64{}, nil
 	}
-	return ms.MockStateFeed
+	return helpers.ActiveValidatorIndices(ms.State, epoch)
+}
+
+// HeadSeed mocks the same method in the chain service.
+func (ms *ChainService) HeadSeed(_ context.Context, epoch uint64) ([32]byte, error) {
+	return helpers.Seed(ms.State, epoch, params.BeaconConfig().DomainBeaconAttester)
+}
+
+// HeadETH1Data provides the current ETH1Data of the head state.
+func (ms *ChainService) HeadETH1Data() *ethpb.Eth1Data {
+	return ms.ETH1Data
+}
+
+// ProtoArrayStore mocks the same method in the chain service.
+func (ms *ChainService) ProtoArrayStore() *protoarray.Store {
+	return ms.ForkChoiceStore
 }
 
 // GenesisTime mocks the same method in the chain service.
 func (ms *ChainService) GenesisTime() time.Time {
 	return ms.Genesis
 }
+
+// GenesisValidatorRoot mocks the same method in the chain service.
+func (ms *ChainService) GenesisValidatorRoot() [32]byte {
+	return ms.ValidatorsRoot
+}
+
+// CurrentSlot mocks the same method in the chain service.
+func (ms *ChainService) CurrentSlot() uint64 {
+	return uint64(time.Now().Unix()-ms.Genesis.Unix()) / params.BeaconConfig().SecondsPerSlot
+}
+
+// Participation mocks the same method in the chain service.
+func (ms *ChainService) Participation(_ uint64) *precompute.Balance {
+	return ms.Balance
+}
+
+// IsValidAttestation always returns true.
+func (ms *ChainService) IsValidAttestation(_ context.Context, _ *ethpb.Attestation) bool {
+	return ms.ValidAttestation
+}
+
+// IsCanonical returns and determines whether a block with the provided root is part of
+// the canonical chain.
+func (ms *ChainService) IsCanonical(_ context.Context, r [32]byte) (bool, error) {
+	if ms.CanonicalRoots != nil {
+		_, ok := ms.CanonicalRoots[r]
+		return ok, nil
+	}
+	return true, nil
+}
+
+// ClearCachedStates does nothing.
+func (ms *ChainService) ClearCachedStates() {}
+
+// HasInitSyncBlock mocks the same method in the chain service.
+func (ms *ChainService) HasInitSyncBlock(_ [32]byte) bool {
+	return false
+}
+
+// HeadGenesisValidatorRoot mocks HeadGenesisValidatorRoot method in chain service.
+func (ms *ChainService) HeadGenesisValidatorRoot() [32]byte {
+	return [32]byte{}
+}
+
+// VerifyBlkDescendant mocks VerifyBlkDescendant and always returns nil.
+func (ms *ChainService) VerifyBlkDescendant(_ context.Context, _ [32]byte) error {
+	return ms.VerifyBlkDescendantErr
+}
+
+// VerifyLmdFfgConsistency mocks VerifyLmdFfgConsistency and always returns nil.
+func (ms *ChainService) VerifyLmdFfgConsistency(_ context.Context, a *ethpb.Attestation) error {
+	if !bytes.Equal(a.Data.BeaconBlockRoot, a.Data.Target.Root) {
+		return errors.New("LMD and FFG miss matched")
+	}
+	return nil
+}
+
+// VerifyFinalizedConsistency mocks VerifyFinalizedConsistency and always returns nil.
+func (ms *ChainService) VerifyFinalizedConsistency(_ context.Context, r []byte) error {
+	if !bytes.Equal(r, ms.FinalizedCheckPoint.Root) {
+		return errors.New("Root and finalized store are not consistent")
+	}
+	return nil
+}
+
+// AttestationCheckPtInfo mocks AttestationCheckPtInfo and always returns nil.
+func (ms *ChainService) AttestationCheckPtInfo(_ context.Context, att *ethpb.Attestation) (*pb.CheckPtInfo, error) {
+	f := ms.State.Fork()
+	g := bytesutil.ToBytes32(ms.State.GenesisValidatorRoot())
+	seed, err := helpers.Seed(ms.State, helpers.SlotToEpoch(att.Data.Slot), params.BeaconConfig().DomainBeaconAttester)
+	if err != nil {
+		return nil, err
+	}
+	indices, err := helpers.ActiveValidatorIndices(ms.State, helpers.SlotToEpoch(att.Data.Slot))
+	if err != nil {
+		return nil, err
+	}
+	validators := ms.State.ValidatorsReadOnly()
+	pks := make([][]byte, len(validators))
+	for i := 0; i < len(pks); i++ {
+		pk := validators[i].PublicKey()
+		pks[i] = pk[:]
+	}
+
+	info := &pb.CheckPtInfo{
+		Fork:          f,
+		GenesisRoot:   g[:],
+		Seed:          seed[:],
+		ActiveIndices: indices,
+		PubKeys:       pks,
+	}
+
+	return info, nil
+}

beacon-chain/blockchain/weak_subjectivity_checks.go

@@ -0,0 +1,60 @@
+package blockchain
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+)
+
+// VerifyWeakSubjectivityRoot verifies the weak subjectivity root in the service struct.
+// Reference design: https://github.com/ethereum/eth2.0-specs/blob/master/specs/phase0/weak-subjectivity.md#weak-subjectivity-sync-procedure
+func (s *Service) VerifyWeakSubjectivityRoot(ctx context.Context) error {
+	// TODO(7342): Remove the following to fully use weak subjectivity in production.
+	if len(s.wsRoot) == 0 || s.wsEpoch == 0 {
+		return nil
+	}
+
+	// Do nothing if the weak subjectivity has previously been verified,
+	// or weak subjectivity epoch is higher than last finalized epoch.
+	if s.wsVerified {
+		return nil
+	}
+	if s.wsEpoch > s.finalizedCheckpt.Epoch {
+		return nil
+	}
+
+	r := bytesutil.ToBytes32(s.wsRoot)
+	log.Infof("Performing weak subjectivity check for root %#x in epoch %d", r, s.wsEpoch)
+	// Save initial sync cached blocks to DB.
+	if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+		return err
+	}
+	// A node should have the weak subjectivity block in the DB.
+	if !s.beaconDB.HasBlock(ctx, r) {
+		return fmt.Errorf("node does not have root in DB: %#x", r)
+	}
+
+	startSlot, err := helpers.StartSlot(s.wsEpoch)
+	if err != nil {
+		return err
+	}
+	// A node should have the weak subjectivity block corresponds to the correct epoch in the DB.
+	filter := filters.NewFilter().SetStartSlot(startSlot).SetEndSlot(startSlot + params.BeaconConfig().SlotsPerEpoch)
+	roots, err := s.beaconDB.BlockRoots(ctx, filter)
+	if err != nil {
+		return err
+	}
+	for _, root := range roots {
+		if r == root {
+			log.Info("Weak subjectivity check has passed")
+			s.wsVerified = true
+			return nil
+		}
+	}
+
+	return fmt.Errorf("node does not have root in db corresponding to epoch: %#x %d", r, s.wsEpoch)
+}

beacon-chain/blockchain/weak_subjectivity_checks_test.go

@@ -0,0 +1,85 @@
+package blockchain
+
+import (
+	"context"
+	"testing"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+)
+
+func TestService_VerifyWeakSubjectivityRoot(t *testing.T) {
+	db, _ := testDB.SetupDB(t)
+
+	b := testutil.NewBeaconBlock()
+	b.Block.Slot = 32
+	require.NoError(t, db.SaveBlock(context.Background(), b))
+	r, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+	tests := []struct {
+		wsVerified     bool
+		wantErr        bool
+		wsRoot         [32]byte
+		wsEpoch        uint64
+		finalizedEpoch uint64
+		errString      string
+		name           string
+	}{
+		{
+			name:    "nil root and epoch",
+			wantErr: false,
+		},
+		{
+			name:           "already verified",
+			wsEpoch:        2,
+			finalizedEpoch: 2,
+			wsVerified:     true,
+			wantErr:        false,
+		},
+		{
+			name:           "not yet to verify, ws epoch higher than finalized epoch",
+			wsEpoch:        2,
+			finalizedEpoch: 1,
+			wantErr:        false,
+		},
+		{
+			name:           "can't find the block in DB",
+			wsEpoch:        1,
+			wsRoot:         [32]byte{'a'},
+			finalizedEpoch: 3,
+			wantErr:        true,
+			errString:      "node does not have root in DB",
+		},
+		{
+			name:           "can't find the block corresponds to ws epoch in DB",
+			wsEpoch:        2,
+			wsRoot:         r, // Root belongs in epoch 1.
+			finalizedEpoch: 3,
+			wantErr:        true,
+			errString:      "node does not have root in db corresponding to epoch",
+		},
+		{
+			name:           "can verify and pass",
+			wsEpoch:        1,
+			wsRoot:         r,
+			finalizedEpoch: 3,
+			wantErr:        false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := &Service{
+				beaconDB:         db,
+				wsRoot:           tt.wsRoot[:],
+				wsEpoch:          tt.wsEpoch,
+				wsVerified:       tt.wsVerified,
+				finalizedCheckpt: &ethpb.Checkpoint{Epoch: tt.finalizedEpoch},
+			}
+			if err := s.VerifyWeakSubjectivityRoot(context.Background()); (err != nil) != tt.wantErr {
+				require.ErrorContains(t, tt.errString, err)
+			}
+		})
+	}
+}

beacon-chain/cache/BUILD.bazel

@@ -1,30 +1,51 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
+# gazelle:ignore committee_disabled.go
+# gazelle:ignore proposer_indices_disabled.go
 go_library(
     name = "go_default_library",
     srcs = [
-        "active_count.go",
-        "active_indices.go",
         "attestation_data.go",
         "checkpoint_state.go",
-        "committee.go",
+        "committees.go",
         "common.go",
-        "eth1_data.go",
-    ],
+        "doc.go",
+        "hot_state_cache.go",
+        "skip_slot_cache.go",
+        "state_summary.go",
+        "subnet_ids.go",
+        "proposer_indices_type.go",
+    ] + select({
+        "//fuzz:fuzzing_enabled": [
+            "committee_disabled.go",
+            "proposer_indices_disabled.go"
+        ],
+        "//conditions:default": [
+            "committee.go",
+            "proposer_indices.go",
+        ],
+    }),
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/cache",
-    visibility = ["//beacon-chain:__subpackages__"],
+    visibility = [
+        "//beacon-chain:__subpackages__",
+        "//fuzz:__pkg__",
+        "//tools:__subpackages__",
+    ],
     deps = [
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
-        "//proto/beacon/rpc/v1:go_default_library",
-        "//proto/eth/v1alpha1:go_default_library",
         "//shared/featureconfig:go_default_library",
         "//shared/hashutil:go_default_library",
         "//shared/params:go_default_library",
         "//shared/sliceutil:go_default_library",
-        "@com_github_gogo_protobuf//proto:go_default_library",
+        "@com_github_hashicorp_golang_lru//:go_default_library",
+        "@com_github_patrickmn_go_cache//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
+        "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
         "@io_k8s_client_go//tools/cache:go_default_library",
+        "@io_opencensus_go//trace:go_default_library",
     ],
 )
 
@@ -32,24 +53,27 @@ go_test(
     name = "go_default_test",
     size = "small",
     srcs = [
-        "active_count_test.go",
-        "active_indices_test.go",
         "attestation_data_test.go",
-        "benchmarks_test.go",
         "checkpoint_state_test.go",
+        "committee_fuzz_test.go",
         "committee_test.go",
-        "eth1_data_test.go",
         "feature_flag_test.go",
+        "hot_state_cache_test.go",
+        "skip_slot_cache_test.go",
+        "subnet_ids_test.go",
+        "proposer_indices_test.go"
     ],
     embed = [":go_default_library"],
-    race = "on",
     deps = [
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
-        "//proto/beacon/rpc/v1:go_default_library",
-        "//proto/eth/v1alpha1:go_default_library",
+        "//shared/bytesutil:go_default_library",
         "//shared/featureconfig:go_default_library",
-        "//shared/hashutil:go_default_library",
         "//shared/params:go_default_library",
+        "//shared/testutil/assert:go_default_library",
+        "//shared/testutil/require:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
+        "@com_github_google_gofuzz//:go_default_library",
+        "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
     ],
 )

beacon-chain/cache/active_count.go

@@ -1,102 +0,0 @@
-package cache
-
-import (
-	"errors"
-	"strconv"
-	"sync"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"k8s.io/client-go/tools/cache"
-)
-
-var (
-	// ErrNotActiveCountInfo will be returned when a cache object is not a pointer to
-	// a ActiveCountByEpoch struct.
-	ErrNotActiveCountInfo = errors.New("object is not a active count obj")
-
-	// maxActiveCountListSize defines the max number of active count can cache.
-	maxActiveCountListSize = 1000
-
-	// Metrics.
-	activeCountCacheMiss = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "active_validator_count_cache_miss",
-		Help: "The number of active validator count requests that aren't present in the cache.",
-	})
-	activeCountCacheHit = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "active_validator_count_cache_hit",
-		Help: "The number of active validator count requests that are present in the cache.",
-	})
-)
-
-// ActiveCountByEpoch defines the active validator count per epoch.
-type ActiveCountByEpoch struct {
-	Epoch       uint64
-	ActiveCount uint64
-}
-
-// ActiveCountCache is a struct with 1 queue for looking up active count by epoch.
-type ActiveCountCache struct {
-	activeCountCache *cache.FIFO
-	lock             sync.RWMutex
-}
-
-// activeCountKeyFn takes the epoch as the key for the active count of a given epoch.
-func activeCountKeyFn(obj interface{}) (string, error) {
-	aInfo, ok := obj.(*ActiveCountByEpoch)
-	if !ok {
-		return "", ErrNotActiveCountInfo
-	}
-
-	return strconv.Itoa(int(aInfo.Epoch)), nil
-}
-
-// NewActiveCountCache creates a new active count cache for storing/accessing active validator count.
-func NewActiveCountCache() *ActiveCountCache {
-	return &ActiveCountCache{
-		activeCountCache: cache.NewFIFO(activeCountKeyFn),
-	}
-}
-
-// ActiveCountInEpoch fetches ActiveCountByEpoch by epoch. Returns true with a
-// reference to the ActiveCountInEpoch info, if exists. Otherwise returns false, nil.
-func (c *ActiveCountCache) ActiveCountInEpoch(epoch uint64) (uint64, error) {
-	if !featureconfig.Get().EnableActiveCountCache {
-		return params.BeaconConfig().FarFutureEpoch, nil
-	}
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-	obj, exists, err := c.activeCountCache.GetByKey(strconv.Itoa(int(epoch)))
-	if err != nil {
-		return params.BeaconConfig().FarFutureEpoch, err
-	}
-
-	if exists {
-		activeCountCacheHit.Inc()
-	} else {
-		activeCountCacheMiss.Inc()
-		return params.BeaconConfig().FarFutureEpoch, nil
-	}
-
-	aInfo, ok := obj.(*ActiveCountByEpoch)
-	if !ok {
-		return params.BeaconConfig().FarFutureEpoch, ErrNotActiveCountInfo
-	}
-
-	return aInfo.ActiveCount, nil
-}
-
-// AddActiveCount adds ActiveCountByEpoch object to the cache. This method also trims the least
-// recently added ActiveCountByEpoch object if the cache size has ready the max cache size limit.
-func (c *ActiveCountCache) AddActiveCount(activeCount *ActiveCountByEpoch) error {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-	if err := c.activeCountCache.AddIfNotPresent(activeCount); err != nil {
-		return err
-	}
-
-	trim(c.activeCountCache, maxActiveCountListSize)
-	return nil
-}

beacon-chain/cache/active_count_test.go

@@ -1,83 +0,0 @@
-package cache
-
-import (
-	"reflect"
-	"strconv"
-	"testing"
-
-	"github.com/prysmaticlabs/prysm/shared/params"
-)
-
-func TestActiveCountKeyFn_OK(t *testing.T) {
-	aInfo := &ActiveCountByEpoch{
-		Epoch:       999,
-		ActiveCount: 10,
-	}
-
-	key, err := activeCountKeyFn(aInfo)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if key != strconv.Itoa(int(aInfo.Epoch)) {
-		t.Errorf("Incorrect hash key: %s, expected %s", key, strconv.Itoa(int(aInfo.Epoch)))
-	}
-}
-
-func TestActiveCountKeyFn_InvalidObj(t *testing.T) {
-	_, err := activeCountKeyFn("bad")
-	if err != ErrNotActiveCountInfo {
-		t.Errorf("Expected error %v, got %v", ErrNotActiveCountInfo, err)
-	}
-}
-
-func TestActiveCountCache_ActiveCountByEpoch(t *testing.T) {
-	cache := NewActiveCountCache()
-
-	aInfo := &ActiveCountByEpoch{
-		Epoch:       99,
-		ActiveCount: 11,
-	}
-	activeCount, err := cache.ActiveCountInEpoch(aInfo.Epoch)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if activeCount != params.BeaconConfig().FarFutureEpoch {
-		t.Error("Expected active count not to exist in empty cache")
-	}
-
-	if err := cache.AddActiveCount(aInfo); err != nil {
-		t.Fatal(err)
-	}
-	activeCount, err = cache.ActiveCountInEpoch(aInfo.Epoch)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(activeCount, aInfo.ActiveCount) {
-		t.Errorf(
-			"Expected fetched active count to be %v, got %v",
-			aInfo.ActiveCount,
-			activeCount,
-		)
-	}
-}
-
-func TestActiveCount_MaxSize(t *testing.T) {
-	cache := NewActiveCountCache()
-
-	for i := uint64(0); i < 1001; i++ {
-		aInfo := &ActiveCountByEpoch{
-			Epoch: i,
-		}
-		if err := cache.AddActiveCount(aInfo); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	if len(cache.activeCountCache.ListKeys()) != maxActiveCountListSize {
-		t.Errorf(
-			"Expected hash cache key size to be %d, got %d",
-			maxActiveCountListSize,
-			len(cache.activeCountCache.ListKeys()),
-		)
-	}
-}

beacon-chain/cache/active_indices.go

@@ -1,106 +0,0 @@
-package cache
-
-import (
-	"errors"
-	"strconv"
-	"sync"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"k8s.io/client-go/tools/cache"
-)
-
-var (
-	// ErrNotActiveIndicesInfo will be returned when a cache object is not a pointer to
-	// a ActiveIndicesByEpoch struct.
-	ErrNotActiveIndicesInfo = errors.New("object is not a active indices list")
-
-	// maxActiveIndicesListSize defines the max number of active indices can cache.
-	maxActiveIndicesListSize = 4
-
-	// Metrics.
-	activeIndicesCacheMiss = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "active_validator_indices_cache_miss",
-		Help: "The number of active validator indices requests that aren't present in the cache.",
-	})
-	activeIndicesCacheHit = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "active_validator_indices_cache_hit",
-		Help: "The number of active validator indices requests that are present in the cache.",
-	})
-)
-
-// ActiveIndicesByEpoch defines the active validator indices per epoch.
-type ActiveIndicesByEpoch struct {
-	Epoch         uint64
-	ActiveIndices []uint64
-}
-
-// ActiveIndicesCache is a struct with 1 queue for looking up active indices by epoch.
-type ActiveIndicesCache struct {
-	activeIndicesCache *cache.FIFO
-	lock               sync.RWMutex
-}
-
-// activeIndicesKeyFn takes the epoch as the key for the active indices of a given epoch.
-func activeIndicesKeyFn(obj interface{}) (string, error) {
-	aInfo, ok := obj.(*ActiveIndicesByEpoch)
-	if !ok {
-		return "", ErrNotActiveIndicesInfo
-	}
-
-	return strconv.Itoa(int(aInfo.Epoch)), nil
-}
-
-// NewActiveIndicesCache creates a new active indices cache for storing/accessing active validator indices.
-func NewActiveIndicesCache() *ActiveIndicesCache {
-	return &ActiveIndicesCache{
-		activeIndicesCache: cache.NewFIFO(activeIndicesKeyFn),
-	}
-}
-
-// ActiveIndicesInEpoch fetches ActiveIndicesByEpoch by epoch. Returns true with a
-// reference to the ActiveIndicesInEpoch info, if exists. Otherwise returns false, nil.
-func (c *ActiveIndicesCache) ActiveIndicesInEpoch(epoch uint64) ([]uint64, error) {
-	if !featureconfig.Get().EnableActiveIndicesCache {
-		return nil, nil
-	}
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-	obj, exists, err := c.activeIndicesCache.GetByKey(strconv.Itoa(int(epoch)))
-	if err != nil {
-		return nil, err
-	}
-
-	if exists {
-		activeIndicesCacheHit.Inc()
-	} else {
-		activeIndicesCacheMiss.Inc()
-		return nil, nil
-	}
-
-	aInfo, ok := obj.(*ActiveIndicesByEpoch)
-	if !ok {
-		return nil, ErrNotActiveIndicesInfo
-	}
-
-	return aInfo.ActiveIndices, nil
-}
-
-// AddActiveIndicesList adds ActiveIndicesByEpoch object to the cache. This method also trims the least
-// recently added ActiveIndicesByEpoch object if the cache size has ready the max cache size limit.
-func (c *ActiveIndicesCache) AddActiveIndicesList(activeIndices *ActiveIndicesByEpoch) error {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-	if err := c.activeIndicesCache.AddIfNotPresent(activeIndices); err != nil {
-		return err
-	}
-
-	trim(c.activeIndicesCache, maxActiveIndicesListSize)
-	return nil
-}
-
-// ActiveIndicesKeys returns the keys of the active indices cache.
-func (c *ActiveIndicesCache) ActiveIndicesKeys() []string {
-	return c.activeIndicesCache.ListKeys()
-}

beacon-chain/cache/active_indices_test.go

@@ -1,82 +0,0 @@
-package cache
-
-import (
-	"reflect"
-	"strconv"
-	"testing"
-)
-
-func TestActiveIndicesKeyFn_OK(t *testing.T) {
-	aInfo := &ActiveIndicesByEpoch{
-		Epoch:         999,
-		ActiveIndices: []uint64{1, 2, 3, 4, 5},
-	}
-
-	key, err := activeIndicesKeyFn(aInfo)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if key != strconv.Itoa(int(aInfo.Epoch)) {
-		t.Errorf("Incorrect hash key: %s, expected %s", key, strconv.Itoa(int(aInfo.Epoch)))
-	}
-}
-
-func TestActiveIndicesKeyFn_InvalidObj(t *testing.T) {
-	_, err := activeIndicesKeyFn("bad")
-	if err != ErrNotActiveIndicesInfo {
-		t.Errorf("Expected error %v, got %v", ErrNotActiveIndicesInfo, err)
-	}
-}
-
-func TestActiveIndicesCache_ActiveIndicesByEpoch(t *testing.T) {
-	cache := NewActiveIndicesCache()
-
-	aInfo := &ActiveIndicesByEpoch{
-		Epoch:         99,
-		ActiveIndices: []uint64{1, 2, 3, 4},
-	}
-
-	activeIndices, err := cache.ActiveIndicesInEpoch(aInfo.Epoch)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if activeIndices != nil {
-		t.Error("Expected active indices not to exist in empty cache")
-	}
-
-	if err := cache.AddActiveIndicesList(aInfo); err != nil {
-		t.Fatal(err)
-	}
-	activeIndices, err = cache.ActiveIndicesInEpoch(aInfo.Epoch)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(activeIndices, aInfo.ActiveIndices) {
-		t.Errorf(
-			"Expected fetched active indices to be %v, got %v",
-			aInfo.ActiveIndices,
-			activeIndices,
-		)
-	}
-}
-
-func TestActiveIndices_MaxSize(t *testing.T) {
-	cache := NewActiveIndicesCache()
-
-	for i := uint64(0); i < 100; i++ {
-		aInfo := &ActiveIndicesByEpoch{
-			Epoch: i,
-		}
-		if err := cache.AddActiveIndicesList(aInfo); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	if len(cache.activeIndicesCache.ListKeys()) != maxActiveIndicesListSize {
-		t.Errorf(
-			"Expected hash cache key size to be %d, got %d",
-			maxActiveIndicesListSize,
-			len(cache.activeIndicesCache.ListKeys()),
-		)
-	}
-}

beacon-chain/cache/attestation_data.go

@@ -10,9 +10,8 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/rpc/v1"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"k8s.io/client-go/tools/cache"
 )
 
@@ -59,13 +58,7 @@ func NewAttestationCache() *AttestationCache {
 
 // Get waits for any in progress calculation to complete before returning a
 // cached response, if any.
-func (c *AttestationCache) Get(ctx context.Context, req *pb.AttestationRequest) (*ethpb.AttestationData, error) {
-	if !featureconfig.Get().EnableAttestationCache {
-		// Return a miss result if cache is not enabled.
-		attestationCacheMiss.Inc()
-		return nil, nil
-	}
-
+func (c *AttestationCache) Get(ctx context.Context, req *ethpb.AttestationDataRequest) (*ethpb.AttestationData, error) {
 	if req == nil {
 		return nil, errors.New("nil attestation data request")
 	}
@@ -105,7 +98,7 @@ func (c *AttestationCache) Get(ctx context.Context, req *pb.AttestationRequest)
 
 	if exists && item != nil && item.(*attestationReqResWrapper).res != nil {
 		attestationCacheHit.Inc()
-		return item.(*attestationReqResWrapper).res, nil
+		return state.CopyAttestationData(item.(*attestationReqResWrapper).res), nil
 	}
 	attestationCacheMiss.Inc()
 	return nil, nil
@@ -113,11 +106,7 @@ func (c *AttestationCache) Get(ctx context.Context, req *pb.AttestationRequest)
 
 // MarkInProgress a request so that any other similar requests will block on
 // Get until MarkNotInProgress is called.
-func (c *AttestationCache) MarkInProgress(req *pb.AttestationRequest) error {
-	if !featureconfig.Get().EnableAttestationCache {
-		return nil
-	}
-
+func (c *AttestationCache) MarkInProgress(req *ethpb.AttestationDataRequest) error {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 	s, e := reqToKey(req)
@@ -127,19 +116,13 @@ func (c *AttestationCache) MarkInProgress(req *pb.AttestationRequest) error {
 	if c.inProgress[s] {
 		return ErrAlreadyInProgress
 	}
-	if featureconfig.Get().EnableAttestationCache {
-		c.inProgress[s] = true
-	}
+	c.inProgress[s] = true
 	return nil
 }
 
 // MarkNotInProgress will release the lock on a given request. This should be
 // called after put.
-func (c *AttestationCache) MarkNotInProgress(req *pb.AttestationRequest) error {
-	if !featureconfig.Get().EnableAttestationCache {
-		return nil
-	}
-
+func (c *AttestationCache) MarkNotInProgress(req *ethpb.AttestationDataRequest) error {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 	s, e := reqToKey(req)
@@ -151,11 +134,7 @@ func (c *AttestationCache) MarkNotInProgress(req *pb.AttestationRequest) error {
 }
 
 // Put the response in the cache.
-func (c *AttestationCache) Put(ctx context.Context, req *pb.AttestationRequest, res *ethpb.AttestationData) error {
-	if !featureconfig.Get().EnableAttestationCache {
-		return nil
-	}
-
+func (c *AttestationCache) Put(_ context.Context, req *ethpb.AttestationDataRequest, res *ethpb.AttestationData) error {
 	data := &attestationReqResWrapper{
 		req,
 		res,
@@ -170,7 +149,10 @@ func (c *AttestationCache) Put(ctx context.Context, req *pb.AttestationRequest,
 }
 
 func wrapperToKey(i interface{}) (string, error) {
-	w := i.(*attestationReqResWrapper)
+	w, ok := i.(*attestationReqResWrapper)
+	if !ok {
+		return "", errors.New("key is not of type *attestationReqResWrapper")
+	}
 	if w == nil {
 		return "", errors.New("nil wrapper")
 	}
@@ -180,11 +162,11 @@ func wrapperToKey(i interface{}) (string, error) {
 	return reqToKey(w.req)
 }
 
-func reqToKey(req *pb.AttestationRequest) (string, error) {
-	return fmt.Sprintf("%d-%d", req.CommitteeIndex, req.Slot), nil
+func reqToKey(req *ethpb.AttestationDataRequest) (string, error) {
+	return fmt.Sprintf("%d", req.Slot), nil
 }
 
 type attestationReqResWrapper struct {
-	req *pb.AttestationRequest
+	req *ethpb.AttestationDataRequest
 	res *ethpb.AttestationData
 }

beacon-chain/cache/attestation_data_test.go

@@ -5,49 +5,35 @@ import (
 	"testing"
 
 	"github.com/gogo/protobuf/proto"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/rpc/v1"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
 )
 
 func TestAttestationCache_RoundTrip(t *testing.T) {
 	ctx := context.Background()
 	c := cache.NewAttestationCache()
 
-	req := &pb.AttestationRequest{
+	req := &ethpb.AttestationDataRequest{
 		CommitteeIndex: 0,
 		Slot:           1,
 	}
 
 	response, err := c.Get(ctx, req)
-	if err != nil {
-		t.Error(err)
-	}
+	assert.NoError(t, err)
+	assert.Equal(t, (*ethpb.AttestationData)(nil), response)
 
-	if response != nil {
-		t.Errorf("Empty cache returned an object: %v", response)
-	}
-
-	if err := c.MarkInProgress(req); err != nil {
-		t.Error(err)
-	}
+	assert.NoError(t, c.MarkInProgress(req))
 
 	res := &ethpb.AttestationData{
-		Target: &ethpb.Checkpoint{Epoch: 5},
+		Target: &ethpb.Checkpoint{Epoch: 5, Root: make([]byte, 32)},
 	}
 
-	if err = c.Put(ctx, req, res); err != nil {
-		t.Error(err)
-	}
-
-	if err := c.MarkNotInProgress(req); err != nil {
-		t.Error(err)
-	}
+	assert.NoError(t, c.Put(ctx, req, res))
+	assert.NoError(t, c.MarkNotInProgress(req))
 
 	response, err = c.Get(ctx, req)
-	if err != nil {
-		t.Error(err)
-	}
+	assert.NoError(t, err)
 
 	if !proto.Equal(response, res) {
 		t.Error("Expected equal protos to return from cache")

beacon-chain/cache/benchmarks_test.go

@@ -1,45 +0,0 @@
-package cache
-
-import (
-	"testing"
-)
-
-var indices300k = createIndices(300000)
-var epoch = uint64(1)
-
-func createIndices(count int) *ActiveIndicesByEpoch {
-	indices := make([]uint64, 0, count)
-	for i := 0; i < count; i++ {
-		indices = append(indices, uint64(i))
-	}
-	return &ActiveIndicesByEpoch{
-		Epoch:         epoch,
-		ActiveIndices: indices,
-	}
-}
-
-func BenchmarkCachingAddRetrieve(b *testing.B) {
-
-	c := NewActiveIndicesCache()
-
-	b.Run("ADD300K", func(b *testing.B) {
-		b.N = 10
-		b.ResetTimer()
-		for i := 0; i < b.N; i++ {
-			if err := c.AddActiveIndicesList(indices300k); err != nil {
-				b.Fatal(err)
-			}
-		}
-	})
-
-	b.Run("RETR300K", func(b *testing.B) {
-		b.N = 10
-		b.ResetTimer()
-		for i := 0; i < b.N; i++ {
-			if _, err := c.ActiveIndicesInEpoch(epoch); err != nil {
-				b.Fatal(err)
-			}
-		}
-	})
-
-}

beacon-chain/cache/checkpoint_state.go

@@ -1,29 +1,25 @@
 package cache
 
 import (
-	"errors"
 	"sync"
 
-	"github.com/gogo/protobuf/proto"
+	lru "github.com/hashicorp/golang-lru"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/hashutil"
-	"k8s.io/client-go/tools/cache"
 )
 
 var (
-	// ErrNotCheckpointState will be returned when a cache object is not a pointer to
-	// a CheckpointState struct.
-	ErrNotCheckpointState = errors.New("object is not a state by check point struct")
-
 	// maxCheckpointStateSize defines the max number of entries check point to state cache can contain.
-	maxCheckpointStateSize = 4
+	// Choosing 10 to account for multiple forks, this allows 5 forks per epoch boundary with 2 epochs
+	// window to accept attestation based on latest spec.
+	maxCheckpointStateSize = 10
 
 	// Metrics.
 	checkpointStateMiss = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "check_point_statecache_miss",
+		Name: "check_point_state_cache_miss",
 		Help: "The number of check point state requests that aren't present in the cache.",
 	})
 	checkpointStateHit = promauto.NewCounter(prometheus.CounterOpts{
@@ -32,42 +28,26 @@ var (
 	})
 )
 
-// CheckpointState defines the active validator indices per epoch.
-type CheckpointState struct {
-	Checkpoint *ethpb.Checkpoint
-	State      *pb.BeaconState
-}
-
 // CheckpointStateCache is a struct with 1 queue for looking up state by checkpoint.
 type CheckpointStateCache struct {
-	cache *cache.FIFO
+	cache *lru.Cache
 	lock  sync.RWMutex
 }
 
-// checkpointState takes the checkpoint as the key of the resulting state.
-func checkpointState(obj interface{}) (string, error) {
-	info, ok := obj.(*CheckpointState)
-	if !ok {
-		return "", ErrNotCheckpointState
-	}
-
-	h, err := hashutil.HashProto(info.Checkpoint)
-	if err != nil {
-		return "", err
-	}
-	return string(h[:]), nil
-}
-
 // NewCheckpointStateCache creates a new checkpoint state cache for storing/accessing processed state.
 func NewCheckpointStateCache() *CheckpointStateCache {
+	cache, err := lru.New(maxCheckpointStateSize)
+	if err != nil {
+		panic(err)
+	}
 	return &CheckpointStateCache{
-		cache: cache.NewFIFO(checkpointState),
+		cache: cache,
 	}
 }
 
 // StateByCheckpoint fetches state by checkpoint. Returns true with a
 // reference to the CheckpointState info, if exists. Otherwise returns false, nil.
-func (c *CheckpointStateCache) StateByCheckpoint(cp *ethpb.Checkpoint) (*pb.BeaconState, error) {
+func (c *CheckpointStateCache) StateByCheckpoint(cp *ethpb.Checkpoint) (*stateTrie.BeaconState, error) {
 	c.lock.RLock()
 	defer c.lock.RUnlock()
 	h, err := hashutil.HashProto(cp)
@@ -75,40 +55,27 @@ func (c *CheckpointStateCache) StateByCheckpoint(cp *ethpb.Checkpoint) (*pb.Beac
 		return nil, err
 	}
 
-	obj, exists, err := c.cache.GetByKey(string(h[:]))
-	if err != nil {
-		return nil, err
-	}
+	item, exists := c.cache.Get(h)
 
-	if exists {
+	if exists && item != nil {
 		checkpointStateHit.Inc()
-	} else {
-		checkpointStateMiss.Inc()
-		return nil, nil
+		// Copy here is unnecessary since the return will only be used to verify attestation signature.
+		return item.(*stateTrie.BeaconState), nil
 	}
 
-	info, ok := obj.(*CheckpointState)
-	if !ok {
-		return nil, ErrNotCheckpointState
-	}
-
-	return proto.Clone(info.State).(*pb.BeaconState), nil
+	checkpointStateMiss.Inc()
+	return nil, nil
 }
 
 // AddCheckpointState adds CheckpointState object to the cache. This method also trims the least
 // recently added CheckpointState object if the cache size has ready the max cache size limit.
-func (c *CheckpointStateCache) AddCheckpointState(cp *CheckpointState) error {
+func (c *CheckpointStateCache) AddCheckpointState(cp *ethpb.Checkpoint, s *stateTrie.BeaconState) error {
 	c.lock.Lock()
 	defer c.lock.Unlock()
-	if err := c.cache.AddIfNotPresent(cp); err != nil {
+	h, err := hashutil.HashProto(cp)
+	if err != nil {
 		return err
 	}
-
-	trim(c.cache, maxCheckpointStateSize)
+	c.cache.Add(h, s)
 	return nil
 }
-
-// CheckpointStateKeys returns the keys of the state in cache.
-func (c *CheckpointStateCache) CheckpointStateKeys() []string {
-	return c.cache.ListKeys()
-}

beacon-chain/cache/checkpoint_state_test.go

@@ -1,110 +1,68 @@
 package cache
 
 import (
-	"reflect"
 	"testing"
 
+	"github.com/gogo/protobuf/proto"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/shared/hashutil"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 )
 
-func TestCheckpointStateCacheKeyFn_OK(t *testing.T) {
-	cp := &ethpb.Checkpoint{Epoch: 1, Root: []byte{'A'}}
-	info := &CheckpointState{
-		Checkpoint: cp,
-		State:      &pb.BeaconState{Slot: 64},
-	}
-	key, err := checkpointState(info)
-	if err != nil {
-		t.Fatal(err)
-	}
-	wantedKey, err := hashutil.HashProto(cp)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if key != string(wantedKey[:]) {
-		t.Errorf("Incorrect hash key: %s, expected %s", key, string(wantedKey[:]))
-	}
-}
-
-func TestCheckpointStateCacheKeyFn_InvalidObj(t *testing.T) {
-	_, err := checkpointState("bad")
-	if err != ErrNotCheckpointState {
-		t.Errorf("Expected error %v, got %v", ErrNotCheckpointState, err)
-	}
-}
-
 func TestCheckpointStateCache_StateByCheckpoint(t *testing.T) {
 	cache := NewCheckpointStateCache()
 
-	cp1 := &ethpb.Checkpoint{Epoch: 1, Root: []byte{'A'}}
-	info1 := &CheckpointState{
-		Checkpoint: cp1,
-		State:      &pb.BeaconState{Slot: 64},
-	}
+	cp1 := &ethpb.Checkpoint{Epoch: 1, Root: bytesutil.PadTo([]byte{'A'}, 32)}
+	st, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		GenesisValidatorsRoot: params.BeaconConfig().ZeroHash[:],
+		Slot:                  64,
+	})
+	require.NoError(t, err)
+
 	state, err := cache.StateByCheckpoint(cp1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if state != nil {
-		t.Error("Expected state not to exist in empty cache")
-	}
+	require.NoError(t, err)
+	assert.Equal(t, (*stateTrie.BeaconState)(nil), state, "Expected state not to exist in empty cache")
+
+	require.NoError(t, cache.AddCheckpointState(cp1, st))
 
-	if err := cache.AddCheckpointState(info1); err != nil {
-		t.Fatal(err)
-	}
 	state, err = cache.StateByCheckpoint(cp1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(state, info1.State) {
+	require.NoError(t, err)
+
+	if !proto.Equal(state.InnerStateUnsafe(), st.InnerStateUnsafe()) {
 		t.Error("incorrectly cached state")
 	}
 
-	cp2 := &ethpb.Checkpoint{Epoch: 2, Root: []byte{'B'}}
-	info2 := &CheckpointState{
-		Checkpoint: cp2,
-		State:      &pb.BeaconState{Slot: 128},
-	}
-	if err := cache.AddCheckpointState(info2); err != nil {
-		t.Fatal(err)
-	}
+	cp2 := &ethpb.Checkpoint{Epoch: 2, Root: bytesutil.PadTo([]byte{'B'}, 32)}
+	st2, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot: 128,
+	})
+	require.NoError(t, err)
+	require.NoError(t, cache.AddCheckpointState(cp2, st2))
+
 	state, err = cache.StateByCheckpoint(cp2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(state, info2.State) {
-		t.Error("incorrectly cached state")
-	}
+	require.NoError(t, err)
+	assert.DeepEqual(t, st2.CloneInnerState(), state.CloneInnerState(), "incorrectly cached state")
 
 	state, err = cache.StateByCheckpoint(cp1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(state, info1.State) {
-		t.Error("incorrectly cached state")
-	}
+	require.NoError(t, err)
+	assert.DeepEqual(t, st.CloneInnerState(), state.CloneInnerState(), "incorrectly cached state")
 }
 
-func TestCheckpointStateCache__MaxSize(t *testing.T) {
+func TestCheckpointStateCache_MaxSize(t *testing.T) {
 	c := NewCheckpointStateCache()
+	st, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot: 0,
+	})
+	require.NoError(t, err)
 
-	for i := 0; i < maxCheckpointStateSize+100; i++ {
-		info := &CheckpointState{
-			Checkpoint: &ethpb.Checkpoint{Epoch: uint64(i)},
-			State:      &pb.BeaconState{Slot: uint64(i)},
-		}
-		if err := c.AddCheckpointState(info); err != nil {
-			t.Fatal(err)
-		}
+	for i := uint64(0); i < uint64(maxCheckpointStateSize+100); i++ {
+		require.NoError(t, st.SetSlot(i))
+		require.NoError(t, c.AddCheckpointState(&ethpb.Checkpoint{Epoch: i, Root: make([]byte, 32)}, st))
 	}
 
-	if len(c.cache.ListKeys()) != maxCheckpointStateSize {
-		t.Errorf(
-			"Expected hash cache key size to be %d, got %d",
-			maxCheckpointStateSize,
-			len(c.cache.ListKeys()),
-		)
-	}
+	assert.Equal(t, maxCheckpointStateSize, len(c.cache.Keys()))
 }

beacon-chain/cache/committee.go

@@ -1,26 +1,22 @@
+// +build !libfuzzer
+
 package cache
 
 import (
 	"errors"
-	"strconv"
 	"sync"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/sliceutil"
 	"k8s.io/client-go/tools/cache"
 )
 
 var (
-	// ErrNotCommittee will be returned when a cache object is not a pointer to
-	// a Committee struct.
-	ErrNotCommittee = errors.New("object is not a committee struct")
-
-	// maxShuffledIndicesSize defines the max number of shuffled indices list can cache.
-	// 3 for previous, current epoch and next epoch.
-	maxShuffledIndicesSize = 3
+	// maxCommitteesCacheSize defines the max number of shuffled committees on per randao basis can cache.
+	// Due to reorgs and long finality, it's good to keep the old cache around for quickly switch over.
+	maxCommitteesCacheSize = uint64(32)
 
 	// CommitteeCacheMiss tracks the number of committee requests that aren't present in the cache.
 	CommitteeCacheMiss = promauto.NewCounter(prometheus.CounterOpts{
@@ -34,47 +30,36 @@ var (
 	})
 )
 
-// Committee defines the committee per epoch and index.
-type Committee struct {
-	CommitteeCount uint64
-	Epoch          uint64
-	Committee      []uint64
-}
-
-// CommitteeCache is a struct with 1 queue for looking up shuffled indices list by epoch and committee index.
+// CommitteeCache is a struct with 1 queue for looking up shuffled indices list by seed.
 type CommitteeCache struct {
 	CommitteeCache *cache.FIFO
 	lock           sync.RWMutex
 }
 
-// committeeKeyFn takes the epoch as the key to retrieve shuffled indices of a committee in a given epoch.
+// committeeKeyFn takes the seed as the key to retrieve shuffled indices of a committee in a given epoch.
 func committeeKeyFn(obj interface{}) (string, error) {
-	info, ok := obj.(*Committee)
+	info, ok := obj.(*Committees)
 	if !ok {
 		return "", ErrNotCommittee
 	}
 
-	return strconv.Itoa(int(info.Epoch)), nil
+	return key(info.Seed), nil
 }
 
-// NewCommitteeCache creates a new committee cache for storing/accessing shuffled indices of a committee.
-func NewCommitteeCache() *CommitteeCache {
+// NewCommitteesCache creates a new committee cache for storing/accessing shuffled indices of a committee.
+func NewCommitteesCache() *CommitteeCache {
 	return &CommitteeCache{
 		CommitteeCache: cache.NewFIFO(committeeKeyFn),
 	}
 }
 
-// ShuffledIndices fetches the shuffled indices by slot and committee index. Every list of indices
+// Committee fetches the shuffled indices by slot and committee index. Every list of indices
 // represent one committee. Returns true if the list exists with slot and committee index. Otherwise returns false, nil.
-func (c *CommitteeCache) ShuffledIndices(slot uint64, index uint64) ([]uint64, error) {
-	if !featureconfig.Get().EnableShuffledIndexCache && !featureconfig.Get().EnableNewCache {
-		return nil, nil
-	}
+func (c *CommitteeCache) Committee(slot uint64, seed [32]byte, index uint64) ([]uint64, error) {
 	c.lock.RLock()
 	defer c.lock.RUnlock()
 
-	epoch := int(slot / params.BeaconConfig().SlotsPerEpoch)
-	obj, exists, err := c.CommitteeCache.GetByKey(strconv.Itoa(epoch))
+	obj, exists, err := c.CommitteeCache.GetByKey(key(seed))
 	if err != nil {
 		return nil, err
 	}
@@ -86,7 +71,7 @@ func (c *CommitteeCache) ShuffledIndices(slot uint64, index uint64) ([]uint64, e
 		return nil, nil
 	}
 
-	item, ok := obj.(*Committee)
+	item, ok := obj.(*Committees)
 	if !ok {
 		return nil, ErrNotCommittee
 	}
@@ -98,100 +83,32 @@ func (c *CommitteeCache) ShuffledIndices(slot uint64, index uint64) ([]uint64, e
 
 	indexOffSet := index + (slot%params.BeaconConfig().SlotsPerEpoch)*committeeCountPerSlot
 	start, end := startEndIndices(item, indexOffSet)
-	return item.Committee[start:end], nil
+
+	if end > uint64(len(item.ShuffledIndices)) || end < start {
+		return nil, errors.New("requested index out of bound")
+	}
+
+	return item.ShuffledIndices[start:end], nil
 }
 
 // AddCommitteeShuffledList adds Committee shuffled list object to the cache. T
 // his method also trims the least recently list if the cache size has ready the max cache size limit.
-func (c *CommitteeCache) AddCommitteeShuffledList(committee *Committee) error {
-	if !featureconfig.Get().EnableShuffledIndexCache && !featureconfig.Get().EnableNewCache {
-		return nil
-	}
+func (c *CommitteeCache) AddCommitteeShuffledList(committees *Committees) error {
 	c.lock.Lock()
 	defer c.lock.Unlock()
-	if err := c.CommitteeCache.AddIfNotPresent(committee); err != nil {
+
+	if err := c.CommitteeCache.AddIfNotPresent(committees); err != nil {
 		return err
 	}
-	trim(c.CommitteeCache, maxShuffledIndicesSize)
+	trim(c.CommitteeCache, maxCommitteesCacheSize)
 	return nil
 }
 
-// Epochs returns the epochs stored in the committee cache. These are the keys to the cache.
-func (c *CommitteeCache) Epochs() ([]uint64, error) {
-	if !featureconfig.Get().EnableShuffledIndexCache {
-		return nil, nil
-	}
+// ActiveIndices returns the active indices of a given seed stored in cache.
+func (c *CommitteeCache) ActiveIndices(seed [32]byte) ([]uint64, error) {
 	c.lock.RLock()
 	defer c.lock.RUnlock()
-
-	epochs := make([]uint64, len(c.CommitteeCache.ListKeys()))
-	for i, s := range c.CommitteeCache.ListKeys() {
-		epoch, err := strconv.Atoi(s)
-		if err != nil {
-			return nil, err
-		}
-		epochs[i] = uint64(epoch)
-	}
-	return epochs, nil
-}
-
-// EpochInCache returns true if an input epoch is part of keys in cache.
-func (c *CommitteeCache) EpochInCache(wantedEpoch uint64) (bool, error) {
-	if !featureconfig.Get().EnableShuffledIndexCache && !featureconfig.Get().EnableNewCache {
-		return false, nil
-	}
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-
-	for _, s := range c.CommitteeCache.ListKeys() {
-		epoch, err := strconv.Atoi(s)
-		if err != nil {
-			return false, err
-		}
-		if wantedEpoch == uint64(epoch) {
-			return true, nil
-		}
-	}
-	return false, nil
-}
-
-// CommitteeCountPerSlot returns the number of committees in a given slot as stored in cache.
-func (c *CommitteeCache) CommitteeCountPerSlot(slot uint64) (uint64, bool, error) {
-	if !featureconfig.Get().EnableShuffledIndexCache && !featureconfig.Get().EnableNewCache {
-		return 0, false, nil
-	}
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-	epoch := int(slot / params.BeaconConfig().SlotsPerEpoch)
-	obj, exists, err := c.CommitteeCache.GetByKey(strconv.Itoa(int(epoch)))
-	if err != nil {
-		return 0, false, err
-	}
-
-	if exists {
-		CommitteeCacheHit.Inc()
-	} else {
-		CommitteeCacheMiss.Inc()
-		return 0, false, nil
-	}
-
-	item, ok := obj.(*Committee)
-	if !ok {
-		return 0, false, ErrNotCommittee
-	}
-
-	return item.CommitteeCount / params.BeaconConfig().SlotsPerEpoch, true, nil
-}
-
-// ActiveIndices returns the active indices of a given epoch stored in cache.
-func (c *CommitteeCache) ActiveIndices(epoch uint64) ([]uint64, error) {
-	if !featureconfig.Get().EnableShuffledIndexCache && !featureconfig.Get().EnableNewCache {
-		return nil, nil
-	}
-
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-	obj, exists, err := c.CommitteeCache.GetByKey(strconv.Itoa(int(epoch)))
+	obj, exists, err := c.CommitteeCache.GetByKey(key(seed))
 	if err != nil {
 		return nil, err
 	}
@@ -203,18 +120,55 @@ func (c *CommitteeCache) ActiveIndices(epoch uint64) ([]uint64, error) {
 		return nil, nil
 	}
 
-	item, ok := obj.(*Committee)
+	item, ok := obj.(*Committees)
 	if !ok {
 		return nil, ErrNotCommittee
 	}
 
-	return item.Committee, nil
+	return item.SortedIndices, nil
 }
 
-func startEndIndices(c *Committee, index uint64) (uint64, uint64) {
-	validatorCount := uint64(len(c.Committee))
+// ActiveIndicesCount returns the active indices count of a given seed stored in cache.
+func (c *CommitteeCache) ActiveIndicesCount(seed [32]byte) (int, error) {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	obj, exists, err := c.CommitteeCache.GetByKey(key(seed))
+	if err != nil {
+		return 0, err
+	}
+
+	if exists {
+		CommitteeCacheHit.Inc()
+	} else {
+		CommitteeCacheMiss.Inc()
+		return 0, nil
+	}
+
+	item, ok := obj.(*Committees)
+	if !ok {
+		return 0, ErrNotCommittee
+	}
+
+	return len(item.SortedIndices), nil
+}
+
+// HasEntry returns true if the committee cache has a value.
+func (c *CommitteeCache) HasEntry(seed string) bool {
+	_, ok, err := c.CommitteeCache.GetByKey(seed)
+	return err == nil && ok
+}
+
+func startEndIndices(c *Committees, index uint64) (uint64, uint64) {
+	validatorCount := uint64(len(c.ShuffledIndices))
 	start := sliceutil.SplitOffset(validatorCount, c.CommitteeCount, index)
 	end := sliceutil.SplitOffset(validatorCount, c.CommitteeCount, index+1)
-
 	return start, end
 }
+
+// Using seed as source for key to handle reorgs in the same epoch.
+// The seed is derived from state's array of randao mixes and epoch value
+// hashed together. This avoids collisions on different validator set. Spec definition:
+// https://github.com/ethereum/eth2.0-specs/blob/v0.9.3/specs/core/0_beacon-chain.md#get_seed
+func key(seed [32]byte) string {
+	return string(seed[:])
+}

beacon-chain/cache/committee_disabled.go

@@ -0,0 +1,50 @@
+// +build libfuzzer
+
+// This file is used in fuzzer builds to bypass global committee caches.
+package cache
+
+// FakeCommitteeCache is a struct with 1 queue for looking up shuffled indices list by seed.
+type FakeCommitteeCache struct {
+}
+
+// NewCommitteesCache creates a new committee cache for storing/accessing shuffled indices of a committee.
+func NewCommitteesCache() *FakeCommitteeCache {
+	return &FakeCommitteeCache{}
+}
+
+// Committee fetches the shuffled indices by slot and committee index. Every list of indices
+// represent one committee. Returns true if the list exists with slot and committee index. Otherwise returns false, nil.
+func (c *FakeCommitteeCache) Committee(slot uint64, seed [32]byte, index uint64) ([]uint64, error) {
+	return nil, nil
+}
+
+// AddCommitteeShuffledList adds Committee shuffled list object to the cache. T
+// his method also trims the least recently list if the cache size has ready the max cache size limit.
+func (c *FakeCommitteeCache) AddCommitteeShuffledList(committees *Committees) error {
+	return nil
+}
+
+// AddProposerIndicesList updates the committee shuffled list with proposer indices.
+func (c *FakeCommitteeCache) AddProposerIndicesList(seed [32]byte, indices []uint64) error {
+	return nil
+}
+
+// ActiveIndices returns the active indices of a given seed stored in cache.
+func (c *FakeCommitteeCache) ActiveIndices(seed [32]byte) ([]uint64, error) {
+	return nil, nil
+}
+
+// ActiveIndicesCount returns the active indices count of a given seed stored in cache.
+func (c *FakeCommitteeCache) ActiveIndicesCount(seed [32]byte) (int, error) {
+	return 0, nil
+}
+
+// ProposerIndices returns the proposer indices of a given seed.
+func (c *FakeCommitteeCache) ProposerIndices(seed [32]byte) ([]uint64, error) {
+	return nil, nil
+}
+
+// HasEntry returns true if the committee cache has a value.
+func (c *FakeCommitteeCache) HasEntry(string) bool {
+	return false
+}

beacon-chain/cache/committee_fuzz_test.go

@@ -0,0 +1,53 @@
+package cache
+
+import (
+	"testing"
+
+	fuzz "github.com/google/gofuzz"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+)
+
+func TestCommitteeKeyFuzz_OK(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	c := &Committees{}
+
+	for i := 0; i < 100000; i++ {
+		fuzzer.Fuzz(c)
+		k, err := committeeKeyFn(c)
+		require.NoError(t, err)
+		assert.Equal(t, key(c.Seed), k)
+	}
+}
+
+func TestCommitteeCache_FuzzCommitteesByEpoch(t *testing.T) {
+	cache := NewCommitteesCache()
+	fuzzer := fuzz.NewWithSeed(0)
+	c := &Committees{}
+
+	for i := 0; i < 100000; i++ {
+		fuzzer.Fuzz(c)
+		require.NoError(t, cache.AddCommitteeShuffledList(c))
+		_, err := cache.Committee(0, c.Seed, 0)
+		require.NoError(t, err)
+	}
+
+	assert.Equal(t, maxCommitteesCacheSize, uint64(len(cache.CommitteeCache.ListKeys())), "Incorrect key size")
+}
+
+func TestCommitteeCache_FuzzActiveIndices(t *testing.T) {
+	cache := NewCommitteesCache()
+	fuzzer := fuzz.NewWithSeed(0)
+	c := &Committees{}
+
+	for i := 0; i < 100000; i++ {
+		fuzzer.Fuzz(c)
+		require.NoError(t, cache.AddCommitteeShuffledList(c))
+
+		indices, err := cache.ActiveIndices(c.Seed)
+		require.NoError(t, err)
+		assert.DeepEqual(t, c.SortedIndices, indices)
+	}
+
+	assert.Equal(t, maxCommitteesCacheSize, uint64(len(cache.CommitteeCache.ListKeys())), "Incorrect key size")
+}

beacon-chain/cache/committee_test.go

@@ -1,179 +1,129 @@
 package cache
 
 import (
-	"reflect"
+	"math"
+	"sort"
 	"strconv"
 	"testing"
 
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 )
 
 func TestCommitteeKeyFn_OK(t *testing.T) {
-	item := &Committee{
-		Epoch:          999,
-		CommitteeCount: 1,
-		Committee:      []uint64{1, 2, 3, 4, 5},
+	item := &Committees{
+		CommitteeCount:  1,
+		Seed:            [32]byte{'A'},
+		ShuffledIndices: []uint64{1, 2, 3, 4, 5},
 	}
 
-	key, err := committeeKeyFn(item)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if key != strconv.Itoa(int(item.Epoch)) {
-		t.Errorf("Incorrect hash key: %s, expected %s", key, strconv.Itoa(int(item.Epoch)))
-	}
+	k, err := committeeKeyFn(item)
+	require.NoError(t, err)
+	assert.Equal(t, key(item.Seed), k)
 }
 
 func TestCommitteeKeyFn_InvalidObj(t *testing.T) {
 	_, err := committeeKeyFn("bad")
-	if err != ErrNotCommittee {
-		t.Errorf("Expected error %v, got %v", ErrNotCommittee, err)
-	}
+	assert.Equal(t, ErrNotCommittee, err)
 }
 
 func TestCommitteeCache_CommitteesByEpoch(t *testing.T) {
-	cache := NewCommitteeCache()
+	cache := NewCommitteesCache()
 
-	item := &Committee{
-		Epoch:          1,
-		Committee:      []uint64{1, 2, 3, 4, 5, 6},
-		CommitteeCount: 3,
+	item := &Committees{
+		ShuffledIndices: []uint64{1, 2, 3, 4, 5, 6},
+		Seed:            [32]byte{'A'},
+		CommitteeCount:  3,
 	}
 
-	slot := uint64(item.Epoch * params.BeaconConfig().SlotsPerEpoch)
+	slot := params.BeaconConfig().SlotsPerEpoch
 	committeeIndex := uint64(1)
-	indices, err := cache.ShuffledIndices(slot, committeeIndex)
-	if err != nil {
-		t.Fatal(err)
+	indices, err := cache.Committee(slot, item.Seed, committeeIndex)
+	require.NoError(t, err)
+	if indices != nil {
+		t.Error("Expected committee not to exist in empty cache")
 	}
+	require.NoError(t, cache.AddCommitteeShuffledList(item))
+
+	wantedIndex := uint64(0)
+	indices, err = cache.Committee(slot, item.Seed, wantedIndex)
+	require.NoError(t, err)
+
+	start, end := startEndIndices(item, wantedIndex)
+	assert.DeepEqual(t, item.ShuffledIndices[start:end], indices)
+}
+
+func TestCommitteeCache_ActiveIndices(t *testing.T) {
+	cache := NewCommitteesCache()
+
+	item := &Committees{Seed: [32]byte{'A'}, SortedIndices: []uint64{1, 2, 3, 4, 5, 6}}
+	indices, err := cache.ActiveIndices(item.Seed)
+	require.NoError(t, err)
 	if indices != nil {
 		t.Error("Expected committee not to exist in empty cache")
 	}
 
-	if err := cache.AddCommitteeShuffledList(item); err != nil {
-		t.Fatal(err)
-	}
-	wantedIndex := uint64(0)
-	indices, err = cache.ShuffledIndices(slot, wantedIndex)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, cache.AddCommitteeShuffledList(item))
 
-	start, end := startEndIndices(item, wantedIndex)
-	if !reflect.DeepEqual(indices, item.Committee[start:end]) {
-		t.Errorf(
-			"Expected fetched active indices to be %v, got %v",
-			indices,
-			item.Committee[start:end],
-		)
-	}
+	indices, err = cache.ActiveIndices(item.Seed)
+	require.NoError(t, err)
+	assert.DeepEqual(t, item.SortedIndices, indices)
+}
+
+func TestCommitteeCache_ActiveCount(t *testing.T) {
+	cache := NewCommitteesCache()
+
+	item := &Committees{Seed: [32]byte{'A'}, SortedIndices: []uint64{1, 2, 3, 4, 5, 6}}
+	count, err := cache.ActiveIndicesCount(item.Seed)
+	require.NoError(t, err)
+	assert.Equal(t, 0, count, "Expected active count not to exist in empty cache")
+
+	require.NoError(t, cache.AddCommitteeShuffledList(item))
+
+	count, err = cache.ActiveIndicesCount(item.Seed)
+	require.NoError(t, err)
+	assert.Equal(t, len(item.SortedIndices), count)
 }
 
 func TestCommitteeCache_CanRotate(t *testing.T) {
-	cache := NewCommitteeCache()
-	item1 := &Committee{Epoch: 1}
-	if err := cache.AddCommitteeShuffledList(item1); err != nil {
-		t.Fatal(err)
-	}
-	item2 := &Committee{Epoch: 2}
-	if err := cache.AddCommitteeShuffledList(item2); err != nil {
-		t.Fatal(err)
-	}
-	epochs, err := cache.Epochs()
-	if err != nil {
-		t.Fatal(err)
-	}
-	wanted := item1.Epoch + item2.Epoch
-	if sum(epochs) != wanted {
-		t.Errorf("Wanted: %v, got: %v", wanted, sum(epochs))
+	cache := NewCommitteesCache()
+
+	// Should rotate out all the epochs except 190 through 199.
+	start := 100
+	end := 200
+	for i := start; i < end; i++ {
+		s := []byte(strconv.Itoa(i))
+		item := &Committees{Seed: bytesutil.ToBytes32(s)}
+		require.NoError(t, cache.AddCommitteeShuffledList(item))
 	}
 
-	item3 := &Committee{Epoch: 4}
-	if err := cache.AddCommitteeShuffledList(item3); err != nil {
-		t.Fatal(err)
-	}
-	epochs, err = cache.Epochs()
-	if err != nil {
-		t.Fatal(err)
-	}
-	wanted = item1.Epoch + item2.Epoch + item3.Epoch
-	if sum(epochs) != wanted {
-		t.Errorf("Wanted: %v, got: %v", wanted, sum(epochs))
-	}
+	k := cache.CommitteeCache.ListKeys()
+	assert.Equal(t, maxCommitteesCacheSize, uint64(len(k)))
 
-	item4 := &Committee{Epoch: 6}
-	if err := cache.AddCommitteeShuffledList(item4); err != nil {
-		t.Fatal(err)
-	}
-	epochs, err = cache.Epochs()
-	if err != nil {
-		t.Fatal(err)
-	}
-	wanted = item2.Epoch + item3.Epoch + item4.Epoch
-	if sum(epochs) != wanted {
-		t.Errorf("Wanted: %v, got: %v", wanted, sum(epochs))
-	}
+	sort.Slice(k, func(i, j int) bool {
+		return k[i] < k[j]
+	})
+	wanted := end - int(maxCommitteesCacheSize)
+	s := bytesutil.ToBytes32([]byte(strconv.Itoa(wanted)))
+	assert.Equal(t, key(s), k[0], "incorrect key received for slot 190")
+
+	s = bytesutil.ToBytes32([]byte(strconv.Itoa(199)))
+	assert.Equal(t, key(s), k[len(k)-1], "incorrect key received for slot 199")
 }
 
-func TestCommitteeCache_EpochInCache(t *testing.T) {
-	cache := NewCommitteeCache()
-	if err := cache.AddCommitteeShuffledList(&Committee{Epoch: 1}); err != nil {
-		t.Fatal(err)
-	}
-	if err := cache.AddCommitteeShuffledList(&Committee{Epoch: 2}); err != nil {
-		t.Fatal(err)
-	}
-	if err := cache.AddCommitteeShuffledList(&Committee{Epoch: 99}); err != nil {
-		t.Fatal(err)
-	}
-	if err := cache.AddCommitteeShuffledList(&Committee{Epoch: 100}); err != nil {
-		t.Fatal(err)
-	}
-	inCache, err := cache.EpochInCache(1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if inCache {
-		t.Error("Epoch shouldn't be in cache")
-	}
-	inCache, err = cache.EpochInCache(100)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !inCache {
-		t.Error("Epoch should be in cache")
-	}
-}
-
-func TestCommitteeCache_ActiveIndices(t *testing.T) {
-	cache := NewCommitteeCache()
-
-	item := &Committee{Epoch: 1, Committee: []uint64{1, 2, 3, 4, 5, 6}}
-	indices, err := cache.ActiveIndices(1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if indices != nil {
-		t.Error("Expected committee count not to exist in empty cache")
-	}
-
-	if err := cache.AddCommitteeShuffledList(item); err != nil {
-		t.Fatal(err)
-	}
-
-	indices, err = cache.ActiveIndices(1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(indices, item.Committee) {
-		t.Error("Did not receive correct active indices from cache")
-	}
-}
-
-func sum(values []uint64) uint64 {
-	sum := uint64(0)
-	for _, v := range values {
-		sum = v + sum
-	}
-	return sum
+func TestCommitteeCacheOutOfRange(t *testing.T) {
+	cache := NewCommitteesCache()
+	seed := bytesutil.ToBytes32([]byte("foo"))
+	err := cache.CommitteeCache.Add(&Committees{
+		CommitteeCount:  1,
+		Seed:            seed,
+		ShuffledIndices: []uint64{0},
+		SortedIndices:   []uint64{},
+	})
+	require.NoError(t, err)
+
+	_, err = cache.Committee(0, seed, math.MaxUint64) // Overflow!
+	require.NotNil(t, err, "Did not fail as expected")
 }

beacon-chain/cache/committees.go

@@ -0,0 +1,15 @@
+package cache
+
+import "errors"
+
+// ErrNotCommittee will be returned when a cache object is not a pointer to
+// a Committee struct.
+var ErrNotCommittee = errors.New("object is not a committee struct")
+
+// Committees defines the shuffled committees seed.
+type Committees struct {
+	CommitteeCount  uint64
+	Seed            [32]byte
+	ShuffledIndices []uint64
+	SortedIndices   []uint64
+}

beacon-chain/cache/common.go

@@ -8,18 +8,22 @@ import (
 var (
 	// maxCacheSize is 4x of the epoch length for additional cache padding.
 	// Requests should be only accessing committees within defined epoch length.
-	maxCacheSize = int(4 * params.BeaconConfig().SlotsPerEpoch)
+	maxCacheSize = 4 * params.BeaconConfig().SlotsPerEpoch
 )
 
 // trim the FIFO queue to the maxSize.
-func trim(queue *cache.FIFO, maxSize int) {
-	for s := len(queue.ListKeys()); s > maxSize; s-- {
-		// #nosec G104 popProcessNoopFunc never returns an error
-		_, _ = queue.Pop(popProcessNoopFunc)
+func trim(queue *cache.FIFO, maxSize uint64) {
+	for s := uint64(len(queue.ListKeys())); s > maxSize; s-- {
+		_, err := queue.Pop(popProcessNoopFunc)
+		if err != nil {
+			// popProcessNoopFunc never returns an error, but we handle this anyway to make linter
+			// happy.
+			return
+		}
 	}
 }
 
 // popProcessNoopFunc is a no-op function that never returns an error.
-func popProcessNoopFunc(obj interface{}) error {
+func popProcessNoopFunc(_ interface{}) error {
 	return nil
 }

beacon-chain/cache/depositcache/BUILD.bazel

@@ -1,4 +1,5 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -9,10 +10,14 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache",
     visibility = ["//beacon-chain:__subpackages__"],
     deps = [
-        "//proto/eth/v1alpha1:go_default_library",
+        "//proto/beacon/db:go_default_library",
+        "//shared/bytesutil:go_default_library",
         "//shared/hashutil:go_default_library",
+        "//shared/params:go_default_library",
+        "//shared/trieutil:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
+        "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@io_opencensus_go//trace:go_default_library",
     ],
@@ -21,14 +26,19 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = [
-        "deposits_test.go",
+        "deposits_cache_test.go",
         "pending_deposits_test.go",
     ],
     embed = [":go_default_library"],
     deps = [
-        "//proto/eth/v1alpha1:go_default_library",
+        "//proto/beacon/db:go_default_library",
         "//shared/bytesutil:go_default_library",
+        "//shared/params:go_default_library",
+        "//shared/testutil/assert:go_default_library",
+        "//shared/testutil/require:go_default_library",
+        "//shared/trieutil:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
+        "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",
     ],
 )

beacon-chain/cache/depositcache/deposits_cache.go

@@ -1,3 +1,7 @@
+// Package depositcache is the source of validator deposits maintained
+// in-memory by the beacon node – deposits processed from the
+// eth1 powchain are then stored in this cache to be accessed by
+// any other service during a beacon node's runtime.
 package depositcache
 
 import (
@@ -10,7 +14,11 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	dbpb "github.com/prysmaticlabs/prysm/proto/beacon/db"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/trieutil"
 	log "github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
 )
@@ -24,48 +32,56 @@ var (
 
 // DepositFetcher defines a struct which can retrieve deposit information from a store.
 type DepositFetcher interface {
-	AllDeposits(ctx context.Context, beforeBlk *big.Int) []*ethpb.Deposit
+	AllDeposits(ctx context.Context, untilBlk *big.Int) []*ethpb.Deposit
 	DepositByPubkey(ctx context.Context, pubKey []byte) (*ethpb.Deposit, *big.Int)
 	DepositsNumberAndRootAtHeight(ctx context.Context, blockHeight *big.Int) (uint64, [32]byte)
+	FinalizedDeposits(ctx context.Context) *FinalizedDeposits
+	NonFinalizedDeposits(ctx context.Context, untilBlk *big.Int) []*ethpb.Deposit
+}
+
+// FinalizedDeposits stores the trie of deposits that have been included
+// in the beacon state up to the latest finalized checkpoint.
+type FinalizedDeposits struct {
+	Deposits        *trieutil.SparseMerkleTrie
+	MerkleTrieIndex int64
 }
 
 // DepositCache stores all in-memory deposit objects. This
 // stores all the deposit related data that is required by the beacon-node.
 type DepositCache struct {
 	// Beacon chain deposits in memory.
-	pendingDeposits       []*DepositContainer
-	deposits              []*DepositContainer
-	depositsLock          sync.RWMutex
-	chainStartDeposits    []*ethpb.Deposit
-	chainstartPubkeys     map[string]bool
-	chainstartPubkeysLock sync.RWMutex
+	pendingDeposits    []*dbpb.DepositContainer
+	deposits           []*dbpb.DepositContainer
+	finalizedDeposits  *FinalizedDeposits
+	depositsLock       sync.RWMutex
+	chainStartDeposits []*ethpb.Deposit
+	chainStartPubkeys  map[string]bool
 }
 
-// DepositContainer object for holding the deposit and a reference to the block in
-// which the deposit transaction was included in the proof of work chain.
-type DepositContainer struct {
-	Deposit     *ethpb.Deposit
-	Block       *big.Int
-	Index       int
-	depositRoot [32]byte
-}
-
-// NewDepositCache instantiates a new deposit cache
-func NewDepositCache() *DepositCache {
-	return &DepositCache{
-		pendingDeposits:    []*DepositContainer{},
-		deposits:           []*DepositContainer{},
-		chainstartPubkeys:  make(map[string]bool),
-		chainStartDeposits: make([]*ethpb.Deposit, 0),
+// New instantiates a new deposit cache
+func New() (*DepositCache, error) {
+	finalizedDepositsTrie, err := trieutil.NewTrie(params.BeaconConfig().DepositContractTreeDepth)
+	if err != nil {
+		return nil, err
 	}
+
+	// finalizedDeposits.MerkleTrieIndex is initialized to -1 because it represents the index of the last trie item.
+	// Inserting the first item into the trie will set the value of the index to 0.
+	return &DepositCache{
+		pendingDeposits:    []*dbpb.DepositContainer{},
+		deposits:           []*dbpb.DepositContainer{},
+		finalizedDeposits:  &FinalizedDeposits{Deposits: finalizedDepositsTrie, MerkleTrieIndex: -1},
+		chainStartPubkeys:  make(map[string]bool),
+		chainStartDeposits: make([]*ethpb.Deposit, 0),
+	}, nil
 }
 
 // InsertDeposit into the database. If deposit or block number are nil
 // then this method does nothing.
-func (dc *DepositCache) InsertDeposit(ctx context.Context, d *ethpb.Deposit, blockNum *big.Int, index int, depositRoot [32]byte) {
-	ctx, span := trace.StartSpan(ctx, "BeaconDB.InsertDeposit")
+func (dc *DepositCache) InsertDeposit(ctx context.Context, d *ethpb.Deposit, blockNum uint64, index int64, depositRoot [32]byte) {
+	ctx, span := trace.StartSpan(ctx, "DepositsCache.InsertDeposit")
 	defer span.End()
-	if d == nil || blockNum == nil {
+	if d == nil {
 		log.WithFields(log.Fields{
 			"block":        blockNum,
 			"deposit":      d,
@@ -76,46 +92,95 @@ func (dc *DepositCache) InsertDeposit(ctx context.Context, d *ethpb.Deposit, blo
 	}
 	dc.depositsLock.Lock()
 	defer dc.depositsLock.Unlock()
-	// keep the slice sorted on insertion in order to avoid costly sorting on retrival.
+	// Keep the slice sorted on insertion in order to avoid costly sorting on retrieval.
 	heightIdx := sort.Search(len(dc.deposits), func(i int) bool { return dc.deposits[i].Index >= index })
-	newDeposits := append([]*DepositContainer{{Deposit: d, Block: blockNum, depositRoot: depositRoot, Index: index}}, dc.deposits[heightIdx:]...)
+	newDeposits := append([]*dbpb.DepositContainer{{Deposit: d, Eth1BlockHeight: blockNum, DepositRoot: depositRoot[:], Index: index}}, dc.deposits[heightIdx:]...)
 	dc.deposits = append(dc.deposits[:heightIdx], newDeposits...)
 	historicalDepositsCount.Inc()
 }
 
+// InsertDepositContainers inserts a set of deposit containers into our deposit cache.
+func (dc *DepositCache) InsertDepositContainers(ctx context.Context, ctrs []*dbpb.DepositContainer) {
+	ctx, span := trace.StartSpan(ctx, "DepositsCache.InsertDepositContainers")
+	defer span.End()
+	dc.depositsLock.Lock()
+	defer dc.depositsLock.Unlock()
+
+	sort.SliceStable(ctrs, func(i int, j int) bool { return ctrs[i].Index < ctrs[j].Index })
+	dc.deposits = ctrs
+	historicalDepositsCount.Add(float64(len(ctrs)))
+}
+
+// InsertFinalizedDeposits inserts deposits up to eth1DepositIndex (inclusive) into the finalized deposits cache.
+func (dc *DepositCache) InsertFinalizedDeposits(ctx context.Context, eth1DepositIndex int64) {
+	ctx, span := trace.StartSpan(ctx, "DepositsCache.InsertFinalizedDeposits")
+	defer span.End()
+	dc.depositsLock.Lock()
+	defer dc.depositsLock.Unlock()
+
+	depositTrie := dc.finalizedDeposits.Deposits
+	insertIndex := int(dc.finalizedDeposits.MerkleTrieIndex + 1)
+	for _, d := range dc.deposits {
+		if d.Index <= dc.finalizedDeposits.MerkleTrieIndex {
+			continue
+		}
+		if d.Index > eth1DepositIndex {
+			break
+		}
+		depHash, err := d.Deposit.Data.HashTreeRoot()
+		if err != nil {
+			log.WithError(err).Error("Could not hash deposit data. Finalized deposit cache not updated.")
+			return
+		}
+		depositTrie.Insert(depHash[:], insertIndex)
+		insertIndex++
+	}
+
+	dc.finalizedDeposits = &FinalizedDeposits{
+		Deposits:        depositTrie,
+		MerkleTrieIndex: eth1DepositIndex,
+	}
+}
+
+// AllDepositContainers returns all historical deposit containers.
+func (dc *DepositCache) AllDepositContainers(ctx context.Context) []*dbpb.DepositContainer {
+	ctx, span := trace.StartSpan(ctx, "DepositsCache.AllDepositContainers")
+	defer span.End()
+	dc.depositsLock.RLock()
+	defer dc.depositsLock.RUnlock()
+
+	return dc.deposits
+}
+
 // MarkPubkeyForChainstart sets the pubkey deposit status to true.
 func (dc *DepositCache) MarkPubkeyForChainstart(ctx context.Context, pubkey string) {
-	ctx, span := trace.StartSpan(ctx, "BeaconDB.MarkPubkeyForChainstart")
+	ctx, span := trace.StartSpan(ctx, "DepositsCache.MarkPubkeyForChainstart")
 	defer span.End()
-	dc.chainstartPubkeysLock.Lock()
-	defer dc.chainstartPubkeysLock.Unlock()
-	dc.chainstartPubkeys[pubkey] = true
+	dc.chainStartPubkeys[pubkey] = true
 }
 
 // PubkeyInChainstart returns bool for whether the pubkey passed in has deposited.
 func (dc *DepositCache) PubkeyInChainstart(ctx context.Context, pubkey string) bool {
-	ctx, span := trace.StartSpan(ctx, "BeaconDB.PubkeyInChainstart")
+	ctx, span := trace.StartSpan(ctx, "DepositsCache.PubkeyInChainstart")
 	defer span.End()
-	dc.chainstartPubkeysLock.Lock()
-	defer dc.chainstartPubkeysLock.Unlock()
-	if dc.chainstartPubkeys != nil {
-		return dc.chainstartPubkeys[pubkey]
+	if dc.chainStartPubkeys != nil {
+		return dc.chainStartPubkeys[pubkey]
 	}
-	dc.chainstartPubkeys = make(map[string]bool)
+	dc.chainStartPubkeys = make(map[string]bool)
 	return false
 }
 
-// AllDeposits returns a list of deposits all historical deposits until the given block number
+// AllDeposits returns a list of historical deposits until the given block number
 // (inclusive). If no block is specified then this method returns all historical deposits.
-func (dc *DepositCache) AllDeposits(ctx context.Context, beforeBlk *big.Int) []*ethpb.Deposit {
-	ctx, span := trace.StartSpan(ctx, "BeaconDB.AllDeposits")
+func (dc *DepositCache) AllDeposits(ctx context.Context, untilBlk *big.Int) []*ethpb.Deposit {
+	ctx, span := trace.StartSpan(ctx, "DepositsCache.AllDeposits")
 	defer span.End()
 	dc.depositsLock.RLock()
 	defer dc.depositsLock.RUnlock()
 
 	var deposits []*ethpb.Deposit
 	for _, ctnr := range dc.deposits {
-		if beforeBlk == nil || beforeBlk.Cmp(ctnr.Block) > -1 {
+		if untilBlk == nil || untilBlk.Uint64() >= ctnr.Eth1BlockHeight {
 			deposits = append(deposits, ctnr.Deposit)
 		}
 	}
@@ -125,23 +190,23 @@ func (dc *DepositCache) AllDeposits(ctx context.Context, beforeBlk *big.Int) []*
 // DepositsNumberAndRootAtHeight returns number of deposits made prior to blockheight and the
 // root that corresponds to the latest deposit at that blockheight.
 func (dc *DepositCache) DepositsNumberAndRootAtHeight(ctx context.Context, blockHeight *big.Int) (uint64, [32]byte) {
-	ctx, span := trace.StartSpan(ctx, "Beacondb.DepositsNumberAndRootAtHeight")
+	ctx, span := trace.StartSpan(ctx, "DepositsCache.DepositsNumberAndRootAtHeight")
 	defer span.End()
 	dc.depositsLock.RLock()
 	defer dc.depositsLock.RUnlock()
-	heightIdx := sort.Search(len(dc.deposits), func(i int) bool { return dc.deposits[i].Block.Cmp(blockHeight) > 0 })
+	heightIdx := sort.Search(len(dc.deposits), func(i int) bool { return dc.deposits[i].Eth1BlockHeight > blockHeight.Uint64() })
 	// send the deposit root of the empty trie, if eth1follow distance is greater than the time of the earliest
 	// deposit.
 	if heightIdx == 0 {
 		return 0, [32]byte{}
 	}
-	return uint64(heightIdx), dc.deposits[heightIdx-1].depositRoot
+	return uint64(heightIdx), bytesutil.ToBytes32(dc.deposits[heightIdx-1].DepositRoot)
 }
 
 // DepositByPubkey looks through historical deposits and finds one which contains
 // a certain public key within its deposit data.
 func (dc *DepositCache) DepositByPubkey(ctx context.Context, pubKey []byte) (*ethpb.Deposit, *big.Int) {
-	ctx, span := trace.StartSpan(ctx, "BeaconDB.DepositByPubkey")
+	ctx, span := trace.StartSpan(ctx, "DepositsCache.DepositByPubkey")
 	defer span.End()
 	dc.depositsLock.RLock()
 	defer dc.depositsLock.RUnlock()
@@ -151,9 +216,70 @@ func (dc *DepositCache) DepositByPubkey(ctx context.Context, pubKey []byte) (*et
 	for _, ctnr := range dc.deposits {
 		if bytes.Equal(ctnr.Deposit.Data.PublicKey, pubKey) {
 			deposit = ctnr.Deposit
-			blockNum = ctnr.Block
+			blockNum = big.NewInt(int64(ctnr.Eth1BlockHeight))
 			break
 		}
 	}
 	return deposit, blockNum
 }
+
+// FinalizedDeposits returns the finalized deposits trie.
+func (dc *DepositCache) FinalizedDeposits(ctx context.Context) *FinalizedDeposits {
+	ctx, span := trace.StartSpan(ctx, "DepositsCache.FinalizedDeposits")
+	defer span.End()
+	dc.depositsLock.RLock()
+	defer dc.depositsLock.RUnlock()
+
+	return &FinalizedDeposits{
+		Deposits:        dc.finalizedDeposits.Deposits.Copy(),
+		MerkleTrieIndex: dc.finalizedDeposits.MerkleTrieIndex,
+	}
+}
+
+// NonFinalizedDeposits returns the list of non-finalized deposits until the given block number (inclusive).
+// If no block is specified then this method returns all non-finalized deposits.
+func (dc *DepositCache) NonFinalizedDeposits(ctx context.Context, untilBlk *big.Int) []*ethpb.Deposit {
+	ctx, span := trace.StartSpan(ctx, "DepositsCache.NonFinalizedDeposits")
+	defer span.End()
+	dc.depositsLock.RLock()
+	defer dc.depositsLock.RUnlock()
+
+	if dc.finalizedDeposits == nil {
+		return dc.AllDeposits(ctx, untilBlk)
+	}
+
+	lastFinalizedDepositIndex := dc.finalizedDeposits.MerkleTrieIndex
+	var deposits []*ethpb.Deposit
+	for _, d := range dc.deposits {
+		if (d.Index > lastFinalizedDepositIndex) && (untilBlk == nil || untilBlk.Uint64() >= d.Eth1BlockHeight) {
+			deposits = append(deposits, d.Deposit)
+		}
+	}
+
+	return deposits
+}
+
+// PruneProofs removes proofs from all deposits whose index is equal or less than untilDepositIndex.
+func (dc *DepositCache) PruneProofs(ctx context.Context, untilDepositIndex int64) error {
+	ctx, span := trace.StartSpan(ctx, "DepositsCache.PruneProofs")
+	defer span.End()
+	dc.depositsLock.Lock()
+	defer dc.depositsLock.Unlock()
+
+	if untilDepositIndex > int64(len(dc.deposits)) {
+		untilDepositIndex = int64(len(dc.deposits) - 1)
+	}
+
+	for i := untilDepositIndex; i >= 0; i-- {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		// Finding a nil proof means that all proofs up to this deposit have been already pruned.
+		if dc.deposits[i].Deposit.Proof == nil {
+			break
+		}
+		dc.deposits[i].Deposit.Proof = nil
+	}
+
+	return nil
+}

beacon-chain/cache/depositcache/deposits_cache_test.go

@@ -0,0 +1,709 @@
+package depositcache
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"math/big"
+	"testing"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	dbpb "github.com/prysmaticlabs/prysm/proto/beacon/db"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+	"github.com/prysmaticlabs/prysm/shared/trieutil"
+	logTest "github.com/sirupsen/logrus/hooks/test"
+)
+
+const nilDepositErr = "Ignoring nil deposit insertion"
+
+var _ DepositFetcher = (*DepositCache)(nil)
+
+func TestInsertDeposit_LogsOnNilDepositInsertion(t *testing.T) {
+	hook := logTest.NewGlobal()
+	dc, err := New()
+	require.NoError(t, err)
+
+	dc.InsertDeposit(context.Background(), nil, 1, 0, [32]byte{})
+
+	require.Equal(t, 0, len(dc.deposits), "Number of deposits changed")
+	assert.Equal(t, nilDepositErr, hook.LastEntry().Message)
+}
+
+func TestInsertDeposit_MaintainsSortedOrderByIndex(t *testing.T) {
+	dc, err := New()
+	require.NoError(t, err)
+
+	insertions := []struct {
+		blkNum  uint64
+		deposit *ethpb.Deposit
+		index   int64
+	}{
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{},
+			index:   0,
+		},
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{},
+			index:   3,
+		},
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{},
+			index:   1,
+		},
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{},
+			index:   4,
+		},
+	}
+
+	for _, ins := range insertions {
+		dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{})
+	}
+
+	expectedIndices := []int64{0, 1, 3, 4}
+	for i, ei := range expectedIndices {
+		assert.Equal(t, ei, dc.deposits[i].Index,
+			fmt.Sprintf("dc.deposits[%d].Index = %d, wanted %d", i, dc.deposits[i].Index, ei))
+	}
+}
+
+func TestAllDeposits_ReturnsAllDeposits(t *testing.T) {
+	dc, err := New()
+	require.NoError(t, err)
+
+	deposits := []*dbpb.DepositContainer{
+		{
+			Eth1BlockHeight: 10,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 10,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 10,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 11,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 11,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 12,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 12,
+			Deposit:         &ethpb.Deposit{},
+		},
+	}
+	dc.deposits = deposits
+
+	d := dc.AllDeposits(context.Background(), nil)
+	assert.Equal(t, len(deposits), len(d))
+}
+
+func TestAllDeposits_FiltersDepositUpToAndIncludingBlockNumber(t *testing.T) {
+	dc, err := New()
+	require.NoError(t, err)
+
+	deposits := []*dbpb.DepositContainer{
+		{
+			Eth1BlockHeight: 10,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 10,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 10,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 11,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 11,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 12,
+			Deposit:         &ethpb.Deposit{},
+		},
+		{
+			Eth1BlockHeight: 12,
+			Deposit:         &ethpb.Deposit{},
+		},
+	}
+	dc.deposits = deposits
+
+	d := dc.AllDeposits(context.Background(), big.NewInt(11))
+	assert.Equal(t, 5, len(d))
+}
+
+func TestDepositsNumberAndRootAtHeight_ReturnsAppropriateCountAndRoot(t *testing.T) {
+	dc, err := New()
+	require.NoError(t, err)
+
+	dc.deposits = []*dbpb.DepositContainer{
+		{
+			Eth1BlockHeight: 10,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             make([]byte, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+		},
+		{
+			Eth1BlockHeight: 10,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             make([]byte, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+		},
+		{
+			Eth1BlockHeight: 10,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             make([]byte, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+		},
+		{
+			Eth1BlockHeight: 10,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             make([]byte, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+		},
+		{
+			Eth1BlockHeight: 11,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             make([]byte, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			DepositRoot: bytesutil.PadTo([]byte("root"), 32),
+		},
+		{
+			Eth1BlockHeight: 12,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             make([]byte, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+		},
+		{
+			Eth1BlockHeight: 12,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             make([]byte, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+		},
+	}
+
+	n, root := dc.DepositsNumberAndRootAtHeight(context.Background(), big.NewInt(11))
+	assert.Equal(t, 5, int(n))
+	assert.Equal(t, bytesutil.ToBytes32([]byte("root")), root)
+}
+
+func TestDepositsNumberAndRootAtHeight_ReturnsEmptyTrieIfBlockHeightLessThanOldestDeposit(t *testing.T) {
+	dc, err := New()
+	require.NoError(t, err)
+
+	dc.deposits = []*dbpb.DepositContainer{
+		{
+			Eth1BlockHeight: 10,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             make([]byte, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			DepositRoot: bytesutil.PadTo([]byte("root"), 32),
+		},
+		{
+			Eth1BlockHeight: 11,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             make([]byte, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			DepositRoot: bytesutil.PadTo([]byte("root"), 32),
+		},
+	}
+
+	n, root := dc.DepositsNumberAndRootAtHeight(context.Background(), big.NewInt(2))
+	assert.Equal(t, 0, int(n))
+	assert.Equal(t, [32]byte{}, root)
+}
+
+func TestDepositByPubkey_ReturnsFirstMatchingDeposit(t *testing.T) {
+	dc, err := New()
+	require.NoError(t, err)
+
+	dc.deposits = []*dbpb.DepositContainer{
+		{
+			Eth1BlockHeight: 9,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte("pk0"), 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+		},
+		{
+			Eth1BlockHeight: 10,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte("pk1"), 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+		},
+		{
+			Eth1BlockHeight: 11,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte("pk1"), 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+		},
+		{
+			Eth1BlockHeight: 12,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte("pk2"), 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+		},
+	}
+
+	pk1 := bytesutil.PadTo([]byte("pk1"), 48)
+	dep, blkNum := dc.DepositByPubkey(context.Background(), pk1)
+
+	if dep == nil || !bytes.Equal(dep.Data.PublicKey, pk1) {
+		t.Error("Returned wrong deposit")
+	}
+	assert.Equal(t, 0, blkNum.Cmp(big.NewInt(10)),
+		fmt.Sprintf("Returned wrong block number %v", blkNum))
+}
+
+func TestFinalizedDeposits_DepositsCachedCorrectly(t *testing.T) {
+	dc, err := New()
+	require.NoError(t, err)
+
+	finalizedDeposits := []*dbpb.DepositContainer{
+		{
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{0}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 0,
+		},
+		{
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{1}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 1,
+		},
+		{
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{2}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 2,
+		},
+	}
+	dc.deposits = append(finalizedDeposits, &dbpb.DepositContainer{
+		Deposit: &ethpb.Deposit{
+			Data: &ethpb.Deposit_Data{
+				PublicKey:             bytesutil.PadTo([]byte{3}, 48),
+				WithdrawalCredentials: make([]byte, 32),
+				Signature:             make([]byte, 96),
+			},
+		},
+		Index: 3,
+	})
+
+	dc.InsertFinalizedDeposits(context.Background(), 2)
+
+	cachedDeposits := dc.FinalizedDeposits(context.Background())
+	require.NotNil(t, cachedDeposits, "Deposits not cached")
+	assert.Equal(t, int64(2), cachedDeposits.MerkleTrieIndex)
+
+	var deps [][]byte
+	for _, d := range finalizedDeposits {
+		hash, err := d.Deposit.Data.HashTreeRoot()
+		require.NoError(t, err, "Could not hash deposit data")
+		deps = append(deps, hash[:])
+	}
+	trie, err := trieutil.GenerateTrieFromItems(deps, params.BeaconConfig().DepositContractTreeDepth)
+	require.NoError(t, err, "Could not generate deposit trie")
+	assert.Equal(t, trie.HashTreeRoot(), cachedDeposits.Deposits.HashTreeRoot())
+}
+
+func TestFinalizedDeposits_UtilizesPreviouslyCachedDeposits(t *testing.T) {
+	dc, err := New()
+	require.NoError(t, err)
+
+	oldFinalizedDeposits := []*dbpb.DepositContainer{
+		{
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{0}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 0,
+		},
+		{
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{1}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 1,
+		},
+	}
+	newFinalizedDeposit := dbpb.DepositContainer{
+		Deposit: &ethpb.Deposit{
+			Data: &ethpb.Deposit_Data{
+				PublicKey:             bytesutil.PadTo([]byte{2}, 48),
+				WithdrawalCredentials: make([]byte, 32),
+				Signature:             make([]byte, 96),
+			},
+		},
+		Index: 2,
+	}
+	dc.deposits = oldFinalizedDeposits
+	dc.InsertFinalizedDeposits(context.Background(), 1)
+	// Artificially exclude old deposits so that they can only be retrieved from previously finalized deposits.
+	dc.deposits = []*dbpb.DepositContainer{&newFinalizedDeposit}
+
+	dc.InsertFinalizedDeposits(context.Background(), 2)
+
+	cachedDeposits := dc.FinalizedDeposits(context.Background())
+	require.NotNil(t, cachedDeposits, "Deposits not cached")
+	assert.Equal(t, int64(2), cachedDeposits.MerkleTrieIndex)
+
+	var deps [][]byte
+	for _, d := range append(oldFinalizedDeposits, &newFinalizedDeposit) {
+		hash, err := d.Deposit.Data.HashTreeRoot()
+		require.NoError(t, err, "Could not hash deposit data")
+		deps = append(deps, hash[:])
+	}
+	trie, err := trieutil.GenerateTrieFromItems(deps, params.BeaconConfig().DepositContractTreeDepth)
+	require.NoError(t, err, "Could not generate deposit trie")
+	assert.Equal(t, trie.HashTreeRoot(), cachedDeposits.Deposits.HashTreeRoot())
+}
+
+func TestFinalizedDeposits_InitializedCorrectly(t *testing.T) {
+	dc, err := New()
+	require.NoError(t, err)
+
+	finalizedDeposits := dc.finalizedDeposits
+	assert.NotNil(t, finalizedDeposits)
+	assert.NotNil(t, finalizedDeposits.Deposits)
+	assert.Equal(t, int64(-1), finalizedDeposits.MerkleTrieIndex)
+}
+
+func TestNonFinalizedDeposits_ReturnsAllNonFinalizedDeposits(t *testing.T) {
+	dc, err := New()
+	require.NoError(t, err)
+
+	finalizedDeposits := []*dbpb.DepositContainer{
+		{
+			Eth1BlockHeight: 10,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{0}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 0,
+		},
+		{
+			Eth1BlockHeight: 10,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{1}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 1,
+		},
+	}
+	dc.deposits = append(finalizedDeposits,
+		&dbpb.DepositContainer{
+			Eth1BlockHeight: 10,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{2}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 2,
+		},
+		&dbpb.DepositContainer{
+			Eth1BlockHeight: 11,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{3}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 3,
+		})
+	dc.InsertFinalizedDeposits(context.Background(), 1)
+
+	deps := dc.NonFinalizedDeposits(context.Background(), nil)
+	assert.Equal(t, 2, len(deps))
+}
+
+func TestNonFinalizedDeposits_ReturnsNonFinalizedDepositsUpToBlockNumber(t *testing.T) {
+	dc, err := New()
+	require.NoError(t, err)
+
+	finalizedDeposits := []*dbpb.DepositContainer{
+		{
+			Eth1BlockHeight: 10,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{0}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 0,
+		},
+		{
+			Eth1BlockHeight: 10,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{1}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 1,
+		},
+	}
+	dc.deposits = append(finalizedDeposits,
+		&dbpb.DepositContainer{
+			Eth1BlockHeight: 10,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{2}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 2,
+		},
+		&dbpb.DepositContainer{
+			Eth1BlockHeight: 11,
+			Deposit: &ethpb.Deposit{
+				Data: &ethpb.Deposit_Data{
+					PublicKey:             bytesutil.PadTo([]byte{3}, 48),
+					WithdrawalCredentials: make([]byte, 32),
+					Signature:             make([]byte, 96),
+				},
+			},
+			Index: 3,
+		})
+	dc.InsertFinalizedDeposits(context.Background(), 1)
+
+	deps := dc.NonFinalizedDeposits(context.Background(), big.NewInt(10))
+	assert.Equal(t, 1, len(deps))
+}
+
+func TestPruneProofs_Ok(t *testing.T) {
+	dc, err := New()
+	require.NoError(t, err)
+
+	deposits := []struct {
+		blkNum  uint64
+		deposit *ethpb.Deposit
+		index   int64
+	}{
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{Proof: makeDepositProof()},
+			index:   0,
+		},
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{Proof: makeDepositProof()},
+			index:   1,
+		},
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{Proof: makeDepositProof()},
+			index:   2,
+		},
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{Proof: makeDepositProof()},
+			index:   3,
+		},
+	}
+
+	for _, ins := range deposits {
+		dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{})
+	}
+
+	require.NoError(t, dc.PruneProofs(context.Background(), 1))
+
+	assert.DeepEqual(t, ([][]byte)(nil), dc.deposits[0].Deposit.Proof)
+	assert.DeepEqual(t, ([][]byte)(nil), dc.deposits[1].Deposit.Proof)
+	assert.NotNil(t, dc.deposits[2].Deposit.Proof)
+	assert.NotNil(t, dc.deposits[3].Deposit.Proof)
+}
+
+func TestPruneProofs_SomeAlreadyPruned(t *testing.T) {
+	dc, err := New()
+	require.NoError(t, err)
+
+	deposits := []struct {
+		blkNum  uint64
+		deposit *ethpb.Deposit
+		index   int64
+	}{
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{Proof: nil},
+			index:   0,
+		},
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{Proof: nil},
+			index:   1,
+		},
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{Proof: makeDepositProof()},
+			index:   2,
+		},
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{Proof: makeDepositProof()},
+			index:   3,
+		},
+	}
+
+	for _, ins := range deposits {
+		dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{})
+	}
+
+	require.NoError(t, dc.PruneProofs(context.Background(), 2))
+
+	assert.DeepEqual(t, ([][]byte)(nil), dc.deposits[2].Deposit.Proof)
+}
+
+func TestPruneProofs_PruneAllWhenDepositIndexTooBig(t *testing.T) {
+	dc, err := New()
+	require.NoError(t, err)
+
+	deposits := []struct {
+		blkNum  uint64
+		deposit *ethpb.Deposit
+		index   int64
+	}{
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{Proof: makeDepositProof()},
+			index:   0,
+		},
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{Proof: makeDepositProof()},
+			index:   1,
+		},
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{Proof: makeDepositProof()},
+			index:   2,
+		},
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{Proof: makeDepositProof()},
+			index:   3,
+		},
+	}
+
+	for _, ins := range deposits {
+		dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{})
+	}
+
+	require.NoError(t, dc.PruneProofs(context.Background(), 99))
+
+	assert.DeepEqual(t, ([][]byte)(nil), dc.deposits[0].Deposit.Proof)
+	assert.DeepEqual(t, ([][]byte)(nil), dc.deposits[1].Deposit.Proof)
+	assert.DeepEqual(t, ([][]byte)(nil), dc.deposits[2].Deposit.Proof)
+	assert.DeepEqual(t, ([][]byte)(nil), dc.deposits[3].Deposit.Proof)
+}
+
+func makeDepositProof() [][]byte {
+	proof := make([][]byte, int(params.BeaconConfig().DepositContractTreeDepth)+1)
+	for i := range proof {
+		proof[i] = make([]byte, 32)
+	}
+	return proof
+}

beacon-chain/cache/depositcache/deposits_test.go

@@ -1,288 +0,0 @@
-package depositcache
-
-import (
-	"bytes"
-	"context"
-	"math/big"
-	"testing"
-
-	ethpb "github.com/prysmaticlabs/prysm/proto/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	logTest "github.com/sirupsen/logrus/hooks/test"
-)
-
-const nilDepositErr = "Ignoring nil deposit insertion"
-
-var _ = DepositFetcher(&DepositCache{})
-
-func TestBeaconDB_InsertDeposit_LogsOnNilDepositInsertion(t *testing.T) {
-	hook := logTest.NewGlobal()
-	dc := DepositCache{}
-
-	dc.InsertDeposit(context.Background(), nil, big.NewInt(1), 0, [32]byte{})
-
-	if len(dc.deposits) != 0 {
-		t.Fatal("Number of deposits changed")
-	}
-	if hook.LastEntry().Message != nilDepositErr {
-		t.Errorf("Did not log correct message, wanted \"Ignoring nil deposit insertion\", got \"%s\"", hook.LastEntry().Message)
-	}
-}
-
-func TestBeaconDB_InsertDeposit_LogsOnNilBlockNumberInsertion(t *testing.T) {
-	hook := logTest.NewGlobal()
-	dc := DepositCache{}
-
-	dc.InsertDeposit(context.Background(), &ethpb.Deposit{}, nil, 0, [32]byte{})
-
-	if len(dc.deposits) != 0 {
-		t.Fatal("Number of deposits changed")
-	}
-	if hook.LastEntry().Message != nilDepositErr {
-		t.Errorf("Did not log correct message, wanted \"Ignoring nil deposit insertion\", got \"%s\"", hook.LastEntry().Message)
-	}
-}
-
-func TestBeaconDB_InsertDeposit_MaintainsSortedOrderByIndex(t *testing.T) {
-	dc := DepositCache{}
-
-	insertions := []struct {
-		blkNum  *big.Int
-		deposit *ethpb.Deposit
-		index   int
-	}{
-		{
-			blkNum:  big.NewInt(0),
-			deposit: &ethpb.Deposit{},
-			index:   0,
-		},
-		{
-			blkNum:  big.NewInt(0),
-			deposit: &ethpb.Deposit{},
-			index:   3,
-		},
-		{
-			blkNum:  big.NewInt(0),
-			deposit: &ethpb.Deposit{},
-			index:   1,
-		},
-		{
-			blkNum:  big.NewInt(0),
-			deposit: &ethpb.Deposit{},
-			index:   4,
-		},
-	}
-
-	for _, ins := range insertions {
-		dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{})
-	}
-
-	expectedIndices := []int{0, 1, 3, 4}
-	for i, ei := range expectedIndices {
-		if dc.deposits[i].Index != ei {
-			t.Errorf("dc.deposits[%d].Index = %d, wanted %d", i, dc.deposits[i].Index, ei)
-		}
-	}
-}
-
-func TestBeaconDB_AllDeposits_ReturnsAllDeposits(t *testing.T) {
-	dc := DepositCache{}
-
-	deposits := []*DepositContainer{
-		{
-			Block:   big.NewInt(10),
-			Deposit: &ethpb.Deposit{},
-		},
-		{
-			Block:   big.NewInt(10),
-			Deposit: &ethpb.Deposit{},
-		},
-		{
-			Block:   big.NewInt(10),
-			Deposit: &ethpb.Deposit{},
-		},
-		{
-			Block:   big.NewInt(11),
-			Deposit: &ethpb.Deposit{},
-		},
-		{
-			Block:   big.NewInt(11),
-			Deposit: &ethpb.Deposit{},
-		},
-		{
-			Block:   big.NewInt(12),
-			Deposit: &ethpb.Deposit{},
-		},
-		{
-			Block:   big.NewInt(12),
-			Deposit: &ethpb.Deposit{},
-		},
-	}
-	dc.deposits = deposits
-
-	d := dc.AllDeposits(context.Background(), nil)
-	if len(d) != len(deposits) {
-		t.Errorf("Return the wrong number of deposits (%d) wanted %d", len(d), len(deposits))
-	}
-}
-
-func TestBeaconDB_AllDeposits_FiltersDepositUpToAndIncludingBlockNumber(t *testing.T) {
-	dc := DepositCache{}
-
-	deposits := []*DepositContainer{
-		{
-			Block:   big.NewInt(10),
-			Deposit: &ethpb.Deposit{},
-		},
-		{
-			Block:   big.NewInt(10),
-			Deposit: &ethpb.Deposit{},
-		},
-		{
-			Block:   big.NewInt(10),
-			Deposit: &ethpb.Deposit{},
-		},
-		{
-			Block:   big.NewInt(11),
-			Deposit: &ethpb.Deposit{},
-		},
-		{
-			Block:   big.NewInt(11),
-			Deposit: &ethpb.Deposit{},
-		},
-		{
-			Block:   big.NewInt(12),
-			Deposit: &ethpb.Deposit{},
-		},
-		{
-			Block:   big.NewInt(12),
-			Deposit: &ethpb.Deposit{},
-		},
-	}
-	dc.deposits = deposits
-
-	d := dc.AllDeposits(context.Background(), big.NewInt(11))
-	expected := 5
-	if len(d) != expected {
-		t.Errorf("Return the wrong number of deposits (%d) wanted %d", len(d), expected)
-	}
-}
-
-func TestBeaconDB_DepositsNumberAndRootAtHeight_ReturnsAppropriateCountAndRoot(t *testing.T) {
-	dc := DepositCache{}
-
-	dc.deposits = []*DepositContainer{
-		{
-			Block:   big.NewInt(10),
-			Deposit: &ethpb.Deposit{},
-		},
-		{
-			Block:   big.NewInt(10),
-			Deposit: &ethpb.Deposit{},
-		},
-		{
-			Block:   big.NewInt(10),
-			Deposit: &ethpb.Deposit{},
-		},
-		{
-			Block:   big.NewInt(11),
-			Deposit: &ethpb.Deposit{},
-		},
-		{
-			Block:       big.NewInt(11),
-			Deposit:     &ethpb.Deposit{},
-			depositRoot: bytesutil.ToBytes32([]byte("root")),
-		},
-		{
-			Block:   big.NewInt(12),
-			Deposit: &ethpb.Deposit{},
-		},
-		{
-			Block:   big.NewInt(12),
-			Deposit: &ethpb.Deposit{},
-		},
-	}
-
-	n, root := dc.DepositsNumberAndRootAtHeight(context.Background(), big.NewInt(11))
-	if int(n) != 5 {
-		t.Errorf("Returned unexpected deposits number %d wanted %d", n, 5)
-	}
-
-	if root != bytesutil.ToBytes32([]byte("root")) {
-		t.Errorf("Returned unexpected root: %v", root)
-	}
-}
-
-func TestBeaconDB_DepositsNumberAndRootAtHeight_ReturnsEmptyTrieIfBlockHeightLessThanOldestDeposit(t *testing.T) {
-	dc := DepositCache{}
-
-	dc.deposits = []*DepositContainer{
-		{
-			Block:       big.NewInt(10),
-			Deposit:     &ethpb.Deposit{},
-			depositRoot: bytesutil.ToBytes32([]byte("root")),
-		},
-		{
-			Block:       big.NewInt(11),
-			Deposit:     &ethpb.Deposit{},
-			depositRoot: bytesutil.ToBytes32([]byte("root")),
-		},
-	}
-
-	n, root := dc.DepositsNumberAndRootAtHeight(context.Background(), big.NewInt(2))
-	if int(n) != 0 {
-		t.Errorf("Returned unexpected deposits number %d wanted %d", n, 0)
-	}
-
-	if root != [32]byte{} {
-		t.Errorf("Returned unexpected root: %v", root)
-	}
-}
-
-func TestBeaconDB_DepositByPubkey_ReturnsFirstMatchingDeposit(t *testing.T) {
-	dc := DepositCache{}
-
-	dc.deposits = []*DepositContainer{
-		{
-			Block: big.NewInt(9),
-			Deposit: &ethpb.Deposit{
-				Data: &ethpb.Deposit_Data{
-					PublicKey: []byte("pk0"),
-				},
-			},
-		},
-		{
-			Block: big.NewInt(10),
-			Deposit: &ethpb.Deposit{
-				Data: &ethpb.Deposit_Data{
-					PublicKey: []byte("pk1"),
-				},
-			},
-		},
-		{
-			Block: big.NewInt(11),
-			Deposit: &ethpb.Deposit{
-				Data: &ethpb.Deposit_Data{
-					PublicKey: []byte("pk1"),
-				},
-			},
-		},
-		{
-			Block: big.NewInt(12),
-			Deposit: &ethpb.Deposit{
-				Data: &ethpb.Deposit_Data{
-					PublicKey: []byte("pk2"),
-				},
-			},
-		},
-	}
-
-	dep, blkNum := dc.DepositByPubkey(context.Background(), []byte("pk1"))
-
-	if !bytes.Equal(dep.Data.PublicKey, []byte("pk1")) {
-		t.Error("Returned wrong deposit")
-	}
-	if blkNum.Cmp(big.NewInt(10)) != 0 {
-		t.Errorf("Returned wrong block number %v", blkNum)
-	}
-}