Reduce attester state copies (#6025)

* Test

* Fix

* Removed extra line

* Feature flag

* Gaz

.bazelrc

@@ -15,7 +15,7 @@ run --host_force_python=PY2
 # Network sandboxing only works on linux.
 --experimental_sandbox_default_allow_network=false
 
-# Use minimal protobufs at runtime
+# Use mainnet protobufs at runtime
 run --define ssz=mainnet
 test --define ssz=mainnet
 build --define ssz=mainnet
@@ -24,3 +24,202 @@ build --define ssz=mainnet
 build --incompatible_strict_action_env
 test --incompatible_strict_action_env
 run --incompatible_strict_action_env
+
+# Disable kafka by default, it takes a long time to build...
+build --define kafka_enabled=false
+test --define kafka_enabled=false
+run --define kafka_enabled=false
+
+# Release flags
+build:release --workspace_status_command=./scripts/workspace_status.sh
+build:release --stamp
+build:release --compilation_mode=opt
+
+# LLVM compiler for building C/C++ dependencies.
+build:llvm --crosstool_top=@llvm_toolchain//:toolchain
+build:llvm --define compiler=llvm
+build:llvm --copt -fno-sanitize=vptr,function
+build:llvm --linkopt -fno-sanitize=vptr,function
+
+build:asan --copt -fsanitize=address,undefined
+build:asan --copt -fno-omit-frame-pointer
+build:asan --linkopt -fsanitize=address,undefined
+build:asan --copt -fno-sanitize=vptr,function
+build:asan --linkopt -fno-sanitize=vptr,function
+build:asan --copt -DADDRESS_SANITIZER=1
+build:asan --copt -D__SANITIZE_ADDRESS__
+build:asan --linkopt -ldl
+
+build:llvm-asan --config=llvm
+build:llvm-asan --config=asan
+build:llvm-asan --linkopt -fuse-ld=ld.lld
+
+build:fuzz --define=gotags=libfuzzer
+build:fuzz --config=llvm-asan
+build:fuzz --copt=-fsanitize=fuzzer-no-link
+build:fuzz --linkopt=-fsanitize=fuzzer
+build:fuzz --copt=-fno-omit-frame-pointer
+build:fuzz --define=FUZZING_ENGINE=libfuzzer
+build:fuzz --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+build:fuzz --linkopt -Wl,--no-as-needed
+build:fuzz --define=gc_goopts=-d=libfuzzer
+build:fuzz --run_under=//tools:fuzz_wrapper
+build:fuzz --compilation_mode=opt
+
+test:fuzz --local_test_jobs="HOST_CPUS*.5"
+
+test:fuzzit --config=fuzz
+test:fuzzit --test_env=FUZZIT_API_KEY
+test:fuzzit --test_env=PRYSM_BUILD_IMAGE=gcr.io/prysmaticlabs/prysm-fuzzit:v0.11.0
+test:fuzzit --test_timeout=1200
+test:fuzzit --run_under=//tools:fuzzit_wrapper
+
+# Build binary with cgo symbolizer for debugging / profiling.
+build:cgo_symbolizer --config=llvm
+build:cgo_symbolizer --copt=-g
+build:cgo_symbolizer --define=USE_CGO_SYMBOLIZER=true
+build:cgo_symbolizer -c dbg
+
+# multi-arch cross-compiling toolchain configs:
+-----------------------------------------------
+build:cross --crosstool_top=@prysm_toolchains//:multiarch_toolchain
+build:cross --host_platform=@io_bazel_rules_go//go/toolchain:linux_amd64
+build:cross --host_crosstool_top=@prysm_toolchains//:hostonly_toolchain
+
+# linux_amd64 config for cross compiler toolchain, not strictly necessary since host/exec env is amd64
+build:linux_amd64 --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64_cgo
+
+# osx_amd64 config for cross compiler toolchain
+build:osx_amd64 --config=cross
+build:osx_amd64 --platforms=@io_bazel_rules_go//go/toolchain:darwin_amd64_cgo
+build:osx_amd64 --compiler=osxcross
+
+# windows
+build:windows_amd64 --config=cross
+build:windows_amd64  --platforms=@io_bazel_rules_go//go/toolchain:windows_amd64_cgo
+build:windows_amd64 --compiler=mingw-w64
+
+# linux_arm64 conifg for cross compiler toolchain
+build:linux_arm64 --config=cross
+build:linux_arm64 --platforms=@io_bazel_rules_go//go/toolchain:linux_arm64_cgo
+build:linux_arm64 --copt=-funsafe-math-optimizations
+build:linux_arm64 --copt=-ftree-vectorize
+build:linux_arm64 --copt=-fomit-frame-pointer
+build:linux_arm64 --cpu=aarch64
+build:linux_arm64 --compiler=clang
+build:linux_arm64 --copt=-march=armv8-a
+
+
+# toolchain build debug configs
+#------------------------------
+build:debug --sandbox_debug
+build:debug --toolchain_resolution_debug
+build:debug --verbose_failures
+build:debug -s
+
+# windows debug
+build:windows_amd64_debug --config=windows_amd64
+build:windows_amd64_debug --config=debug
+
+# osx_amd64 debug config
+build:osx_amd64_debug --config=debug
+build:osx_amd64_debug --config=osx_amd64
+
+# linux_arm64_debug
+build:linux_arm64_debug --config=linux_arm64
+build:linux_arm64_debug --config=debug
+
+# linux_amd64_debug
+build:linux_amd64_debug --config=linux_amd64
+build:linux_amd64_debug --config=debug
+
+
+# Docker Sandbox Configs
+#-----------------------
+# Note all docker sandbox configs must run from a linux x86_64 host
+# build:docker-sandbox --experimental_docker_image=gcr.io/prysmaticlabs/rbe-worker:latest
+build:docker-sandbox --spawn_strategy=docker --strategy=Javac=docker --genrule_strategy=docker
+build:docker-sandbox --define=EXECUTOR=remote
+build:docker-sandbox --experimental_docker_verbose
+build:docker-sandbox --experimental_enable_docker_sandbox
+build:docker-sandbox --crosstool_top=@rbe_ubuntu_clang//cc:toolchain 
+build:docker-sandbox --host_javabase=@rbe_ubuntu_clang//java:jdk 
+build:docker-sandbox --javabase=@rbe_ubuntu_clang//java:jdk 
+build:docker-sandbox --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 
+build:docker-sandbox --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 
+build:docker-sandbox --extra_execution_platforms=@rbe_ubuntu_clang//config:platform 
+build:docker-sandbox --host_platform=@rbe_ubuntu_clang//config:platform 
+build:docker-sandbox --platforms=@rbe_ubuntu_clang//config:platform
+build:docker-sandbox --extra_toolchains=@prysm_toolchains//:cc-toolchain-multiarch
+
+# windows_amd64 docker sandbox build config
+build:windows_amd64_docker --config=docker-sandbox --config=windows_amd64
+build:windows_amd64_docker_debug --config=windows_amd64_docker --config=debug
+
+# osx_amd64 docker sandbox build config
+build:osx_amd64_docker --config=docker-sandbox --config=osx_amd64
+build:osx_amd64_docker_debug --config=osx_amd64_docker --config=debug
+
+# linux_arm64 docker sandbox build config
+build:linux_arm64_docker --config=docker-sandbox --config=linux_arm64
+build:linux_arm64_docker_debug --config=linux_arm64_docker --config=debug
+
+# linux_amd64 docker sandbox build config
+build:linux_amd64_docker --config=docker-sandbox --config=linux_amd64
+build:linux_amd64_docker_debug --config=linux_amd64_docker --config=debug
+
+
+# Remote Build Execution
+#-----------------------
+# Originally from https://github.com/bazelbuild/bazel-toolchains/blob/master/bazelrc/bazel-2.0.0.bazelrc
+#
+# Depending on how many machines are in the remote execution instance, setting
+# this higher can make builds faster by allowing more jobs to run in parallel.
+# Setting it too high can result in jobs that timeout, however, while waiting
+# for a remote machine to execute them.
+build:remote --jobs=50
+
+# Set several flags related to specifying the platform, toolchain and java
+# properties.
+# These flags should only be used as is for the rbe-ubuntu16-04 container
+# and need to be adapted to work with other toolchain containers.
+build:remote --host_javabase=@rbe_ubuntu_clang//java:jdk
+build:remote --javabase=@rbe_ubuntu_clang//java:jdk
+build:remote --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
+build:remote --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
+build:remote --crosstool_top=@rbe_ubuntu_clang//cc:toolchain
+build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
+# Platform flags:
+# The toolchain container used for execution is defined in the target indicated
+# by "extra_execution_platforms", "host_platform" and "platforms".
+# More about platforms: https://docs.bazel.build/versions/master/platforms.html
+build:remote --extra_toolchains=@rbe_ubuntu_clang//config:cc-toolchain
+build:remote --extra_execution_platforms=@rbe_ubuntu_clang//config:platform
+build:remote --host_platform=@rbe_ubuntu_clang//config:platform
+build:remote --platforms=@rbe_ubuntu_clang//config:platform
+
+# Starting with Bazel 0.27.0 strategies do not need to be explicitly
+# defined. See https://github.com/bazelbuild/bazel/issues/7480
+build:remote --define=EXECUTOR=remote
+
+# Enable remote execution so actions are performed on the remote systems.
+# build:remote --remote_executor=grpcs://remotebuildexecution.googleapis.com
+
+# Enforce stricter environment rules, which eliminates some non-hermetic
+# behavior and therefore improves both the remote cache hit rate and the
+# correctness and repeatability of the build.
+build:remote --incompatible_strict_action_env=true
+
+# Set a higher timeout value, just in case.
+build:remote --remote_timeout=3600
+
+# Enable authentication. This will pick up application default credentials by
+# default. You can use --google_credentials=some_file.json to use a service
+# account credential instead.
+# build:remote --google_default_credentials=true
+
+# Enable build without the bytes
+# See: https://github.com/bazelbuild/bazel/issues/6862
+build:remote --experimental_remote_download_outputs=toplevel --experimental_inmemory_jdeps_files --experimental_inmemory_dotd_files
+
+build:remote --remote_local_fallback

.bazelversion

@@ -0,0 +1 @@
+3.0.0

.buildkite-bazelrc

@@ -20,15 +20,16 @@ build:remote-cache --strategy=Genrule=standalone
 build:remote-cache --disk_cache=
 build:remote-cache --host_platform_remote_properties_override='properties:{name:\"cache-silo-key\" value:\"prysm\"}' 
 build:remote-cache --remote_instance_name=projects/prysmaticlabs/instances/default_instance
-
-build:remote-cache --experimental_remote_download_outputs=minimal
-build:remote-cache --experimental_inmemory_jdeps_files
-build:remote-cache --experimental_inmemory_dotd_files
+build:remote-cache --remote_download_minimal
 
 # Import workspace options.
 import %workspace%/.bazelrc
 
 startup --host_jvm_args=-Xmx1000m --host_jvm_args=-Xms1000m
+query --repository_cache=/tmp/repositorycache
+query --experimental_repository_cache_hardlinks
+build --repository_cache=/tmp/repositorycache
+build --experimental_repository_cache_hardlinks
 build --experimental_strict_action_env
 build --disk_cache=/tmp/bazelbuilds
 build --experimental_multi_threaded_digest
@@ -36,12 +37,23 @@ build --sandbox_tmpfs_path=/tmp
 build --verbose_failures
 build --announce_rc
 build --show_progress_rate_limit=5
-build --curses=yes --color=yes
+build --curses=yes --color=no
 build --keep_going
 build --test_output=errors
 build --flaky_test_attempts=5
-build --jobs=50
-build --stamp
-test --local_test_jobs=2
 # Disabled race detection due to unstable test results under constrained environment build kite
 # build --features=race
+
+# Enable kafka for CI tests only.
+test --define kafka_enabled=true
+
+build --bes_backend=grpcs://builds.prylabs.net:1985
+build --bes_results_url=https://builds.prylabs.net/invocation/
+
+# Disable flaky test detection for fuzzing.
+test:fuzz --flaky_test_attempts=1
+
+# Expose test environment variables in CI
+test:fuzzit --test_env=GITHUB_REF
+test:fuzzit --test_env=GITHUB_SHA
+test:fuzzit --test_env=DOCKER_HOST

.github/CODEOWNERS

@@ -0,0 +1,5 @@
+# Automatically require code review from core-team.
+* @prysmaticlabs/core-team
+
+# Starlark code owners
+*.bzl @prestonvanloon

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,59 @@
+---
+name: "\U0001F41EBug report"
+about: Report a bug or problem with running Prysm
+---
+<!--💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎
+
+Hellooo! 😄 
+
+To help us tend to your issue faster, please search our currently open issues before submitting a new one.
+Existing issues often contain information about workarounds, resolution, or progress updates.
+
+💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎-->
+
+# 🐞 Bug Report
+
+### Description
+
+<!-- ✍️--> A clear and concise description of the problem...
+
+### Has this worked before in a previous version?
+
+<!-- Did this behavior use to work in the previous version? -->
+<!-- ✍️--> Yes, the previous version in which this bug was not present was: ....
+
+## 🔬 Minimal Reproduction
+
+<!--
+Please let us know how we can reproduce this issue. Include the exact method you used to run Prysm along with any flags used in your beacon chain and/or validator. Make sure you don't upload any confidential files or private keys.
+-->
+
+## 🔥 Error
+
+<pre><code>
+<!-- If the issue is accompanied by an error, please share the error logs with us below. If you have a lot of logs, place make a paste bin with your logs and share the link with us here: -->
+<!-- ✍️-->
+
+</code></pre>
+
+
+## 🌍  Your Environment
+
+**Operating System:**
+
+<pre>
+  <code>
+
+  </code>
+</pre>
+
+**What version of Prysm are you running? (Which release)**
+
+<pre>
+  <code>
+
+  </code>
+</pre>
+
+**Anything else relevant (validator index / public key)?**
+

.github/ISSUE_TEMPLATE/discussion.md

@@ -0,0 +1,21 @@
+---
+name: "\U0001F48EGeneral issue / discussion"
+about: Any other type of general issue or discussion
+
+---
+<!--💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎
+
+Hellooo! 😄 
+
+To help us tend to your issue faster, please search our currently open issues before submitting a new one.
+Existing issues often contain information about workarounds, resolution, or progress updates.
+
+💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎-->
+
+# 💎 Issue
+
+### Background
+
+<!-- ✍️--> Context and background information on the discussion...
+
+### Description

.github/ISSUE_TEMPLATE/feature_flag.md

@@ -0,0 +1,38 @@
+---
+name: "\U0001F984Feature Flag Tracking"
+about: Track a new feature, in development, in Prysm. This issue template should only be used by developers or contributors!
+labels: Tracking
+---
+<!--💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎
+
+Hellooo! 😄 
+
+Thanks for taking the time to file a tracking issue for your new feature. These issues really help
+us track progress of features as they work their way through development. Be sure to review 
+shared/featureconfig/README.md for the latest documentation around feature flags.
+
+💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎-->
+
+# 🦄 Feature Tracking
+
+**Current status** (opt-in or opt-out): 
+
+**Opt-in feature flag name**:
+
+**Opt-out feature flag name**: 
+
+### Feature Description
+
+<!-- ✍️--> A clear and concise description of the new feature. Provide links to your technical
+design document or other supporting information.
+
+### Tasks
+
+- [ ] Feature flag created as opt-in
+- [ ] Feature implemented / code complete
+- [ ] Feature tested in production for adequate amount of time
+- [ ] Feature flag is inverted to be opt-out and the opt-in flag is deprecated
+- [ ] Feature has made it to a tagged production release as an opt-out flag
+- [ ] Opt-out feature flag is deprecated and old code paths are cleaned up
+
+This issue should be closed after all of the above tasks are complete.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,27 @@
+---
+name: "\U0001F680Feature request"
+about: Suggest a feature for Prysm
+
+---
+<!--💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎
+
+Hellooo! 😄 
+
+To help us tend to your issue faster, please search our currently open issues before submitting a new one.
+Existing issues often contain information about workarounds, resolution, or progress updates.
+
+💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎-->
+
+# 🚀 Feature Request
+
+### Description
+
+<!-- ✍️--> A clear and concise description of the problem or missing capability...
+
+### Describe the solution you'd like
+
+<!-- ✍️--> If you have a solution in mind, please describe it.
+
+### Describe alternatives you've considered
+
+<!-- ✍️--> Have you considered any alternative solutions or workarounds?

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,30 @@
+<!-- Thanks for sending a PR! Before submitting:
+
+1. If this is your first PR, please read CONTRIBUTING.md and sign the CLA
+   first. We cannot review code without a signed CLA.
+2. Please file an issue *first*. All features and most bug fixes should have
+   an associated issue with a design discussed and decided upon. Small bug
+   fixes and documentation improvements don't need issues.
+3. New features and bug fixes must have tests. Documentation may need to
+   be updated. If you're unsure what to update, send the PR, and we'll discuss
+   in review.
+4. Note that PRs updating dependencies and new Go versions are not accepted.
+   Please file an issue instead.
+-->
+
+**What type of PR is this?**
+
+> Uncomment one line below and remove others.
+>
+> Bug fix
+> Feature
+> Documentation
+> Other
+
+**What does this PR do? Why is it needed?**
+
+**Which issues(s) does this PR fix?**
+
+Fixes #
+
+**Other notes for review**

.github/no-response.yml

@@ -0,0 +1,13 @@
+# Configuration for probot-no-response - https://github.com/probot/no-response
+
+# Number of days of inactivity before an Issue is closed for lack of response
+daysUntilClose: 14
+# Label requiring a response
+responseRequiredLabel: Need-Info
+# Comment to post when closing an Issue for lack of response. Set to `false` to disable
+closeComment: >
+  This issue has been automatically closed because there has been no response
+  to our request for more information from the original author. With only the
+  information that is currently in the issue, we don't have enough information
+  to take action. Please reach out if you have or find the answers we need so
+  that we can investigate further.

.github/workflows/dappnode-release-trigger.yml

@@ -0,0 +1,41 @@
+name: Update DAppNodePackages
+
+on:
+  push:
+    tags:
+    - '*'
+
+jobs:
+  dappnode-update-beacon-chain:
+    name: Trigger a beacon-chain release
+    runs-on: ubuntu-latest
+    steps:
+    - name: Get latest tag
+      id: get_tag
+      run: echo ::set-output name=TAG::${GITHUB_REF/refs\/tags\//}
+    - name: Send dispatch event to DAppNodePackage-prysm-beacon-chain
+      env: 
+        DISPATCH_REPO: dappnode/DAppNodePackage-prysm-beacon-chain
+      run: |
+        curl -v -X POST -u "${{ secrets.PAT_GITHUB }}" \
+        -H "Accept: application/vnd.github.everest-preview+json" \
+        -H "Content-Type: application/json" \
+        --data '{"event_type":"new_release", "client_payload": { "tag":"${{ steps.get_tag.outputs.TAG }}"}}' \
+        https://api.github.com/repos/$DISPATCH_REPO/dispatches
+        
+  dappnode-update-validator:
+    name: Trigger a validator release
+    runs-on: ubuntu-latest
+    steps:
+    - name: Get latest tag
+      id: get_tag
+      run: echo ::set-output name=TAG::${GITHUB_REF/refs\/tags\//}
+    - name: Send dispatch event to DAppNodePackage validator repository
+      env: 
+        DISPATCH_REPO: dappnode/DAppNodePackage-prysm-validator
+      run: |
+        curl -v -X POST -u "${{ secrets.PAT_GITHUB }}" \
+        -H "Accept: application/vnd.github.everest-preview+json" \
+        -H "Content-Type: application/json" \
+        --data '{"event_type":"new_release", "client_payload": { "tag":"${{ steps.get_tag.outputs.TAG }}"}}' \
+        https://api.github.com/repos/$DISPATCH_REPO/dispatches

.gitignore

@@ -4,6 +4,9 @@ bazel-*
 .DS_Store
 .swp
 
+# Ignore VI/Vim swapfiles
+.*.sw?
+
 # IntelliJ
 .idea
 .ijwb
@@ -14,6 +17,7 @@ bazel-*
 # Coverage outputs
 coverage.txt
 profile.out
+profile.grind
 
 # Nodejs
 node_modules
@@ -26,3 +30,6 @@ password.txt
 # go dependancy
 /go.mod
 /go.sum
+
+# Dist files
+dist

.well-known/security.txt

@@ -4,23 +4,23 @@ Hash: SHA512
 Contact: mailto:security@prysmaticlabs.com
 Encryption: openpgp4fpr:0AE0051D647BA3C1A917AF4072E33E4DF1A5036E
 Encryption: openpgp4fpr:341396BAFACC28C5082327F889725027FC8EC0D4
-Encryption: openpgp4fpr:8B7814F1B221A8E8AA465FC7BDBF744ADE1A0033
+Encryption: openpgp4fpr:FEE44615A19049DF0CA0C2735E2B7E5734DFADCB
 Preferred-Languages: en
 Canonical: https://github.com/prysmaticlabs/prysm/tree/master/.well-known/security.txt
 
 -----BEGIN PGP SIGNATURE-----
 
-iQIzBAEBCgAdFiEECuAFHWR7o8GpF69AcuM+TfGlA24FAlzi0WgACgkQcuM+TfGl
-A241pw/+Ks3Hxx8eGbjRIeuncuK811FkCiofNJS+MY2p4W2/tIrk48DtLRx8/k5L
-Dh1QyypZsqUgofrK7PbGVdEin6oEb2jYbTWUarAVTbhlsUdM4YcxwpgmGVslW7+C
-Hm8wMasQZhCkFfakzhfKX5hIQoFaFI/OvtVKIQsodP8dAieCDaGmtfq1Bs1LgFqi
-KrpeEdC2XbBQs33ADheC5SdGT1mnatP3VX8cOhLsfoPksYgTSpwK0clkoWs1eZOQ
-l1ImfW/FJCpSndBWgBR503ZgaU3Ic+5qxmAIuUP4chl0DFRMlPFEM5OWC6JkkCOd
-5kKrXGRmrhgtQg+pA3zqJnFItRj7gxPBA/ypxCkKPrLEkRvbdpdZEl5vAlYkeBL6
-iKSLHnMswGKldiYxy7ofam5bM3myhYYNFb25boV5pRptrnoUmWOACHioBGQHwWNt
-B0XktD0j7+pCCiJyyYxmOnElsk/Y/u4Tv5pYWvfFuxTF2XOg+P/EH64AIFLWgB1U
-VnITxhakxqejCBxZkuVCFNSzt+TXG0NS9EIj/UOYBY+wxrBZ62ITjdA16RS/3n3z
-DuIDtxOOwUumbOO32+a5zIb+ARmnocYJviI7FuENb01/U6qb+nm9hQI6oIpSCNsv
-Pb4O/ZlOx70U/7mt4Xn/dTKH9bnKOOVhOw00KJWFfAce73AVnLA=
-=Uhqg
+iQIzBAEBCgAdFiEECuAFHWR7o8GpF69AcuM+TfGlA24FAl6HrcwACgkQcuM+TfGl
+A26voQ/8DFB5wUHP0uyY8k7FGbxhLzSeImxomnUHJaUGfdczbCdYPMEHc9zI1iZP
+6LRiy9wS6qhqj/GSKVwvDPr+ZymXuV3L22GOP2lRhl7Z9Mm21ZJNOcoQBFOZnyHu
+DAy9HeTmeuJxYkf8weqZYXyzEoKJBDmfuWmEFjrtMcFXUfT3aJn1E2A/AQdcVQIC
+9L+iGWwFwjsPhcfaMuwcB7QMheDO6KSB7XPPCbrZ036Np8UTZ4qbZ5y73tlfkcOc
+tYTrMSPtS4eNutiDOP5Np36cLzRtNpm/BziAK+7ZKiYY0HI5h9IkCTLO4x2UmAMX
+sPoeaAB5z2QLIwmU9J2NhJrwiNMGTpJ+0bowy8U4cgzAX20CXVjRqGhy+cir8Ewg
+DjEGjWINUw6W0yzJp0mKSKzuOhdTTmzIYBeMBsyce+pgN1KGFCxeIwxGxyJzADdw
+mYQdljRXn4yEYP/KEpu/F2o8L4ptRO2jZWKvTvdzSSGGSyKyF4HsIRJ7m98DaB6S
+0oGq1KpbKKTbQi5g8UShGV2gPeMCs5ZIIqK2b/cRzUet18aUuofLmR4lkKZa9yEG
+rbzuJq/gB2vgQwExUEgVQ3/DfVc+y80e3YZ5s+rzV0vbLxl4Gh4yExpLo7hRf9iY
+EFvMzH+BEEb5VfCwByZyV1BmesZVIosr7K6UmVtPe0bZGvv3uIg=
+=5qpD
 -----END PGP SIGNATURE-----

BUILD.bazel

@@ -4,7 +4,7 @@ load("@com_github_atlassian_bazel_tools//goimports:def.bzl", "goimports")
 load("@io_kubernetes_build//defs:run_in_workspace.bzl", "workspace_binary")
 load("@io_bazel_rules_go//go:def.bzl", "nogo")
 load("@graknlabs_bazel_distribution//common:rules.bzl", "assemble_targz", "assemble_versioned")
-load("//tools:binary_targets.bzl", "binary_targets", "determine_targets")
+load("@bazel_skylib//rules:common_settings.bzl", "string_setting")
 
 prefix = "github.com/prysmaticlabs/prysm"
 
@@ -13,6 +13,7 @@ exports_files([
 ])
 
 # gazelle:prefix github.com/prysmaticlabs/prysm
+# gazelle:map_kind go_library go_library @prysm//tools/go:def.bzl
 gazelle(
     name = "gazelle",
     prefix = prefix,
@@ -104,42 +105,29 @@ nogo(
         "@org_golang_x_tools//go/analysis/passes/assign:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/inspect:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/asmdecl:go_tool_library",
+        "//tools/analyzers/maligned:go_tool_library",
+        "//tools/analyzers/roughtime:go_tool_library",
+        "//tools/analyzers/errcheck:go_tool_library",
+        "//tools/analyzers/featureconfig:go_tool_library",
     ],
 )
 
-assemble_versioned(
-    name = "assemble-versioned-all",
-    tags = ["manual"],
-    targets = [
-        ":assemble-{}-{}-targz".format(
-            pair[0],
-            pair[1],
-        )
-        for pair in binary_targets
-    ],
-    version_file = "//:VERSION",
-)
-
 common_files = {
     "//:LICENSE.md": "LICENSE.md",
     "//:README.md": "README.md",
 }
 
-[assemble_targz(
-    name = "assemble-{}-{}-targz".format(
-        pair[0],
-        pair[1],
-    ),
-    additional_files = determine_targets(pair, common_files),
-    output_filename = "prysm-{}-{}".format(
-        pair[0],
-        pair[1],
-    ),
-    tags = ["manual"],
-) for pair in binary_targets]
-
 toolchain(
     name = "built_cmake_toolchain",
     toolchain = "@rules_foreign_cc//tools/build_defs/native_tools:built_cmake",
     toolchain_type = "@rules_foreign_cc//tools/build_defs:cmake_toolchain",
 )
+
+string_setting(
+    name = "gotags",
+    build_setting_default = "",
+    values = [
+        "",
+        "libfuzzer",
+    ],
+)

INTEROP.md

@@ -57,7 +57,7 @@ the system with 64 validators and the genesis time set to the current unix times
 Wait a bit until your beacon chain starts, and in the other window:
 
 ```
-bazel run //validator -- --interop-num-validators 64
+bazel run //validator -- --keymanager=interop --keymanageropts='{"keys":64}'
 ```
 
 This will launch and kickstart the system with your 64 validators performing their duties accordingly.
@@ -78,13 +78,7 @@ Assuming you generated a `genesis.ssz` file with 64 validators, open up two term
 Wait a bit until your beacon chain starts, and in the other window:
 
 ```
-bazel run //validator -- --interop-num-validators 64
+bazel run //validator -- --keymanager=interop --keymanageropts='{"keys":64}'
 ```
 
 This will launch and kickstart the system with your 64 validators performing their duties accordingly.
-
-
-
-
-
-

PULL_REQUEST_TEMPLATE.md

@@ -1,13 +0,0 @@
-TODO(you): choose "part of" or "resolves" and the associated github issue #.
-
-[Part of|Resolves] #531 
-
----
-
-# Description
-
-**Write why you are making the changes in this pull request**
-
-**Write a summary of the changes you are making**
-
-**Link anything that would be helpful or relevant to the reviewers**

README.md

@@ -1,236 +1,21 @@
 # Prysm: An Ethereum 2.0 Client Written in Go
 
 [![Build status](https://badge.buildkite.com/b555891daf3614bae4284dcf365b2340cefc0089839526f096.svg?branch=master)](https://buildkite.com/prysmatic-labs/prysm)
-[![ETH2.0_Spec_Version 0.9.3](https://img.shields.io/badge/ETH2.0%20Spec%20Version-v0.9.3-blue.svg)](https://github.com/ethereum/eth2.0-specs/tree/v0.9.3)
+[![fuzzit](https://app.fuzzit.dev/badge?org_id=prysmaticlabs-gh)](https://app.fuzzit.dev/orgs/prysmaticlabs-gh/dashboard)
+[![ETH2.0_Spec_Version 0.11.3](https://img.shields.io/badge/ETH2.0%20Spec%20Version-v0.11.3-blue.svg)](https://github.com/ethereum/eth2.0-specs/tree/v0.11.3)
 [![Discord](https://user-images.githubusercontent.com/7288322/34471967-1df7808a-efbb-11e7-9088-ed0b04151291.png)](https://discord.gg/KSA7rPr)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/prysmaticlabs/geth-sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 
 This is the core repository for Prysm, a [Golang](https://golang.org/) implementation of the Ethereum 2.0 client specifications developed by [Prysmatic Labs](https://prysmaticlabs.com).
 
-### Need assistance?
-A more detailed set of installation and usage instructions as well as breakdowns of each individual component are available in the [official documentation portal](https://prysmaticlabs.gitbook.io/prysm/). If you still have questions, feel free to stop by either our [Discord](https://discord.gg/KSA7rPr) or [Gitter](https://gitter.im/prysmaticlabs/geth-sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) and a member of the team or our community will be happy to assist you.
+### Getting Started
+A detailed set of installation and usage instructions as well as breakdowns of each individual component are available in the [official documentation portal](https://docs.prylabs.network). If you still have questions, feel free to stop by either our [Discord](https://discord.gg/KSA7rPr) or [Gitter](https://gitter.im/prysmaticlabs/geth-sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) and a member of the team or our community will be happy to assist you.
 
 ### Come join the testnet!
-Participation is now open to the public for our Ethereum 2.0 phase 0 testnet release. Visit [prylabs.net](https://prylabs.net) for more information on the project or to sign up as a validator on the network.
-
-# Table of Contents
-
-- [Dependencies](#dependencies)
-- [Installation](#installing-prysm)
-    - [Build via Docker](#build-via-docker)
-    - [Build via Bazel](#build-via-bazel)
-- [Connecting to the public testnet: running a beacon node](#connecting-to-the-testnet-running-a-beacon-node)
-    - [Running via Docker](#running-via-docker)
-    - [Running via Bazel](#running-via-bazel)
-- [Staking ETH: running a validator client](#staking-eth-running-a-validator-client)
-    - [Activating your validator: depositing 3.2 Goerli ETH](#activating-your-validator-depositing-32-göerli-eth)
-    - [Starting the validator with Bazel](#starting-the-validator-with-bazel)
-- [Setting up a local ETH2 development chain](#setting-up-a-local-eth2-development-chain)
-    - [Installation and dependencies](#installation-and-dependencies) 
-    - [Running a local beacon node and validator client](#running-a-local-beacon-node-and-validator-client)   
--   [Testing Prysm](#testing-prysm)
--   [Contributing](#contributing)
--   [License](#license)
-
-## Dependencies
-
-Prysm can be installed either with Docker **\(recommended\)** or using our build tool, Bazel. The below instructions include sections for performing both.
-
-#### **For Docker installations:**
-
-* The latest release of [Docker](https://docs.docker.com/install/)
-
-#### **For Bazel installations:**
-
-* The latest release of [Bazel](https://docs.bazel.build/versions/master/install.html)
-* The latest release of `cmake`
-* The latest release of `git`
-* A modern UNIX operating system \(macOS included\)
-
-## Installing Prysm
-
-### Build via Docker
-
-1. Ensure you are running the most recent version of Docker by issuing the command:
-
-```text
-docker -v
-```
-
-2. To pull the Prysm images, issue the following commands:
-
-```text
-docker pull gcr.io/prysmaticlabs/prysm/validator:latest
-docker pull gcr.io/prysmaticlabs/prysm/beacon-chain:latest
-```
-
-This process will also install any related dependencies.
-
-### Build via Bazel
-
-1. Open a terminal window. Ensure you are running the most recent version of Bazel by issuing the command:
-
-```text
-bazel version
-```
-
-2. Clone Prysm's [main repository](https://github.com/prysmaticlabs/prysm) and enter the directory:
-
-```text
-git clone https://github.com/prysmaticlabs/prysm
-cd prysm
-```
-
-3. Build both the beacon chain node and the validator client:
-
-```text
-bazel build //beacon-chain:beacon-chain
-bazel build //validator:validator
-```
-
-Bazel will automatically pull and install any dependencies as well, including Go and necessary compilers.
-
-## Connecting to the testnet: running a beacon node
-
-Below are instructions for initialising a beacon node and connecting to the public testnet. To further understand the role that the beacon node plays in Prysm, see [this section of the documentation.](https://prysmaticlabs.gitbook.io/prysm/how-prysm-works/overview-technical)
-
-
-**NOTE:** It is recommended to open up port 13000 on your local router to improve connectivity and receive more peers from the network. To do so, navigate to `192.168.0.1` in your browser and login if required. Follow along with the interface to modify your routers firewall settings. When this task is completed, append the parameter`--p2p-host-ip=$(curl -s ident.me)` to your selected beacon startup command presented in this section to use the newly opened port.
-
-### Running via Docker
-
-#### **Docker on Linux/macOS:**
-
-To start your beacon node, issue the following command:
-
-```text
-docker run -it -v $HOME/prysm:/data -p 4000:4000 --name beacon-node \
-  gcr.io/prysmaticlabs/prysm/beacon-chain:latest \
-  --datadir=/data \
-  --init-sync-no-verify
-```
-
-The beacon node can be halted by either using `Ctrl+c` or with the command:
-
-```text
-docker stop beacon-node
-```
-
-To restart the beacon node, issue the following command:
-
-```text
-docker start -ai beacon-node
-```
-
-To delete a corrupted container, issue the following command:
-
-```text
-docker rm beacon-node
-```
-
-To recreate a deleted container and refresh the chain database, issue the start command with an additional `--clear-db` parameter:
-
-```text
-docker run -it -v $HOME/prysm:/data -p 4000:4000 --name beacon-node \
-  gcr.io/prysmaticlabs/prysm/beacon-chain:latest \
-  --datadir=/data \
-  --clear-db
-```
-
-#### **Docker on Windows:**
-
-1. You will need to 'share' the local drive you wish to mount to \(e.g. C:\).
-   1. Enter Docker settings \(right click the tray icon\)
-   2. Click 'Shared Drives'
-   3. Select a drive to share
-   4. Click 'Apply'
-   
-2. You will next need to create a directory named `/prysm/` within your selected shared Drive. This folder will be used as a local data directory for Beacon Node chain data as well as account and keystore information required by the validator. Docker will **not** create this directory if it does not exist already. For the purposes of these instructions, it is assumed that `C:` is your prior-selected shared Drive.
-3. To run the beacon node, issue the following command:
-
-```text
-docker run -it -v c:/prysm/:/data -p 4000:4000 gcr.io/prysmaticlabs/prysm/beacon-chain:latest --datadir=/data --init-sync-no-verify --clear-db
-```
-
-### Running via Bazel
-
-To start your Beacon Node with Bazel, issue the following command:
-
-```text
-bazel run //beacon-chain -- --clear-db --datadir=$HOME/prysm
-```
-
-This will sync up the beacon node with the latest head block in the network.
-
-
-**NOTE:** The beacon node must be **completely synced** before attempting to initialise a validator client, otherwise the validator will not be able to complete the deposit and **funds will lost**.
-
-
-## Staking ETH: Running a validator client
-
-Once your beacon node is up, the chain will be waiting for you to deposit 3.2 Goerli ETH into a [validator deposit contract](how-prysm-works/validator-deposit-contract.md) in order to activate your validator \(discussed in the section below\). First though, you will need to create this validator and connect to this node to participate in consensus.
-
-Each validator represents 3.2 Goerli ETH being staked in the system, and it is possible to spin up as many as you desire in order to have more stake in the network.
-
-### Activating your validator: depositing 3.2 Göerli ETH
-
-To begin setting up a validator, follow the instructions found on [prylabs.net](https://prylabs.net) to use the Göerli ETH faucet and make a deposit. For step-by-step assistance with the deposit page, see the [Activating a Validator ](activating-a-validator.md)section of this documentation.
-
-It will take a while for the nodes in the network to process a deposit. Once the node is active, the validator will immediately begin performing its responsibilities.
-
-In your validator client, you will be able to frequently see your validator balance as it goes up over time. Note that, should your node ever go offline for a long period, a validator will start gradually losing its deposit until it is removed from the network entirely.
-
-**Congratulations, you are now running Ethereum 2.0 Phase 0!**
-
-## Setting up a local ETH2 development chain
-
-This section outlines the process of setting up Prysm for local testing with other Ethereum 2.0 client implementations. See the [INTEROP.md](https://github.com/prysmaticlabs/prysm/blob/master/INTEROP.md) file for advanced configuration options. For more background information on interoperability development, see [this blog post](https://blog.ethereum.org/2019/09/19/eth2-interop-in-review/).
-
-### Installation and dependencies
-
-To begin setting up a local ETH2 development chain, follow the **Bazel** instructions found in the [dependencies](https://github.com/prysmaticlabs/prysm#dependencies) and [installation](https://github.com/prysmaticlabs/prysm#installation) sections respectively.
-
-### Running a local beacon node and validator client
-
-The example below will generate a beacon genesis state and initiate Prysm with 64 validators with the genesis time set to your machines UNIX time.
-
-Open up two terminal windows. In the first, issue the command:
-
-```text
-bazel run //beacon-chain -- \
---no-genesis-delay \
---bootstrap-node= \
---deposit-contract $(curl https://prylabs.net/contract) \
---clear-db \
---interop-num-validators 64 \
---interop-eth1data-votes
-```
-
-Wait a moment for the beacon chain to start. In the other terminal, issue the command:
-
-```text
-bazel run //validator -- --interop-num-validators 64
-```
-
-This command will kickstart the system with your 64 validators performing their duties accordingly.
-
-## Testing Prysm
-
-To run the unit tests of our system, issue the command:
-
-```text
-bazel test //...
-```
-
-To run our linter, make sure you have [golangci-lint](https://github.com/golangci/golangci-lint) installed and then issue the command:
-
-```text
-golangci-lint run
-```
-
+Participation is now open to the public for our Ethereum 2.0 phase 0 testnet release. Visit [prylabs.net](https://prylabs.net) for more information on the project or to sign up as a validator on the network. You can visualize the nodes in the network on [eth2stats.io](https://eth2stats.io), explore validator rewards/penalties via Bitfly's block explorer: [beaconcha.in](https://beaconcha.in), and follow the latest blocks added to the chain on [Etherscan](https://beacon.etherscan.io).
 
 ## Contributing
-Want to get involved? Check out our [Contribution Guide](https://prysmaticlabs.gitbook.io/prysm/getting-involved/contribution-guidelines) to learn more!
+Want to get involved? Check out our [Contribution Guide](https://docs.prylabs.network/docs/contribute/contribution-guidelines/) to learn more!
 
 ## License
 [GNU General Public License v3.0](https://www.gnu.org/licenses/gpl-3.0.en.html)

WORKSPACE

@@ -4,18 +4,66 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 
 http_archive(
-    name = "bazel_skylib",
-    sha256 = "2ea8a5ed2b448baf4a6855d3ce049c4c452a6470b1efd1504fdb7c1c134d220a",
-    strip_prefix = "bazel-skylib-0.8.0",
-    url = "https://github.com/bazelbuild/bazel-skylib/archive/0.8.0.tar.gz",
+    name = "bazel_toolchains",
+    sha256 = "144290c4166bd67e76a54f96cd504ed86416ca3ca82030282760f0823c10be48",
+    strip_prefix = "bazel-toolchains-3.1.1",
+    urls = [
+        "https://github.com/bazelbuild/bazel-toolchains/releases/download/3.1.1/bazel-toolchains-3.1.1.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/3.1.1.tar.gz",
+    ],
 )
 
 http_archive(
-    name = "bazel_gazelle",
-    sha256 = "86c6d481b3f7aedc1d60c1c211c6f76da282ae197c3b3160f54bd3a8f847896f",
+    name = "com_grail_bazel_toolchain",
+    sha256 = "0bec89e35d8a141c87f28cfc506d6d344785c8eb2ff3a453140a1fe972ada79d",
+    strip_prefix = "bazel-toolchain-77a87103145f86f03f90475d19c2c8854398a444",
+    urls = ["https://github.com/grailbio/bazel-toolchain/archive/77a87103145f86f03f90475d19c2c8854398a444.tar.gz"],
+)
+
+load("@com_grail_bazel_toolchain//toolchain:deps.bzl", "bazel_toolchain_dependencies")
+
+bazel_toolchain_dependencies()
+
+load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "llvm_toolchain")
+
+llvm_toolchain(
+    name = "llvm_toolchain",
+    llvm_version = "9.0.0",
+)
+
+load("@llvm_toolchain//:toolchains.bzl", "llvm_register_toolchains")
+
+llvm_register_toolchains()
+
+load("@prysm//tools/cross-toolchain:prysm_toolchains.bzl", "configure_prysm_toolchains")
+
+configure_prysm_toolchains()
+
+load("@prysm//tools/cross-toolchain:rbe_toolchains_config.bzl", "rbe_toolchains_config")
+
+rbe_toolchains_config()
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "bazel_skylib",
+    sha256 = "97e70364e9249702246c0e9444bccdc4b847bed1eb03c5a3ece4f83dfe6abc44",
     urls = [
-        "https://storage.googleapis.com/bazel-mirror/github.com/bazelbuild/bazel-gazelle/releases/download/v0.19.1/bazel-gazelle-v0.19.1.tar.gz",
-        "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.19.1/bazel-gazelle-v0.19.1.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz",
+        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz",
+    ],
+)
+
+load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
+
+bazel_skylib_workspace()
+
+http_archive(
+    name = "bazel_gazelle",
+    sha256 = "d8c45ee70ec39a57e7a05e5027c32b1576cc7f16d9dd37135b0eddde45cf1b10",
+    urls = [
+        "https://storage.googleapis.com/bazel-mirror/github.com/bazelbuild/bazel-gazelle/releases/download/v0.20.0/bazel-gazelle-v0.20.0.tar.gz",
+        "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.20.0/bazel-gazelle-v0.20.0.tar.gz",
     ],
 )
 
@@ -28,46 +76,46 @@ http_archive(
 
 http_archive(
     name = "io_bazel_rules_docker",
-    #    sha256 = "9ff889216e28c918811b77999257d4ac001c26c1f7c7fb17a79bc28abf74182e",
-    strip_prefix = "rules_docker-0.12.1",
-    url = "https://github.com/bazelbuild/rules_docker/archive/v0.12.1.tar.gz",
+    sha256 = "dc97fccceacd4c6be14e800b2a00693d5e8d07f69ee187babfd04a80a9f8e250",
+    strip_prefix = "rules_docker-0.14.1",
+    url = "https://github.com/bazelbuild/rules_docker/archive/v0.14.1.tar.gz",
 )
 
 http_archive(
     name = "io_bazel_rules_go",
-    sha256 = "e88471aea3a3a4f19ec1310a55ba94772d087e9ce46e41ae38ecebe17935de7b",
+    sha256 = "7b9bbe3ea1fccb46dcfa6c3f3e29ba7ec740d8733370e21cdc8937467b4a4349",
     urls = [
-        "https://storage.googleapis.com/bazel-mirror/github.com/bazelbuild/rules_go/releases/download/v0.20.3/rules_go-v0.20.3.tar.gz",
-        "https://github.com/bazelbuild/rules_go/releases/download/v0.20.3/rules_go-v0.20.3.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.22.4/rules_go-v0.22.4.tar.gz",
+        "https://github.com/bazelbuild/rules_go/releases/download/v0.22.4/rules_go-v0.22.4.tar.gz",
     ],
 )
 
 http_archive(
-    name = "build_bazel_rules_nodejs",
-    sha256 = "0942d188f4d0de6ddb743b9f6642a26ce1ad89f09c0035a9a5ca5ba9615c96aa",
-    urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/0.38.1/rules_nodejs-0.38.1.tar.gz"],
+    name = "fuzzit_linux",
+    build_file_content = "exports_files([\"fuzzit\"])",
+    sha256 = "9ca76ac1c22d9360936006efddf992977ebf8e4788ded8e5f9d511285c9ac774",
+    urls = ["https://github.com/fuzzitdev/fuzzit/releases/download/v2.4.76/fuzzit_Linux_x86_64.zip"],
 )
 
 git_repository(
     name = "graknlabs_bazel_distribution",
     commit = "962f3a7e56942430c0ec120c24f9e9f2a9c2ce1a",
     remote = "https://github.com/graknlabs/bazel-distribution",
-    shallow_since = "1563544980 +0300",
+    shallow_since = "1569509514 +0300",
 )
 
 # Override default import in rules_go with special patch until
 # https://github.com/gogo/protobuf/pull/582 is merged.
 git_repository(
     name = "com_github_gogo_protobuf",
-    # v1.3.0 (latest) as of 2019-10-05
-    commit = "0ca988a254f991240804bf9821f3450d87ccbb1b",
+    commit = "5628607bb4c51c3157aacc3a50f0ab707582b805",
     patch_args = ["-p1"],
     patches = [
         "@io_bazel_rules_go//third_party:com_github_gogo_protobuf-gazelle.patch",
         "//third_party:com_github_gogo_protobuf-equal.patch",
     ],
     remote = "https://github.com/gogo/protobuf",
-    shallow_since = "1567336231 +0200",
+    shallow_since = "1571033717 +0200",
     # gazelle args: -go_prefix github.com/gogo/protobuf -proto legacy
 )
 
@@ -78,13 +126,37 @@ load(
 
 container_repositories()
 
+load(
+    "@io_bazel_rules_docker//container:container.bzl",
+    "container_pull",
+)
+
+container_pull(
+    name = "alpine_cc_linux_amd64",
+    digest = "sha256:3f7f4dfcb6dceac3a902f36609cc232262e49f5656a6dc4bb3da89e35fecc8a5",
+    registry = "index.docker.io",
+    repository = "fasibio/alpine-libgcc",
+)
+
+container_pull(
+    name = "fuzzit_base",
+    digest = "sha256:24a39a4360b07b8f0121eb55674a2e757ab09f0baff5569332fefd227ee4338f",
+    registry = "gcr.io",
+    repository = "fuzzit-public/stretch-llvm8",
+)
+
+load("@prysm//third_party/herumi:herumi.bzl", "bls_dependencies")
+
+bls_dependencies()
+
 load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
 
 go_rules_dependencies()
 
 go_register_toolchains(nogo = "@//:nogo")
 
-load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository")
+load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
+load("@prysm//tools/go:def.bzl", "go_repository")
 
 gazelle_dependencies()
 
@@ -97,15 +169,8 @@ load(
     _go_image_repos = "repositories",
 )
 
-_go_image_repos()
-
 # Golang images
 # This is using gcr.io/distroless/base
-load(
-    "@io_bazel_rules_docker//go:image.bzl",
-    _go_image_repos = "repositories",
-)
-
 _go_image_repos()
 
 # CC images
@@ -125,16 +190,16 @@ proto_library(
   srcs = ["src/proto/faucet.proto"],
   visibility = ["//visibility:public"],
 )""",
-    sha256 = "1184e44a7a9b8b172e68e82c02cc3b15a80122340e05a92bd1edeafe5e68debe",
-    strip_prefix = "prysm-testnet-site-ec6a4a4e421bf4445845969167d06e93ee8d7acc",
-    url = "https://github.com/prestonvanloon/prysm-testnet-site/archive/ec6a4a4e421bf4445845969167d06e93ee8d7acc.tar.gz",
+    sha256 = "29742136ff9faf47343073c4569a7cf21b8ed138f726929e09e3c38ab83544f7",
+    strip_prefix = "prysm-testnet-site-5c711600f0a77fc553b18cf37b880eaffef4afdb",
+    url = "https://github.com/prestonvanloon/prysm-testnet-site/archive/5c711600f0a77fc553b18cf37b880eaffef4afdb.tar.gz",
 )
 
 http_archive(
     name = "io_kubernetes_build",
-    sha256 = "5ab110312cd7665a1940ba0523b67b9fbb6053beb9dd4e147643867bebd7e809",
-    strip_prefix = "repo-infra-db6ceb5f992254db76af7c25db2edc5469b5ea82",
-    url = "https://github.com/kubernetes/repo-infra/archive/db6ceb5f992254db76af7c25db2edc5469b5ea82.tar.gz",
+    sha256 = "b84fbd1173acee9d02a7d3698ad269fdf4f7aa081e9cecd40e012ad0ad8cfa2a",
+    strip_prefix = "repo-infra-6537f2101fb432b679f3d103ee729dd8ac5d30a0",
+    url = "https://github.com/kubernetes/repo-infra/archive/6537f2101fb432b679f3d103ee729dd8ac5d30a0.tar.gz",
 )
 
 http_archive(
@@ -149,8 +214,8 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "72c6ee3c20d19736b1203f364a6eb0ddee2c173073e20bee2beccd288fdc42be",
-    url = "https://github.com/ethereum/eth2.0-spec-tests/releases/download/v0.9.4/general.tar.gz",
+    sha256 = "489f85d7c17a901b9069c95f656154fdf1385db00f3aeb3e0319aed8745f9453",
+    url = "https://github.com/ethereum/eth2.0-spec-tests/releases/download/v0.11.3/general.tar.gz",
 )
 
 http_archive(
@@ -165,8 +230,8 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "a3cc860a3679f6f62ee57b65677a9b48a65fdebb151cdcbf50f23852632845ef",
-    url = "https://github.com/ethereum/eth2.0-spec-tests/releases/download/v0.9.4/minimal.tar.gz",
+    sha256 = "b83000fbcb60b7a5b8c0e805f3fee6953b17bfe0fe6658416e7d99e6d261f284",
+    url = "https://github.com/ethereum/eth2.0-spec-tests/releases/download/v0.11.3/minimal.tar.gz",
 )
 
 http_archive(
@@ -181,8 +246,8 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "8fc1b6220973ca30fa4ddc4ed24d66b1719abadca8bedb5e06c3bd9bc0df28e9",
-    url = "https://github.com/ethereum/eth2.0-spec-tests/releases/download/v0.9.4/mainnet.tar.gz",
+    sha256 = "ae0c09ab49afa69085c91f9e2f2f4de6526d43b927609839b1597c674b4dccde",
+    url = "https://github.com/ethereum/eth2.0-spec-tests/releases/download/v0.11.3/mainnet.tar.gz",
 )
 
 http_archive(
@@ -192,13 +257,6 @@ http_archive(
     url = "https://github.com/bazelbuild/buildtools/archive/bf564b4925ab5876a3f64d8b90fab7f769013d42.zip",
 )
 
-http_archive(
-    name = "com_github_herumi_bls_eth_go_binary",
-    sha256 = "15a41ddb0bf7d142ebffae68337f19c16e747676cb56794c5d80dbe388ce004c",
-    strip_prefix = "bls-go-binary-ac038c7cb6d3185c4a46f3bca0c99ebf7b191e16",
-    url = "https://github.com/nisdas/bls-go-binary/archive/ac038c7cb6d3185c4a46f3bca0c99ebf7b191e16.zip",
-)
-
 load("@com_github_bazelbuild_buildtools//buildifier:deps.bzl", "buildifier_dependencies")
 
 buildifier_dependencies()
@@ -211,7 +269,7 @@ go_repository(
 
 git_repository(
     name = "com_google_protobuf",
-    commit = "d09d649aea36f02c03f8396ba39a8d4db8a607e4",
+    commit = "4059c61f27eb1b06c4ee979546a238be792df0a4",
     remote = "https://github.com/protocolbuffers/protobuf",
     shallow_since = "1558721209 -0700",
 )
@@ -225,8 +283,9 @@ all_content = """filegroup(name = "all", srcs = glob(["**"]), visibility = ["//v
 
 http_archive(
     name = "rules_foreign_cc",
-    strip_prefix = "rules_foreign_cc-master",
-    url = "https://github.com/bazelbuild/rules_foreign_cc/archive/master.zip",
+    sha256 = "b85ce66a3410f7370d1a9a61dfe3a29c7532b7637caeb2877d8d0dfd41d77abb",
+    strip_prefix = "rules_foreign_cc-3515b20a2417c4dd51c8a4a8cac1f6ecf3c6d934",
+    url = "https://github.com/bazelbuild/rules_foreign_cc/archive/3515b20a2417c4dd51c8a4a8cac1f6ecf3c6d934.zip",
 )
 
 load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
@@ -238,15 +297,26 @@ rules_foreign_cc_dependencies([
 http_archive(
     name = "librdkafka",
     build_file_content = all_content,
+    sha256 = "f6be27772babfdacbbf2e4c5432ea46c57ef5b7d82e52a81b885e7b804781fd6",
     strip_prefix = "librdkafka-1.2.1",
     urls = ["https://github.com/edenhill/librdkafka/archive/v1.2.1.tar.gz"],
 )
 
+http_archive(
+    name = "sigp_beacon_fuzz_corpora",
+    build_file = "//third_party:beacon-fuzz/corpora.BUILD",
+    sha256 = "42993d0901a316afda45b4ba6d53c7c21f30c551dcec290a4ca131c24453d1ef",
+    strip_prefix = "beacon-fuzz-corpora-bac24ad78d45cc3664c0172241feac969c1ac29b",
+    urls = [
+        "https://github.com/sigp/beacon-fuzz-corpora/archive/bac24ad78d45cc3664c0172241feac969c1ac29b.tar.gz",
+    ],
+)
+
 # External dependencies
 
 go_repository(
     name = "com_github_ethereum_go_ethereum",
-    commit = "40beaeef26d5a2a0918dec2b960c2556c71a90a0",
+    commit = "0beb54b2147b3473a4c55e5ce6f02643ce403b14",
     importpath = "github.com/ethereum/go-ethereum",
     # Note: go-ethereum is not bazel-friendly with regards to cgo. We have a
     # a fork that has resolved these issues by disabling HID/USB support and
@@ -261,12 +331,10 @@ go_repository(
     name = "com_github_prysmaticlabs_go_ssz",
     commit = "e24db4d9e9637cf88ee9e4a779e339a1686a84ee",
     importpath = "github.com/prysmaticlabs/go-ssz",
-)
-
-go_repository(
-    name = "com_github_urfave_cli",
-    commit = "e6cf83ec39f6e1158ced1927d4ed14578fda8edb",  # v1.21.0
-    importpath = "github.com/urfave/cli",
+    patch_args = ["-p1"],
+    patches = [
+        "//third_party:com_github_prysmaticlabs_go_ssz.patch",
+    ],
 )
 
 go_repository(
@@ -295,7 +363,7 @@ go_repository(
 
 go_repository(
     name = "com_github_libp2p_go_libp2p",
-    commit = "c1687281a5c19b61ee5e0dc07fad15697c3bde94",  # v0.4.0
+    commit = "76944c4fc848530530f6be36fb22b70431ca506c",  # v0.5.1
     importpath = "github.com/libp2p/go-libp2p",
 )
 
@@ -314,7 +382,7 @@ go_repository(
 
 go_repository(
     name = "com_github_multiformats_go_multiaddr",
-    commit = "f96df18bf0c217c77f6cc0f9e810a178cea12f38",  # v0.1.1
+    commit = "8c6cee15b340d7210c30a82a19231ee333b69b1d",  # v0.2.0
     importpath = "github.com/multiformats/go-multiaddr",
 )
 
@@ -326,7 +394,7 @@ go_repository(
 
 go_repository(
     name = "com_github_multiformats_go_multihash",
-    commit = "249ead2008065c476a2ee45e8e75e8b85d846a72",  # v0.0.8
+    commit = "6b39927dce4869bc1726861b65ada415ee1f7fc7",  # v0.0.13
     importpath = "github.com/multiformats/go-multihash",
 )
 
@@ -344,13 +412,13 @@ go_repository(
 
 go_repository(
     name = "com_github_libp2p_go_libp2p_peerstore",
-    commit = "f4c9af195c69379f1cf284dba31985482a56f78e",  # v0.1.3
+    commit = "dee88d7532302c001604811fa3fbb5a7f83225e7",  # v0.1.4
     importpath = "github.com/libp2p/go-libp2p-peerstore",
 )
 
 go_repository(
     name = "com_github_libp2p_go_libp2p_circuit",
-    commit = "0305622f3f146485f0ff6df0ae6c010787331ca7",  # v0.1.3
+    commit = "61af9db0dd78e01e53b9fb044be44dcc7255667e",  # v0.1.4
     importpath = "github.com/libp2p/go-libp2p-circuit",
 )
 
@@ -429,7 +497,7 @@ go_repository(
 go_repository(
     name = "com_github_libp2p_go_libp2p_secio",
     build_file_proto_mode = "disable_global",
-    commit = "7c3f577d99debb69c3b68be35fe14d9445a6569c",  # v0.2.0
+    commit = "6f83420d5715a8b1c4082aaf9c5c7785923e702e",  # v0.2.1
     importpath = "github.com/libp2p/go-libp2p-secio",
 )
 
@@ -447,7 +515,7 @@ go_repository(
 
 go_repository(
     name = "com_github_jbenet_goprocess",
-    commit = "1dc239722b2ba3784472fb5301f62640fa5a8bc3",  # v0.1.3
+    commit = "7f9d9ed286badffcf2122cfeb383ec37daf92508",
     importpath = "github.com/jbenet/goprocess",
 )
 
@@ -465,7 +533,7 @@ go_repository(
 
 go_repository(
     name = "com_github_libp2p_go_libp2p_nat",
-    commit = "c50c291a61bceccb914366d93eb24f58594e9134",  # v0.0.4
+    commit = "873ef75f6ab6273821d77197660c1fb3af4cc02e",  # v0.0.5
     importpath = "github.com/libp2p/go-libp2p-nat",
 )
 
@@ -483,7 +551,7 @@ go_repository(
 
 go_repository(
     name = "com_github_mattn_go_isatty",
-    commit = "e1f7b56ace729e4a73a29a6b4fac6cd5fcda7ab3",  # v0.0.9
+    commit = "7b513a986450394f7bbf1476909911b3aa3a55ce",
     importpath = "github.com/mattn/go-isatty",
 )
 
@@ -567,7 +635,7 @@ go_repository(
 
 go_repository(
     name = "com_github_libp2p_go_flow_metrics",
-    commit = "1f5b3acc846b2c8ce4c4e713296af74f5c24df55",  # v0.0.1
+    commit = "e5a6a4db89199d99b2a74b8da198277a826241d8",  # v0.0.3
     importpath = "github.com/libp2p/go-flow-metrics",
 )
 
@@ -591,14 +659,15 @@ go_repository(
 
 go_repository(
     name = "com_github_libp2p_go_ws_transport",
-    commit = "8cca0dbc7f3533b122bd2cbeaa4a9b07c2913b9d",  # v0.1.2
+    commit = "370d1a3a7420e27423417c37630cad3754ad5702",  # v0.2.0
     importpath = "github.com/libp2p/go-ws-transport",
 )
 
 go_repository(
     name = "org_golang_x_crypto",
-    commit = "4def268fd1a49955bfb3dda92fe3db4f924f2285",
     importpath = "golang.org/x/crypto",
+    sum = "h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw=",
+    version = "v0.0.0-20200221231518-2aa609cf4a9d",
 )
 
 go_repository(
@@ -631,6 +700,12 @@ go_repository(
     importpath = "github.com/syndtr/goleveldb",
 )
 
+go_repository(
+    name = "com_github_emicklei_dot",
+    commit = "5810de2f2ab7aac98cd7bcbd59147a7ca6071768",
+    importpath = "github.com/emicklei/dot",
+)
+
 go_repository(
     name = "com_github_libp2p_go_libp2p_blankhost",
     commit = "da3b45205dfce3ef3926054ffd5dee76f5903382",  # v0.1.4
@@ -690,19 +765,19 @@ go_repository(
 
 go_repository(
     name = "com_github_prometheus_client_model",
-    commit = "fd36f4220a901265f90734c3183c5f0c91daa0b8",
+    commit = "7bc5445566f0fe75b15de23e6b93886e982d7bf9",
     importpath = "github.com/prometheus/client_model",
 )
 
 go_repository(
     name = "com_github_prometheus_common",
-    commit = "287d3e634a1e550c9e463dd7e5a75a422c614505",  # v0.7.0
+    commit = "d978bcb1309602d68bb4ba69cf3f8ed900e07308",
     importpath = "github.com/prometheus/common",
 )
 
 go_repository(
     name = "com_github_prometheus_procfs",
-    commit = "499c85531f756d1129edd26485a5f73871eeb308",  # v0.0.5
+    commit = "6d489fc7f1d9cd890a250f3ea3431b1744b9623f",
     importpath = "github.com/prometheus/procfs",
 )
 
@@ -718,12 +793,6 @@ go_repository(
     importpath = "github.com/matttproud/golang_protobuf_extensions",
 )
 
-go_repository(
-    name = "com_github_boltdb_bolt",
-    commit = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8",  # v1.3.1
-    importpath = "github.com/boltdb/bolt",
-)
-
 go_repository(
     name = "com_github_pborman_uuid",
     commit = "8b1b92947f46224e3b97bb1a3a5b0382be00d31e",  # v1.2.0
@@ -771,7 +840,7 @@ go_repository(
 
 go_repository(
     name = "com_github_ipfs_go_datastore",
-    commit = "d0ca9bc39f9d5b77bd602abe1a897473e105be7f",  # v0.1.1
+    commit = "e7a498916ccca1b0b40fb08630659cd4d68a01e8",  # v0.3.1
     importpath = "github.com/ipfs/go-datastore",
 )
 
@@ -783,14 +852,14 @@ go_repository(
 
 go_repository(
     name = "com_github_ipfs_go_cid",
-    commit = "9bb7ea69202c6c9553479eb355ab8a8a97d43a2e",  # v0.0.3
+    commit = "3da5bbbe45260437a44f777e6b2e5effa2606901",  # v0.0.4
     importpath = "github.com/ipfs/go-cid",
 )
 
 go_repository(
     name = "com_github_libp2p_go_libp2p_record",
     build_file_proto_mode = "disable_global",
-    commit = "3f535b1abcdf698e11ac16f618c2e64c4e5a114a",  # v0.1.1
+    commit = "8ccbca30634f70a8f03d133ac64cbf245d079e1e",  # v0.1.2
     importpath = "github.com/libp2p/go-libp2p-record",
 )
 
@@ -802,7 +871,7 @@ go_repository(
 
 go_repository(
     name = "com_github_libp2p_go_libp2p_kbucket",
-    commit = "8b77351e0f784a5f71749d23000897c8aee71a76",  # v0.2.1
+    commit = "a0cac6f63c491504b18eeba24be2ac0bbbfa0e5c",  # v0.2.3
     importpath = "github.com/libp2p/go-libp2p-kbucket",
 )
 
@@ -826,7 +895,7 @@ go_repository(
 
 go_repository(
     name = "com_github_hashicorp_golang_lru",
-    commit = "7f827b33c0f158ec5dfbba01bb0b14a4541fd81d",  # v0.5.3
+    commit = "14eae340515388ca95aa8e7b86f0de668e981f54",  # v0.5.4
     importpath = "github.com/hashicorp/golang-lru",
 )
 
@@ -845,7 +914,7 @@ go_repository(
 
 go_repository(
     name = "com_github_libp2p_go_libp2p_autonat",
-    commit = "3464f9b4f7bfbd7bb008813eacb626c7ab7fb9a3",  # v0.1.0
+    commit = "60bf479cf6bc73c939f4db97ad711756e949e522",  # v0.1.1
     importpath = "github.com/libp2p/go-libp2p-autonat",
 )
 
@@ -868,10 +937,17 @@ go_repository(
     importpath = "k8s.io/client-go",
 )
 
+go_repository(
+    name = "io_etcd_go_bbolt",
+    importpath = "go.etcd.io/bbolt",
+    sum = "h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=",
+    version = "v1.3.4",
+)
+
 go_repository(
     name = "io_k8s_apimachinery",
     build_file_proto_mode = "disable_global",
-    commit = "bfcf53abc9f82bad3e534fcb1c36599d3c989ebf",
+    commit = "79c2a76c473a20cdc4ce59cae4b72529b5d9d16b",  # v0.17.2
     importpath = "k8s.io/apimachinery",
 )
 
@@ -884,8 +960,8 @@ go_repository(
 go_repository(
     name = "com_github_google_gofuzz",
     importpath = "github.com/google/gofuzz",
-    sum = "h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=",
-    version = "v1.0.0",
+    sum = "h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=",
+    version = "v1.1.0",
 )
 
 go_repository(
@@ -953,7 +1029,7 @@ go_repository(
 
 go_repository(
     name = "com_google_cloud_go",
-    commit = "264def2dd949cdb8a803bb9f50fa29a67b798a6a",  # v0.46.3
+    commit = "6daa679260d92196ffca2362d652c924fdcb7a22",  # v0.52.0
     importpath = "cloud.google.com/go",
 )
 
@@ -995,7 +1071,7 @@ go_repository(
 
 go_repository(
     name = "com_github_pkg_errors",
-    commit = "ba968bfe8b2f7e042a574c888954fccecfa385b4",  # v0.8.1
+    commit = "614d223910a179a466c1767a985424175c39b465",  # v0.9.1
     importpath = "github.com/pkg/errors",
 )
 
@@ -1037,7 +1113,7 @@ go_repository(
 
 go_repository(
     name = "com_github_apache_thrift",
-    commit = "384647d290e2e4a55a14b1b7ef1b7e66293a2c33",  # v0.12.0
+    commit = "cecee50308fc7e6f77f55b3fd906c1c6c471fa2f",  # v0.13.0
     importpath = "github.com/apache/thrift",
 )
 
@@ -1049,7 +1125,7 @@ go_repository(
 
 go_repository(
     name = "com_github_libp2p_go_libp2p_connmgr",
-    commit = "b46e9bdbcd8436b4fe4b30a53ec913c07e5e09c9",  # v0.1.1
+    commit = "273839464339f1885413b385feee35301c5cb76f",  # v0.2.1
     importpath = "github.com/libp2p/go-libp2p-connmgr",
 )
 
@@ -1061,8 +1137,9 @@ go_repository(
 
 go_repository(
     name = "grpc_ecosystem_grpc_gateway",
-    commit = "da7a886035e25b2f274f89b6f3c64bf70a9f6780",
     importpath = "github.com/grpc-ecosystem/grpc-gateway",
+    sum = "h1:IOPK2xMPP3aV6/NPt4jt//ELFo3Vv8sDVD8j3+tleDU=",
+    version = "v1.14.4",
 )
 
 go_repository(
@@ -1080,13 +1157,13 @@ go_repository(
 go_repository(
     name = "com_github_libp2p_go_libp2p_core",
     build_file_proto_mode = "disable_global",
-    commit = "26b960839df84e2783f8f6125fa822a9978c2b8f",  # v0.2.3
+    commit = "f7f724862d85ec9f9ee7c58b0f79836abdee8cd9",  # v0.3.0
     importpath = "github.com/libp2p/go-libp2p-core",
 )
 
 go_repository(
     name = "com_github_libp2p_go_libp2p_testing",
-    commit = "1fa303da162dc57872d8fc553497f7602aa11c10",  # v0.1.0
+    commit = "82713a62880a5fe72d438bd58d737f0d3c4b7f36",  # v0.1.1
     importpath = "github.com/libp2p/go-libp2p-testing",
 )
 
@@ -1114,6 +1191,12 @@ go_repository(
     importpath = "github.com/multiformats/go-multiaddr-fmt",
 )
 
+go_repository(
+    name = "com_github_multiformats_go_varint",
+    commit = "0aa688902217dff2cba0f678c7e4a0f547b4983e",
+    importpath = "github.com/multiformats/go-varint",
+)
+
 go_repository(
     name = "com_github_libp2p_go_yamux",
     commit = "663972181d409e7263040f0b668462f87c85e1bd",  # v1.2.3
@@ -1122,7 +1205,7 @@ go_repository(
 
 go_repository(
     name = "com_github_libp2p_go_nat",
-    commit = "d13fdefb3bbb2fde2c6fc090a7ea992cec8b26df",  # v0.0.3
+    commit = "4b355d438085545df006ad9349686f30d8d37a27",  # v0.0.4
     importpath = "github.com/libp2p/go-nat",
 )
 
@@ -1146,7 +1229,7 @@ go_repository(
 
 go_repository(
     name = "com_github_prysmaticlabs_go_bitfield",
-    commit = "dbb55b15e92f897ee230360c8d9695e2f224b117",
+    commit = "62c2aee7166951c456888f92237aee4303ba1b9d",
     importpath = "github.com/prysmaticlabs/go-bitfield",
 )
 
@@ -1157,8 +1240,9 @@ go_ssz_dependencies()
 go_repository(
     name = "org_golang_google_grpc",
     build_file_proto_mode = "disable",
-    commit = "1d89a3c832915b2314551c1d2a506874d62e53f7",  # v1.22.0
     importpath = "google.golang.org/grpc",
+    sum = "h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk=",
+    version = "v1.27.1",
 )
 
 go_repository(
@@ -1187,7 +1271,7 @@ go_repository(
 
 go_repository(
     name = "com_github_googleapis_gnostic",
-    commit = "ab0dd09aa10e2952b28e12ecd35681b20463ebab",  # v0.3.1
+    commit = "896953e6749863beec38e27029c804e88c3144b8",  # v0.4.1
     importpath = "github.com/googleapis/gnostic",
 )
 
@@ -1211,7 +1295,7 @@ go_repository(
 
 go_repository(
     name = "com_github_google_go_cmp",
-    commit = "2d0692c2e9617365a95b295612ac0d4415ba4627",  # v0.3.1
+    commit = "5a6f75716e1203a923a78c9efb94089d857df0f6",  # v0.4.0
     importpath = "github.com/google/go-cmp",
 )
 
@@ -1241,12 +1325,6 @@ go_repository(
     importpath = "k8s.io/utils",
 )
 
-go_repository(
-    name = "com_github_googleapis_gnostic",
-    commit = "25d8b0b6698593f520d9d8dc5a88e6b16ca9ecc0",
-    importpath = "github.com/googleapis/gnostic",
-)
-
 go_repository(
     name = "com_github_patrickmn_go_cache",
     commit = "46f407853014144407b6c2ec7ccc76bf67958d93",
@@ -1255,18 +1333,15 @@ go_repository(
 
 go_repository(
     name = "com_github_prysmaticlabs_ethereumapis",
-    commit = "87118fb893cc6f32b25793d819790fd3bcce3221",
+    commit = "3444ffb75440161e4d6b9d38edef65212474cc48",
     importpath = "github.com/prysmaticlabs/ethereumapis",
-    patch_args = ["-p1"],
-    patches = [
-        "//third_party:com_github_prysmaticlabs_ethereumapis-tags.patch",
-    ],
 )
 
 go_repository(
     name = "com_github_cloudflare_roughtime",
-    commit = "d41fdcee702eb3e5c3296288a453b9340184d37e",
     importpath = "github.com/cloudflare/roughtime",
+    sum = "h1:jeSxE3fepJdhASERvBHI6RFkMhISv6Ir2JUybYLIVXs=",
+    version = "v0.0.0-20200205191924-a69ef1dab727",
 )
 
 go_repository(
@@ -1289,13 +1364,6 @@ go_repository(
     version = "v0.0.4",
 )
 
-go_repository(
-    name = "com_github_mdlayher_prombolt",
-    importpath = "github.com/mdlayher/prombolt",
-    sum = "h1:N257g6TTx0LxYoskSDFxvkSJ3NOZpy9IF1xQ7Gu+K8I=",
-    version = "v0.0.0-20161005185022-dfcf01d20ee9",
-)
-
 go_repository(
     name = "com_github_minio_highwayhash",
     importpath = "github.com/minio/highwayhash",
@@ -1333,13 +1401,6 @@ go_repository(
     version = "v0.10.5",
 )
 
-go_repository(
-    name = "in_gopkg_urfave_cli_v1",
-    importpath = "gopkg.in/urfave/cli.v1",
-    sum = "h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0=",
-    version = "v1.20.0",
-)
-
 go_repository(
     name = "com_github_naoina_go_stringutil",
     importpath = "github.com/naoina/go-stringutil",
@@ -1417,9 +1478,9 @@ go_repository(
 )
 
 go_repository(
-    name = "com_github_emicklei_dot",
-    commit = "f4a04130244d60cef56086d2f649b4b55e9624aa",
-    importpath = "github.com/emicklei/dot",
+    name = "com_github_googleapis_gnostic",
+    commit = "25d8b0b6698593f520d9d8dc5a88e6b16ca9ecc0",
+    importpath = "github.com/googleapis/gnostic",
 )
 
 go_repository(
@@ -1445,7 +1506,7 @@ go_repository(
 
 go_repository(
     name = "com_github_dgraph_io_ristretto",
-    commit = "99d1bbbf28e64530eb246be0568fc7709a35ebdd",
+    commit = "99d1bbbf28e64530eb246be0568fc7709a35ebdd",  # v0.0.1
     importpath = "github.com/dgraph-io/ristretto",
 )
 
@@ -1463,13 +1524,223 @@ go_repository(
 )
 
 go_repository(
-    name = "com_github_dgraph_io_ristretto",
-    commit = "99d1bbbf28e64530eb246be0568fc7709a35ebdd",
-    importpath = "github.com/dgraph-io/ristretto",
+    name = "com_github_kevinms_leakybucket_go",
+    importpath = "github.com/kevinms/leakybucket-go",
+    sum = "h1:oq6BiN7v0MfWCRcJAxSV+hesVMAAV8COrQbTjYNnso4=",
+    version = "v0.0.0-20190611015032-8a3d0352aa79",
 )
 
 go_repository(
-    name = "com_github_cespare_xxhash",
-    commit = "d7df74196a9e781ede915320c11c378c1b2f3a1f",
-    importpath = "github.com/cespare/xxhash",
+    name = "com_github_wealdtech_go_eth2_wallet",
+    importpath = "github.com/wealdtech/go-eth2-wallet",
+    sum = "h1:Hna/w4EKBJIs86VprIq7ez063A6kwk31d/O3Gs+MpYc=",
+    version = "v1.9.3",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_eth2_wallet_hd_v2",
+    importpath = "github.com/wealdtech/go-eth2-wallet-hd/v2",
+    sum = "h1:kiCvdexK3zRC2GwZHSHq+hS+irVNtMs5pNADyumeeRM=",
+    version = "v2.0.1",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_eth2_wallet_nd_v2",
+    importpath = "github.com/wealdtech/go-eth2-wallet-nd/v2",
+    sum = "h1:4lB6GY5oHQn5xwn/Sxm1e9SeVCaxa7q/0hqXUQYUNwU=",
+    version = "v2.0.1",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_eth2_wallet_store_filesystem",
+    importpath = "github.com/wealdtech/go-eth2-wallet-store-filesystem",
+    sum = "h1:Lc6wVTjIYeD+2hLAIzq1SugTWR527vEX4tEr5v3zxJc=",
+    version = "v1.7.2",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_eth2_wallet_store_s3",
+    importpath = "github.com/wealdtech/go-eth2-wallet-store-s3",
+    sum = "h1:xzyQDxbe5nr7xG0ByevTV2S8qkeOZvvjp+leBJcpxXQ=",
+    version = "v1.6.2",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_eth2_wallet_encryptor_keystorev4",
+    importpath = "github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4",
+    sum = "h1:IcpS4VpXhYz+TVupB5n6C6IQzaKwG+Rc8nvgCa/da4c=",
+    version = "v1.0.0",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_eth2_wallet_types_v2",
+    importpath = "github.com/wealdtech/go-eth2-wallet-types/v2",
+    sum = "h1:Ct3RrNJTapBiG2GxVl53Kfgy96f0GEUV7bediTu91u8=",
+    version = "v2.0.1",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_eth2_types_v2",
+    build_directives = [
+        "gazelle:resolve go github.com/herumi/bls-eth-go-binary/bls @herumi_bls_eth_go_binary//:go_default_library",
+    ],
+    importpath = "github.com/wealdtech/go-eth2-types/v2",
+    sum = "h1:qfmgaCBkH2N11LHCXsRWYz7OOxc+1QXrKHlS9yDnFsw=",
+    version = "v2.3.0",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_eth2_util",
+    importpath = "github.com/wealdtech/go-eth2-util",
+    sum = "h1:MyM16V7Qhd9q2ZaRa0WteBg2bWb8UplIKjZr8aeBZP0=",
+    version = "v1.1.4",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_ecodec",
+    importpath = "github.com/wealdtech/go-ecodec",
+    sum = "h1:yggrTSckcPJRaxxOxQF7FPm21kgE8WA6+f5jdq5Kr8o=",
+    version = "v1.1.0",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_bytesutil",
+    importpath = "github.com/wealdtech/go-bytesutil",
+    sum = "h1:ocEg3Ke2GkZ4vQw5lp46rmO+pfqCCTgq35gqOy8JKVc=",
+    version = "v1.1.1",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_indexer",
+    importpath = "github.com/wealdtech/go-indexer",
+    sum = "h1:/S4rfWQbSOnnYmwnvuTVatDibZ8o1s9bmTCHO16XINg=",
+    version = "v1.0.0",
+)
+
+go_repository(
+    name = "com_github_shibukawa_configdir",
+    commit = "e180dbdc8da04c4fa04272e875ce64949f38bd3e",
+    importpath = "github.com/shibukawa/configdir",
+)
+
+go_repository(
+    name = "com_github_libp2p_go_libp2p_noise",
+    importpath = "github.com/libp2p/go-libp2p-noise",
+    sum = "h1:J1gHJRNFEk7NdiaPQQqAvxEy+7hhCsVv3uzduWybmqY=",
+    version = "v0.0.0-20200302201340-8c54356e12c9",
+)
+
+go_repository(
+    name = "com_github_ferranbt_fastssz",
+    commit = "06015a5d84f9e4eefe2c21377ca678fa8f1a1b09",
+    importpath = "github.com/ferranbt/fastssz",
+    nofuzz = True,
+)
+
+http_archive(
+    name = "sszgen",  # Hack because we don't want to build this binary with libfuzzer, but need it to build.
+    build_file_content = """
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_binary")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "sszgen/main.go",
+        "sszgen/marshal.go",
+        "sszgen/size.go",
+        "sszgen/unmarshal.go",
+    ],
+    importpath = "github.com/ferranbt/fastssz/sszgen",
+    visibility = ["//visibility:private"],
+)
+
+go_binary(
+    name = "sszgen",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)
+    """,
+    strip_prefix = "fastssz-06015a5d84f9e4eefe2c21377ca678fa8f1a1b09",
+    urls = ["https://github.com/ferranbt/fastssz/archive/06015a5d84f9e4eefe2c21377ca678fa8f1a1b09.tar.gz"],
+)
+
+go_repository(
+    name = "com_github_burntsushi_toml",
+    importpath = "github.com/BurntSushi/toml",
+    sum = "h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=",
+    version = "v0.3.1",
+)
+
+go_repository(
+    name = "com_github_cpuguy83_go_md2man_v2",
+    importpath = "github.com/cpuguy83/go-md2man/v2",
+    sum = "h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=",
+    version = "v2.0.0",
+)
+
+go_repository(
+    name = "com_github_russross_blackfriday_v2",
+    importpath = "github.com/russross/blackfriday/v2",
+    sum = "h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=",
+    version = "v2.0.1",
+)
+
+go_repository(
+    name = "com_github_shurcool_sanitized_anchor_name",
+    importpath = "github.com/shurcooL/sanitized_anchor_name",
+    sum = "h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=",
+    version = "v1.0.0",
+)
+
+go_repository(
+    name = "in_gopkg_urfave_cli_v2",
+    importpath = "gopkg.in/urfave/cli.v2",
+    sum = "h1:OvXt/p4cdwNl+mwcWMq/AxaKFkhdxcjx+tx+qf4EOvY=",
+    version = "v2.0.0-20190806201727-b62605953717",
+)
+
+go_repository(
+    name = "in_gopkg_urfave_cli_v1",
+    importpath = "gopkg.in/urfave/cli.v1",
+    sum = "h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0=",
+    version = "v1.20.0",
+)
+
+go_repository(
+    name = "com_github_wealdtech_eth2_signer_api",
+    build_file_proto_mode = "disable_global",
+    importpath = "github.com/wealdtech/eth2-signer-api",
+    sum = "h1:Fs0GfrdhboBKW7zaMvIvUHJaOB1ibpAmRG3lkB53in4=",
+    version = "v1.3.0",
+)
+
+go_repository(
+    name = "com_github_prysmaticlabs_prombbolt",
+    importpath = "github.com/prysmaticlabs/prombbolt",
+    sum = "h1:bVD46NhbqEE6bsIqj42TCS3ELUdumti3WfAw9DXNtkg=",
+    version = "v0.0.0-20200324184628-09789ef63796",
+)
+
+load("@com_github_prysmaticlabs_prombbolt//:repositories.bzl", "prombbolt_dependencies")
+
+prombbolt_dependencies()
+
+go_repository(
+    name = "com_github_ianlancetaylor_cgosymbolizer",
+    importpath = "github.com/ianlancetaylor/cgosymbolizer",
+    sum = "h1:GWsU1WjSE2rtvyTYGcndqmPPkQkBNV7pEuZdnGtwtu4=",
+    version = "v0.0.0-20200321040036-d43e30eacb43",
+)
+
+go_repository(
+    name = "org_golang_x_mod",
+    importpath = "golang.org/x/mod",
+    sum = "h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=",
+    version = "v0.2.0",
+)
+
+go_repository(
+    name = "com_github_golang_gddo",
+    commit = "3c2cc9a6329d9842b3bbdaf307a8110d740cf94c",
+    importpath = "github.com/golang/gddo",
 )

bazel.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# This script serves as a wrapper around bazel to limit the scope of environment variables that
+# may change the action output. Using this script should result in a higher cache hit ratio for
+# cached actions with a more heremtic build.
+
+env -i \
+ PATH=/usr/bin:/bin \
+ HOME=$HOME \
+ GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS \
+ bazel "$@"

beacon-chain/BUILD.bazel

@@ -1,7 +1,8 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_test")
 load("@io_bazel_rules_docker//go:image.bzl", "go_image")
 load("@io_bazel_rules_docker//container:container.bzl", "container_bundle")
-load("//tools:binary_targets.bzl", "binary_targets")
+load("//tools:go_image.bzl", "go_image_alpine", "go_image_debug")
 load("@io_bazel_rules_docker//contrib:push-all.bzl", "docker_push")
 
 go_library(
@@ -20,12 +21,14 @@ go_library(
         "//shared/featureconfig:go_default_library",
         "//shared/logutil:go_default_library",
         "//shared/version:go_default_library",
+        "@com_github_ethereum_go_ethereum//log:go_default_library",
         "@com_github_ipfs_go_log//:go_default_library",
         "@com_github_joonix_log//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_urfave_cli//:go_default_library",
         "@com_github_whyrusleeping_go_logging//:go_default_library",
         "@com_github_x_cray_logrus_prefixed_formatter//:go_default_library",
+        "@in_gopkg_urfave_cli_v2//:go_default_library",
+        "@in_gopkg_urfave_cli_v2//altsrc:go_default_library",
         "@org_uber_go_automaxprocs//:go_default_library",
     ],
 )
@@ -36,7 +39,11 @@ go_image(
         "main.go",
         "usage.go",
     ],
-    base = "//tools:cc_image",
+    base = select({
+        "//tools:base_image_alpine": "//tools:alpine_cc_image",
+        "//tools:base_image_cc": "//tools:cc_image",
+        "//conditions:default": "//tools:cc_image",
+    }),
     goarch = "amd64",
     goos = "linux",
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain",
@@ -52,13 +59,17 @@ go_image(
         "//shared/featureconfig:go_default_library",
         "//shared/logutil:go_default_library",
         "//shared/version:go_default_library",
+        "//shared/params:go_default_library",
+        "@com_github_ethereum_go_ethereum//log:go_default_library",
         "@com_github_ipfs_go_log//:go_default_library",
         "@com_github_joonix_log//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_urfave_cli//:go_default_library",
         "@com_github_whyrusleeping_go_logging//:go_default_library",
         "@com_github_x_cray_logrus_prefixed_formatter//:go_default_library",
+        "@in_gopkg_urfave_cli_v2//:go_default_library",
+        "@in_gopkg_urfave_cli_v2//altsrc:go_default_library",
         "@org_uber_go_automaxprocs//:go_default_library",
+        "@in_gopkg_yaml_v2//:go_default_library",
     ],
 )
 
@@ -71,12 +82,54 @@ container_bundle(
     tags = ["manual"],
 )
 
+go_image_debug(
+    name = "image_debug",
+    image = ":image",
+    tags = ["manual"],
+)
+
+container_bundle(
+    name = "image_bundle_debug",
+    images = {
+        "gcr.io/prysmaticlabs/prysm/beacon-chain:latest-debug": ":image_debug",
+        "gcr.io/prysmaticlabs/prysm/beacon-chain:{DOCKER_TAG}-debug": ":image_debug",
+    },
+    tags = ["manual"],
+)
+
+go_image_alpine(
+    name = "image_alpine",
+    image = ":image",
+    tags = ["manual"],
+)
+
+container_bundle(
+    name = "image_bundle_alpine",
+    images = {
+        "gcr.io/prysmaticlabs/prysm/beacon-chain:latest-alpine": ":image_alpine",
+        "gcr.io/prysmaticlabs/prysm/beacon-chain:{DOCKER_TAG}-alpine": ":image_alpine",
+    },
+    tags = ["manual"],
+)
+
 docker_push(
     name = "push_images",
     bundle = ":image_bundle",
     tags = ["manual"],
 )
 
+docker_push(
+    name = "push_images_debug",
+    bundle = ":image_bundle_debug",
+    tags = ["manual"],
+)
+
+docker_push(
+    name = "push_images_alpine",
+    bundle = ":image_bundle_alpine",
+    tags = ["manual"],
+)
+
 go_binary(
     name = "beacon-chain",
     embed = [":go_default_library"],
@@ -91,17 +144,8 @@ go_test(
     size = "small",
     srcs = ["usage_test.go"],
     embed = [":go_default_library"],
-    deps = ["@com_github_urfave_cli//:go_default_library"],
+    deps = [
+        "//shared/featureconfig:go_default_library",
+        "@in_gopkg_urfave_cli_v2//:go_default_library",
+    ],
 )
-
-[go_binary(
-    name = "beacon-chain-{}-{}".format(
-        pair[0],
-        pair[1],
-    ),
-    embed = [":go_default_library"],
-    goarch = pair[1],
-    goos = pair[0],
-    tags = ["manual"],
-    visibility = ["//visibility:public"],
-) for pair in binary_targets]

beacon-chain/README.md

@@ -5,6 +5,6 @@ This is the main project folder for the beacon chain implementation of Ethereum
 [![Discord](https://user-images.githubusercontent.com/7288322/34471967-1df7808a-efbb-11e7-9088-ed0b04151291.png)](https://discord.gg/KSA7rPr)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/prysmaticlabs/prysm?badge&utm_medium=badge&utm_campaign=pr-badge)
 
-Also, read the latest beacon chain [design spec](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md), this design spec serves as a source of truth for the beacon chain implementation we follow at prysmatic labs.
+Also, read the latest beacon chain [design spec](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/phase0/beacon-chain.md), this design spec serves as a source of truth for the beacon chain implementation we follow at prysmatic labs.
 Check out the [FAQs](https://notes.ethereum.org/9MMuzWeFTTSg-3Tz_YeiBA?view). Refer this page on [why](http://email.mg2.substack.com/c/eJwlj9GOhCAMRb9G3jRQQPGBh5mM8xsbhKrsDGIAM9m_X9xN2qZtbpt7rCm4xvSjj5gLOTOmL-809CMbKXFaOKakIl4DZYr2AGyQIGjHOnWH22OiYnoIxmDijaBhhS6fcy7GvjobA9m0mSXOcnZq5GBqLkilXBZhBsus5ZK89VbKkRt-a-BZI6DzZ7iur1lQ953KJ9bemnxgahuQU9XJu6pFPdu8meT8vragzEjpMCwMGLlgLo6h5z1JumQTu4IJd4v15xqMf_8ZLP_Y1bSLdbnrD-LL71i2Kj7DLxaWWF4)
 we are combining sharding and casper together.

beacon-chain/archiver/BUILD.bazel

@@ -1,4 +1,5 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -12,6 +13,7 @@ go_library(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/validators:go_default_library",
         "//beacon-chain/db:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/params:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
@@ -32,12 +34,12 @@ go_test(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/db:go_default_library",
         "//beacon-chain/db/testing:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/params:go_default_library",
         "//shared/testutil:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
-        "@com_github_prysmaticlabs_go_bitfield//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",
     ],

beacon-chain/archiver/service.go

@@ -12,6 +12,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/validators"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/sirupsen/logrus"
@@ -24,7 +25,7 @@ var log = logrus.WithField("prefix", "archiver")
 type Service struct {
 	ctx                  context.Context
 	cancel               context.CancelFunc
-	beaconDB             db.Database
+	beaconDB             db.NoHeadAccessDatabase
 	headFetcher          blockchain.HeadFetcher
 	participationFetcher blockchain.ParticipationFetcher
 	stateNotifier        statefeed.Notifier
@@ -33,7 +34,7 @@ type Service struct {
 
 // Config options for the archiver service.
 type Config struct {
-	BeaconDB             db.Database
+	BeaconDB             db.NoHeadAccessDatabase
 	HeadFetcher          blockchain.HeadFetcher
 	ParticipationFetcher blockchain.ParticipationFetcher
 	StateNotifier        statefeed.Notifier
@@ -70,7 +71,7 @@ func (s *Service) Status() error {
 }
 
 // We archive committee information pertaining to the head state's epoch.
-func (s *Service) archiveCommitteeInfo(ctx context.Context, headState *pb.BeaconState, epoch uint64) error {
+func (s *Service) archiveCommitteeInfo(ctx context.Context, headState *state.BeaconState, epoch uint64) error {
 	proposerSeed, err := helpers.Seed(headState, epoch, params.BeaconConfig().DomainBeaconProposer)
 	if err != nil {
 		return errors.Wrap(err, "could not generate seed")
@@ -91,15 +92,16 @@ func (s *Service) archiveCommitteeInfo(ctx context.Context, headState *pb.Beacon
 }
 
 // We archive active validator set changes that happened during the previous epoch.
-func (s *Service) archiveActiveSetChanges(ctx context.Context, headState *pb.BeaconState, epoch uint64) error {
+func (s *Service) archiveActiveSetChanges(ctx context.Context, headState *state.BeaconState, epoch uint64) error {
 	prevEpoch := epoch - 1
-	activations := validators.ActivatedValidatorIndices(prevEpoch, headState.Validators)
-	slashings := validators.SlashedValidatorIndices(prevEpoch, headState.Validators)
+	vals := headState.Validators()
+	activations := validators.ActivatedValidatorIndices(prevEpoch, vals)
+	slashings := validators.SlashedValidatorIndices(prevEpoch, vals)
 	activeValidatorCount, err := helpers.ActiveValidatorCount(headState, prevEpoch)
 	if err != nil {
 		return errors.Wrap(err, "could not get active validator count")
 	}
-	exited, err := validators.ExitedValidatorIndices(prevEpoch, headState.Validators, activeValidatorCount)
+	exited, err := validators.ExitedValidatorIndices(prevEpoch, vals, activeValidatorCount)
 	if err != nil {
 		return errors.Wrap(err, "could not determine exited validator indices")
 	}
@@ -117,21 +119,20 @@ func (s *Service) archiveActiveSetChanges(ctx context.Context, headState *pb.Bea
 // We compute participation metrics by first retrieving the head state and
 // matching validator attestations during the epoch.
 func (s *Service) archiveParticipation(ctx context.Context, epoch uint64) error {
-	p := s.participationFetcher.Participation(epoch)
+	pBal := s.participationFetcher.Participation(epoch)
 	participation := &ethpb.ValidatorParticipation{}
-	if p != nil {
+	if pBal != nil {
 		participation = &ethpb.ValidatorParticipation{
-			EligibleEther:           p.PrevEpoch,
-			VotedEther:              p.PrevEpochTargetAttesters,
-			GlobalParticipationRate: float32(p.PrevEpochTargetAttesters) / float32(p.PrevEpoch),
+			EligibleEther:           pBal.ActivePrevEpoch,
+			VotedEther:              pBal.PrevEpochTargetAttested,
+			GlobalParticipationRate: float32(pBal.PrevEpochTargetAttested) / float32(pBal.ActivePrevEpoch),
 		}
 	}
 	return s.beaconDB.SaveArchivedValidatorParticipation(ctx, epoch, participation)
 }
 
 // We archive validator balances and active indices.
-func (s *Service) archiveBalances(ctx context.Context, headState *pb.BeaconState, epoch uint64) error {
-	balances := headState.Balances
+func (s *Service) archiveBalances(ctx context.Context, balances []uint64, epoch uint64) error {
 	if err := s.beaconDB.SaveArchivedBalances(ctx, epoch, balances); err != nil {
 		return errors.Wrap(err, "could not archive balances")
 	}
@@ -146,19 +147,24 @@ func (s *Service) run(ctx context.Context) {
 		select {
 		case event := <-stateChannel:
 			if event.Type == statefeed.BlockProcessed {
-				data := event.Data.(*statefeed.BlockProcessedData)
+				data, ok := event.Data.(*statefeed.BlockProcessedData)
+				if !ok {
+					log.Error("Event feed data is not type *statefeed.BlockProcessedData")
+					continue
+				}
 				log.WithField("headRoot", fmt.Sprintf("%#x", data.BlockRoot)).Debug("Received block processed event")
 				headState, err := s.headFetcher.HeadState(ctx)
 				if err != nil {
 					log.WithError(err).Error("Head state is not available")
 					continue
 				}
-				currentEpoch := helpers.CurrentEpoch(headState)
-				if !helpers.IsEpochEnd(headState.Slot) && currentEpoch <= s.lastArchivedEpoch {
+				slot := headState.Slot()
+				currentEpoch := helpers.SlotToEpoch(slot)
+				if !helpers.IsEpochEnd(slot) && currentEpoch <= s.lastArchivedEpoch {
 					continue
 				}
 				epochToArchive := currentEpoch
-				if !helpers.IsEpochEnd(headState.Slot) {
+				if !helpers.IsEpochEnd(slot) {
 					epochToArchive--
 				}
 				if err := s.archiveCommitteeInfo(ctx, headState, epochToArchive); err != nil {
@@ -173,7 +179,7 @@ func (s *Service) run(ctx context.Context) {
 					log.WithError(err).Error("Could not archive validator participation")
 					continue
 				}
-				if err := s.archiveBalances(ctx, headState, epochToArchive); err != nil {
+				if err := s.archiveBalances(ctx, headState.Balances(), epochToArchive); err != nil {
 					log.WithError(err).Error("Could not archive validator balances and active indices")
 					continue
 				}

beacon-chain/archiver/service_test.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-bitfield"
 	mock "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
@@ -17,6 +16,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
 	dbutil "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
@@ -27,15 +27,17 @@ import (
 func init() {
 	logrus.SetLevel(logrus.DebugLevel)
 	logrus.SetOutput(ioutil.Discard)
-	params.OverrideBeaconConfig(params.MinimalSpecConfig())
 }
 
 func TestArchiverService_ReceivesBlockProcessedEvent(t *testing.T) {
 	hook := logTest.NewGlobal()
-	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
+	svc, _ := setupService(t)
+	st := testutil.NewBeaconState()
+	if err := st.SetSlot(1); err != nil {
+		t.Fatal(err)
+	}
 	svc.headFetcher = &mock.ChainService{
-		State: &pb.BeaconState{Slot: 1},
+		State: st,
 	}
 
 	event := &feed.Event{
@@ -52,11 +54,14 @@ func TestArchiverService_ReceivesBlockProcessedEvent(t *testing.T) {
 
 func TestArchiverService_OnlyArchiveAtEpochEnd(t *testing.T) {
 	hook := logTest.NewGlobal()
-	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
+	svc, _ := setupService(t)
 	// The head state is NOT an epoch end.
+	st := testutil.NewBeaconState()
+	if err := st.SetSlot(params.BeaconConfig().SlotsPerEpoch - 2); err != nil {
+		t.Fatal(err)
+	}
 	svc.headFetcher = &mock.ChainService{
-		State: &pb.BeaconState{Slot: params.BeaconConfig().SlotsPerEpoch - 2},
+		State: st,
 	}
 	event := &feed.Event{
 		Type: statefeed.BlockProcessed,
@@ -79,10 +84,12 @@ func TestArchiverService_OnlyArchiveAtEpochEnd(t *testing.T) {
 
 func TestArchiverService_ArchivesEvenThroughSkipSlot(t *testing.T) {
 	hook := logTest.NewGlobal()
-	svc, beaconDB := setupService(t)
+	svc, _ := setupService(t)
 	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
-	defer dbutil.TeardownDB(t, beaconDB)
+	headState, err := setupState(validatorCount)
+	if err != nil {
+		t.Fatal(err)
+	}
 	event := &feed.Event{
 		Type: statefeed.BlockProcessed,
 		Data: &statefeed.BlockProcessedData{
@@ -99,7 +106,9 @@ func TestArchiverService_ArchivesEvenThroughSkipSlot(t *testing.T) {
 
 	// Send out an event every slot, skipping the end slot of the epoch.
 	for i := uint64(0); i < params.BeaconConfig().SlotsPerEpoch+1; i++ {
-		headState.Slot = i
+		if err := headState.SetSlot(i); err != nil {
+			t.Fatal(err)
+		}
 		svc.headFetcher = &mock.ChainService{
 			State: headState,
 		}
@@ -130,9 +139,11 @@ func TestArchiverService_ArchivesEvenThroughSkipSlot(t *testing.T) {
 func TestArchiverService_ComputesAndSavesParticipation(t *testing.T) {
 	hook := logTest.NewGlobal()
 	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
-	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
+	headState, err := setupState(validatorCount)
+	if err != nil {
+		t.Fatal(err)
+	}
+	svc, _ := setupService(t)
 	svc.headFetcher = &mock.ChainService{
 		State: headState,
 	}
@@ -168,9 +179,11 @@ func TestArchiverService_ComputesAndSavesParticipation(t *testing.T) {
 func TestArchiverService_SavesIndicesAndBalances(t *testing.T) {
 	hook := logTest.NewGlobal()
 	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
-	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
+	headState, err := setupState(validatorCount)
+	if err != nil {
+		t.Fatal(err)
+	}
+	svc, _ := setupService(t)
 	svc.headFetcher = &mock.ChainService{
 		State: headState,
 	}
@@ -187,11 +200,11 @@ func TestArchiverService_SavesIndicesAndBalances(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(headState.Balances, retrieved) {
+	if !reflect.DeepEqual(headState.Balances(), retrieved) {
 		t.Errorf(
 			"Wanted balances for epoch %d %v, retrieved %v",
 			helpers.CurrentEpoch(headState),
-			headState.Balances,
+			headState.Balances(),
 			retrieved,
 		)
 	}
@@ -201,9 +214,11 @@ func TestArchiverService_SavesIndicesAndBalances(t *testing.T) {
 func TestArchiverService_SavesCommitteeInfo(t *testing.T) {
 	hook := logTest.NewGlobal()
 	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
-	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
+	headState, err := setupState(validatorCount)
+	if err != nil {
+		t.Fatal(err)
+	}
+	svc, _ := setupService(t)
 	svc.headFetcher = &mock.ChainService{
 		State: headState,
 	}
@@ -248,16 +263,32 @@ func TestArchiverService_SavesCommitteeInfo(t *testing.T) {
 func TestArchiverService_SavesActivatedValidatorChanges(t *testing.T) {
 	hook := logTest.NewGlobal()
 	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
+	headState, err := setupState(validatorCount)
+	if err != nil {
+		t.Fatal(err)
+	}
 	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
 	svc.headFetcher = &mock.ChainService{
 		State: headState,
 	}
 	prevEpoch := helpers.PrevEpoch(headState)
-	delayedActEpoch := helpers.DelayedActivationExitEpoch(prevEpoch)
-	headState.Validators[4].ActivationEpoch = delayedActEpoch
-	headState.Validators[5].ActivationEpoch = delayedActEpoch
+	delayedActEpoch := helpers.ActivationExitEpoch(prevEpoch)
+	val1, err := headState.ValidatorAtIndex(4)
+	if err != nil {
+		t.Fatal(err)
+	}
+	val1.ActivationEpoch = delayedActEpoch
+	val2, err := headState.ValidatorAtIndex(5)
+	if err != nil {
+		t.Fatal(err)
+	}
+	val2.ActivationEpoch = delayedActEpoch
+	if err := headState.UpdateValidatorAtIndex(4, val1); err != nil {
+		t.Fatal(err)
+	}
+	if err := headState.UpdateValidatorAtIndex(5, val1); err != nil {
+		t.Fatal(err)
+	}
 	event := &feed.Event{
 		Type: statefeed.BlockProcessed,
 		Data: &statefeed.BlockProcessedData{
@@ -274,8 +305,8 @@ func TestArchiverService_SavesActivatedValidatorChanges(t *testing.T) {
 	if retrieved == nil {
 		t.Fatal("Retrieved indices are nil")
 	}
-	if !reflect.DeepEqual(retrieved.Activated, []uint64{4, 5}) {
-		t.Errorf("Wanted indices 4 5 activated, received %v", retrieved.Activated)
+	if len(retrieved.Activated) != 98 {
+		t.Error("Did not get wanted active length")
 	}
 	testutil.AssertLogsContain(t, hook, "Successfully archived")
 }
@@ -283,15 +314,31 @@ func TestArchiverService_SavesActivatedValidatorChanges(t *testing.T) {
 func TestArchiverService_SavesSlashedValidatorChanges(t *testing.T) {
 	hook := logTest.NewGlobal()
 	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
+	headState, err := setupState(validatorCount)
+	if err != nil {
+		t.Fatal(err)
+	}
 	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
 	svc.headFetcher = &mock.ChainService{
 		State: headState,
 	}
 	prevEpoch := helpers.PrevEpoch(headState)
-	headState.Validators[95].Slashed = true
-	headState.Validators[96].Slashed = true
+	val1, err := headState.ValidatorAtIndex(95)
+	if err != nil {
+		t.Fatal(err)
+	}
+	val1.Slashed = true
+	val2, err := headState.ValidatorAtIndex(96)
+	if err != nil {
+		t.Fatal(err)
+	}
+	val2.Slashed = true
+	if err := headState.UpdateValidatorAtIndex(95, val1); err != nil {
+		t.Fatal(err)
+	}
+	if err := headState.UpdateValidatorAtIndex(96, val1); err != nil {
+		t.Fatal(err)
+	}
 	event := &feed.Event{
 		Type: statefeed.BlockProcessed,
 		Data: &statefeed.BlockProcessedData{
@@ -317,15 +364,24 @@ func TestArchiverService_SavesSlashedValidatorChanges(t *testing.T) {
 func TestArchiverService_SavesExitedValidatorChanges(t *testing.T) {
 	hook := logTest.NewGlobal()
 	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
+	headState, err := setupState(validatorCount)
+	if err != nil {
+		t.Fatal(err)
+	}
 	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
 	svc.headFetcher = &mock.ChainService{
 		State: headState,
 	}
 	prevEpoch := helpers.PrevEpoch(headState)
-	headState.Validators[95].ExitEpoch = prevEpoch
-	headState.Validators[95].WithdrawableEpoch = prevEpoch + params.BeaconConfig().MinValidatorWithdrawabilityDelay
+	val, err := headState.ValidatorAtIndex(95)
+	if err != nil {
+		t.Fatal(err)
+	}
+	val.ExitEpoch = prevEpoch
+	val.WithdrawableEpoch = prevEpoch + params.BeaconConfig().MinValidatorWithdrawabilityDelay
+	if err := headState.UpdateValidatorAtIndex(95, val); err != nil {
+		t.Fatal(err)
+	}
 	event := &feed.Event{
 		Type: statefeed.BlockProcessed,
 		Data: &statefeed.BlockProcessedData{
@@ -347,7 +403,7 @@ func TestArchiverService_SavesExitedValidatorChanges(t *testing.T) {
 	}
 }
 
-func setupState(t *testing.T, validatorCount uint64) *pb.BeaconState {
+func setupState(validatorCount uint64) (*stateTrie.BeaconState, error) {
 	validators := make([]*ethpb.Validator, validatorCount)
 	balances := make([]uint64, validatorCount)
 	for i := 0; i < len(validators); i++ {
@@ -363,18 +419,20 @@ func setupState(t *testing.T, validatorCount uint64) *pb.BeaconState {
 
 	// We initialize a head state that has attestations from participated
 	// validators in a simulated fashion.
-	return &pb.BeaconState{
-		Slot:                       (2 * params.BeaconConfig().SlotsPerEpoch) - 1,
-		Validators:                 validators,
-		Balances:                   balances,
-		BlockRoots:                 make([][]byte, 128),
-		Slashings:                  []uint64{0, 1e9, 1e9},
-		RandaoMixes:                make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
-		CurrentEpochAttestations:   atts,
-		FinalizedCheckpoint:        &ethpb.Checkpoint{},
-		JustificationBits:          bitfield.Bitvector4{0x00},
-		CurrentJustifiedCheckpoint: &ethpb.Checkpoint{},
+	st := testutil.NewBeaconState()
+	if err := st.SetSlot((2 * params.BeaconConfig().SlotsPerEpoch) - 1); err != nil {
+		return nil, err
 	}
+	if err := st.SetValidators(validators); err != nil {
+		return nil, err
+	}
+	if err := st.SetBalances(balances); err != nil {
+		return nil, err
+	}
+	if err := st.SetCurrentEpochAttestations(atts); err != nil {
+		return nil, err
+	}
+	return st, nil
 }
 
 func setupService(t *testing.T) (*Service, db.Database) {
@@ -389,7 +447,7 @@ func setupService(t *testing.T) (*Service, db.Database) {
 		cancel:        cancel,
 		stateNotifier: mockChainService.StateNotifier(),
 		participationFetcher: &mock.ChainService{
-			Balance: &precompute.Balance{PrevEpoch: totalBalance, PrevEpochTargetAttesters: 1}},
+			Balance: &precompute.Balance{ActivePrevEpoch: totalBalance, PrevEpochTargetAttested: 1}},
 	}, beaconDB
 }
 

beacon-chain/blockchain/BUILD.bazel

@@ -1,12 +1,19 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_library(
     name = "go_default_library",
     srcs = [
         "chain_info.go",
+        "head.go",
         "info.go",
+        "init_sync_process_block.go",
         "log.go",
         "metrics.go",
+        "process_attestation.go",
+        "process_attestation_helpers.go",
+        "process_block.go",
+        "process_block_helpers.go",
         "receive_attestation.go",
         "receive_block.go",
         "service.go",
@@ -14,7 +21,7 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/blockchain",
     visibility = ["//beacon-chain:__subpackages__"],
     deps = [
-        "//beacon-chain/blockchain/forkchoice:go_default_library",
+        "//beacon-chain/cache:go_default_library",
         "//beacon-chain/cache/depositcache:go_default_library",
         "//beacon-chain/core/blocks:go_default_library",
         "//beacon-chain/core/epoch/precompute:go_default_library",
@@ -23,16 +30,27 @@ go_library(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/state:go_default_library",
         "//beacon-chain/db:go_default_library",
+        "//beacon-chain/db/filters:go_default_library",
+        "//beacon-chain/flags:go_default_library",
+        "//beacon-chain/forkchoice:go_default_library",
+        "//beacon-chain/forkchoice/protoarray:go_default_library",
         "//beacon-chain/operations/attestations:go_default_library",
+        "//beacon-chain/operations/slashings:go_default_library",
+        "//beacon-chain/operations/voluntaryexits:go_default_library",
         "//beacon-chain/p2p:go_default_library",
         "//beacon-chain/powchain:go_default_library",
+        "//beacon-chain/state:go_default_library",
+        "//beacon-chain/state/stategen:go_default_library",
+        "//beacon-chain/state/stateutil:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
+        "//shared/attestationutil:go_default_library",
         "//shared/bytesutil:go_default_library",
         "//shared/featureconfig:go_default_library",
         "//shared/params:go_default_library",
+        "//shared/roughtime:go_default_library",
         "//shared/slotutil:go_default_library",
         "//shared/traceutil:go_default_library",
-        "@com_github_gogo_protobuf//proto:go_default_library",
+        "@com_github_emicklei_dot//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
@@ -56,8 +74,11 @@ go_test(
     size = "medium",
     srcs = [
         "chain_info_test.go",
+        "head_test.go",
+        "init_sync_process_block_test.go",
+        "process_attestation_test.go",
+        "process_block_test.go",
         "receive_attestation_test.go",
-        "receive_block_test.go",
         "service_test.go",
     ],
     embed = [":go_default_library"],
@@ -70,11 +91,12 @@ go_test(
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/p2p:go_default_library",
         "//beacon-chain/powchain:go_default_library",
+        "//beacon-chain/state/stateutil:go_default_library",
+        "//proto/beacon/db:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/bytesutil:go_default_library",
         "//shared/event:go_default_library",
         "//shared/params:go_default_library",
-        "//shared/stateutil:go_default_library",
         "//shared/testutil:go_default_library",
         "@com_github_ethereum_go_ethereum//:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
@@ -84,6 +106,7 @@ go_test(
         "@com_github_prysmaticlabs_go_ssz//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",
+        "@in_gopkg_d4l3k_messagediff_v1//:go_default_library",
         "@org_golang_x_net//context:go_default_library",
     ],
 )
@@ -95,6 +118,13 @@ go_test(
         "service_norace_test.go",
     ],
     embed = [":go_default_library"],
+    gc_goopts = [
+        # Go 1.14 enables checkptr by default when building with -race or -msan. There is a pointer
+        # issue in boltdb, so must disable checkptr at compile time. This flag can be removed once
+        # the project is migrated to etcd's version of boltdb and the issue has been fixed.
+        # See: https://github.com/etcd-io/bbolt/issues/187.
+        "-d=checkptr=0",
+    ],
     race = "on",
     tags = ["race_on"],
     deps = [

beacon-chain/blockchain/chain_info.go

@@ -5,11 +5,14 @@ import (
 	"context"
 	"time"
 
-	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
@@ -17,30 +20,33 @@ import (
 // directly retrieves chain info related data.
 type ChainInfoFetcher interface {
 	HeadFetcher
-	CanonicalRootFetcher
 	FinalizationFetcher
+	GenesisFetcher
+	CanonicalFetcher
 }
 
-// GenesisTimeFetcher retrieves the Eth2 genesis timestamp.
-type GenesisTimeFetcher interface {
+// TimeFetcher retrieves the Eth2 data that's related to time.
+type TimeFetcher interface {
 	GenesisTime() time.Time
+	CurrentSlot() uint64
+}
+
+// GenesisFetcher retrieves the eth2 data related to its genesis.
+type GenesisFetcher interface {
+	GenesisValidatorRoot() [32]byte
 }
 
 // HeadFetcher defines a common interface for methods in blockchain service which
 // directly retrieves head related data.
 type HeadFetcher interface {
 	HeadSlot() uint64
-	HeadRoot() []byte
-	HeadBlock() *ethpb.SignedBeaconBlock
-	HeadState(ctx context.Context) (*pb.BeaconState, error)
+	HeadRoot(ctx context.Context) ([]byte, error)
+	HeadBlock(ctx context.Context) (*ethpb.SignedBeaconBlock, error)
+	HeadState(ctx context.Context) (*state.BeaconState, error)
 	HeadValidatorsIndices(epoch uint64) ([]uint64, error)
 	HeadSeed(epoch uint64) ([32]byte, error)
-}
-
-// CanonicalRootFetcher defines a common interface for methods in blockchain service which
-// directly retrieves canonical roots related data.
-type CanonicalRootFetcher interface {
-	CanonicalRoot(slot uint64) []byte
+	HeadGenesisValidatorRoot() [32]byte
+	ProtoArrayStore() *protoarray.Store
 }
 
 // ForkFetcher retrieves the current fork information of the Ethereum beacon chain.
@@ -48,6 +54,11 @@ type ForkFetcher interface {
 	CurrentFork() *pb.Fork
 }
 
+// CanonicalFetcher retrieves the current chain's canonical information.
+type CanonicalFetcher interface {
+	IsCanonical(ctx context.Context, blockRoot [32]byte) (bool, error)
+}
+
 // FinalizationFetcher defines a common interface for methods in blockchain service which
 // directly retrieves finalization and justification related data.
 type FinalizationFetcher interface {
@@ -64,115 +75,132 @@ type ParticipationFetcher interface {
 
 // FinalizedCheckpt returns the latest finalized checkpoint from head state.
 func (s *Service) FinalizedCheckpt() *ethpb.Checkpoint {
-	if s.headState == nil || s.headState.FinalizedCheckpoint == nil {
+	if s.finalizedCheckpt == nil {
 		return &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
 	}
 
 	// If head state exists but there hasn't been a finalized check point,
 	// the check point's root should refer to genesis block root.
-	if bytes.Equal(s.headState.FinalizedCheckpoint.Root, params.BeaconConfig().ZeroHash[:]) {
+	if bytes.Equal(s.finalizedCheckpt.Root, params.BeaconConfig().ZeroHash[:]) {
 		return &ethpb.Checkpoint{Root: s.genesisRoot[:]}
 	}
 
-	return s.headState.FinalizedCheckpoint
+	return state.CopyCheckpoint(s.finalizedCheckpt)
 }
 
 // CurrentJustifiedCheckpt returns the current justified checkpoint from head state.
 func (s *Service) CurrentJustifiedCheckpt() *ethpb.Checkpoint {
-	if s.headState == nil || s.headState.CurrentJustifiedCheckpoint == nil {
+	if s.justifiedCheckpt == nil {
 		return &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
 	}
 
 	// If head state exists but there hasn't been a justified check point,
 	// the check point root should refer to genesis block root.
-	if bytes.Equal(s.headState.CurrentJustifiedCheckpoint.Root, params.BeaconConfig().ZeroHash[:]) {
+	if bytes.Equal(s.justifiedCheckpt.Root, params.BeaconConfig().ZeroHash[:]) {
 		return &ethpb.Checkpoint{Root: s.genesisRoot[:]}
 	}
 
-	return s.headState.CurrentJustifiedCheckpoint
+	return state.CopyCheckpoint(s.justifiedCheckpt)
 }
 
 // PreviousJustifiedCheckpt returns the previous justified checkpoint from head state.
 func (s *Service) PreviousJustifiedCheckpt() *ethpb.Checkpoint {
-	if s.headState == nil || s.headState.PreviousJustifiedCheckpoint == nil {
+	if s.prevJustifiedCheckpt == nil {
 		return &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
 	}
 
 	// If head state exists but there hasn't been a justified check point,
 	// the check point root should refer to genesis block root.
-	if bytes.Equal(s.headState.PreviousJustifiedCheckpoint.Root, params.BeaconConfig().ZeroHash[:]) {
+	if bytes.Equal(s.prevJustifiedCheckpt.Root, params.BeaconConfig().ZeroHash[:]) {
 		return &ethpb.Checkpoint{Root: s.genesisRoot[:]}
 	}
 
-	return s.headState.PreviousJustifiedCheckpoint
+	return state.CopyCheckpoint(s.prevJustifiedCheckpt)
 }
 
 // HeadSlot returns the slot of the head of the chain.
 func (s *Service) HeadSlot() uint64 {
-	s.headLock.RLock()
-	defer s.headLock.RUnlock()
+	if !s.hasHeadState() {
+		return 0
+	}
 
-	return s.headSlot
+	return s.headSlot()
 }
 
 // HeadRoot returns the root of the head of the chain.
-func (s *Service) HeadRoot() []byte {
-	s.headLock.RLock()
-	defer s.headLock.RUnlock()
-
-	root := s.canonicalRoots[s.headSlot]
-	if len(root) != 0 {
-		return root
+func (s *Service) HeadRoot(ctx context.Context) ([]byte, error) {
+	if s.headRoot() != params.BeaconConfig().ZeroHash {
+		r := s.headRoot()
+		return r[:], nil
 	}
 
-	return params.BeaconConfig().ZeroHash[:]
+	b, err := s.beaconDB.HeadBlock(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if b == nil {
+		return params.BeaconConfig().ZeroHash[:], nil
+	}
+
+	r, err := stateutil.BlockRoot(b.Block)
+	if err != nil {
+		return nil, err
+	}
+
+	return r[:], nil
 }
 
 // HeadBlock returns the head block of the chain.
-func (s *Service) HeadBlock() *ethpb.SignedBeaconBlock {
-	s.headLock.RLock()
-	defer s.headLock.RUnlock()
+// If the head state is nil from service struct,
+// it will attempt to get the head block from DB.
+func (s *Service) HeadBlock(ctx context.Context) (*ethpb.SignedBeaconBlock, error) {
+	if s.hasHeadState() {
+		return s.headBlock(), nil
+	}
 
-	return proto.Clone(s.headBlock).(*ethpb.SignedBeaconBlock)
+	return s.beaconDB.HeadBlock(ctx)
 }
 
 // HeadState returns the head state of the chain.
 // If the head state is nil from service struct,
-// it will attempt to get from DB and error if nil again.
-func (s *Service) HeadState(ctx context.Context) (*pb.BeaconState, error) {
-	s.headLock.RLock()
-	defer s.headLock.RUnlock()
-
-	if s.headState == nil {
-		return s.beaconDB.HeadState(ctx)
+// it will attempt to get the head state from DB.
+func (s *Service) HeadState(ctx context.Context) (*state.BeaconState, error) {
+	if s.hasHeadState() {
+		return s.headState(), nil
 	}
 
-	return proto.Clone(s.headState).(*pb.BeaconState), nil
+	return s.beaconDB.HeadState(ctx)
 }
 
 // HeadValidatorsIndices returns a list of active validator indices from the head view of a given epoch.
 func (s *Service) HeadValidatorsIndices(epoch uint64) ([]uint64, error) {
-	if s.headState == nil {
+	if !s.hasHeadState() {
 		return []uint64{}, nil
 	}
-	return helpers.ActiveValidatorIndices(s.headState, epoch)
+	return helpers.ActiveValidatorIndices(s.headState(), epoch)
 }
 
 // HeadSeed returns the seed from the head view of a given epoch.
 func (s *Service) HeadSeed(epoch uint64) ([32]byte, error) {
-	if s.headState == nil {
+	if !s.hasHeadState() {
 		return [32]byte{}, nil
 	}
 
-	return helpers.Seed(s.headState, epoch, params.BeaconConfig().DomainBeaconAttester)
+	return helpers.Seed(s.headState(), epoch, params.BeaconConfig().DomainBeaconAttester)
 }
 
-// CanonicalRoot returns the canonical root of a given slot.
-func (s *Service) CanonicalRoot(slot uint64) []byte {
-	s.headLock.RLock()
-	defer s.headLock.RUnlock()
+// HeadGenesisValidatorRoot returns genesis validator root of the head state.
+func (s *Service) HeadGenesisValidatorRoot() [32]byte {
+	if !s.hasHeadState() {
+		return [32]byte{}
+	}
 
-	return s.canonicalRoots[slot]
+	return s.headGenesisValidatorRoot()
+}
+
+// ProtoArrayStore returns the proto array store object.
+func (s *Service) ProtoArrayStore() *protoarray.Store {
+	return s.forkChoiceStore.Store()
 }
 
 // GenesisTime returns the genesis time of beacon chain.
@@ -180,15 +208,24 @@ func (s *Service) GenesisTime() time.Time {
 	return s.genesisTime
 }
 
+// GenesisValidatorRoot returns the genesis validator
+// root of the chain.
+func (s *Service) GenesisValidatorRoot() [32]byte {
+	if !s.hasHeadState() {
+		return [32]byte{}
+	}
+	return bytesutil.ToBytes32(s.head.state.GenesisValidatorRoot())
+}
+
 // CurrentFork retrieves the latest fork information of the beacon chain.
 func (s *Service) CurrentFork() *pb.Fork {
-	if s.headState == nil {
+	if !s.hasHeadState() {
 		return &pb.Fork{
 			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
 			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
 		}
 	}
-	return proto.Clone(s.headState.Fork).(*pb.Fork)
+	return s.head.state.Fork()
 }
 
 // Participation returns the participation stats of a given epoch.
@@ -198,3 +235,17 @@ func (s *Service) Participation(epoch uint64) *precompute.Balance {
 
 	return s.epochParticipation[epoch]
 }
+
+// IsCanonical returns true if the input block root is part of the canonical chain.
+func (s *Service) IsCanonical(ctx context.Context, blockRoot [32]byte) (bool, error) {
+	// If the block has been finalized, the block will always be part of the canonical chain.
+	if s.beaconDB.IsFinalizedBlock(ctx, blockRoot) {
+		return true, nil
+	}
+
+	// If the block has not been finalized, the block must be recent. Check recent canonical roots
+	// mapping which uses proto array fork choice.
+	s.recentCanonicalBlocksLock.RLock()
+	defer s.recentCanonicalBlocksLock.RUnlock()
+	return s.recentCanonicalBlocks[blockRoot], nil
+}

beacon-chain/blockchain/chain_info_norace_test.go

@@ -5,73 +5,70 @@ import (
 	"testing"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 )
 
 func TestHeadSlot_DataRace(t *testing.T) {
 	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
 	s := &Service{
-		beaconDB:       db,
-		canonicalRoots: make(map[uint64][]byte),
+		beaconDB: db,
 	}
 	go func() {
-		s.saveHead(
-			context.Background(),
-			&ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 777}},
-			[32]byte{},
-		)
+		if err := s.saveHead(context.Background(), [32]byte{}); err != nil {
+			t.Fatal(err)
+		}
 	}()
 	s.HeadSlot()
 }
 
 func TestHeadRoot_DataRace(t *testing.T) {
 	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
 	s := &Service{
-		beaconDB:       db,
-		canonicalRoots: make(map[uint64][]byte),
+		beaconDB: db,
+		head:     &head{root: [32]byte{'A'}},
+		stateGen: stategen.New(db, cache.NewStateSummaryCache()),
 	}
 	go func() {
-		s.saveHead(
-			context.Background(),
-			&ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 777}},
-			[32]byte{},
-		)
+		if err := s.saveHead(context.Background(), [32]byte{}); err != nil {
+			t.Fatal(err)
+		}
 	}()
-	s.HeadRoot()
+	if _, err := s.HeadRoot(context.Background()); err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestHeadBlock_DataRace(t *testing.T) {
 	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
 	s := &Service{
-		beaconDB:       db,
-		canonicalRoots: make(map[uint64][]byte),
+		beaconDB: db,
+		head:     &head{block: &ethpb.SignedBeaconBlock{}},
+		stateGen: stategen.New(db, cache.NewStateSummaryCache()),
 	}
 	go func() {
-		s.saveHead(
-			context.Background(),
-			&ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 777}},
-			[32]byte{},
-		)
+		if err := s.saveHead(context.Background(), [32]byte{}); err != nil {
+			t.Fatal(err)
+		}
 	}()
-	s.HeadBlock()
+	if _, err := s.HeadBlock(context.Background()); err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestHeadState_DataRace(t *testing.T) {
 	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
 	s := &Service{
-		beaconDB:       db,
-		canonicalRoots: make(map[uint64][]byte),
+		beaconDB: db,
+		stateGen: stategen.New(db, cache.NewStateSummaryCache()),
 	}
 	go func() {
-		s.saveHead(
-			context.Background(),
-			&ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 777}},
-			[32]byte{},
-		)
+		if err := s.saveHead(context.Background(), [32]byte{}); err != nil {
+			t.Fatal(err)
+		}
 	}()
-	s.HeadState(context.Background())
+	if _, err := s.HeadState(context.Background()); err != nil {
+		t.Fatal(err)
+	}
 }

beacon-chain/blockchain/chain_info_test.go

@@ -7,23 +7,22 @@ import (
 	"testing"
 	"time"
 
+	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
 
 // Ensure Service implements chain info interface.
 var _ = ChainInfoFetcher(&Service{})
-var _ = GenesisTimeFetcher(&Service{})
+var _ = TimeFetcher(&Service{})
 var _ = ForkFetcher(&Service{})
 
 func TestFinalizedCheckpt_Nil(t *testing.T) {
 	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
 	c := setupBeaconChain(t, db)
-	c.headState, _ = testutil.DeterministicGenesisState(t, 1)
 	if !bytes.Equal(c.FinalizedCheckpt().Root, params.BeaconConfig().ZeroHash[:]) {
 		t.Error("Incorrect pre chain start value")
 	}
@@ -31,20 +30,22 @@ func TestFinalizedCheckpt_Nil(t *testing.T) {
 
 func TestHeadRoot_Nil(t *testing.T) {
 	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
 	c := setupBeaconChain(t, db)
-	if !bytes.Equal(c.HeadRoot(), params.BeaconConfig().ZeroHash[:]) {
+	headRoot, err := c.HeadRoot(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(headRoot, params.BeaconConfig().ZeroHash[:]) {
 		t.Error("Incorrect pre chain start value")
 	}
 }
 
 func TestFinalizedCheckpt_CanRetrieve(t *testing.T) {
 	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
 
-	cp := &ethpb.Checkpoint{Epoch: 5}
+	cp := &ethpb.Checkpoint{Epoch: 5, Root: []byte("foo")}
 	c := setupBeaconChain(t, db)
-	c.headState = &pb.BeaconState{FinalizedCheckpoint: cp}
+	c.finalizedCheckpt = cp
 
 	if c.FinalizedCheckpt().Epoch != cp.Epoch {
 		t.Errorf("Finalized epoch at genesis should be %d, got: %d", cp.Epoch, c.FinalizedCheckpt().Epoch)
@@ -53,12 +54,12 @@ func TestFinalizedCheckpt_CanRetrieve(t *testing.T) {
 
 func TestFinalizedCheckpt_GenesisRootOk(t *testing.T) {
 	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
 
-	cp := &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	genesisRoot := [32]byte{'A'}
+	cp := &ethpb.Checkpoint{Root: genesisRoot[:]}
 	c := setupBeaconChain(t, db)
-	c.headState = &pb.BeaconState{FinalizedCheckpoint: cp}
-	c.genesisRoot = [32]byte{'A'}
+	c.finalizedCheckpt = cp
+	c.genesisRoot = genesisRoot
 
 	if !bytes.Equal(c.FinalizedCheckpt().Root, c.genesisRoot[:]) {
 		t.Errorf("Got: %v, wanted: %v", c.FinalizedCheckpt().Root, c.genesisRoot[:])
@@ -67,11 +68,10 @@ func TestFinalizedCheckpt_GenesisRootOk(t *testing.T) {
 
 func TestCurrentJustifiedCheckpt_CanRetrieve(t *testing.T) {
 	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
 
-	cp := &ethpb.Checkpoint{Epoch: 6}
+	cp := &ethpb.Checkpoint{Epoch: 6, Root: []byte("foo")}
 	c := setupBeaconChain(t, db)
-	c.headState = &pb.BeaconState{CurrentJustifiedCheckpoint: cp}
+	c.justifiedCheckpt = cp
 
 	if c.CurrentJustifiedCheckpt().Epoch != cp.Epoch {
 		t.Errorf("Current Justifiied epoch at genesis should be %d, got: %d", cp.Epoch, c.CurrentJustifiedCheckpt().Epoch)
@@ -80,12 +80,12 @@ func TestCurrentJustifiedCheckpt_CanRetrieve(t *testing.T) {
 
 func TestJustifiedCheckpt_GenesisRootOk(t *testing.T) {
 	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
 
-	cp := &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	genesisRoot := [32]byte{'B'}
+	cp := &ethpb.Checkpoint{Root: genesisRoot[:]}
 	c := setupBeaconChain(t, db)
-	c.headState = &pb.BeaconState{CurrentJustifiedCheckpoint: cp}
-	c.genesisRoot = [32]byte{'B'}
+	c.justifiedCheckpt = cp
+	c.genesisRoot = genesisRoot
 
 	if !bytes.Equal(c.CurrentJustifiedCheckpt().Root, c.genesisRoot[:]) {
 		t.Errorf("Got: %v, wanted: %v", c.CurrentJustifiedCheckpt().Root, c.genesisRoot[:])
@@ -94,11 +94,10 @@ func TestJustifiedCheckpt_GenesisRootOk(t *testing.T) {
 
 func TestPreviousJustifiedCheckpt_CanRetrieve(t *testing.T) {
 	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
 
-	cp := &ethpb.Checkpoint{Epoch: 7}
+	cp := &ethpb.Checkpoint{Epoch: 7, Root: []byte("foo")}
 	c := setupBeaconChain(t, db)
-	c.headState = &pb.BeaconState{PreviousJustifiedCheckpoint: cp}
+	c.prevJustifiedCheckpt = cp
 
 	if c.PreviousJustifiedCheckpt().Epoch != cp.Epoch {
 		t.Errorf("Previous Justifiied epoch at genesis should be %d, got: %d", cp.Epoch, c.PreviousJustifiedCheckpt().Epoch)
@@ -107,12 +106,12 @@ func TestPreviousJustifiedCheckpt_CanRetrieve(t *testing.T) {
 
 func TestPrevJustifiedCheckpt_GenesisRootOk(t *testing.T) {
 	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
 
-	cp := &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	genesisRoot := [32]byte{'C'}
+	cp := &ethpb.Checkpoint{Root: genesisRoot[:]}
 	c := setupBeaconChain(t, db)
-	c.headState = &pb.BeaconState{PreviousJustifiedCheckpoint: cp}
-	c.genesisRoot = [32]byte{'C'}
+	c.prevJustifiedCheckpt = cp
+	c.genesisRoot = genesisRoot
 
 	if !bytes.Equal(c.PreviousJustifiedCheckpt().Root, c.genesisRoot[:]) {
 		t.Errorf("Got: %v, wanted: %v", c.PreviousJustifiedCheckpt().Root, c.genesisRoot[:])
@@ -121,37 +120,55 @@ func TestPrevJustifiedCheckpt_GenesisRootOk(t *testing.T) {
 
 func TestHeadSlot_CanRetrieve(t *testing.T) {
 	c := &Service{}
-	c.headSlot = 100
+	s, err := state.InitializeFromProto(&pb.BeaconState{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	c.head = &head{slot: 100, state: s}
 	if c.HeadSlot() != 100 {
 		t.Errorf("Wanted head slot: %d, got: %d", 100, c.HeadSlot())
 	}
 }
 
 func TestHeadRoot_CanRetrieve(t *testing.T) {
-	c := &Service{canonicalRoots: make(map[uint64][]byte)}
-	c.headSlot = 100
-	c.canonicalRoots[c.headSlot] = []byte{'A'}
-	if !bytes.Equal([]byte{'A'}, c.HeadRoot()) {
-		t.Errorf("Wanted head root: %v, got: %d", []byte{'A'}, c.HeadRoot())
+	c := &Service{}
+	c.head = &head{root: [32]byte{'A'}}
+	if [32]byte{'A'} != c.headRoot() {
+		t.Errorf("Wanted head root: %v, got: %d", []byte{'A'}, c.headRoot())
 	}
 }
 
 func TestHeadBlock_CanRetrieve(t *testing.T) {
 	b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1}}
-	c := &Service{headBlock: b}
-	if !reflect.DeepEqual(b, c.HeadBlock()) {
+	s, err := state.InitializeFromProto(&pb.BeaconState{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	c := &Service{}
+	c.head = &head{block: b, state: s}
+
+	recevied, err := c.HeadBlock(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(b, recevied) {
 		t.Error("incorrect head block received")
 	}
 }
 
 func TestHeadState_CanRetrieve(t *testing.T) {
-	s := &pb.BeaconState{Slot: 2}
-	c := &Service{headState: s}
+	s, err := state.InitializeFromProto(&pb.BeaconState{Slot: 2, GenesisValidatorsRoot: params.BeaconConfig().ZeroHash[:]})
+	if err != nil {
+		t.Fatal(err)
+	}
+	c := &Service{}
+	c.head = &head{state: s}
 	headState, err := c.HeadState(context.Background())
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(s, headState) {
+	if !proto.Equal(s.InnerStateUnsafe(), headState.InnerStateUnsafe()) {
 		t.Error("incorrect head state received")
 	}
 }
@@ -166,19 +183,30 @@ func TestGenesisTime_CanRetrieve(t *testing.T) {
 
 func TestCurrentFork_CanRetrieve(t *testing.T) {
 	f := &pb.Fork{Epoch: 999}
-	s := &pb.BeaconState{Fork: f}
-	c := &Service{headState: s}
-	if !reflect.DeepEqual(c.CurrentFork(), f) {
-		t.Error("Recieved incorrect fork version")
+	s, err := state.InitializeFromProto(&pb.BeaconState{Fork: f})
+	if err != nil {
+		t.Fatal(err)
+	}
+	c := &Service{}
+	c.head = &head{state: s}
+	if !proto.Equal(c.CurrentFork(), f) {
+		t.Error("Received incorrect fork version")
 	}
 }
 
-func TestCanonicalRoot_CanRetrieve(t *testing.T) {
-	c := &Service{canonicalRoots: make(map[uint64][]byte)}
-	slot := uint64(123)
-	r := []byte{'B'}
-	c.canonicalRoots[slot] = r
-	if !bytes.Equal(r, c.CanonicalRoot(slot)) {
-		t.Errorf("Wanted head root: %v, got: %d", []byte{'A'}, c.CanonicalRoot(slot))
+func TestGenesisValidatorRoot_CanRetrieve(t *testing.T) {
+	// Should not panic if head state is nil.
+	c := &Service{}
+	if c.GenesisValidatorRoot() != [32]byte{} {
+		t.Error("Did not get correct genesis validator root")
+	}
+
+	s, err := state.InitializeFromProto(&pb.BeaconState{GenesisValidatorsRoot: []byte{'a'}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	c.head = &head{state: s}
+	if c.GenesisValidatorRoot() != [32]byte{'a'} {
+		t.Error("Did not get correct genesis validator root")
 	}
 }

beacon-chain/blockchain/forkchoice/benchmark_test.go

@@ -1,168 +0,0 @@
-package forkchoice
-
-import (
-	"context"
-	"testing"
-
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-)
-
-func BenchmarkForkChoiceTree1(b *testing.B) {
-	ctx := context.Background()
-	db := testDB.SetupDB(b)
-	defer testDB.TeardownDB(b, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db, []byte{'g'})
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	// Benchmark fork choice with 1024 validators
-	validators := make([]*ethpb.Validator, 1024)
-	for i := 0; i < len(validators); i++ {
-		validators[i] = &ethpb.Validator{ExitEpoch: 2, EffectiveBalance: 1e9}
-	}
-	s := &pb.BeaconState{Validators: validators}
-
-	if err := store.GenesisStore(ctx, &ethpb.Checkpoint{}, &ethpb.Checkpoint{}); err != nil {
-		b.Fatal(err)
-	}
-
-	store.justifiedCheckpt.Root = roots[0]
-	if err := store.db.SaveState(ctx, s, bytesutil.ToBytes32(roots[0])); err != nil {
-		b.Fatal(err)
-	}
-
-	if err := store.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: store.justifiedCheckpt,
-		State:      s,
-	}); err != nil {
-		b.Fatal(err)
-	}
-
-	// Spread out the votes evenly for all 3 leaf nodes
-	for i := 0; i < len(validators); i++ {
-		switch {
-		case i < 256:
-			store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[1]}
-		case i > 768:
-			store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[7]}
-		default:
-			store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[8]}
-		}
-	}
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := store.Head(ctx)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func BenchmarkForkChoiceTree2(b *testing.B) {
-	ctx := context.Background()
-	db := testDB.SetupDB(b)
-	defer testDB.TeardownDB(b, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree2(db)
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	// Benchmark fork choice with 1024 validators
-	validators := make([]*ethpb.Validator, 1024)
-	for i := 0; i < len(validators); i++ {
-		validators[i] = &ethpb.Validator{ExitEpoch: 2, EffectiveBalance: 1e9}
-	}
-	s := &pb.BeaconState{Validators: validators}
-	if err := store.GenesisStore(ctx, &ethpb.Checkpoint{}, &ethpb.Checkpoint{}); err != nil {
-		b.Fatal(err)
-	}
-
-	store.justifiedCheckpt.Root = roots[0]
-	if err := store.db.SaveState(ctx, s, bytesutil.ToBytes32(roots[0])); err != nil {
-		b.Fatal(err)
-	}
-
-	if err := store.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: store.justifiedCheckpt,
-		State:      s,
-	}); err != nil {
-		b.Fatal(err)
-	}
-
-	// Spread out the votes evenly for all the leaf nodes. 8 to 15
-	nodeIndex := 8
-	for i := 0; i < len(validators); i++ {
-		store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[nodeIndex]}
-		if i%155 == 0 {
-			nodeIndex++
-		}
-	}
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := store.Head(ctx)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func BenchmarkForkChoiceTree3(b *testing.B) {
-	ctx := context.Background()
-	db := testDB.SetupDB(b)
-	defer testDB.TeardownDB(b, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree3(db)
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	// Benchmark fork choice with 1024 validators
-	validators := make([]*ethpb.Validator, 1024)
-	for i := 0; i < len(validators); i++ {
-		validators[i] = &ethpb.Validator{ExitEpoch: 2, EffectiveBalance: 1e9}
-	}
-	s := &pb.BeaconState{Validators: validators}
-	if err := store.GenesisStore(ctx, &ethpb.Checkpoint{}, &ethpb.Checkpoint{}); err != nil {
-		b.Fatal(err)
-	}
-
-	store.justifiedCheckpt.Root = roots[0]
-	if err := store.db.SaveState(ctx, s, bytesutil.ToBytes32(roots[0])); err != nil {
-		b.Fatal(err)
-	}
-
-	if err := store.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: store.justifiedCheckpt,
-		State:      s,
-	}); err != nil {
-		b.Fatal(err)
-	}
-
-	// All validators vote on the same head
-	for i := 0; i < len(validators); i++ {
-		store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[len(roots)-1]}
-	}
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := store.Head(ctx)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}

beacon-chain/blockchain/forkchoice/doc.go

@@ -1,9 +0,0 @@
-/*
-Package forkchoice implements the Latest Message Driven GHOST (Greediest Heaviest Observed
-Sub-Tree) algorithm as the Ethereum Serenity beacon chain fork choice rule. This algorithm is designed to
-properly detect the canonical chain based on validator votes even in the presence of high network
-latency, network partitions, and many conflicting blocks. To read more about fork choice, read the
-official accompanying document:
-https://github.com/ethereum/eth2.0-specs/blob/v0.9.0/specs/core/0_fork-choice.md
-*/
-package forkchoice

beacon-chain/blockchain/forkchoice/lmd_ghost_test.yaml

@@ -1,59 +0,0 @@
-test_cases:
-  # GHOST chooses b3 with the heaviest weight
-  - blocks:
-      - id: 'b0'
-        parent: 'b0'
-      - id: 'b1'
-        parent: 'b0'
-      - id: 'b2'
-        parent: 'b1'
-      - id: 'b3'
-        parent: 'b1'
-    weights:
-      b0: 0
-      b1: 0
-      b2: 5
-      b3: 10
-    head: 'b3'
-  # GHOST chooses b1 with the heaviest weight
-  - blocks:
-      - id: 'b0'
-        parent: 'b0'
-      - id: 'b1'
-        parent: 'b0'
-      - id: 'b2'
-        parent: 'b0'
-      - id: 'b3'
-        parent: 'b0'
-    weights:
-      b1: 5
-      b2: 4
-      b3: 3
-    head: 'b1'
-  # Equal weights children, GHOST chooses b2 because it is higher lexicographically than b3
-  - blocks:
-      - id: 'b0'
-        parent: 'b0'
-      - id: 'b1'
-        parent: 'b0'
-      - id: 'b2'
-        parent: 'b0'
-      - id: 'b3'
-        parent: 'b0'
-    weights:
-      b1: 5
-      b2: 6
-      b3: 6
-    head: 'b3'
-  # Equal weights children, GHOST chooses b2 because it is higher lexicographically than b1
-  - blocks:
-      - id: 'b0'
-        parent: 'b0'
-      - id: 'b1'
-        parent: 'b0'
-      - id: 'b2'
-        parent: 'b0'
-    weights:
-      b1: 0
-      b2: 0
-    head: 'b2'

beacon-chain/blockchain/forkchoice/lmd_ghost_yaml_test.go

@@ -1,140 +0,0 @@
-package forkchoice
-
-import (
-	"bytes"
-	"context"
-	"io/ioutil"
-	"path/filepath"
-	"strconv"
-	"testing"
-
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"gopkg.in/yaml.v2"
-)
-
-type Config struct {
-	TestCases []struct {
-		Blocks []struct {
-			ID     string `yaml:"id"`
-			Parent string `yaml:"parent"`
-		} `yaml:"blocks"`
-		Weights map[string]int `yaml:"weights"`
-		Head    string         `yaml:"head"`
-	} `yaml:"test_cases"`
-}
-
-func TestGetHeadFromYaml(t *testing.T) {
-	ctx := context.Background()
-	filename, _ := filepath.Abs("./lmd_ghost_test.yaml")
-	yamlFile, err := ioutil.ReadFile(filename)
-	if err != nil {
-		t.Fatal(err)
-	}
-	var c *Config
-	err = yaml.Unmarshal(yamlFile, &c)
-
-	params.UseMainnetConfig()
-
-	for _, test := range c.TestCases {
-		db := testDB.SetupDB(t)
-		defer testDB.TeardownDB(t, db)
-
-		blksRoot := make(map[int][]byte)
-		// Construct block tree from yaml.
-		for _, blk := range test.Blocks {
-			// genesis block condition
-			if blk.ID == blk.Parent {
-				b := &ethpb.BeaconBlock{Slot: 0, ParentRoot: []byte{'g'}}
-				if err := db.SaveBlock(ctx, &ethpb.SignedBeaconBlock{Block: b}); err != nil {
-					t.Fatal(err)
-				}
-				root, err := ssz.HashTreeRoot(b)
-				if err != nil {
-					t.Fatal(err)
-				}
-				blksRoot[0] = root[:]
-			} else {
-				slot, err := strconv.Atoi(blk.ID[1:])
-				if err != nil {
-					t.Fatal(err)
-				}
-				parentSlot, err := strconv.Atoi(blk.Parent[1:])
-				if err != nil {
-					t.Fatal(err)
-				}
-				b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: uint64(slot), ParentRoot: blksRoot[parentSlot]}}
-				if err := db.SaveBlock(ctx, b); err != nil {
-					t.Fatal(err)
-				}
-				root, err := ssz.HashTreeRoot(b.Block)
-				if err != nil {
-					t.Fatal(err)
-				}
-				blksRoot[slot] = root[:]
-				if err := db.SaveState(ctx, &pb.BeaconState{}, root); err != nil {
-					t.Fatal(err)
-				}
-			}
-		}
-
-		store := NewForkChoiceService(ctx, db)
-
-		// Assign validator votes to the blocks as weights.
-		count := 0
-		for blk, votes := range test.Weights {
-			slot, err := strconv.Atoi(blk[1:])
-			if err != nil {
-				t.Fatal(err)
-			}
-			max := count + votes
-			for i := count; i < max; i++ {
-				store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: blksRoot[slot]}
-				count++
-			}
-		}
-
-		validators := make([]*ethpb.Validator, count)
-		for i := 0; i < len(validators); i++ {
-			validators[i] = &ethpb.Validator{ExitEpoch: 2, EffectiveBalance: 1e9}
-		}
-
-		s := &pb.BeaconState{Validators: validators}
-
-		if err := store.db.SaveState(ctx, s, bytesutil.ToBytes32(blksRoot[0])); err != nil {
-			t.Fatal(err)
-		}
-		if err := store.GenesisStore(ctx, &ethpb.Checkpoint{Root: blksRoot[0]}, &ethpb.Checkpoint{Root: blksRoot[0]}); err != nil {
-			t.Fatal(err)
-		}
-
-		if err := store.checkpointState.AddCheckpointState(&cache.CheckpointState{
-			Checkpoint: store.justifiedCheckpt,
-			State:      s,
-		}); err != nil {
-			t.Fatal(err)
-		}
-
-		head, err := store.Head(ctx)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		headSlot, err := strconv.Atoi(test.Head[1:])
-		if err != nil {
-			t.Fatal(err)
-		}
-		wantedHead := blksRoot[headSlot]
-
-		if !bytes.Equal(head, wantedHead) {
-			t.Errorf("wanted root %#x, got root %#x", wantedHead, head)
-		}
-
-		testDB.TeardownDB(t, db)
-	}
-}

beacon-chain/blockchain/forkchoice/log.go

@@ -1,40 +0,0 @@
-package forkchoice
-
-import (
-	"fmt"
-
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/sirupsen/logrus"
-)
-
-var log = logrus.WithField("prefix", "forkchoice")
-
-// logs epoch related data during epoch boundary.
-func logEpochData(beaconState *pb.BeaconState) {
-	log.WithFields(logrus.Fields{
-		"epoch":                  helpers.CurrentEpoch(beaconState),
-		"finalizedEpoch":         beaconState.FinalizedCheckpoint.Epoch,
-		"justifiedEpoch":         beaconState.CurrentJustifiedCheckpoint.Epoch,
-		"previousJustifiedEpoch": beaconState.PreviousJustifiedCheckpoint.Epoch,
-	}).Info("Starting next epoch")
-	activeVals, err := helpers.ActiveValidatorIndices(beaconState, helpers.CurrentEpoch(beaconState))
-	if err != nil {
-		log.WithError(err).Error("Could not get active validator indices")
-		return
-	}
-	log.WithFields(logrus.Fields{
-		"totalValidators":  len(beaconState.Validators),
-		"activeValidators": len(activeVals),
-		"averageBalance":   fmt.Sprintf("%.5f ETH", averageBalance(beaconState.Balances)),
-	}).Info("Validator registry information")
-}
-
-func averageBalance(balances []uint64) float64 {
-	total := uint64(0)
-	for i := 0; i < len(balances); i++ {
-		total += balances[i]
-	}
-	return float64(total) / float64(len(balances)) / float64(params.BeaconConfig().GweiPerEth)
-}

beacon-chain/blockchain/forkchoice/metrics.go

@@ -1,161 +0,0 @@
-package forkchoice
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/params"
-)
-
-var (
-	beaconFinalizedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_finalized_epoch",
-		Help: "Last finalized epoch of the processed state",
-	})
-	beaconFinalizedRoot = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_finalized_root",
-		Help: "Last finalized root of the processed state",
-	})
-	cacheFinalizedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "cache_finalized_epoch",
-		Help: "Last cached finalized epoch",
-	})
-	cacheFinalizedRoot = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "cache_finalized_root",
-		Help: "Last cached finalized root",
-	})
-	beaconCurrentJustifiedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_current_justified_epoch",
-		Help: "Current justified epoch of the processed state",
-	})
-	beaconCurrentJustifiedRoot = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_current_justified_root",
-		Help: "Current justified root of the processed state",
-	})
-	beaconPrevJustifiedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_previous_justified_epoch",
-		Help: "Previous justified epoch of the processed state",
-	})
-	beaconPrevJustifiedRoot = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_previous_justified_root",
-		Help: "Previous justified root of the processed state",
-	})
-	sigFailsToVerify = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "att_signature_failed_to_verify_with_cache",
-		Help: "Number of attestation signatures that failed to verify with cache on, but succeeded without cache",
-	})
-	validatorsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Name: "validator_count",
-		Help: "The total number of validators, in GWei",
-	}, []string{"state"})
-	validatorsBalance = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Name: "validators_total_balance",
-		Help: "The total balance of validators, in GWei",
-	}, []string{"state"})
-	validatorsEffectiveBalance = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Name: "validators_total_effective_balance",
-		Help: "The total effective balance of validators, in GWei",
-	}, []string{"state"})
-	currentEth1DataDepositCount = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "current_eth1_data_deposit_count",
-		Help: "The current eth1 deposit count in the last processed state eth1data field.",
-	})
-	totalEligibleBalances = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "total_eligible_balances",
-		Help: "The total amount of ether, in gwei, that has been used in voting attestation target of previous epoch",
-	})
-	totalVotedTargetBalances = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "total_voted_target_balances",
-		Help: "The total amount of ether, in gwei, that is eligible for voting of previous epoch",
-	})
-)
-
-func reportEpochMetrics(state *pb.BeaconState) {
-	currentEpoch := state.Slot / params.BeaconConfig().SlotsPerEpoch
-
-	// Validator instances
-	pendingInstances := 0
-	activeInstances := 0
-	slashingInstances := 0
-	slashedInstances := 0
-	exitingInstances := 0
-	exitedInstances := 0
-	// Validator balances
-	pendingBalance := uint64(0)
-	activeBalance := uint64(0)
-	activeEffectiveBalance := uint64(0)
-	exitingBalance := uint64(0)
-	exitingEffectiveBalance := uint64(0)
-	slashingBalance := uint64(0)
-	slashingEffectiveBalance := uint64(0)
-
-	for i, validator := range state.Validators {
-		if validator.Slashed {
-			if currentEpoch < validator.ExitEpoch {
-				slashingInstances++
-				slashingBalance += state.Balances[i]
-				slashingEffectiveBalance += validator.EffectiveBalance
-			} else {
-				slashedInstances++
-			}
-			continue
-		}
-		if validator.ExitEpoch != params.BeaconConfig().FarFutureEpoch {
-			if currentEpoch < validator.ExitEpoch {
-				exitingInstances++
-				exitingBalance += state.Balances[i]
-				exitingEffectiveBalance += validator.EffectiveBalance
-			} else {
-				exitedInstances++
-			}
-			continue
-		}
-		if currentEpoch < validator.ActivationEpoch {
-			pendingInstances++
-			pendingBalance += state.Balances[i]
-			continue
-		}
-		activeInstances++
-		activeBalance += state.Balances[i]
-		activeEffectiveBalance += validator.EffectiveBalance
-	}
-	validatorsCount.WithLabelValues("Pending").Set(float64(pendingInstances))
-	validatorsCount.WithLabelValues("Active").Set(float64(activeInstances))
-	validatorsCount.WithLabelValues("Exiting").Set(float64(exitingInstances))
-	validatorsCount.WithLabelValues("Exited").Set(float64(exitedInstances))
-	validatorsCount.WithLabelValues("Slashing").Set(float64(slashingInstances))
-	validatorsCount.WithLabelValues("Slashed").Set(float64(slashedInstances))
-	validatorsBalance.WithLabelValues("Pending").Set(float64(pendingBalance))
-	validatorsBalance.WithLabelValues("Active").Set(float64(activeBalance))
-	validatorsBalance.WithLabelValues("Exiting").Set(float64(exitingBalance))
-	validatorsBalance.WithLabelValues("Slashing").Set(float64(slashingBalance))
-	validatorsEffectiveBalance.WithLabelValues("Active").Set(float64(activeEffectiveBalance))
-	validatorsEffectiveBalance.WithLabelValues("Exiting").Set(float64(exitingEffectiveBalance))
-	validatorsEffectiveBalance.WithLabelValues("Slashing").Set(float64(slashingEffectiveBalance))
-
-	// Last justified slot
-	if state.CurrentJustifiedCheckpoint != nil {
-		beaconCurrentJustifiedEpoch.Set(float64(state.CurrentJustifiedCheckpoint.Epoch))
-		beaconCurrentJustifiedRoot.Set(float64(bytesutil.ToLowInt64(state.CurrentJustifiedCheckpoint.Root)))
-	}
-	// Last previous justified slot
-	if state.PreviousJustifiedCheckpoint != nil {
-		beaconPrevJustifiedEpoch.Set(float64(state.PreviousJustifiedCheckpoint.Epoch))
-		beaconPrevJustifiedRoot.Set(float64(bytesutil.ToLowInt64(state.PreviousJustifiedCheckpoint.Root)))
-	}
-	// Last finalized slot
-	if state.FinalizedCheckpoint != nil {
-		beaconFinalizedEpoch.Set(float64(state.FinalizedCheckpoint.Epoch))
-		beaconFinalizedRoot.Set(float64(bytesutil.ToLowInt64(state.FinalizedCheckpoint.Root)))
-	}
-	if state.Eth1Data != nil {
-		currentEth1DataDepositCount.Set(float64(state.Eth1Data.DepositCount))
-	}
-
-	if precompute.Balances != nil {
-		totalEligibleBalances.Set(float64(precompute.Balances.PrevEpoch))
-		totalVotedTargetBalances.Set(float64(precompute.Balances.PrevEpochTargetAttesters))
-	}
-}

beacon-chain/blockchain/forkchoice/process_attestation.go

@@ -1,309 +0,0 @@
-package forkchoice
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"github.com/gogo/protobuf/proto"
-	"github.com/pkg/errors"
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/sirupsen/logrus"
-	"go.opencensus.io/trace"
-)
-
-// ErrTargetRootNotInDB returns when the target block root of an attestation cannot be found in the
-// beacon database.
-var ErrTargetRootNotInDB = errors.New("target root does not exist in db")
-
-// OnAttestation is called whenever an attestation is received, it updates validators latest vote,
-// as well as the fork choice store struct.
-//
-// Spec pseudocode definition:
-//   def on_attestation(store: Store, attestation: Attestation) -> None:
-//    """
-//    Run ``on_attestation`` upon receiving a new ``attestation`` from either within a block or directly on the wire.
-//
-//    An ``attestation`` that is asserted as invalid may be valid at a later time,
-//    consider scheduling it for later processing in such case.
-//    """
-//    target = attestation.data.target
-//
-//    # Attestations must be from the current or previous epoch
-//    current_epoch = compute_epoch_at_slot(get_current_slot(store))
-//    # Use GENESIS_EPOCH for previous when genesis to avoid underflow
-//    previous_epoch = current_epoch - 1 if current_epoch > GENESIS_EPOCH else GENESIS_EPOCH
-//    assert target.epoch in [current_epoch, previous_epoch]
-//    assert target.epoch == compute_epoch_at_slot(attestation.data.slot)
-//
-//    # Attestations target be for a known block. If target block is unknown, delay consideration until the block is found
-//    assert target.root in store.blocks
-//    # Attestations cannot be from future epochs. If they are, delay consideration until the epoch arrives
-//    base_state = store.block_states[target.root].copy()
-//    assert store.time >= base_state.genesis_time + compute_start_slot_at_epoch(target.epoch) * SECONDS_PER_SLOT
-//
-//    # Attestations must be for a known block. If block is unknown, delay consideration until the block is found
-//    assert attestation.data.beacon_block_root in store.blocks
-//    # Attestations must not be for blocks in the future. If not, the attestation should not be considered
-//    assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot
-//
-//    # Store target checkpoint state if not yet seen
-//    if target not in store.checkpoint_states:
-//        process_slots(base_state, compute_start_slot_at_epoch(target.epoch))
-//        store.checkpoint_states[target] = base_state
-//    target_state = store.checkpoint_states[target]
-//
-//    # Attestations can only affect the fork choice of subsequent slots.
-//    # Delay consideration in the fork choice until their slot is in the past.
-//    assert store.time >= (attestation.data.slot + 1) * SECONDS_PER_SLOT
-//
-//    # Get state at the `target` to validate attestation and calculate the committees
-//    indexed_attestation = get_indexed_attestation(target_state, attestation)
-//    assert is_valid_indexed_attestation(target_state, indexed_attestation)
-//
-//    # Update latest messages
-//    for i in indexed_attestation.attesting_indices:
-//        if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch:
-//            store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=attestation.data.beacon_block_root)
-func (s *Store) OnAttestation(ctx context.Context, a *ethpb.Attestation) error {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.onAttestation")
-	defer span.End()
-
-	tgt := proto.Clone(a.Data.Target).(*ethpb.Checkpoint)
-	tgtSlot := helpers.StartSlot(tgt.Epoch)
-
-	if helpers.SlotToEpoch(a.Data.Slot) != a.Data.Target.Epoch {
-		return fmt.Errorf("data slot is not in the same epoch as target %d != %d", helpers.SlotToEpoch(a.Data.Slot), a.Data.Target.Epoch)
-	}
-
-	// Verify beacon node has seen the target block before.
-	if !s.db.HasBlock(ctx, bytesutil.ToBytes32(tgt.Root)) {
-		return ErrTargetRootNotInDB
-	}
-
-	// Verify attestation target has had a valid pre state produced by the target block.
-	baseState, err := s.verifyAttPreState(ctx, tgt)
-	if err != nil {
-		return err
-	}
-
-	// Verify attestation target is from current epoch or previous epoch.
-	if err := s.verifyAttTargetEpoch(ctx, baseState.GenesisTime, uint64(time.Now().Unix()), tgt); err != nil {
-		return err
-	}
-
-	// Verify Attestations cannot be from future epochs.
-	if err := helpers.VerifySlotTime(baseState.GenesisTime, tgtSlot); err != nil {
-		return errors.Wrap(err, "could not verify attestation target slot")
-	}
-
-	// Verify attestation beacon block is known and not from the future.
-	if err := s.verifyBeaconBlock(ctx, a.Data); err != nil {
-		return errors.Wrap(err, "could not verify attestation beacon block")
-	}
-
-	// Store target checkpoint state if not yet seen.
-	baseState, err = s.saveCheckpointState(ctx, baseState, tgt)
-	if err != nil {
-		return err
-	}
-
-	// Verify attestations can only affect the fork choice of subsequent slots.
-	if err := helpers.VerifySlotTime(baseState.GenesisTime, a.Data.Slot+1); err != nil {
-		return err
-	}
-
-	// Use the target state to to validate attestation and calculate the committees.
-	indexedAtt, err := s.verifyAttestation(ctx, baseState, a)
-	if err != nil {
-		return err
-	}
-
-	// Update every validator's latest vote.
-	if err := s.updateAttVotes(ctx, indexedAtt, tgt.Root, tgt.Epoch); err != nil {
-		return err
-	}
-
-	if err := s.db.SaveAttestation(ctx, a); err != nil {
-		return err
-	}
-
-	log := log.WithFields(logrus.Fields{
-		"Slot":               a.Data.Slot,
-		"Index":              a.Data.CommitteeIndex,
-		"AggregatedBitfield": fmt.Sprintf("%08b", a.AggregationBits),
-		"BeaconBlockRoot":    fmt.Sprintf("%#x", bytesutil.Trunc(a.Data.BeaconBlockRoot)),
-	})
-	log.Debug("Updated latest votes")
-
-	return nil
-}
-
-// verifyAttPreState validates input attested check point has a valid pre-state.
-func (s *Store) verifyAttPreState(ctx context.Context, c *ethpb.Checkpoint) (*pb.BeaconState, error) {
-	baseState, err := s.db.State(ctx, bytesutil.ToBytes32(c.Root))
-	if err != nil {
-		return nil, errors.Wrapf(err, "could not get pre state for slot %d", helpers.StartSlot(c.Epoch))
-	}
-	if baseState == nil {
-		return nil, fmt.Errorf("pre state of target block %d does not exist", helpers.StartSlot(c.Epoch))
-	}
-	return baseState, nil
-}
-
-// verifyAttTargetEpoch validates attestation is from the current or previous epoch.
-func (s *Store) verifyAttTargetEpoch(ctx context.Context, genesisTime uint64, nowTime uint64, c *ethpb.Checkpoint) error {
-	currentSlot := (nowTime - genesisTime) / params.BeaconConfig().SecondsPerSlot
-	currentEpoch := helpers.SlotToEpoch(currentSlot)
-	var prevEpoch uint64
-	// Prevents previous epoch under flow
-	if currentEpoch > 1 {
-		prevEpoch = currentEpoch - 1
-	}
-	if c.Epoch != prevEpoch && c.Epoch != currentEpoch {
-		return fmt.Errorf("target epoch %d does not match current epoch %d or prev epoch %d", c.Epoch, currentEpoch, prevEpoch)
-	}
-	return nil
-}
-
-// verifyBeaconBlock verifies beacon head block is known and not from the future.
-func (s *Store) verifyBeaconBlock(ctx context.Context, data *ethpb.AttestationData) error {
-	b, err := s.db.Block(ctx, bytesutil.ToBytes32(data.BeaconBlockRoot))
-	if err != nil {
-		return err
-	}
-	if b == nil || b.Block == nil {
-		return fmt.Errorf("beacon block %#x does not exist", bytesutil.Trunc(data.BeaconBlockRoot))
-	}
-	if b.Block.Slot > data.Slot {
-		return fmt.Errorf("could not process attestation for future block, %d > %d", b.Block.Slot, data.Slot)
-	}
-	return nil
-}
-
-// saveCheckpointState saves and returns the processed state with the associated check point.
-func (s *Store) saveCheckpointState(ctx context.Context, baseState *pb.BeaconState, c *ethpb.Checkpoint) (*pb.BeaconState, error) {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.saveCheckpointState")
-	defer span.End()
-
-	s.checkpointStateLock.Lock()
-	defer s.checkpointStateLock.Unlock()
-	cachedState, err := s.checkpointState.StateByCheckpoint(c)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not get cached checkpoint state")
-	}
-	if cachedState != nil {
-		return cachedState, nil
-	}
-
-	// Advance slots only when it's higher than current state slot.
-	if helpers.StartSlot(c.Epoch) > baseState.Slot {
-		stateCopy := proto.Clone(baseState).(*pb.BeaconState)
-		stateCopy, err = state.ProcessSlots(ctx, stateCopy, helpers.StartSlot(c.Epoch))
-		if err != nil {
-			return nil, errors.Wrapf(err, "could not process slots up to %d", helpers.StartSlot(c.Epoch))
-		}
-
-		if err := s.checkpointState.AddCheckpointState(&cache.CheckpointState{
-			Checkpoint: c,
-			State:      stateCopy,
-		}); err != nil {
-			return nil, errors.Wrap(err, "could not saved checkpoint state to cache")
-		}
-
-		return stateCopy, nil
-	}
-
-	return baseState, nil
-}
-
-// verifyAttestation validates input attestation is valid.
-func (s *Store) verifyAttestation(ctx context.Context, baseState *pb.BeaconState, a *ethpb.Attestation) (*ethpb.IndexedAttestation, error) {
-	committee, err := helpers.BeaconCommitteeFromState(baseState, a.Data.Slot, a.Data.CommitteeIndex)
-	if err != nil {
-		return nil, err
-	}
-	indexedAtt, err := blocks.ConvertToIndexed(ctx, a, committee)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not convert attestation to indexed attestation")
-	}
-
-	if err := blocks.VerifyIndexedAttestation(ctx, baseState, indexedAtt); err != nil {
-
-		// TODO(3603): Delete the following signature verify fallback when issue 3603 closes.
-		// When signature fails to verify with committee cache enabled at run time,
-		// the following re-runs the same signature verify routine without cache in play.
-		// This provides extra assurance that committee cache can't break run time.
-		if err == blocks.ErrSigFailedToVerify {
-			committee, err = helpers.BeaconCommitteeWithoutCache(baseState, a.Data.Slot, a.Data.CommitteeIndex)
-			if err != nil {
-				return nil, errors.Wrap(err, "could not convert attestation to indexed attestation without cache")
-			}
-			indexedAtt, err = blocks.ConvertToIndexed(ctx, a, committee)
-			if err != nil {
-				return nil, errors.Wrap(err, "could not convert attestation to indexed attestation")
-			}
-			if err := blocks.VerifyIndexedAttestation(ctx, baseState, indexedAtt); err != nil {
-				return nil, errors.Wrap(err, "could not verify indexed attestation without cache")
-			}
-			sigFailsToVerify.Inc()
-			return indexedAtt, nil
-		}
-
-		return nil, errors.Wrap(err, "could not verify indexed attestation")
-	}
-	return indexedAtt, nil
-}
-
-// updateAttVotes updates validator's latest votes based on the incoming attestation.
-func (s *Store) updateAttVotes(
-	ctx context.Context,
-	indexedAtt *ethpb.IndexedAttestation,
-	tgtRoot []byte,
-	tgtEpoch uint64) error {
-
-	indices := indexedAtt.AttestingIndices
-	s.voteLock.Lock()
-	defer s.voteLock.Unlock()
-	for _, i := range indices {
-		vote, ok := s.latestVoteMap[i]
-		if !ok || tgtEpoch > vote.Epoch {
-			s.latestVoteMap[i] = &pb.ValidatorLatestVote{
-				Epoch: tgtEpoch,
-				Root:  tgtRoot,
-			}
-		}
-	}
-	return nil
-}
-
-// aggregatedAttestation returns the aggregated attestation after checking saved one in db.
-func (s *Store) aggregatedAttestations(ctx context.Context, att *ethpb.Attestation) ([]*ethpb.Attestation, error) {
-	r, err := ssz.HashTreeRoot(att.Data)
-	if err != nil {
-		return nil, err
-	}
-	saved, err := s.db.AttestationsByDataRoot(ctx, r)
-	if err != nil {
-		return nil, err
-	}
-
-	if saved == nil {
-		return []*ethpb.Attestation{att}, nil
-	}
-
-	aggregated, err := helpers.AggregateAttestations(append(saved, att))
-	if err != nil {
-		return nil, err
-	}
-
-	return aggregated, nil
-}

beacon-chain/blockchain/forkchoice/process_attestation_test.go

@@ -1,369 +0,0 @@
-package forkchoice
-
-import (
-	"context"
-	"reflect"
-	"strings"
-	"testing"
-
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-bitfield"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
-	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/prysmaticlabs/prysm/shared/testutil"
-)
-
-func TestStore_OnAttestation(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	_, err := blockTree1(db, []byte{'g'})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	BlkWithOutState := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 0}}
-	if err := db.SaveBlock(ctx, BlkWithOutState); err != nil {
-		t.Fatal(err)
-	}
-	BlkWithOutStateRoot, _ := ssz.HashTreeRoot(BlkWithOutState.Block)
-
-	BlkWithStateBadAtt := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1}}
-	if err := db.SaveBlock(ctx, BlkWithStateBadAtt); err != nil {
-		t.Fatal(err)
-	}
-	BlkWithStateBadAttRoot, _ := ssz.HashTreeRoot(BlkWithStateBadAtt.Block)
-	if err := store.db.SaveState(ctx, &pb.BeaconState{}, BlkWithStateBadAttRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	BlkWithValidState := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 2}}
-	if err := db.SaveBlock(ctx, BlkWithValidState); err != nil {
-		t.Fatal(err)
-	}
-	BlkWithValidStateRoot, _ := ssz.HashTreeRoot(BlkWithValidState.Block)
-	if err := store.db.SaveState(ctx, &pb.BeaconState{
-		Fork: &pb.Fork{
-			Epoch:           0,
-			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
-			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
-		},
-		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
-	}, BlkWithValidStateRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	tests := []struct {
-		name          string
-		a             *ethpb.Attestation
-		s             *pb.BeaconState
-		wantErr       bool
-		wantErrString string
-	}{
-		{
-			name:          "attestation's data slot not aligned with target vote",
-			a:             &ethpb.Attestation{Data: &ethpb.AttestationData{Slot: params.BeaconConfig().SlotsPerEpoch, Target: &ethpb.Checkpoint{}}},
-			s:             &pb.BeaconState{},
-			wantErr:       true,
-			wantErrString: "data slot is not in the same epoch as target 1 != 0",
-		},
-		{
-			name:          "attestation's target root not in db",
-			a:             &ethpb.Attestation{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{Root: []byte{'A'}}}},
-			s:             &pb.BeaconState{},
-			wantErr:       true,
-			wantErrString: "target root does not exist in db",
-		},
-		{
-			name:          "no pre state for attestations's target block",
-			a:             &ethpb.Attestation{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{Root: BlkWithOutStateRoot[:]}}},
-			s:             &pb.BeaconState{},
-			wantErr:       true,
-			wantErrString: "pre state of target block 0 does not exist",
-		},
-		{
-			name: "process attestation doesn't match current epoch",
-			a: &ethpb.Attestation{Data: &ethpb.AttestationData{Slot: 100 * params.BeaconConfig().SlotsPerEpoch, Target: &ethpb.Checkpoint{Epoch: 100,
-				Root: BlkWithStateBadAttRoot[:]}}},
-			s:             &pb.BeaconState{},
-			wantErr:       true,
-			wantErrString: "does not match current epoch",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if err := store.GenesisStore(
-				ctx,
-				&ethpb.Checkpoint{Root: BlkWithValidStateRoot[:]},
-				&ethpb.Checkpoint{Root: BlkWithValidStateRoot[:]}); err != nil {
-				t.Fatal(err)
-			}
-
-			err := store.OnAttestation(ctx, tt.a)
-			if tt.wantErr {
-				if !strings.Contains(err.Error(), tt.wantErrString) {
-					t.Errorf("Store.OnAttestation() error = %v, wantErr = %v", err, tt.wantErrString)
-				}
-			} else {
-				t.Error(err)
-			}
-		})
-	}
-}
-
-func TestStore_SaveCheckpointState(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	params.UseDemoBeaconConfig()
-
-	store := NewForkChoiceService(ctx, db)
-
-	s := &pb.BeaconState{
-		Fork: &pb.Fork{
-			Epoch:           0,
-			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
-			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
-		},
-		RandaoMixes:         make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
-		StateRoots:          make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
-		BlockRoots:          make([][]byte, params.BeaconConfig().SlotsPerHistoricalRoot),
-		LatestBlockHeader:   &ethpb.BeaconBlockHeader{},
-		JustificationBits:   []byte{0},
-		Slashings:           make([]uint64, params.BeaconConfig().EpochsPerSlashingsVector),
-		FinalizedCheckpoint: &ethpb.Checkpoint{},
-	}
-	r := [32]byte{'g'}
-	if err := store.db.SaveState(ctx, s, r); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.GenesisStore(ctx, &ethpb.Checkpoint{Root: r[:]}, &ethpb.Checkpoint{Root: r[:]}); err != nil {
-		t.Fatal(err)
-	}
-
-	cp1 := &ethpb.Checkpoint{Epoch: 1, Root: []byte{'A'}}
-	s1, err := store.saveCheckpointState(ctx, s, cp1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if s1.Slot != 1*params.BeaconConfig().SlotsPerEpoch {
-		t.Errorf("Wanted state slot: %d, got: %d", 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot)
-	}
-
-	cp2 := &ethpb.Checkpoint{Epoch: 2, Root: []byte{'B'}}
-	s2, err := store.saveCheckpointState(ctx, s, cp2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if s2.Slot != 2*params.BeaconConfig().SlotsPerEpoch {
-		t.Errorf("Wanted state slot: %d, got: %d", 2*params.BeaconConfig().SlotsPerEpoch, s2.Slot)
-	}
-
-	s1, err = store.saveCheckpointState(ctx, nil, cp1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if s1.Slot != 1*params.BeaconConfig().SlotsPerEpoch {
-		t.Errorf("Wanted state slot: %d, got: %d", 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot)
-	}
-
-	s1, err = store.checkpointState.StateByCheckpoint(cp1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if s1.Slot != 1*params.BeaconConfig().SlotsPerEpoch {
-		t.Errorf("Wanted state slot: %d, got: %d", 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot)
-	}
-
-	s2, err = store.checkpointState.StateByCheckpoint(cp2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if s2.Slot != 2*params.BeaconConfig().SlotsPerEpoch {
-		t.Errorf("Wanted state slot: %d, got: %d", 2*params.BeaconConfig().SlotsPerEpoch, s2.Slot)
-	}
-
-	s.Slot = params.BeaconConfig().SlotsPerEpoch + 1
-	if err := store.GenesisStore(ctx, &ethpb.Checkpoint{Root: r[:]}, &ethpb.Checkpoint{Root: r[:]}); err != nil {
-		t.Fatal(err)
-	}
-	cp3 := &ethpb.Checkpoint{Epoch: 1, Root: []byte{'C'}}
-	s3, err := store.saveCheckpointState(ctx, s, cp3)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if s3.Slot != s.Slot {
-		t.Errorf("Wanted state slot: %d, got: %d", s.Slot, s3.Slot)
-	}
-}
-
-func TestStore_ReturnAggregatedAttestation(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	a1 := &ethpb.Attestation{Data: &ethpb.AttestationData{}, AggregationBits: bitfield.Bitlist{0x02}}
-	err := store.db.SaveAttestation(ctx, a1)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	a2 := &ethpb.Attestation{Data: &ethpb.AttestationData{}, AggregationBits: bitfield.Bitlist{0x03}}
-	saved, err := store.aggregatedAttestations(ctx, a2)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if !reflect.DeepEqual([]*ethpb.Attestation{a2}, saved) {
-		t.Error("did not retrieve saved attestation")
-	}
-}
-
-func TestStore_UpdateCheckpointState(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	epoch := uint64(1)
-	baseState, _ := testutil.DeterministicGenesisState(t, 1)
-	baseState.Slot = epoch * params.BeaconConfig().SlotsPerEpoch
-	checkpoint := &ethpb.Checkpoint{Epoch: epoch}
-	returned, err := store.saveCheckpointState(ctx, baseState, checkpoint)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(baseState, returned) {
-		t.Error("Incorrectly returned base state")
-	}
-
-	cached, err := store.checkpointState.StateByCheckpoint(checkpoint)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if cached != nil {
-		t.Error("State shouldn't have been cached")
-	}
-
-	epoch = uint64(2)
-	newCheckpoint := &ethpb.Checkpoint{Epoch: epoch}
-	returned, err = store.saveCheckpointState(ctx, baseState, newCheckpoint)
-	if err != nil {
-		t.Fatal(err)
-	}
-	baseState, err = state.ProcessSlots(ctx, baseState, helpers.StartSlot(newCheckpoint.Epoch))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(baseState, returned) {
-		t.Error("Incorrectly returned base state")
-	}
-
-	cached, err = store.checkpointState.StateByCheckpoint(newCheckpoint)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(returned, cached) {
-		t.Error("Incorrectly cached base state")
-	}
-}
-
-func TestAttEpoch_MatchPrevEpoch(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	if err := store.verifyAttTargetEpoch(
-		ctx,
-		0,
-		params.BeaconConfig().SlotsPerEpoch*params.BeaconConfig().SecondsPerSlot,
-		&ethpb.Checkpoint{}); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestAttEpoch_MatchCurrentEpoch(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	if err := store.verifyAttTargetEpoch(
-		ctx,
-		0,
-		params.BeaconConfig().SlotsPerEpoch*params.BeaconConfig().SecondsPerSlot,
-		&ethpb.Checkpoint{Epoch: 1}); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestAttEpoch_NotMatch(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	if err := store.verifyAttTargetEpoch(
-		ctx,
-		0,
-		2*params.BeaconConfig().SlotsPerEpoch*params.BeaconConfig().SecondsPerSlot,
-		&ethpb.Checkpoint{}); !strings.Contains(err.Error(),
-		"target epoch 0 does not match current epoch 2 or prev epoch 1") {
-		t.Error("Did not receive wanted error")
-	}
-}
-
-func TestVerifyBeaconBlock_NoBlock(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	s := NewForkChoiceService(ctx, db)
-	d := &ethpb.AttestationData{}
-	if err := s.verifyBeaconBlock(ctx, d); !strings.Contains(err.Error(), "beacon block  does not exist") {
-		t.Error("Did not receive the wanted error")
-	}
-}
-
-func TestVerifyBeaconBlock_futureBlock(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	s := NewForkChoiceService(ctx, db)
-	b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 2}}
-	s.db.SaveBlock(ctx, b)
-	r, _ := ssz.HashTreeRoot(b.Block)
-	d := &ethpb.AttestationData{Slot: 1, BeaconBlockRoot: r[:]}
-
-	if err := s.verifyBeaconBlock(ctx, d); !strings.Contains(err.Error(), "could not process attestation for future block") {
-		t.Error("Did not receive the wanted error")
-	}
-}
-
-func TestVerifyBeaconBlock_OK(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	s := NewForkChoiceService(ctx, db)
-	b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 2}}
-	s.db.SaveBlock(ctx, b)
-	r, _ := ssz.HashTreeRoot(b.Block)
-	d := &ethpb.AttestationData{Slot: 2, BeaconBlockRoot: r[:]}
-
-	if err := s.verifyBeaconBlock(ctx, d); err != nil {
-		t.Error("Did not receive the wanted error")
-	}
-}

beacon-chain/blockchain/forkchoice/process_block.go

@@ -1,572 +0,0 @@
-package forkchoice
-
-import (
-	"bytes"
-	"context"
-	"encoding/hex"
-	"fmt"
-	"time"
-
-	"github.com/gogo/protobuf/proto"
-	"github.com/pkg/errors"
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
-	"github.com/prysmaticlabs/prysm/beacon-chain/flags"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/prysmaticlabs/prysm/shared/traceutil"
-	"github.com/sirupsen/logrus"
-	"go.opencensus.io/trace"
-)
-
-// OnBlock is called when a gossip block is received. It runs regular state transition on the block and
-// update fork choice store.
-//
-// Spec pseudocode definition:
-//   def on_block(store: Store, block: BeaconBlock) -> None:
-//    # Make a copy of the state to avoid mutability issues
-//    assert block.parent_root in store.block_states
-//    pre_state = store.block_states[block.parent_root].copy()
-//    # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past.
-//    assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT
-//    # Add new block to the store
-//    store.blocks[signing_root(block)] = block
-//    # Check block is a descendant of the finalized block
-//    assert (
-//        get_ancestor(store, signing_root(block), store.blocks[store.finalized_checkpoint.root].slot) ==
-//        store.finalized_checkpoint.root
-//    )
-//    # Check that block is later than the finalized epoch slot
-//    assert block.slot > compute_start_slot_of_epoch(store.finalized_checkpoint.epoch)
-//    # Check the block is valid and compute the post-state
-//    state = state_transition(pre_state, block)
-//    # Add new state for this block to the store
-//    store.block_states[signing_root(block)] = state
-//
-//    # Update justified checkpoint
-//    if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch:
-//        if state.current_justified_checkpoint.epoch > store.best_justified_checkpoint.epoch:
-//            store.best_justified_checkpoint = state.current_justified_checkpoint
-//
-//    # Update finalized checkpoint
-//    if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch:
-//        store.finalized_checkpoint = state.finalized_checkpoint
-func (s *Store) OnBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock) error {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.onBlock")
-	defer span.End()
-
-	if signed == nil || signed.Block == nil {
-		return errors.New("nil block")
-	}
-
-	b := signed.Block
-
-	// Retrieve incoming block's pre state.
-	preState, err := s.getBlockPreState(ctx, b)
-	if err != nil {
-		return err
-	}
-	preStateValidatorCount := len(preState.Validators)
-
-	root, err := ssz.HashTreeRoot(b)
-	if err != nil {
-		return errors.Wrapf(err, "could not get signing root of block %d", b.Slot)
-	}
-	log.WithFields(logrus.Fields{
-		"slot": b.Slot,
-		"root": fmt.Sprintf("0x%s...", hex.EncodeToString(root[:])[:8]),
-	}).Info("Executing state transition on block")
-	postState, err := state.ExecuteStateTransition(ctx, preState, signed)
-	if err != nil {
-		return errors.Wrap(err, "could not execute state transition")
-	}
-
-	if err := s.db.SaveBlock(ctx, signed); err != nil {
-		return errors.Wrapf(err, "could not save block from slot %d", b.Slot)
-	}
-	if err := s.db.SaveState(ctx, postState, root); err != nil {
-		return errors.Wrap(err, "could not save state")
-	}
-
-	// Update justified check point.
-	if postState.CurrentJustifiedCheckpoint.Epoch > s.justifiedCheckpt.Epoch {
-		if err := s.updateJustified(ctx, postState); err != nil {
-			return err
-		}
-	}
-
-	// Update finalized check point.
-	// Prune the block cache and helper caches on every new finalized epoch.
-	if postState.FinalizedCheckpoint.Epoch > s.finalizedCheckpt.Epoch {
-		if err := s.db.SaveFinalizedCheckpoint(ctx, postState.FinalizedCheckpoint); err != nil {
-			return errors.Wrap(err, "could not save finalized checkpoint")
-		}
-
-		startSlot := helpers.StartSlot(s.prevFinalizedCheckpt.Epoch)
-		endSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
-		if endSlot > startSlot {
-			if err := s.rmStatesOlderThanLastFinalized(ctx, startSlot, endSlot); err != nil {
-				return errors.Wrapf(err, "could not delete states prior to finalized check point, range: %d, %d",
-					startSlot, endSlot)
-			}
-		}
-
-		s.prevFinalizedCheckpt = s.finalizedCheckpt
-		s.finalizedCheckpt = postState.FinalizedCheckpoint
-	}
-
-	// Update validator indices in database as needed.
-	if err := s.saveNewValidators(ctx, preStateValidatorCount, postState); err != nil {
-		return errors.Wrap(err, "could not save finalized checkpoint")
-	}
-	// Save the unseen attestations from block to db.
-	if err := s.saveNewBlockAttestations(ctx, b.Body.Attestations); err != nil {
-		return errors.Wrap(err, "could not save attestations")
-	}
-
-	// Epoch boundary bookkeeping such as logging epoch summaries.
-	if postState.Slot >= s.nextEpochBoundarySlot {
-		logEpochData(postState)
-		reportEpochMetrics(postState)
-
-		// Update committees cache at epoch boundary slot.
-		if featureconfig.Get().EnableNewCache {
-			if err := helpers.UpdateCommitteeCache(postState, helpers.CurrentEpoch(postState)); err != nil {
-				return err
-			}
-		}
-
-		s.nextEpochBoundarySlot = helpers.StartSlot(helpers.NextEpoch(postState))
-	}
-
-	return nil
-}
-
-// OnBlockInitialSyncStateTransition is called when an initial sync block is received.
-// It runs state transition on the block and without any BLS verification. The BLS verification
-// includes proposer signature, randao and attestation's aggregated signature. It also does not save
-// attestations.
-func (s *Store) OnBlockInitialSyncStateTransition(ctx context.Context, signed *ethpb.SignedBeaconBlock) error {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.onBlock")
-	defer span.End()
-
-	if signed == nil || signed.Block == nil {
-		return errors.New("nil block")
-	}
-
-	b := signed.Block
-
-	s.initSyncStateLock.Lock()
-	defer s.initSyncStateLock.Unlock()
-
-	// Retrieve incoming block's pre state.
-	preState, err := s.cachedPreState(ctx, b)
-	if err != nil {
-		return err
-	}
-	preStateValidatorCount := len(preState.Validators)
-
-	log.WithField("slot", b.Slot).Debug("Executing state transition on block")
-
-	postState, err := state.ExecuteStateTransitionNoVerify(ctx, preState, signed)
-	if err != nil {
-		return errors.Wrap(err, "could not execute state transition")
-	}
-
-	if err := s.db.SaveBlock(ctx, signed); err != nil {
-		return errors.Wrapf(err, "could not save block from slot %d", b.Slot)
-	}
-	root, err := ssz.HashTreeRoot(b)
-	if err != nil {
-		return errors.Wrapf(err, "could not get signing root of block %d", b.Slot)
-	}
-
-	if featureconfig.Get().InitSyncCacheState {
-		s.initSyncState[root] = postState
-	} else {
-		if err := s.db.SaveState(ctx, postState, root); err != nil {
-			return errors.Wrap(err, "could not save state")
-		}
-	}
-
-	// Update justified check point.
-	if postState.CurrentJustifiedCheckpoint.Epoch > s.justifiedCheckpt.Epoch {
-		if err := s.updateJustified(ctx, postState); err != nil {
-			return err
-		}
-	}
-
-	// Update finalized check point.
-	// Prune the block cache and helper caches on every new finalized epoch.
-	if postState.FinalizedCheckpoint.Epoch > s.finalizedCheckpt.Epoch {
-		startSlot := helpers.StartSlot(s.prevFinalizedCheckpt.Epoch)
-		endSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
-		if endSlot > startSlot {
-			if err := s.rmStatesOlderThanLastFinalized(ctx, startSlot, endSlot); err != nil {
-				return errors.Wrapf(err, "could not delete states prior to finalized check point, range: %d, %d",
-					startSlot, endSlot)
-			}
-		}
-
-		if err := s.saveInitState(ctx, postState); err != nil {
-			return errors.Wrap(err, "could not save init sync finalized state")
-		}
-
-		if err := s.db.SaveFinalizedCheckpoint(ctx, postState.FinalizedCheckpoint); err != nil {
-			return errors.Wrap(err, "could not save finalized checkpoint")
-		}
-
-		s.prevFinalizedCheckpt = s.finalizedCheckpt
-		s.finalizedCheckpt = postState.FinalizedCheckpoint
-	}
-
-	// Update validator indices in database as needed.
-	if err := s.saveNewValidators(ctx, preStateValidatorCount, postState); err != nil {
-		return errors.Wrap(err, "could not save finalized checkpoint")
-	}
-
-	if flags.Get().EnableArchive {
-		// Save the unseen attestations from block to db.
-		if err := s.saveNewBlockAttestations(ctx, b.Body.Attestations); err != nil {
-			return errors.Wrap(err, "could not save attestations")
-		}
-	}
-
-	// Epoch boundary bookkeeping such as logging epoch summaries.
-	if postState.Slot >= s.nextEpochBoundarySlot {
-		reportEpochMetrics(postState)
-
-		s.nextEpochBoundarySlot = helpers.StartSlot(helpers.NextEpoch(postState))
-	}
-
-	return nil
-}
-
-// getBlockPreState returns the pre state of an incoming block. It uses the parent root of the block
-// to retrieve the state in DB. It verifies the pre state's validity and the incoming block
-// is in the correct time window.
-func (s *Store) getBlockPreState(ctx context.Context, b *ethpb.BeaconBlock) (*pb.BeaconState, error) {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.getBlockPreState")
-	defer span.End()
-
-	// Verify incoming block has a valid pre state.
-	preState, err := s.verifyBlkPreState(ctx, b)
-	if err != nil {
-		return nil, err
-	}
-
-	// Verify block slot time is not from the feature.
-	if err := helpers.VerifySlotTime(preState.GenesisTime, b.Slot); err != nil {
-		return nil, err
-	}
-
-	// Verify block is a descendent of a finalized block.
-	if err := s.verifyBlkDescendant(ctx, bytesutil.ToBytes32(b.ParentRoot), b.Slot); err != nil {
-		return nil, err
-	}
-
-	// Verify block is later than the finalized epoch slot.
-	if err := s.verifyBlkFinalizedSlot(b); err != nil {
-		return nil, err
-	}
-
-	return preState, nil
-}
-
-// verifyBlkPreState validates input block has a valid pre-state.
-func (s *Store) verifyBlkPreState(ctx context.Context, b *ethpb.BeaconBlock) (*pb.BeaconState, error) {
-	preState, err := s.db.State(ctx, bytesutil.ToBytes32(b.ParentRoot))
-	if err != nil {
-		return nil, errors.Wrapf(err, "could not get pre state for slot %d", b.Slot)
-	}
-	if preState == nil {
-		return nil, fmt.Errorf("pre state of slot %d does not exist", b.Slot)
-	}
-	return preState, nil
-}
-
-// verifyBlkDescendant validates input block root is a descendant of the
-// current finalized block root.
-func (s *Store) verifyBlkDescendant(ctx context.Context, root [32]byte, slot uint64) error {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.verifyBlkDescendant")
-	defer span.End()
-
-	finalizedBlkSigned, err := s.db.Block(ctx, bytesutil.ToBytes32(s.finalizedCheckpt.Root))
-	if err != nil || finalizedBlkSigned == nil || finalizedBlkSigned.Block == nil {
-		return errors.Wrap(err, "could not get finalized block")
-	}
-	finalizedBlk := finalizedBlkSigned.Block
-
-	bFinalizedRoot, err := s.ancestor(ctx, root[:], finalizedBlk.Slot)
-	if err != nil {
-		return errors.Wrap(err, "could not get finalized block root")
-	}
-	if !bytes.Equal(bFinalizedRoot, s.finalizedCheckpt.Root) {
-		err := fmt.Errorf("block from slot %d is not a descendent of the current finalized block slot %d, %#x != %#x",
-			slot, finalizedBlk.Slot, bytesutil.Trunc(bFinalizedRoot), bytesutil.Trunc(s.finalizedCheckpt.Root))
-		traceutil.AnnotateError(span, err)
-		return err
-	}
-	return nil
-}
-
-// verifyBlkFinalizedSlot validates input block is not less than or equal
-// to current finalized slot.
-func (s *Store) verifyBlkFinalizedSlot(b *ethpb.BeaconBlock) error {
-	finalizedSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
-	if finalizedSlot >= b.Slot {
-		return fmt.Errorf("block is equal or earlier than finalized block, slot %d < slot %d", b.Slot, finalizedSlot)
-	}
-	return nil
-}
-
-// saveNewValidators saves newly added validator index from state to db. Does nothing if validator count has not
-// changed.
-func (s *Store) saveNewValidators(ctx context.Context, preStateValidatorCount int, postState *pb.BeaconState) error {
-	postStateValidatorCount := len(postState.Validators)
-	if preStateValidatorCount != postStateValidatorCount {
-		for i := preStateValidatorCount; i < postStateValidatorCount; i++ {
-			pubKey := postState.Validators[i].PublicKey
-			if err := s.db.SaveValidatorIndex(ctx, bytesutil.ToBytes48(pubKey), uint64(i)); err != nil {
-				return errors.Wrapf(err, "could not save activated validator: %d", i)
-			}
-			log.WithFields(logrus.Fields{
-				"index":               i,
-				"pubKey":              hex.EncodeToString(bytesutil.Trunc(pubKey)),
-				"totalValidatorCount": i + 1,
-			}).Info("New validator index saved in DB")
-		}
-	}
-	return nil
-}
-
-// saveNewBlockAttestations saves the new attestations in block to DB.
-func (s *Store) saveNewBlockAttestations(ctx context.Context, atts []*ethpb.Attestation) error {
-	attestations := make([]*ethpb.Attestation, 0, len(atts))
-	for _, att := range atts {
-		aggregated, err := s.aggregatedAttestations(ctx, att)
-		if err != nil {
-			continue
-		}
-		attestations = append(attestations, aggregated...)
-	}
-	if err := s.db.SaveAttestations(ctx, atts); err != nil {
-		return err
-	}
-	return nil
-}
-
-// rmStatesOlderThanLastFinalized deletes the states in db since last finalized check point.
-func (s *Store) rmStatesOlderThanLastFinalized(ctx context.Context, startSlot uint64, endSlot uint64) error {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.rmStatesBySlots")
-	defer span.End()
-
-	// Make sure start slot is not a skipped slot
-	for i := startSlot; i > 0; i-- {
-		filter := filters.NewFilter().SetStartSlot(i).SetEndSlot(i)
-		b, err := s.db.Blocks(ctx, filter)
-		if err != nil {
-			return err
-		}
-		if len(b) > 0 {
-			startSlot = i
-			break
-		}
-	}
-
-	// Make sure finalized slot is not a skipped slot.
-	for i := endSlot; i > 0; i-- {
-		filter := filters.NewFilter().SetStartSlot(i).SetEndSlot(i)
-		b, err := s.db.Blocks(ctx, filter)
-		if err != nil {
-			return err
-		}
-		if len(b) > 0 {
-			endSlot = i - 1
-			break
-		}
-	}
-
-	// Do not remove genesis state
-	if startSlot == 0 {
-		startSlot++
-	}
-	// If end slot comes less than start slot
-	if endSlot < startSlot {
-		endSlot = startSlot
-	}
-
-	filter := filters.NewFilter().SetStartSlot(startSlot).SetEndSlot(endSlot)
-	roots, err := s.db.BlockRoots(ctx, filter)
-	if err != nil {
-		return err
-	}
-
-	roots, err = s.filterBlockRoots(ctx, roots)
-	if err != nil {
-		return err
-	}
-
-	if err := s.db.DeleteStates(ctx, roots); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// shouldUpdateCurrentJustified prevents bouncing attack, by only update conflicting justified
-// checkpoints in the fork choice if in the early slots of the epoch.
-// Otherwise, delay incorporation of new justified checkpoint until next epoch boundary.
-// See https://ethresear.ch/t/prevention-of-bouncing-attack-on-ffg/6114 for more detailed analysis and discussion.
-func (s *Store) shouldUpdateCurrentJustified(ctx context.Context, newJustifiedCheckpt *ethpb.Checkpoint) (bool, error) {
-	if helpers.SlotsSinceEpochStarts(s.currentSlot()) < params.BeaconConfig().SafeSlotsToUpdateJustified {
-		return true, nil
-	}
-	newJustifiedBlockSigned, err := s.db.Block(ctx, bytesutil.ToBytes32(newJustifiedCheckpt.Root))
-	if err != nil {
-		return false, err
-	}
-	if newJustifiedBlockSigned == nil || newJustifiedBlockSigned.Block == nil {
-		return false, errors.New("nil new justified block")
-	}
-	newJustifiedBlock := newJustifiedBlockSigned.Block
-	if newJustifiedBlock.Slot <= helpers.StartSlot(s.justifiedCheckpt.Epoch) {
-		return false, nil
-	}
-	justifiedBlockSigned, err := s.db.Block(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root))
-	if err != nil {
-		return false, err
-	}
-	if justifiedBlockSigned == nil || justifiedBlockSigned.Block == nil {
-		return false, errors.New("nil justified block")
-	}
-	justifiedBlock := justifiedBlockSigned.Block
-	b, err := s.ancestor(ctx, newJustifiedCheckpt.Root, justifiedBlock.Slot)
-	if err != nil {
-		return false, err
-	}
-	if !bytes.Equal(b, s.justifiedCheckpt.Root) {
-		return false, nil
-	}
-	return true, nil
-}
-
-func (s *Store) updateJustified(ctx context.Context, state *pb.BeaconState) error {
-	if state.CurrentJustifiedCheckpoint.Epoch > s.bestJustifiedCheckpt.Epoch {
-		s.bestJustifiedCheckpt = state.CurrentJustifiedCheckpoint
-	}
-	canUpdate, err := s.shouldUpdateCurrentJustified(ctx, state.CurrentJustifiedCheckpoint)
-	if err != nil {
-		return err
-	}
-	if canUpdate {
-		s.justifiedCheckpt = state.CurrentJustifiedCheckpoint
-	}
-
-	if featureconfig.Get().InitSyncCacheState {
-		justifiedRoot := bytesutil.ToBytes32(state.CurrentJustifiedCheckpoint.Root)
-		justifiedState := s.initSyncState[justifiedRoot]
-		if err := s.db.SaveState(ctx, justifiedState, justifiedRoot); err != nil {
-			return errors.Wrap(err, "could not save justified state")
-		}
-	}
-
-	return s.db.SaveJustifiedCheckpoint(ctx, state.CurrentJustifiedCheckpoint)
-}
-
-// currentSlot returns the current slot based on time.
-func (s *Store) currentSlot() uint64 {
-	return (uint64(time.Now().Unix()) - s.genesisTime) / params.BeaconConfig().SecondsPerSlot
-}
-
-// updates justified check point in store if a better check point is known
-func (s *Store) updateJustifiedCheckpoint() {
-	// Update at epoch boundary slot only
-	if !helpers.IsEpochStart(s.currentSlot()) {
-		return
-	}
-	if s.bestJustifiedCheckpt.Epoch > s.justifiedCheckpt.Epoch {
-		s.justifiedCheckpt = s.bestJustifiedCheckpt
-	}
-}
-
-// This receives cached state in memory for initial sync only during initial sync.
-func (s *Store) cachedPreState(ctx context.Context, b *ethpb.BeaconBlock) (*pb.BeaconState, error) {
-	if featureconfig.Get().InitSyncCacheState {
-		preState := s.initSyncState[bytesutil.ToBytes32(b.ParentRoot)]
-		var err error
-		if preState == nil {
-			preState, err = s.db.State(ctx, bytesutil.ToBytes32(b.ParentRoot))
-			if err != nil {
-				return nil, errors.Wrapf(err, "could not get pre state for slot %d", b.Slot)
-			}
-			if preState == nil {
-				return nil, fmt.Errorf("pre state of slot %d does not exist", b.Slot)
-			}
-		}
-		return proto.Clone(preState).(*pb.BeaconState), nil
-	}
-
-	preState, err := s.db.State(ctx, bytesutil.ToBytes32(b.ParentRoot))
-	if err != nil {
-		return nil, errors.Wrapf(err, "could not get pre state for slot %d", b.Slot)
-	}
-	if preState == nil {
-		return nil, fmt.Errorf("pre state of slot %d does not exist", b.Slot)
-	}
-
-	return preState, nil
-}
-
-// This saves every finalized state in DB during initial sync, needed as part of optimization to
-// use cache state during initial sync in case of restart.
-func (s *Store) saveInitState(ctx context.Context, state *pb.BeaconState) error {
-	if !featureconfig.Get().InitSyncCacheState {
-		return nil
-	}
-	finalizedRoot := bytesutil.ToBytes32(state.FinalizedCheckpoint.Root)
-	fs := s.initSyncState[finalizedRoot]
-
-	if err := s.db.SaveState(ctx, fs, finalizedRoot); err != nil {
-		return errors.Wrap(err, "could not save state")
-	}
-	for r, oldState := range s.initSyncState {
-		if oldState.Slot < state.FinalizedCheckpoint.Epoch*params.BeaconConfig().SlotsPerEpoch {
-			delete(s.initSyncState, r)
-		}
-	}
-	return nil
-}
-
-// This filters block roots that are not known as head root and finalized root in DB.
-// It serves as the last line of defence before we prune states.
-func (s *Store) filterBlockRoots(ctx context.Context, roots [][32]byte) ([][32]byte, error) {
-	f, err := s.db.FinalizedCheckpoint(ctx)
-	if err != nil {
-		return nil, err
-	}
-	fRoot := f.Root
-	h, err := s.db.HeadBlock(ctx)
-	if err != nil {
-		return nil, err
-	}
-	hRoot, err := ssz.SigningRoot(h)
-	if err != nil {
-		return nil, err
-	}
-
-	filtered := make([][32]byte, 0, len(roots))
-	for _, root := range roots {
-		if bytes.Equal(root[:], fRoot[:]) || bytes.Equal(root[:], hRoot[:]) {
-			continue
-		}
-		filtered = append(filtered, root)
-	}
-
-	return filtered, nil
-}

beacon-chain/blockchain/forkchoice/process_block_test.go

@@ -1,610 +0,0 @@
-package forkchoice
-
-import (
-	"bytes"
-	"context"
-	"reflect"
-	"strings"
-	"testing"
-	"time"
-
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-bitfield"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/prysmaticlabs/prysm/shared/stateutil"
-)
-
-func TestStore_OnBlock(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	genesisStateRoot, err := stateutil.HashTreeRootState(&pb.BeaconState{})
-	if err != nil {
-		t.Error(err)
-	}
-	genesis := blocks.NewGenesisBlock(genesisStateRoot[:])
-	if err := db.SaveBlock(ctx, genesis); err != nil {
-		t.Error(err)
-	}
-	validGenesisRoot, err := ssz.HashTreeRoot(genesis.Block)
-	if err != nil {
-		t.Error(err)
-	}
-	if err := store.db.SaveState(ctx, &pb.BeaconState{}, validGenesisRoot); err != nil {
-		t.Fatal(err)
-	}
-	roots, err := blockTree1(db, validGenesisRoot[:])
-	if err != nil {
-		t.Fatal(err)
-	}
-	random := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1, ParentRoot: validGenesisRoot[:]}}
-	if err := db.SaveBlock(ctx, random); err != nil {
-		t.Error(err)
-	}
-	randomParentRoot, err := ssz.HashTreeRoot(random.Block)
-	if err != nil {
-		t.Error(err)
-	}
-	if err := store.db.SaveState(ctx, &pb.BeaconState{}, randomParentRoot); err != nil {
-		t.Fatal(err)
-	}
-	randomParentRoot2 := roots[1]
-	if err := store.db.SaveState(ctx, &pb.BeaconState{}, bytesutil.ToBytes32(randomParentRoot2)); err != nil {
-		t.Fatal(err)
-	}
-
-	tests := []struct {
-		name          string
-		blk           *ethpb.BeaconBlock
-		s             *pb.BeaconState
-		time          uint64
-		wantErrString string
-	}{
-		{
-			name:          "parent block root does not have a state",
-			blk:           &ethpb.BeaconBlock{},
-			s:             &pb.BeaconState{},
-			wantErrString: "pre state of slot 0 does not exist",
-		},
-		{
-			name:          "block is from the feature",
-			blk:           &ethpb.BeaconBlock{ParentRoot: randomParentRoot[:], Slot: params.BeaconConfig().FarFutureEpoch},
-			s:             &pb.BeaconState{},
-			wantErrString: "could not process slot from the future",
-		},
-		{
-			name:          "could not get finalized block",
-			blk:           &ethpb.BeaconBlock{ParentRoot: randomParentRoot[:]},
-			s:             &pb.BeaconState{},
-			wantErrString: "block from slot 0 is not a descendent of the current finalized block",
-		},
-		{
-			name:          "same slot as finalized block",
-			blk:           &ethpb.BeaconBlock{Slot: 0, ParentRoot: randomParentRoot2},
-			s:             &pb.BeaconState{},
-			wantErrString: "block is equal or earlier than finalized block, slot 0 < slot 0",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if err := store.GenesisStore(ctx, &ethpb.Checkpoint{Root: validGenesisRoot[:]}, &ethpb.Checkpoint{Root: validGenesisRoot[:]}); err != nil {
-				t.Fatal(err)
-			}
-			store.finalizedCheckpt.Root = roots[0]
-
-			err := store.OnBlock(ctx, &ethpb.SignedBeaconBlock{Block: tt.blk})
-			if !strings.Contains(err.Error(), tt.wantErrString) {
-				t.Errorf("Store.OnBlock() error = %v, wantErr = %v", err, tt.wantErrString)
-			}
-		})
-	}
-}
-
-func TestStore_SaveNewValidators(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	preCount := 2 // validators 0 and validators 1
-	s := &pb.BeaconState{Validators: []*ethpb.Validator{
-		{PublicKey: []byte{0}}, {PublicKey: []byte{1}},
-		{PublicKey: []byte{2}}, {PublicKey: []byte{3}},
-	}}
-	if err := store.saveNewValidators(ctx, preCount, s); err != nil {
-		t.Fatal(err)
-	}
-
-	if !db.HasValidatorIndex(ctx, bytesutil.ToBytes48([]byte{2})) {
-		t.Error("Wanted validator saved in db")
-	}
-	if !db.HasValidatorIndex(ctx, bytesutil.ToBytes48([]byte{3})) {
-		t.Error("Wanted validator saved in db")
-	}
-	if db.HasValidatorIndex(ctx, bytesutil.ToBytes48([]byte{1})) {
-		t.Error("validator not suppose to be saved in db")
-	}
-}
-
-func TestStore_SavesNewBlockAttestations(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	a1 := &ethpb.Attestation{Data: &ethpb.AttestationData{}, AggregationBits: bitfield.Bitlist{0b101}}
-	a2 := &ethpb.Attestation{Data: &ethpb.AttestationData{BeaconBlockRoot: []byte{'A'}}, AggregationBits: bitfield.Bitlist{0b110}}
-	r1, _ := ssz.HashTreeRoot(a1.Data)
-	r2, _ := ssz.HashTreeRoot(a2.Data)
-
-	if err := store.saveNewBlockAttestations(ctx, []*ethpb.Attestation{a1, a2}); err != nil {
-		t.Fatal(err)
-	}
-
-	saved, err := store.db.AttestationsByDataRoot(ctx, r1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual([]*ethpb.Attestation{a1}, saved) {
-		t.Error("did not retrieve saved attestation")
-	}
-
-	saved, err = store.db.AttestationsByDataRoot(ctx, r2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual([]*ethpb.Attestation{a2}, saved) {
-		t.Error("did not retrieve saved attestation")
-	}
-
-	a1 = &ethpb.Attestation{Data: &ethpb.AttestationData{}, AggregationBits: bitfield.Bitlist{0b111}}
-	a2 = &ethpb.Attestation{Data: &ethpb.AttestationData{BeaconBlockRoot: []byte{'A'}}, AggregationBits: bitfield.Bitlist{0b111}}
-
-	if err := store.saveNewBlockAttestations(ctx, []*ethpb.Attestation{a1, a2}); err != nil {
-		t.Fatal(err)
-	}
-
-	saved, err = store.db.AttestationsByDataRoot(ctx, r1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual([]*ethpb.Attestation{a1}, saved) {
-		t.Error("did not retrieve saved attestation")
-	}
-
-	saved, err = store.db.AttestationsByDataRoot(ctx, r2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual([]*ethpb.Attestation{a2}, saved) {
-		t.Error("did not retrieve saved attestation")
-	}
-}
-
-func TestRemoveStateSinceLastFinalized(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	params.UseMinimalConfig()
-	defer params.UseMainnetConfig()
-
-	store := NewForkChoiceService(ctx, db)
-
-	// Save 100 blocks in DB, each has a state.
-	numBlocks := 100
-	totalBlocks := make([]*ethpb.SignedBeaconBlock, numBlocks)
-	blockRoots := make([][32]byte, 0)
-	for i := 0; i < len(totalBlocks); i++ {
-		totalBlocks[i] = &ethpb.SignedBeaconBlock{
-			Block: &ethpb.BeaconBlock{
-				Slot: uint64(i),
-			},
-		}
-		r, err := ssz.HashTreeRoot(totalBlocks[i].Block)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if err := store.db.SaveState(ctx, &pb.BeaconState{Slot: uint64(i)}, r); err != nil {
-			t.Fatal(err)
-		}
-		if err := store.db.SaveBlock(ctx, totalBlocks[i]); err != nil {
-			t.Fatal(err)
-		}
-		blockRoots = append(blockRoots, r)
-		if err := store.db.SaveHeadBlockRoot(ctx, r); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// New finalized epoch: 1
-	finalizedEpoch := uint64(1)
-	finalizedSlot := finalizedEpoch * params.BeaconConfig().SlotsPerEpoch
-	endSlot := helpers.StartSlot(finalizedEpoch+1) - 1 // Inclusive
-	if err := store.rmStatesOlderThanLastFinalized(ctx, 0, endSlot); err != nil {
-		t.Fatal(err)
-	}
-	for _, r := range blockRoots {
-		s, err := store.db.State(ctx, r)
-		if err != nil {
-			t.Fatal(err)
-		}
-		// Also verifies genesis state didnt get deleted
-		if s != nil && s.Slot != finalizedSlot && s.Slot != 0 && s.Slot < endSlot {
-			t.Errorf("State with slot %d should not be in DB", s.Slot)
-		}
-	}
-
-	// New finalized epoch: 5
-	newFinalizedEpoch := uint64(5)
-	newFinalizedSlot := newFinalizedEpoch * params.BeaconConfig().SlotsPerEpoch
-	endSlot = helpers.StartSlot(newFinalizedEpoch+1) - 1 // Inclusive
-	if err := store.rmStatesOlderThanLastFinalized(ctx, helpers.StartSlot(finalizedEpoch+1)-1, endSlot); err != nil {
-		t.Fatal(err)
-	}
-	for _, r := range blockRoots {
-		s, err := store.db.State(ctx, r)
-		if err != nil {
-			t.Fatal(err)
-		}
-		// Also verifies genesis state didnt get deleted
-		if s != nil && s.Slot != newFinalizedSlot && s.Slot != finalizedSlot && s.Slot != 0 && s.Slot < endSlot {
-			t.Errorf("State with slot %d should not be in DB", s.Slot)
-		}
-	}
-}
-
-func TestRemoveStateSinceLastFinalized_EmptyStartSlot(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	params.UseMinimalConfig()
-	defer params.UseMainnetConfig()
-
-	store := NewForkChoiceService(ctx, db)
-	store.genesisTime = uint64(time.Now().Unix())
-
-	update, err := store.shouldUpdateCurrentJustified(ctx, &ethpb.Checkpoint{})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !update {
-		t.Error("Should be able to update justified, received false")
-	}
-
-	lastJustifiedBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{ParentRoot: []byte{'G'}}}
-	lastJustifiedRoot, _ := ssz.HashTreeRoot(lastJustifiedBlk.Block)
-	newJustifiedBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1, ParentRoot: lastJustifiedRoot[:]}}
-	newJustifiedRoot, _ := ssz.HashTreeRoot(newJustifiedBlk.Block)
-	if err := store.db.SaveBlock(ctx, newJustifiedBlk); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveBlock(ctx, lastJustifiedBlk); err != nil {
-		t.Fatal(err)
-	}
-
-	diff := (params.BeaconConfig().SlotsPerEpoch - 1) * params.BeaconConfig().SecondsPerSlot
-	store.genesisTime = uint64(time.Now().Unix()) - diff
-	store.justifiedCheckpt = &ethpb.Checkpoint{Root: lastJustifiedRoot[:]}
-	update, err = store.shouldUpdateCurrentJustified(ctx, &ethpb.Checkpoint{Root: newJustifiedRoot[:]})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !update {
-		t.Error("Should be able to update justified, received false")
-	}
-}
-
-func TestShouldUpdateJustified_ReturnFalse(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	params.UseMinimalConfig()
-	defer params.UseMainnetConfig()
-
-	store := NewForkChoiceService(ctx, db)
-
-	lastJustifiedBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{ParentRoot: []byte{'G'}}}
-	lastJustifiedRoot, _ := ssz.HashTreeRoot(lastJustifiedBlk.Block)
-	newJustifiedBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{ParentRoot: lastJustifiedRoot[:]}}
-	newJustifiedRoot, _ := ssz.HashTreeRoot(newJustifiedBlk.Block)
-	if err := store.db.SaveBlock(ctx, newJustifiedBlk); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveBlock(ctx, lastJustifiedBlk); err != nil {
-		t.Fatal(err)
-	}
-
-	diff := (params.BeaconConfig().SlotsPerEpoch - 1) * params.BeaconConfig().SecondsPerSlot
-	store.genesisTime = uint64(time.Now().Unix()) - diff
-	store.justifiedCheckpt = &ethpb.Checkpoint{Root: lastJustifiedRoot[:]}
-
-	update, err := store.shouldUpdateCurrentJustified(ctx, &ethpb.Checkpoint{Root: newJustifiedRoot[:]})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if update {
-		t.Error("Should not be able to update justified, received true")
-	}
-}
-
-func TestUpdateJustifiedCheckpoint_Update(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	params.UseMinimalConfig()
-	defer params.UseMainnetConfig()
-
-	store := NewForkChoiceService(ctx, db)
-	store.genesisTime = uint64(time.Now().Unix())
-
-	store.justifiedCheckpt = &ethpb.Checkpoint{Root: []byte{'A'}}
-	store.bestJustifiedCheckpt = &ethpb.Checkpoint{Epoch: 1, Root: []byte{'B'}}
-	store.updateJustifiedCheckpoint()
-
-	if !bytes.Equal(store.justifiedCheckpt.Root, []byte{'B'}) {
-		t.Error("Justified check point root did not update")
-	}
-}
-
-func TestUpdateJustifiedCheckpoint_NoUpdate(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	params.UseMinimalConfig()
-	defer params.UseMainnetConfig()
-
-	store := NewForkChoiceService(ctx, db)
-	store.genesisTime = uint64(time.Now().Unix()) - params.BeaconConfig().SecondsPerSlot
-
-	store.justifiedCheckpt = &ethpb.Checkpoint{Root: []byte{'A'}}
-	store.bestJustifiedCheckpt = &ethpb.Checkpoint{Epoch: 1, Root: []byte{'B'}}
-	store.updateJustifiedCheckpoint()
-
-	if bytes.Equal(store.justifiedCheckpt.Root, []byte{'B'}) {
-		t.Error("Justified check point root was not suppose to update")
-
-		store := NewForkChoiceService(ctx, db)
-
-		// Save 5 blocks in DB, each has a state.
-		numBlocks := 5
-		totalBlocks := make([]*ethpb.SignedBeaconBlock, numBlocks)
-		blockRoots := make([][32]byte, 0)
-		for i := 0; i < len(totalBlocks); i++ {
-			totalBlocks[i] = &ethpb.SignedBeaconBlock{
-				Block: &ethpb.BeaconBlock{
-					Slot: uint64(i),
-				},
-			}
-			r, err := ssz.HashTreeRoot(totalBlocks[i].Block)
-			if err != nil {
-				t.Fatal(err)
-			}
-			if err := store.db.SaveState(ctx, &pb.BeaconState{Slot: uint64(i)}, r); err != nil {
-				t.Fatal(err)
-			}
-			if err := store.db.SaveBlock(ctx, totalBlocks[i]); err != nil {
-				t.Fatal(err)
-			}
-			blockRoots = append(blockRoots, r)
-		}
-		if err := store.db.SaveHeadBlockRoot(ctx, blockRoots[0]); err != nil {
-			t.Fatal(err)
-		}
-		if err := store.rmStatesOlderThanLastFinalized(ctx, 10, 11); err != nil {
-			t.Fatal(err)
-		}
-		// Since 5-10 are skip slots, block with slot 4 should be deleted
-		s, err := store.db.State(ctx, blockRoots[4])
-		if err != nil {
-			t.Fatal(err)
-		}
-		if s != nil {
-			t.Error("Did not delete state for start slot")
-		}
-	}
-}
-
-func TestCachedPreState_CanGetFromCache(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	s := &pb.BeaconState{Slot: 1}
-	r := [32]byte{'A'}
-	b := &ethpb.BeaconBlock{Slot: 1, ParentRoot: r[:]}
-	store.initSyncState[r] = s
-
-	wanted := "pre state of slot 1 does not exist"
-	if _, err := store.cachedPreState(ctx, b); !strings.Contains(err.Error(), wanted) {
-		t.Fatal("Not expected error")
-	}
-}
-
-func TestCachedPreState_CanGetFromCacheWithFeature(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	config := &featureconfig.Flags{
-		InitSyncCacheState: true,
-	}
-	featureconfig.Init(config)
-
-	store := NewForkChoiceService(ctx, db)
-	s := &pb.BeaconState{Slot: 1}
-	r := [32]byte{'A'}
-	b := &ethpb.BeaconBlock{Slot: 1, ParentRoot: r[:]}
-	store.initSyncState[r] = s
-
-	received, err := store.cachedPreState(ctx, b)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(s, received) {
-		t.Error("cached state not the same")
-	}
-}
-
-func TestCachedPreState_CanGetFromDB(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	r := [32]byte{'A'}
-	b := &ethpb.BeaconBlock{Slot: 1, ParentRoot: r[:]}
-
-	_, err := store.cachedPreState(ctx, b)
-	wanted := "pre state of slot 1 does not exist"
-	if err.Error() != wanted {
-		t.Error("Did not get wanted error")
-	}
-
-	s := &pb.BeaconState{Slot: 1}
-	store.db.SaveState(ctx, s, r)
-
-	received, err := store.cachedPreState(ctx, b)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(s, received) {
-		t.Error("cached state not the same")
-	}
-}
-
-func TestSaveInitState_CanSaveDelete(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	config := &featureconfig.Flags{
-		InitSyncCacheState: true,
-	}
-	featureconfig.Init(config)
-
-	for i := uint64(0); i < 64; i++ {
-		b := &ethpb.BeaconBlock{Slot: i}
-		s := &pb.BeaconState{Slot: i}
-		r, _ := ssz.HashTreeRoot(b)
-		store.initSyncState[r] = s
-	}
-
-	// Set finalized root as slot 32
-	finalizedRoot, _ := ssz.HashTreeRoot(&ethpb.BeaconBlock{Slot: 32})
-
-	if err := store.saveInitState(ctx, &pb.BeaconState{FinalizedCheckpoint: &ethpb.Checkpoint{
-		Epoch: 1, Root: finalizedRoot[:]}}); err != nil {
-		t.Fatal(err)
-	}
-
-	// Verify finalized state is saved in DB
-	finalizedState, err := store.db.State(ctx, finalizedRoot)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if finalizedState == nil {
-		t.Error("finalized state can't be nil")
-	}
-
-	// Verify cached state is properly pruned
-	if len(store.initSyncState) != int(params.BeaconConfig().SlotsPerEpoch) {
-		t.Errorf("wanted: %d, got: %d", len(store.initSyncState), params.BeaconConfig().SlotsPerEpoch)
-	}
-}
-
-func TestUpdateJustified_CouldUpdateBest(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	signedBlock := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{}}
-	if err := db.SaveBlock(ctx, signedBlock); err != nil {
-		t.Fatal(err)
-	}
-	r, err := ssz.HashTreeRoot(signedBlock.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	store.justifiedCheckpt = &ethpb.Checkpoint{Root: []byte{'A'}}
-	store.bestJustifiedCheckpt = &ethpb.Checkpoint{Root: []byte{'A'}}
-	store.initSyncState[r] = &pb.BeaconState{}
-	if err := db.SaveState(ctx, &pb.BeaconState{}, r); err != nil {
-		t.Fatal(err)
-	}
-
-	// Could update
-	s := &pb.BeaconState{CurrentJustifiedCheckpoint: &ethpb.Checkpoint{Epoch: 1, Root: r[:]}}
-	if err := store.updateJustified(context.Background(), s); err != nil {
-		t.Fatal(err)
-	}
-
-	if store.bestJustifiedCheckpt.Epoch != s.CurrentJustifiedCheckpoint.Epoch {
-		t.Error("Incorrect justified epoch in store")
-	}
-
-	// Could not update
-	store.bestJustifiedCheckpt.Epoch = 2
-	if err := store.updateJustified(context.Background(), s); err != nil {
-		t.Fatal(err)
-	}
-
-	if store.bestJustifiedCheckpt.Epoch != 2 {
-		t.Error("Incorrect justified epoch in store")
-	}
-}
-
-func TestFilterBlockRoots_CanFilter(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	fBlock := &ethpb.BeaconBlock{}
-	fRoot, _ := ssz.HashTreeRoot(fBlock)
-	hBlock := &ethpb.BeaconBlock{Slot: 1}
-	headRoot, _ := ssz.HashTreeRoot(hBlock)
-	if err := store.db.SaveBlock(ctx, &ethpb.SignedBeaconBlock{Block: fBlock}); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveState(ctx, &pb.BeaconState{}, fRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveFinalizedCheckpoint(ctx, &ethpb.Checkpoint{Root: fRoot[:]}); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveBlock(ctx, &ethpb.SignedBeaconBlock{Block: hBlock}); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveState(ctx, &pb.BeaconState{}, headRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveHeadBlockRoot(ctx, headRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	roots := [][32]byte{{'C'}, {'D'}, headRoot, {'E'}, fRoot, {'F'}}
-	wanted := [][32]byte{{'C'}, {'D'}, {'E'}, {'F'}}
-
-	received, err := store.filterBlockRoots(ctx, roots)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if !reflect.DeepEqual(wanted, received) {
-		t.Error("Did not filter correctly")
-	}
-}

beacon-chain/blockchain/forkchoice/service.go

@@ -1,433 +0,0 @@
-package forkchoice
-
-import (
-	"bytes"
-	"context"
-	"encoding/hex"
-	"fmt"
-	"sync"
-
-	"github.com/gogo/protobuf/proto"
-	"github.com/pkg/errors"
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"github.com/prysmaticlabs/prysm/shared/stateutil"
-	"go.opencensus.io/trace"
-)
-
-// ForkChoicer defines a common interface for methods useful for directly applying fork choice
-// to beacon blocks to compute head.
-type ForkChoicer interface {
-	Head(ctx context.Context) ([]byte, error)
-	OnBlock(ctx context.Context, b *ethpb.SignedBeaconBlock) error
-	OnBlockInitialSyncStateTransition(ctx context.Context, b *ethpb.SignedBeaconBlock) error
-	OnAttestation(ctx context.Context, a *ethpb.Attestation) error
-	GenesisStore(ctx context.Context, justifiedCheckpoint *ethpb.Checkpoint, finalizedCheckpoint *ethpb.Checkpoint) error
-	FinalizedCheckpt() *ethpb.Checkpoint
-}
-
-// Store represents a service struct that handles the forkchoice
-// logic of managing the full PoS beacon chain.
-type Store struct {
-	ctx                   context.Context
-	cancel                context.CancelFunc
-	db                    db.Database
-	justifiedCheckpt      *ethpb.Checkpoint
-	finalizedCheckpt      *ethpb.Checkpoint
-	prevFinalizedCheckpt  *ethpb.Checkpoint
-	checkpointState       *cache.CheckpointStateCache
-	checkpointStateLock   sync.Mutex
-	genesisTime           uint64
-	bestJustifiedCheckpt  *ethpb.Checkpoint
-	latestVoteMap         map[uint64]*pb.ValidatorLatestVote
-	voteLock              sync.RWMutex
-	initSyncState         map[[32]byte]*pb.BeaconState
-	initSyncStateLock     sync.RWMutex
-	nextEpochBoundarySlot uint64
-}
-
-// NewForkChoiceService instantiates a new service instance that will
-// be registered into a running beacon node.
-func NewForkChoiceService(ctx context.Context, db db.Database) *Store {
-	ctx, cancel := context.WithCancel(ctx)
-	return &Store{
-		ctx:             ctx,
-		cancel:          cancel,
-		db:              db,
-		checkpointState: cache.NewCheckpointStateCache(),
-		latestVoteMap:   make(map[uint64]*pb.ValidatorLatestVote),
-		initSyncState:   make(map[[32]byte]*pb.BeaconState),
-	}
-}
-
-// GenesisStore initializes the store struct before beacon chain
-// starts to advance.
-//
-// Spec pseudocode definition:
-//   def get_genesis_store(genesis_state: BeaconState) -> Store:
-//    genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))
-//    root = signing_root(genesis_block)
-//    justified_checkpoint = Checkpoint(epoch=GENESIS_EPOCH, root=root)
-//    finalized_checkpoint = Checkpoint(epoch=GENESIS_EPOCH, root=root)
-//    return Store(
-//        time=genesis_state.genesis_time,
-//        justified_checkpoint=justified_checkpoint,
-//        finalized_checkpoint=finalized_checkpoint,
-//        blocks={root: genesis_block},
-//        block_states={root: genesis_state.copy()},
-//        checkpoint_states={justified_checkpoint: genesis_state.copy()},
-//    )
-func (s *Store) GenesisStore(
-	ctx context.Context,
-	justifiedCheckpoint *ethpb.Checkpoint,
-	finalizedCheckpoint *ethpb.Checkpoint) error {
-
-	s.justifiedCheckpt = proto.Clone(justifiedCheckpoint).(*ethpb.Checkpoint)
-	s.bestJustifiedCheckpt = proto.Clone(justifiedCheckpoint).(*ethpb.Checkpoint)
-	s.finalizedCheckpt = proto.Clone(finalizedCheckpoint).(*ethpb.Checkpoint)
-	s.prevFinalizedCheckpt = proto.Clone(finalizedCheckpoint).(*ethpb.Checkpoint)
-
-	justifiedState, err := s.db.State(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root))
-	if err != nil {
-		return errors.Wrap(err, "could not retrieve last justified state")
-	}
-
-	if err := s.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: s.justifiedCheckpt,
-		State:      justifiedState,
-	}); err != nil {
-		return errors.Wrap(err, "could not save genesis state in check point cache")
-	}
-
-	s.genesisTime = justifiedState.GenesisTime
-	if err := s.cacheGenesisState(ctx); err != nil {
-		return errors.Wrap(err, "could not cache initial sync state")
-	}
-
-	return nil
-}
-
-// This sets up gensis for initial sync state cache.
-func (s *Store) cacheGenesisState(ctx context.Context) error {
-	if !featureconfig.Get().InitSyncCacheState {
-		return nil
-	}
-
-	genesisState, err := s.db.GenesisState(ctx)
-	if err != nil {
-		return err
-	}
-	stateRoot, err := stateutil.HashTreeRootState(genesisState)
-	if err != nil {
-		return errors.Wrap(err, "could not tree hash genesis state")
-	}
-	genesisBlk := blocks.NewGenesisBlock(stateRoot[:])
-	genesisBlkRoot, err := ssz.HashTreeRoot(genesisBlk.Block)
-	if err != nil {
-		return errors.Wrap(err, "could not get genesis block root")
-	}
-	s.initSyncState[genesisBlkRoot] = genesisState
-
-	return nil
-}
-
-// ancestor returns the block root of an ancestry block from the input block root.
-//
-// Spec pseudocode definition:
-//   def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash:
-//    block = store.blocks[root]
-//    if block.slot > slot:
-//      return get_ancestor(store, block.parent_root, slot)
-//    elif block.slot == slot:
-//      return root
-//    else:
-//      return Bytes32()  # root is older than queried slot: no results.
-func (s *Store) ancestor(ctx context.Context, root []byte, slot uint64) ([]byte, error) {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.ancestor")
-	defer span.End()
-
-	// Stop recursive ancestry lookup if context is cancelled.
-	if ctx.Err() != nil {
-		return nil, ctx.Err()
-	}
-
-	signed, err := s.db.Block(ctx, bytesutil.ToBytes32(root))
-	if err != nil {
-		return nil, errors.Wrap(err, "could not get ancestor block")
-	}
-	if signed == nil || signed.Block == nil {
-		return nil, errors.New("nil block")
-	}
-	b := signed.Block
-
-	// If we dont have the ancestor in the DB, simply return nil so rest of fork choice
-	// operation can proceed. This is not an error condition.
-	if b == nil || b.Slot < slot {
-		return nil, nil
-	}
-
-	if b.Slot == slot {
-		return root, nil
-	}
-
-	return s.ancestor(ctx, b.ParentRoot, slot)
-}
-
-// latestAttestingBalance returns the staked balance of a block from the input block root.
-//
-// Spec pseudocode definition:
-//   def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei:
-//    state = store.checkpoint_states[store.justified_checkpoint]
-//    active_indices = get_active_validator_indices(state, get_current_epoch(state))
-//    return Gwei(sum(
-//        state.validators[i].effective_balance for i in active_indices
-//        if (i in store.latest_messages
-//            and get_ancestor(store, store.latest_messages[i].root, store.blocks[root].slot) == root)
-//    ))
-func (s *Store) latestAttestingBalance(ctx context.Context, root []byte) (uint64, error) {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.latestAttestingBalance")
-	defer span.End()
-
-	lastJustifiedState, err := s.checkpointState.StateByCheckpoint(s.JustifiedCheckpt())
-	if err != nil {
-		return 0, errors.Wrap(err, "could not retrieve cached state via last justified check point")
-	}
-	if lastJustifiedState == nil {
-		return 0, errors.Wrapf(err, "could not get justified state at epoch %d", s.JustifiedCheckpt().Epoch)
-	}
-
-	lastJustifiedEpoch := helpers.CurrentEpoch(lastJustifiedState)
-	activeIndices, err := helpers.ActiveValidatorIndices(lastJustifiedState, lastJustifiedEpoch)
-	if err != nil {
-		return 0, errors.Wrap(err, "could not get active indices for last justified checkpoint")
-	}
-
-	wantedBlkSigned, err := s.db.Block(ctx, bytesutil.ToBytes32(root))
-	if err != nil {
-		return 0, errors.Wrap(err, "could not get target block")
-	}
-	if wantedBlkSigned == nil || wantedBlkSigned.Block == nil {
-		return 0, errors.New("nil wanted block")
-	}
-	wantedBlk := wantedBlkSigned.Block
-
-	balances := uint64(0)
-	s.voteLock.RLock()
-	defer s.voteLock.RUnlock()
-	for _, i := range activeIndices {
-		vote, ok := s.latestVoteMap[i]
-		if !ok {
-			continue
-		}
-
-		wantedRoot, err := s.ancestor(ctx, vote.Root, wantedBlk.Slot)
-		if err != nil {
-			return 0, errors.Wrapf(err, "could not get ancestor root for slot %d", wantedBlk.Slot)
-		}
-		if bytes.Equal(wantedRoot, root) {
-			balances += lastJustifiedState.Validators[i].EffectiveBalance
-		}
-	}
-	return balances, nil
-}
-
-// Head returns the head of the beacon chain.
-//
-// Spec pseudocode definition:
-//   def get_head(store: Store) -> Root:
-//    # Get filtered block tree that only includes viable branches
-//    blocks = get_filtered_block_tree(store)
-//    # Execute the LMD-GHOST fork choice
-//    head = store.justified_checkpoint.root
-//    justified_slot = compute_start_slot_at_epoch(store.justified_checkpoint.epoch)
-//    while True:
-//        children = [
-//            root for root in blocks.keys()
-//            if blocks[root].parent_root == head and blocks[root].slot > justified_slot
-//        ]
-//        if len(children) == 0:
-//            return head
-//        # Sort by latest attesting balance with ties broken lexicographically
-//        head = max(children, key=lambda root: (get_latest_attesting_balance(store, root), root))
-func (s *Store) Head(ctx context.Context) ([]byte, error) {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.head")
-	defer span.End()
-
-	head := s.JustifiedCheckpt().Root
-	filteredBlocks, err := s.getFilterBlockTree(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	justifiedSlot := helpers.StartSlot(s.justifiedCheckpt.Epoch)
-	for {
-		children := make([][32]byte, 0, len(filteredBlocks))
-		for root, block := range filteredBlocks {
-			if bytes.Equal(block.ParentRoot, head) && block.Slot > justifiedSlot {
-				children = append(children, root)
-			}
-		}
-
-		if len(children) == 0 {
-			return head, nil
-		}
-
-		// if a block has one child, then we don't have to lookup anything to
-		// know that this child will be the best child.
-		head = children[0][:]
-		if len(children) > 1 {
-			highest, err := s.latestAttestingBalance(ctx, head)
-			if err != nil {
-				return nil, errors.Wrap(err, "could not get latest balance")
-			}
-			for _, child := range children[1:] {
-				balance, err := s.latestAttestingBalance(ctx, child[:])
-				if err != nil {
-					return nil, errors.Wrap(err, "could not get latest balance")
-				}
-				// When there's a tie, it's broken lexicographically to favor the higher one.
-				if balance > highest ||
-					balance == highest && bytes.Compare(child[:], head) > 0 {
-					highest = balance
-					head = child[:]
-				}
-			}
-		}
-	}
-}
-
-// getFilterBlockTree retrieves a filtered block tree from store, it only returns branches
-// whose leaf state's justified and finalized info agrees with what's in the store.
-// Rationale: https://notes.ethereum.org/Fj-gVkOSTpOyUx-zkWjuwg?view
-//
-// Spec pseudocode definition:
-//   def get_filtered_block_tree(store: Store) -> Dict[Root, BeaconBlock]:
-//    """
-//    Retrieve a filtered block true from ``store``, only returning branches
-//    whose leaf state's justified/finalized info agrees with that in ``store``.
-//    """
-//    base = store.justified_checkpoint.root
-//    blocks: Dict[Root, BeaconBlock] = {}
-//    filter_block_tree(store, base, blocks)
-//    return blocks
-func (s *Store) getFilterBlockTree(ctx context.Context) (map[[32]byte]*ethpb.BeaconBlock, error) {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.getFilterBlockTree")
-	defer span.End()
-
-	baseRoot := bytesutil.ToBytes32(s.justifiedCheckpt.Root)
-	filteredBlocks := make(map[[32]byte]*ethpb.BeaconBlock)
-	if _, err := s.filterBlockTree(ctx, baseRoot, filteredBlocks); err != nil {
-		return nil, err
-	}
-
-	return filteredBlocks, nil
-}
-
-// filterBlockTree filters for branches that see latest finalized and justified info as correct on-chain
-// before running Head.
-//
-// Spec pseudocode definition:
-//   def filter_block_tree(store: Store, block_root: Root, blocks: Dict[Root, BeaconBlock]) -> bool:
-//    block = store.blocks[block_root]
-//    children = [
-//        root for root in store.blocks.keys()
-//        if store.blocks[root].parent_root == block_root
-//    ]
-//    # If any children branches contain expected finalized/justified checkpoints,
-//    # add to filtered block-tree and signal viability to parent.
-//    if any(children):
-//        filter_block_tree_result = [filter_block_tree(store, child, blocks) for child in children]
-//        if any(filter_block_tree_result):
-//            blocks[block_root] = block
-//            return True
-//        return False
-//    # If leaf block, check finalized/justified checkpoints as matching latest.
-//    head_state = store.block_states[block_root]
-//    correct_justified = (
-//        store.justified_checkpoint.epoch == GENESIS_EPOCH
-//        or head_state.current_justified_checkpoint == store.justified_checkpoint
-//    )
-//    correct_finalized = (
-//        store.finalized_checkpoint.epoch == GENESIS_EPOCH
-//        or head_state.finalized_checkpoint == store.finalized_checkpoint
-//    )
-//    # If expected finalized/justified, add to viable block-tree and signal viability to parent.
-//    if correct_justified and correct_finalized:
-//        blocks[block_root] = block
-//        return True
-//    # Otherwise, branch not viable
-//    return False
-func (s *Store) filterBlockTree(ctx context.Context, blockRoot [32]byte, filteredBlocks map[[32]byte]*ethpb.BeaconBlock) (bool, error) {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.filterBlockTree")
-	defer span.End()
-	signed, err := s.db.Block(ctx, blockRoot)
-	if err != nil {
-		return false, err
-	}
-	if signed == nil || signed.Block == nil {
-		return false, errors.New("nil block")
-	}
-	block := signed.Block
-
-	filter := filters.NewFilter().SetParentRoot(blockRoot[:])
-	childrenRoots, err := s.db.BlockRoots(ctx, filter)
-	if err != nil {
-		return false, err
-	}
-
-	if len(childrenRoots) != 0 {
-		var filtered bool
-		for _, childRoot := range childrenRoots {
-			didFilter, err := s.filterBlockTree(ctx, childRoot, filteredBlocks)
-			if err != nil {
-				return false, err
-			}
-			if didFilter {
-				filtered = true
-			}
-		}
-		if filtered {
-			filteredBlocks[blockRoot] = block
-			return true, nil
-		}
-		return false, nil
-	}
-
-	headState, err := s.db.State(ctx, blockRoot)
-	if err != nil {
-		return false, err
-	}
-
-	if headState == nil {
-		return false, fmt.Errorf("no state matching block root %v", hex.EncodeToString(blockRoot[:]))
-	}
-
-	correctJustified := s.justifiedCheckpt.Epoch == 0 ||
-		proto.Equal(s.justifiedCheckpt, headState.CurrentJustifiedCheckpoint)
-	correctFinalized := s.finalizedCheckpt.Epoch == 0 ||
-		proto.Equal(s.finalizedCheckpt, headState.FinalizedCheckpoint)
-	if correctJustified && correctFinalized {
-		filteredBlocks[blockRoot] = block
-		return true, nil
-	}
-
-	return false, nil
-}
-
-// JustifiedCheckpt returns the latest justified check point from fork choice store.
-func (s *Store) JustifiedCheckpt() *ethpb.Checkpoint {
-	return proto.Clone(s.justifiedCheckpt).(*ethpb.Checkpoint)
-}
-
-// FinalizedCheckpt returns the latest finalized check point from fork choice store.
-func (s *Store) FinalizedCheckpt() *ethpb.Checkpoint {
-	return proto.Clone(s.finalizedCheckpt).(*ethpb.Checkpoint)
-}

beacon-chain/blockchain/forkchoice/service_test.go

@@ -1,517 +0,0 @@
-package forkchoice
-
-import (
-	"bytes"
-	"context"
-	"reflect"
-	"testing"
-	"time"
-
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
-	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"github.com/prysmaticlabs/prysm/shared/stateutil"
-)
-
-func TestStore_GenesisStoreOk(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	genesisTime := time.Unix(9999, 0)
-	genesisState := &pb.BeaconState{GenesisTime: uint64(genesisTime.Unix())}
-	genesisStateRoot, err := stateutil.HashTreeRootState(genesisState)
-	if err != nil {
-		t.Fatal(err)
-	}
-	genesisBlk := blocks.NewGenesisBlock(genesisStateRoot[:])
-	genesisBlkRoot, err := ssz.HashTreeRoot(genesisBlk.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveState(ctx, genesisState, genesisBlkRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveGenesisBlockRoot(ctx, genesisBlkRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	checkPoint := &ethpb.Checkpoint{Root: genesisBlkRoot[:]}
-	if err := store.GenesisStore(ctx, checkPoint, checkPoint); err != nil {
-		t.Fatal(err)
-	}
-
-	if !reflect.DeepEqual(store.justifiedCheckpt, checkPoint) {
-		t.Error("Justified check point from genesis store did not match")
-	}
-	if !reflect.DeepEqual(store.finalizedCheckpt, checkPoint) {
-		t.Error("Finalized check point from genesis store did not match")
-	}
-
-	cachedState, err := store.checkpointState.StateByCheckpoint(checkPoint)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(cachedState, genesisState) {
-		t.Error("Incorrect genesis state cached")
-	}
-}
-
-func TestStore_AncestorOk(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db, []byte{'g'})
-	if err != nil {
-		t.Fatal(err)
-	}
-	type args struct {
-		root []byte
-		slot uint64
-	}
-
-	//    /- B1
-	// B0           /- B5 - B7
-	//    \- B3 - B4 - B6 - B8
-	tests := []struct {
-		args *args
-		want []byte
-	}{
-		{args: &args{roots[1], 0}, want: roots[0]},
-		{args: &args{roots[8], 0}, want: roots[0]},
-		{args: &args{roots[8], 4}, want: roots[4]},
-		{args: &args{roots[7], 4}, want: roots[4]},
-		{args: &args{roots[7], 0}, want: roots[0]},
-	}
-	for _, tt := range tests {
-		got, err := store.ancestor(ctx, tt.args.root, tt.args.slot)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if !reflect.DeepEqual(got, tt.want) {
-			t.Errorf("Store.ancestor(ctx, ) = %v, want %v", got, tt.want)
-		}
-	}
-}
-
-func TestStore_AncestorNotPartOfTheChain(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db, []byte{'g'})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	//    /- B1
-	// B0           /- B5 - B7
-	//    \- B3 - B4 - B6 - B8
-	root, err := store.ancestor(ctx, roots[8], 1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if root != nil {
-		t.Error("block at slot 1 is not part of the chain")
-	}
-	root, err = store.ancestor(ctx, roots[8], 2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if root != nil {
-		t.Error("block at slot 2 is not part of the chain")
-	}
-}
-
-func TestStore_LatestAttestingBalance(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db, []byte{'g'})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	validators := make([]*ethpb.Validator, 100)
-	for i := 0; i < len(validators); i++ {
-		validators[i] = &ethpb.Validator{ExitEpoch: 2, EffectiveBalance: 1e9}
-	}
-
-	s := &pb.BeaconState{Validators: validators}
-	stateRoot, err := stateutil.HashTreeRootState(s)
-	if err != nil {
-		t.Fatal(err)
-	}
-	b := blocks.NewGenesisBlock(stateRoot[:])
-	blkRoot, err := ssz.HashTreeRoot(b.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveState(ctx, s, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveGenesisBlockRoot(ctx, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	checkPoint := &ethpb.Checkpoint{Root: blkRoot[:]}
-	if err := store.GenesisStore(ctx, checkPoint, checkPoint); err != nil {
-		t.Fatal(err)
-	}
-
-	//    /- B1 (33 votes)
-	// B0           /- B5 - B7 (33 votes)
-	//    \- B3 - B4 - B6 - B8 (34 votes)
-	for i := 0; i < len(validators); i++ {
-		switch {
-		case i < 33:
-			store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[1]}
-		case i > 66:
-			store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[7]}
-		default:
-			store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[8]}
-		}
-	}
-
-	tests := []struct {
-		root []byte
-		want uint64
-	}{
-		{root: roots[0], want: 100 * 1e9},
-		{root: roots[1], want: 33 * 1e9},
-		{root: roots[3], want: 67 * 1e9},
-		{root: roots[4], want: 67 * 1e9},
-		{root: roots[7], want: 33 * 1e9},
-		{root: roots[8], want: 34 * 1e9},
-	}
-	for _, tt := range tests {
-		got, err := store.latestAttestingBalance(ctx, tt.root)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if got != tt.want {
-			t.Errorf("Store.latestAttestingBalance(ctx, ) = %v, want %v", got, tt.want)
-		}
-	}
-}
-
-func TestStore_ChildrenBlocksFromParentRoot(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db, []byte{'g'})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	filter := filters.NewFilter().SetParentRoot(roots[0]).SetStartSlot(0)
-	children, err := store.db.BlockRoots(ctx, filter)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(children, [][32]byte{bytesutil.ToBytes32(roots[1]), bytesutil.ToBytes32(roots[3])}) {
-		t.Error("Did not receive correct children roots")
-	}
-
-	filter = filters.NewFilter().SetParentRoot(roots[0]).SetStartSlot(2)
-	children, err = store.db.BlockRoots(ctx, filter)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(children, [][32]byte{bytesutil.ToBytes32(roots[3])}) {
-		t.Error("Did not receive correct children roots")
-	}
-}
-
-func TestStore_GetHead(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db, []byte{'g'})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	validators := make([]*ethpb.Validator, 100)
-	for i := 0; i < len(validators); i++ {
-		validators[i] = &ethpb.Validator{ExitEpoch: 2, EffectiveBalance: 1e9}
-	}
-
-	s := &pb.BeaconState{Validators: validators}
-	stateRoot, err := stateutil.HashTreeRootState(s)
-	if err != nil {
-		t.Fatal(err)
-	}
-	b := blocks.NewGenesisBlock(stateRoot[:])
-	blkRoot, err := ssz.HashTreeRoot(b.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveState(ctx, s, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveGenesisBlockRoot(ctx, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	checkPoint := &ethpb.Checkpoint{Root: blkRoot[:]}
-
-	if err := store.GenesisStore(ctx, checkPoint, checkPoint); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveState(ctx, s, bytesutil.ToBytes32(roots[0])); err != nil {
-		t.Fatal(err)
-	}
-	store.justifiedCheckpt.Root = roots[0]
-	if err := store.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: store.justifiedCheckpt,
-		State:      s,
-	}); err != nil {
-		t.Fatal(err)
-	}
-
-	//    /- B1 (33 votes)
-	// B0           /- B5 - B7 (33 votes)
-	//    \- B3 - B4 - B6 - B8 (34 votes)
-	for i := 0; i < len(validators); i++ {
-		switch {
-		case i < 33:
-			store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[1]}
-		case i > 66:
-			store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[7]}
-		default:
-			store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[8]}
-		}
-	}
-
-	// Default head is B8
-	head, err := store.Head(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !bytes.Equal(head, roots[8]) {
-		t.Error("Incorrect head")
-	}
-
-	// 1 validator switches vote to B7 to gain 34%, enough to switch head
-	store.latestVoteMap[uint64(50)] = &pb.ValidatorLatestVote{Root: roots[7]}
-
-	head, err = store.Head(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !bytes.Equal(head, roots[7]) {
-		t.Error("Incorrect head")
-	}
-
-	// 18 validators switches vote to B1 to gain 51%, enough to switch head
-	for i := 0; i < 18; i++ {
-		idx := 50 + uint64(i)
-		store.latestVoteMap[uint64(idx)] = &pb.ValidatorLatestVote{Root: roots[1]}
-	}
-	head, err = store.Head(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !bytes.Equal(head, roots[1]) {
-		t.Log(head)
-		t.Error("Incorrect head")
-	}
-}
-
-func TestCacheGenesisState_Correct(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	config := &featureconfig.Flags{
-		InitSyncCacheState: true,
-	}
-	featureconfig.Init(config)
-
-	b := &ethpb.BeaconBlock{Slot: 1}
-	r, _ := ssz.HashTreeRoot(b)
-	s := &pb.BeaconState{GenesisTime: 99}
-
-	store.db.SaveState(ctx, s, r)
-	store.db.SaveGenesisBlockRoot(ctx, r)
-
-	if err := store.cacheGenesisState(ctx); err != nil {
-		t.Fatal(err)
-	}
-
-	for _, state := range store.initSyncState {
-		if !reflect.DeepEqual(s, state) {
-			t.Error("Did not get wanted state")
-		}
-	}
-}
-
-func TestStore_GetFilterBlockTree_CorrectLeaf(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db, []byte{'g'})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	s := &pb.BeaconState{}
-	stateRoot, err := stateutil.HashTreeRootState(s)
-	if err != nil {
-		t.Fatal(err)
-	}
-	b := blocks.NewGenesisBlock(stateRoot[:])
-	blkRoot, err := ssz.HashTreeRoot(b.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveState(ctx, s, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveGenesisBlockRoot(ctx, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	checkPoint := &ethpb.Checkpoint{Root: blkRoot[:]}
-
-	if err := store.GenesisStore(ctx, checkPoint, checkPoint); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveState(ctx, s, bytesutil.ToBytes32(roots[0])); err != nil {
-		t.Fatal(err)
-	}
-	store.justifiedCheckpt.Root = roots[0]
-	if err := store.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: store.justifiedCheckpt,
-		State:      s,
-	}); err != nil {
-		t.Fatal(err)
-	}
-
-	tree, err := store.getFilterBlockTree(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	wanted := make(map[[32]byte]*ethpb.BeaconBlock)
-	for _, root := range roots {
-		root32 := bytesutil.ToBytes32(root)
-		b, _ := store.db.Block(ctx, root32)
-		if b != nil {
-			wanted[root32] = b.Block
-		}
-	}
-	if !reflect.DeepEqual(tree, wanted) {
-		t.Error("Did not filter tree correctly")
-	}
-}
-
-func TestStore_GetFilterBlockTree_IncorrectLeaf(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db, []byte{'g'})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	s := &pb.BeaconState{}
-	stateRoot, err := stateutil.HashTreeRootState(s)
-	if err != nil {
-		t.Fatal(err)
-	}
-	b := blocks.NewGenesisBlock(stateRoot[:])
-	blkRoot, err := ssz.HashTreeRoot(b.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveState(ctx, s, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveGenesisBlockRoot(ctx, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	checkPoint := &ethpb.Checkpoint{Root: blkRoot[:]}
-
-	if err := store.GenesisStore(ctx, checkPoint, checkPoint); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveState(ctx, s, bytesutil.ToBytes32(roots[0])); err != nil {
-		t.Fatal(err)
-	}
-	store.justifiedCheckpt.Root = roots[0]
-	if err := store.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: store.justifiedCheckpt,
-		State:      s,
-	}); err != nil {
-		t.Fatal(err)
-	}
-	// Filter for incorrect leaves for 1, 7 and 8
-	store.db.SaveState(ctx, &pb.BeaconState{CurrentJustifiedCheckpoint: &ethpb.Checkpoint{}}, bytesutil.ToBytes32(roots[1]))
-	store.db.SaveState(ctx, &pb.BeaconState{CurrentJustifiedCheckpoint: &ethpb.Checkpoint{}}, bytesutil.ToBytes32(roots[7]))
-	store.db.SaveState(ctx, &pb.BeaconState{CurrentJustifiedCheckpoint: &ethpb.Checkpoint{}}, bytesutil.ToBytes32(roots[8]))
-	store.justifiedCheckpt.Epoch = 1
-	tree, err := store.getFilterBlockTree(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(tree) != 0 {
-		t.Error("filtered tree should be 0 length")
-	}
-
-	// Set leave 1 as correct
-	store.db.SaveState(ctx, &pb.BeaconState{CurrentJustifiedCheckpoint: &ethpb.Checkpoint{Epoch: 1, Root: store.justifiedCheckpt.Root}}, bytesutil.ToBytes32(roots[1]))
-	tree, err = store.getFilterBlockTree(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	wanted := make(map[[32]byte]*ethpb.BeaconBlock)
-	root32 := bytesutil.ToBytes32(roots[0])
-	b, err = store.db.Block(ctx, root32)
-	if err != nil {
-		t.Fatal(err)
-	}
-	wanted[root32] = b.Block
-	root32 = bytesutil.ToBytes32(roots[1])
-	b, err = store.db.Block(ctx, root32)
-	if err != nil {
-		t.Fatal(err)
-	}
-	wanted[root32] = b.Block
-
-	if !reflect.DeepEqual(tree, wanted) {
-		t.Error("Did not filter tree correctly")
-	}
-}

beacon-chain/blockchain/forkchoice/tree_test.go

@@ -1,153 +0,0 @@
-package forkchoice
-
-import (
-	"context"
-
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-)
-
-// blockTree1 constructs the following tree:
-//    /- B1
-// B0           /- B5 - B7
-//    \- B3 - B4 - B6 - B8
-// (B1, and B3 are all from the same slots)
-func blockTree1(db db.Database, genesisRoot []byte) ([][]byte, error) {
-	b0 := &ethpb.BeaconBlock{Slot: 0, ParentRoot: genesisRoot}
-	r0, _ := ssz.HashTreeRoot(b0)
-	b1 := &ethpb.BeaconBlock{Slot: 1, ParentRoot: r0[:]}
-	r1, _ := ssz.HashTreeRoot(b1)
-	b3 := &ethpb.BeaconBlock{Slot: 3, ParentRoot: r0[:]}
-	r3, _ := ssz.HashTreeRoot(b3)
-	b4 := &ethpb.BeaconBlock{Slot: 4, ParentRoot: r3[:]}
-	r4, _ := ssz.HashTreeRoot(b4)
-	b5 := &ethpb.BeaconBlock{Slot: 5, ParentRoot: r4[:]}
-	r5, _ := ssz.HashTreeRoot(b5)
-	b6 := &ethpb.BeaconBlock{Slot: 6, ParentRoot: r4[:]}
-	r6, _ := ssz.HashTreeRoot(b6)
-	b7 := &ethpb.BeaconBlock{Slot: 7, ParentRoot: r5[:]}
-	r7, _ := ssz.HashTreeRoot(b7)
-	b8 := &ethpb.BeaconBlock{Slot: 8, ParentRoot: r6[:]}
-	r8, _ := ssz.HashTreeRoot(b8)
-	for _, b := range []*ethpb.BeaconBlock{b0, b1, b3, b4, b5, b6, b7, b8} {
-		if err := db.SaveBlock(context.Background(), &ethpb.SignedBeaconBlock{Block: b}); err != nil {
-			return nil, err
-		}
-		if err := db.SaveState(context.Background(), &pb.BeaconState{}, bytesutil.ToBytes32(b.ParentRoot)); err != nil {
-			return nil, err
-		}
-	}
-	if err := db.SaveState(context.Background(), &pb.BeaconState{}, r1); err != nil {
-		return nil, err
-	}
-	if err := db.SaveState(context.Background(), &pb.BeaconState{}, r7); err != nil {
-		return nil, err
-	}
-	if err := db.SaveState(context.Background(), &pb.BeaconState{}, r8); err != nil {
-		return nil, err
-	}
-	return [][]byte{r0[:], r1[:], nil, r3[:], r4[:], r5[:], r6[:], r7[:], r8[:]}, nil
-}
-
-// blockTree2 constructs the following tree:
-// Scenario graph: shorturl.at/loyP6
-//
-//digraph G {
-//    rankdir=LR;
-//    node [shape="none"];
-//
-//    subgraph blocks {
-//        rankdir=LR;
-//        node [shape="box"];
-//        a->b;
-//        a->c;
-//        b->d;
-//        b->e;
-//        c->f;
-//        c->g;
-//        d->h
-//        d->i
-//        d->j
-//        d->k
-//        h->l
-//        h->m
-//        g->n
-//        g->o
-//        e->p
-//    }
-//}
-func blockTree2(db db.Database) ([][]byte, error) {
-	b0 := &ethpb.BeaconBlock{Slot: 0, ParentRoot: []byte{'g'}}
-	r0, _ := ssz.HashTreeRoot(b0)
-	b1 := &ethpb.BeaconBlock{Slot: 1, ParentRoot: r0[:]}
-	r1, _ := ssz.HashTreeRoot(b1)
-	b2 := &ethpb.BeaconBlock{Slot: 2, ParentRoot: r0[:]}
-	r2, _ := ssz.HashTreeRoot(b2)
-	b3 := &ethpb.BeaconBlock{Slot: 3, ParentRoot: r1[:]}
-	r3, _ := ssz.HashTreeRoot(b3)
-	b4 := &ethpb.BeaconBlock{Slot: 4, ParentRoot: r1[:]}
-	r4, _ := ssz.HashTreeRoot(b4)
-	b5 := &ethpb.BeaconBlock{Slot: 5, ParentRoot: r2[:]}
-	r5, _ := ssz.HashTreeRoot(b5)
-	b6 := &ethpb.BeaconBlock{Slot: 6, ParentRoot: r2[:]}
-	r6, _ := ssz.HashTreeRoot(b6)
-	b7 := &ethpb.BeaconBlock{Slot: 7, ParentRoot: r3[:]}
-	r7, _ := ssz.HashTreeRoot(b7)
-	b8 := &ethpb.BeaconBlock{Slot: 8, ParentRoot: r3[:]}
-	r8, _ := ssz.HashTreeRoot(b8)
-	b9 := &ethpb.BeaconBlock{Slot: 9, ParentRoot: r3[:]}
-	r9, _ := ssz.HashTreeRoot(b9)
-	b10 := &ethpb.BeaconBlock{Slot: 10, ParentRoot: r3[:]}
-	r10, _ := ssz.HashTreeRoot(b10)
-	b11 := &ethpb.BeaconBlock{Slot: 11, ParentRoot: r4[:]}
-	r11, _ := ssz.HashTreeRoot(b11)
-	b12 := &ethpb.BeaconBlock{Slot: 12, ParentRoot: r6[:]}
-	r12, _ := ssz.HashTreeRoot(b12)
-	b13 := &ethpb.BeaconBlock{Slot: 13, ParentRoot: r6[:]}
-	r13, _ := ssz.HashTreeRoot(b13)
-	b14 := &ethpb.BeaconBlock{Slot: 14, ParentRoot: r7[:]}
-	r14, _ := ssz.HashTreeRoot(b14)
-	b15 := &ethpb.BeaconBlock{Slot: 15, ParentRoot: r7[:]}
-	r15, _ := ssz.HashTreeRoot(b15)
-	for _, b := range []*ethpb.BeaconBlock{b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15} {
-		if err := db.SaveBlock(context.Background(), &ethpb.SignedBeaconBlock{Block: b}); err != nil {
-			return nil, err
-		}
-		if err := db.SaveState(context.Background(), &pb.BeaconState{}, bytesutil.ToBytes32(b.ParentRoot)); err != nil {
-			return nil, err
-		}
-	}
-	return [][]byte{r0[:], r1[:], r2[:], r3[:], r4[:], r5[:], r6[:], r7[:], r8[:], r9[:], r10[:], r11[:], r12[:], r13[:], r14[:], r15[:]}, nil
-}
-
-// blockTree3 constructs a tree that is 512 blocks in a row.
-// B0 - B1 - B2 - B3 - .... - B512
-func blockTree3(db db.Database) ([][]byte, error) {
-	blkCount := 512
-	roots := make([][]byte, 0, blkCount)
-	blks := make([]*ethpb.BeaconBlock, 0, blkCount)
-	b0 := &ethpb.BeaconBlock{Slot: 0, ParentRoot: []byte{'g'}}
-	r0, _ := ssz.HashTreeRoot(b0)
-	roots = append(roots, r0[:])
-	blks = append(blks, b0)
-
-	for i := 1; i < blkCount; i++ {
-		b := &ethpb.BeaconBlock{Slot: uint64(i), ParentRoot: roots[len(roots)-1]}
-		r, _ := ssz.HashTreeRoot(b)
-		roots = append(roots, r[:])
-		blks = append(blks, b)
-	}
-
-	for _, b := range blks {
-		if err := db.SaveBlock(context.Background(), &ethpb.SignedBeaconBlock{Block: b}); err != nil {
-			return nil, err
-		}
-		if err := db.SaveState(context.Background(), &pb.BeaconState{}, bytesutil.ToBytes32(b.ParentRoot)); err != nil {
-			return nil, err
-		}
-	}
-	return roots, nil
-}

beacon-chain/blockchain/head.go

@@ -0,0 +1,286 @@
+package blockchain
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
+	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/sirupsen/logrus"
+	"go.opencensus.io/trace"
+)
+
+// This defines the current chain service's view of head.
+type head struct {
+	slot  uint64                   // current head slot.
+	root  [32]byte                 // current head root.
+	block *ethpb.SignedBeaconBlock // current head block.
+	state *state.BeaconState       // current head state.
+}
+
+// This gets head from the fork choice service and saves head related items
+// (ie root, block, state) to the local service cache.
+func (s *Service) updateHead(ctx context.Context, balances []uint64) error {
+	ctx, span := trace.StartSpan(ctx, "blockchain.updateHead")
+	defer span.End()
+
+	// To get the proper head update, a node first checks its best justified
+	// can become justified. This is designed to prevent bounce attack and
+	// ensure head gets its best justified info.
+	if s.bestJustifiedCheckpt.Epoch > s.justifiedCheckpt.Epoch {
+		s.justifiedCheckpt = s.bestJustifiedCheckpt
+	}
+
+	// Get head from the fork choice service.
+	f := s.finalizedCheckpt
+	j := s.justifiedCheckpt
+	headRoot, err := s.forkChoiceStore.Head(ctx, j.Epoch, bytesutil.ToBytes32(j.Root), balances, f.Epoch)
+	if err != nil {
+		return err
+	}
+
+	if err := s.updateRecentCanonicalBlocks(ctx, headRoot); err != nil {
+		return err
+	}
+
+	// Save head to the local service cache.
+	return s.saveHead(ctx, headRoot)
+}
+
+// This saves head info to the local service cache, it also saves the
+// new head root to the DB.
+func (s *Service) saveHead(ctx context.Context, headRoot [32]byte) error {
+	ctx, span := trace.StartSpan(ctx, "blockchain.saveHead")
+	defer span.End()
+
+	// Do nothing if head hasn't changed.
+	if headRoot == s.headRoot() {
+		return nil
+	}
+
+	// If the head state is not available, just return nil.
+	// There's nothing to cache
+	if featureconfig.Get().NewStateMgmt {
+		if !s.stateGen.StateSummaryExists(ctx, headRoot) {
+			return nil
+		}
+	} else {
+		_, cached := s.initSyncState[headRoot]
+		if !cached && !s.beaconDB.HasState(ctx, headRoot) {
+			return nil
+		}
+	}
+
+	// Get the new head block from DB.
+	newHeadBlock, err := s.beaconDB.Block(ctx, headRoot)
+	if err != nil {
+		return err
+	}
+	if newHeadBlock == nil || newHeadBlock.Block == nil {
+		return errors.New("cannot save nil head block")
+	}
+
+	// Get the new head state from cached state or DB.
+	var newHeadState *state.BeaconState
+	if featureconfig.Get().NewStateMgmt {
+		newHeadState, err = s.stateGen.StateByRoot(ctx, headRoot)
+		if err != nil {
+			return errors.Wrap(err, "could not retrieve head state in DB")
+		}
+	} else {
+		var exists bool
+		newHeadState, exists = s.initSyncState[headRoot]
+		if !exists {
+			newHeadState, err = s.beaconDB.State(ctx, headRoot)
+			if err != nil {
+				return errors.Wrap(err, "could not retrieve head state in DB")
+			}
+		}
+	}
+	if newHeadState == nil {
+		return errors.New("cannot save nil head state")
+	}
+
+	// A chain re-org occurred, so we fire an event notifying the rest of the services.
+	if bytesutil.ToBytes32(newHeadBlock.Block.ParentRoot) != s.headRoot() {
+		log.WithFields(logrus.Fields{
+			"newSlot": fmt.Sprintf("%d", newHeadBlock.Block.Slot),
+			"oldSlot": fmt.Sprintf("%d", s.headSlot()),
+		}).Debug("Chain reorg occurred")
+		s.stateNotifier.StateFeed().Send(&feed.Event{
+			Type: statefeed.Reorg,
+			Data: &statefeed.ReorgData{
+				NewSlot: newHeadBlock.Block.Slot,
+				OldSlot: s.headSlot(),
+			},
+		})
+
+		reorgCount.Inc()
+	}
+
+	// Cache the new head info.
+	s.setHead(headRoot, newHeadBlock, newHeadState)
+
+	// Save the new head root to DB.
+	if err := s.beaconDB.SaveHeadBlockRoot(ctx, headRoot); err != nil {
+		return errors.Wrap(err, "could not save head root in DB")
+	}
+
+	return nil
+}
+
+// This gets called to update canonical root mapping. It does not save head block
+// root in DB. With the inception of inital-sync-cache-state flag, it uses finalized
+// check point as anchors to resume sync therefore head is no longer needed to be saved on per slot basis.
+func (s *Service) saveHeadNoDB(ctx context.Context, b *ethpb.SignedBeaconBlock, r [32]byte) error {
+	if b == nil || b.Block == nil {
+		return errors.New("cannot save nil head block")
+	}
+
+	var headState *state.BeaconState
+	var err error
+	if featureconfig.Get().NewStateMgmt {
+		headState, err = s.stateGen.StateByRootInitialSync(ctx, r)
+		if err != nil {
+			return errors.Wrap(err, "could not retrieve head state in DB")
+		}
+	} else {
+		headState, err = s.beaconDB.State(ctx, r)
+		if err != nil {
+			return errors.Wrap(err, "could not retrieve head state in DB")
+		}
+		if headState == nil {
+			s.initSyncStateLock.RLock()
+			cachedHeadState, ok := s.initSyncState[r]
+			if ok {
+				headState = cachedHeadState
+			}
+			s.initSyncStateLock.RUnlock()
+		}
+	}
+	if headState == nil {
+		return errors.New("nil head state")
+	}
+
+	s.setHeadInitialSync(r, stateTrie.CopySignedBeaconBlock(b), headState)
+
+	return nil
+}
+
+// This sets head view object which is used to track the head slot, root, block and state.
+func (s *Service) setHead(root [32]byte, block *ethpb.SignedBeaconBlock, state *state.BeaconState) {
+	s.headLock.Lock()
+	defer s.headLock.Unlock()
+
+	// This does a full copy of the block and state.
+	s.head = &head{
+		slot:  block.Block.Slot,
+		root:  root,
+		block: stateTrie.CopySignedBeaconBlock(block),
+		state: state.Copy(),
+	}
+}
+
+// This sets head view object which is used to track the head slot, root, block and state. The method
+// assumes that state being passed into the method will not be modified by any other alternate
+// caller which holds the state's reference.
+func (s *Service) setHeadInitialSync(root [32]byte, block *ethpb.SignedBeaconBlock, state *state.BeaconState) {
+	s.headLock.Lock()
+	defer s.headLock.Unlock()
+
+	// This does a full copy of the block only.
+	s.head = &head{
+		slot:  block.Block.Slot,
+		root:  root,
+		block: stateTrie.CopySignedBeaconBlock(block),
+		state: state,
+	}
+}
+
+// This returns the head slot.
+func (s *Service) headSlot() uint64 {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	return s.head.slot
+}
+
+// This returns the head root.
+// It does a full copy on head root for immutability.
+func (s *Service) headRoot() [32]byte {
+	if s.head == nil {
+		return params.BeaconConfig().ZeroHash
+	}
+
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	return s.head.root
+}
+
+// This returns the head block.
+// It does a full copy on head block for immutability.
+func (s *Service) headBlock() *ethpb.SignedBeaconBlock {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	return stateTrie.CopySignedBeaconBlock(s.head.block)
+}
+
+// This returns the head state.
+// It does a full copy on head state for immutability.
+func (s *Service) headState() *state.BeaconState {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	return s.head.state.Copy()
+}
+
+// This returns the genesis validator root of the head state.
+func (s *Service) headGenesisValidatorRoot() [32]byte {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	return bytesutil.ToBytes32(s.head.state.GenesisValidatorRoot())
+}
+
+// Returns true if head state exists.
+func (s *Service) hasHeadState() bool {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	return s.head != nil && s.head.state != nil
+}
+
+// This updates recent canonical block mapping. It uses input head root and retrieves
+// all the canonical block roots that are ancestor of the input head block root.
+func (s *Service) updateRecentCanonicalBlocks(ctx context.Context, headRoot [32]byte) error {
+	s.recentCanonicalBlocksLock.Lock()
+	defer s.recentCanonicalBlocksLock.Unlock()
+
+	s.recentCanonicalBlocks = make(map[[32]byte]bool)
+	s.recentCanonicalBlocks[headRoot] = true
+	nodes := s.forkChoiceStore.Nodes()
+	node := s.forkChoiceStore.Node(headRoot)
+	if node == nil {
+		return nil
+	}
+
+	for node.Parent != protoarray.NonExistentNode {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		node = nodes[node.Parent]
+		s.recentCanonicalBlocks[node.Root] = true
+	}
+
+	return nil
+}

beacon-chain/blockchain/head_test.go

@@ -0,0 +1,200 @@
+package blockchain
+
+import (
+	"bytes"
+	"context"
+	"reflect"
+	"testing"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
+	logTest "github.com/sirupsen/logrus/hooks/test"
+)
+
+func TestSaveHead_Same(t *testing.T) {
+	db := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db)
+
+	r := [32]byte{'A'}
+	service.head = &head{slot: 0, root: r}
+
+	if err := service.saveHead(context.Background(), r); err != nil {
+		t.Fatal(err)
+	}
+
+	if service.headSlot() != 0 {
+		t.Error("Head did not stay the same")
+	}
+
+	if service.headRoot() != r {
+		t.Error("Head did not stay the same")
+	}
+}
+
+func TestSaveHead_Different(t *testing.T) {
+	db := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db)
+
+	oldRoot := [32]byte{'A'}
+	service.head = &head{slot: 0, root: oldRoot}
+
+	newHeadBlock := &ethpb.BeaconBlock{Slot: 1}
+	newHeadSignedBlock := &ethpb.SignedBeaconBlock{Block: newHeadBlock}
+
+	if err := service.beaconDB.SaveBlock(context.Background(), newHeadSignedBlock); err != nil {
+		t.Fatal(err)
+	}
+	newRoot, err := stateutil.BlockRoot(newHeadBlock)
+	if err != nil {
+		t.Fatal(err)
+	}
+	headState := testutil.NewBeaconState()
+	if err := headState.SetSlot(1); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveStateSummary(context.Background(), &pb.StateSummary{Slot: 1, Root: newRoot[:]}); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveState(context.Background(), headState, newRoot); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.saveHead(context.Background(), newRoot); err != nil {
+		t.Fatal(err)
+	}
+
+	if service.HeadSlot() != 1 {
+		t.Error("Head did not change")
+	}
+
+	cachedRoot, err := service.HeadRoot(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(cachedRoot, newRoot[:]) {
+		t.Error("Head did not change")
+	}
+	if !reflect.DeepEqual(service.headBlock(), newHeadSignedBlock) {
+		t.Error("Head did not change")
+	}
+	if !reflect.DeepEqual(service.headState().CloneInnerState(), headState.CloneInnerState()) {
+		t.Error("Head did not change")
+	}
+}
+
+func TestSaveHead_Different_Reorg(t *testing.T) {
+	hook := logTest.NewGlobal()
+	db := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db)
+
+	oldRoot := [32]byte{'A'}
+	service.head = &head{slot: 0, root: oldRoot}
+
+	reorgChainParent := [32]byte{'B'}
+	newHeadBlock := &ethpb.BeaconBlock{
+		Slot:       1,
+		ParentRoot: reorgChainParent[:],
+	}
+	newHeadSignedBlock := &ethpb.SignedBeaconBlock{Block: newHeadBlock}
+
+	if err := service.beaconDB.SaveBlock(context.Background(), newHeadSignedBlock); err != nil {
+		t.Fatal(err)
+	}
+	newRoot, err := stateutil.BlockRoot(newHeadBlock)
+	if err != nil {
+		t.Fatal(err)
+	}
+	headState := testutil.NewBeaconState()
+	if err := headState.SetSlot(1); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveStateSummary(context.Background(), &pb.StateSummary{Slot: 1, Root: newRoot[:]}); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveState(context.Background(), headState, newRoot); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.saveHead(context.Background(), newRoot); err != nil {
+		t.Fatal(err)
+	}
+
+	if service.HeadSlot() != 1 {
+		t.Error("Head did not change")
+	}
+
+	cachedRoot, err := service.HeadRoot(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(cachedRoot, newRoot[:]) {
+		t.Error("Head did not change")
+	}
+	if !reflect.DeepEqual(service.headBlock(), newHeadSignedBlock) {
+		t.Error("Head did not change")
+	}
+	if !reflect.DeepEqual(service.headState().CloneInnerState(), headState.CloneInnerState()) {
+		t.Error("Head did not change")
+	}
+	testutil.AssertLogsContain(t, hook, "Chain reorg occurred")
+}
+
+func TestUpdateRecentCanonicalBlocks_CanUpdateWithoutParent(t *testing.T) {
+	db := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db)
+
+	r := [32]byte{'a'}
+	if err := service.updateRecentCanonicalBlocks(context.Background(), r); err != nil {
+		t.Fatal(err)
+	}
+	canonical, err := service.IsCanonical(context.Background(), r)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !canonical {
+		t.Error("Block should be canonical")
+	}
+}
+
+func TestUpdateRecentCanonicalBlocks_CanUpdateWithParent(t *testing.T) {
+	db := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db)
+	oldHead := [32]byte{'a'}
+	if err := service.forkChoiceStore.ProcessBlock(context.Background(), 1, oldHead, [32]byte{'g'}, [32]byte{}, 0, 0); err != nil {
+		t.Fatal(err)
+	}
+	currentHead := [32]byte{'b'}
+	if err := service.forkChoiceStore.ProcessBlock(context.Background(), 3, currentHead, oldHead, [32]byte{}, 0, 0); err != nil {
+		t.Fatal(err)
+	}
+	forkedRoot := [32]byte{'c'}
+	if err := service.forkChoiceStore.ProcessBlock(context.Background(), 2, forkedRoot, oldHead, [32]byte{}, 0, 0); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := service.updateRecentCanonicalBlocks(context.Background(), currentHead); err != nil {
+		t.Fatal(err)
+	}
+	canonical, err := service.IsCanonical(context.Background(), currentHead)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !canonical {
+		t.Error("Block should be canonical")
+	}
+	canonical, err = service.IsCanonical(context.Background(), oldHead)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !canonical {
+		t.Error("Block should be canonical")
+	}
+	canonical, err = service.IsCanonical(context.Background(), forkedRoot)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if canonical {
+		t.Error("Block should not be canonical")
+	}
+}

beacon-chain/blockchain/info.go

@@ -1,57 +1,85 @@
 package blockchain
 
 import (
-	"bytes"
 	"encoding/hex"
 	"fmt"
 	"net/http"
-	"sort"
+	"strconv"
 
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/sirupsen/logrus"
+	"github.com/emicklei/dot"
 )
 
-const latestSlotCount = 10
+const template = `<html>
+<head>
+    <script src="//cdnjs.cloudflare.com/ajax/libs/viz.js/2.1.2/viz.js"></script>
+    <script src="//cdnjs.cloudflare.com/ajax/libs/viz.js/2.1.2/full.render.js"></script>
+<body>
+    <script type="application/javascript">
+        var graph = ` + "`%s`;" + `
+        var viz = new Viz();
+        viz.renderSVGElement(graph) // reading the graph.
+            .then(function(element) {
+                document.body.appendChild(element); // appends to document.
+            })
+            .catch(error => {
+                // Create a new Viz instance (@see Caveats page for more info)
+                viz = new Viz();
+                // Possibly display the error
+                console.error(error);
+            });
+    </script>
+</head>
+</body>
+</html>`
 
-// HeadsHandler is a handler to serve /heads page in metrics.
-func (s *Service) HeadsHandler(w http.ResponseWriter, _ *http.Request) {
-	buf := new(bytes.Buffer)
-
-	if _, err := fmt.Fprintf(w, "\n %s\t%s\t", "Head slot", "Head root"); err != nil {
-		logrus.WithError(err).Error("Failed to render chain heads page")
-		return
+// TreeHandler is a handler to serve /tree page in metrics.
+func (s *Service) TreeHandler(w http.ResponseWriter, _ *http.Request) {
+	if s.headState() == nil {
+		if _, err := w.Write([]byte("Unavailable during initial syncing")); err != nil {
+			log.WithError(err).Error("Failed to render p2p info page")
+		}
 	}
 
-	if _, err := fmt.Fprintf(w, "\n %s\t%s\t", "---------", "---------"); err != nil {
-		logrus.WithError(err).Error("Failed to render chain heads page")
-		return
+	nodes := s.forkChoiceStore.Nodes()
+
+	graph := dot.NewGraph(dot.Directed)
+	graph.Attr("rankdir", "RL")
+	graph.Attr("labeljust", "l")
+
+	dotNodes := make([]*dot.Node, len(nodes))
+	avgBalance := uint64(averageBalance(s.headState().Balances()))
+
+	for i := len(nodes) - 1; i >= 0; i-- {
+		// Construct label for each node.
+		slot := strconv.Itoa(int(nodes[i].Slot))
+		weight := strconv.Itoa(int(nodes[i].Weight / 1e9)) // Convert unit Gwei to unit ETH.
+		votes := strconv.Itoa(int(nodes[i].Weight / 1e9 / avgBalance))
+		index := strconv.Itoa(int(i))
+		g := nodes[i].Graffiti[:]
+		graffiti := hex.EncodeToString(g[:8])
+		label := "slot: " + slot + "\n votes: " + votes + "\n weight: " + weight + "\n graffiti: " + graffiti
+		var dotN dot.Node
+		if nodes[i].Parent != ^uint64(0) {
+			dotN = graph.Node(index).Box().Attr("label", label)
+		}
+
+		if nodes[i].Slot == s.headSlot() &&
+			nodes[i].BestDescendent == ^uint64(0) {
+			dotN = dotN.Attr("color", "green")
+		}
+
+		dotNodes[i] = &dotN
 	}
 
-	slots := s.latestHeadSlots()
-	for _, slot := range slots {
-		r := hex.EncodeToString(bytesutil.Trunc(s.canonicalRoots[uint64(slot)]))
-		if _, err := fmt.Fprintf(w, "\n %d\t\t%s\t", slot, r); err != nil {
-			logrus.WithError(err).Error("Failed to render chain heads page")
-			return
+	for i := len(nodes) - 1; i >= 0; i-- {
+		if nodes[i].Parent != ^uint64(0) && nodes[i].Parent < uint64(len(dotNodes)) {
+			graph.Edge(*dotNodes[i], *dotNodes[nodes[i].Parent])
 		}
 	}
 
 	w.WriteHeader(http.StatusOK)
-	if _, err := w.Write(buf.Bytes()); err != nil {
-		log.WithError(err).Error("Failed to render chain heads page")
+	w.Header().Set("Content-Type", "text/html")
+	if _, err := fmt.Fprintf(w, template, graph.String()); err != nil {
+		log.WithError(err).Error("Failed to render p2p info page")
 	}
-
-}
-
-// This returns the latest head slots in a slice and up to latestSlotCount
-func (s *Service) latestHeadSlots() []int {
-	slots := make([]int, 0, len(s.canonicalRoots))
-	for k := range s.canonicalRoots {
-		slots = append(slots, int(k))
-	}
-	sort.Ints(slots)
-	if (len(slots)) > latestSlotCount {
-		return slots[len(slots)-latestSlotCount:]
-	}
-	return slots
 }

beacon-chain/blockchain/init_sync_process_block.go

@@ -0,0 +1,261 @@
+package blockchain
+
+import (
+	"context"
+	"sort"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+	"github.com/prysmaticlabs/prysm/shared/params"
+)
+
+const maxCacheSize = 70
+const initialSyncCacheSize = 45
+const minimumCacheSize = initialSyncCacheSize / 3
+
+func (s *Service) persistCachedStates(ctx context.Context, numOfStates int) error {
+	oldStates := make([]*stateTrie.BeaconState, 0, numOfStates)
+
+	// Add slots to the map and add epoch boundary states to the slice.
+	for _, rt := range s.boundaryRoots[:numOfStates-minimumCacheSize] {
+		oldStates = append(oldStates, s.initSyncState[rt])
+	}
+
+	err := s.beaconDB.SaveStates(ctx, oldStates, s.boundaryRoots[:numOfStates-minimumCacheSize])
+	if err != nil {
+		return err
+	}
+	for _, rt := range s.boundaryRoots[:numOfStates-minimumCacheSize] {
+		delete(s.initSyncState, rt)
+	}
+	s.boundaryRoots = s.boundaryRoots[numOfStates-minimumCacheSize:]
+	return nil
+}
+
+// filter out boundary candidates from our currently processed batch of states.
+func (s *Service) filterBoundaryCandidates(ctx context.Context, root [32]byte, postState *stateTrie.BeaconState) {
+	// Only trigger on epoch start.
+	if !helpers.IsEpochStart(postState.Slot()) {
+		return
+	}
+
+	stateSlice := make([][32]byte, 0, len(s.initSyncState))
+	// Add epoch boundary roots to slice.
+	for rt := range s.initSyncState {
+		stateSlice = append(stateSlice, rt)
+	}
+
+	sort.Slice(stateSlice, func(i int, j int) bool {
+		return s.initSyncState[stateSlice[i]].Slot() < s.initSyncState[stateSlice[j]].Slot()
+	})
+	epochLength := params.BeaconConfig().SlotsPerEpoch
+
+	if len(s.boundaryRoots) > 0 {
+		// Retrieve previous boundary root.
+		previousBoundaryRoot := s.boundaryRoots[len(s.boundaryRoots)-1]
+		previousState, ok := s.initSyncState[previousBoundaryRoot]
+		if !ok {
+			// Remove the non-existent root and exit filtering.
+			s.boundaryRoots = s.boundaryRoots[:len(s.boundaryRoots)-1]
+			return
+		}
+		previousSlot := previousState.Slot()
+
+		// Round up slot number to account for skipped slots.
+		previousSlot = helpers.RoundUpToNearestEpoch(previousSlot)
+		if postState.Slot()-previousSlot >= epochLength {
+			targetSlot := postState.Slot()
+			tempRoots := s.loopThroughCandidates(stateSlice, previousBoundaryRoot, previousSlot, targetSlot)
+			s.boundaryRoots = append(s.boundaryRoots, tempRoots...)
+		}
+	}
+	s.boundaryRoots = append(s.boundaryRoots, root)
+	s.pruneOldStates()
+	s.pruneNonBoundaryStates()
+}
+
+// loop-through the provided candidate roots to filter out which would be appropriate boundary roots.
+func (s *Service) loopThroughCandidates(stateSlice [][32]byte, previousBoundaryRoot [32]byte,
+	previousSlot uint64, targetSlot uint64) [][32]byte {
+	tempRoots := [][32]byte{}
+	epochLength := params.BeaconConfig().SlotsPerEpoch
+
+	// Loop through current states to filter for valid boundary states.
+	for i := len(stateSlice) - 1; stateSlice[i] != previousBoundaryRoot && i >= 0; i-- {
+		currentSlot := s.initSyncState[stateSlice[i]].Slot()
+		// Skip if the current slot is larger than the previous epoch
+		// boundary.
+		if currentSlot > targetSlot-epochLength {
+			continue
+		}
+		tempRoots = append(tempRoots, stateSlice[i])
+
+		// Switch target slot if the current slot is greater than
+		// 1 epoch boundary from the previously saved boundary slot.
+		if currentSlot > previousSlot+epochLength {
+			currentSlot = helpers.RoundUpToNearestEpoch(currentSlot)
+			targetSlot = currentSlot
+			continue
+		}
+		break
+	}
+	// Reverse to append the roots in ascending order corresponding
+	// to the respective slots.
+	tempRoots = bytesutil.ReverseBytes32Slice(tempRoots)
+	return tempRoots
+}
+
+// prune for states past the current finalized checkpoint.
+func (s *Service) pruneOldStates() {
+	prunedBoundaryRoots := [][32]byte{}
+	for _, rt := range s.boundaryRoots {
+		st, ok := s.initSyncState[rt]
+		// Skip non-existent roots.
+		if !ok {
+			continue
+		}
+		if st.Slot() < helpers.StartSlot(s.FinalizedCheckpt().Epoch) {
+			delete(s.initSyncState, rt)
+			continue
+		}
+		prunedBoundaryRoots = append(prunedBoundaryRoots, rt)
+	}
+	s.boundaryRoots = prunedBoundaryRoots
+}
+
+// prune cache for non-boundary states.
+func (s *Service) pruneNonBoundaryStates() {
+	boundaryMap := make(map[[32]byte]bool)
+	for i := range s.boundaryRoots {
+		boundaryMap[s.boundaryRoots[i]] = true
+	}
+	for rt := range s.initSyncState {
+		if !boundaryMap[rt] {
+			delete(s.initSyncState, rt)
+		}
+	}
+}
+
+func (s *Service) pruneOldNonFinalizedStates() {
+	stateSlice := make([][32]byte, 0, len(s.initSyncState))
+	// Add epoch boundary roots to slice.
+	for rt := range s.initSyncState {
+		stateSlice = append(stateSlice, rt)
+	}
+
+	// Sort by slots.
+	sort.Slice(stateSlice, func(i int, j int) bool {
+		return s.initSyncState[stateSlice[i]].Slot() < s.initSyncState[stateSlice[j]].Slot()
+	})
+
+	boundaryMap := make(map[[32]byte]bool)
+	for i := range s.boundaryRoots {
+		boundaryMap[s.boundaryRoots[i]] = true
+	}
+	for _, rt := range stateSlice[:initialSyncCacheSize] {
+		if boundaryMap[rt] {
+			continue
+		}
+		delete(s.initSyncState, rt)
+	}
+}
+
+func (s *Service) generateState(ctx context.Context, startRoot [32]byte, endRoot [32]byte) (*stateTrie.BeaconState, error) {
+	preState, err := s.beaconDB.State(ctx, startRoot)
+	if err != nil {
+		return nil, err
+	}
+	if preState == nil {
+		if !s.stateGen.HasState(ctx, startRoot) {
+			if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+				return nil, errors.Wrap(err, "could not save initial sync blocks")
+			}
+			s.clearInitSyncBlocks()
+		}
+		preState, err = s.stateGen.StateByRoot(ctx, startRoot)
+		if err != nil {
+			return nil, err
+		}
+		if preState == nil {
+			return nil, errors.New("finalized state does not exist in db")
+		}
+	}
+
+	if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+		return nil, err
+	}
+	var endBlock *ethpb.SignedBeaconBlock
+	if !featureconfig.Get().NoInitSyncBatchSaveBlocks && s.hasInitSyncBlock(endRoot) {
+		endBlock = s.getInitSyncBlock(endRoot)
+		s.clearInitSyncBlocks()
+	} else {
+		endBlock, err = s.beaconDB.Block(ctx, endRoot)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if endBlock == nil {
+		return nil, errors.New("provided block root does not have block saved in the db")
+	}
+	log.Warnf("Generating missing state of slot %d and root %#x", endBlock.Block.Slot, endRoot)
+
+	blocks, err := s.stateGen.LoadBlocks(ctx, preState.Slot()+1, endBlock.Block.Slot, endRoot)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not load the required blocks")
+	}
+	postState, err := s.stateGen.ReplayBlocks(ctx, preState, blocks, endBlock.Block.Slot)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not replay the blocks to generate the resultant state")
+	}
+	return postState, nil
+}
+
+// This saves a beacon block to the initial sync blocks cache.
+func (s *Service) saveInitSyncBlock(r [32]byte, b *ethpb.SignedBeaconBlock) {
+	s.initSyncBlocksLock.Lock()
+	defer s.initSyncBlocksLock.Unlock()
+	s.initSyncBlocks[r] = b
+}
+
+// This checks if a beacon block exists in the initial sync blocks cache using the root
+// of the block.
+func (s *Service) hasInitSyncBlock(r [32]byte) bool {
+	s.initSyncBlocksLock.RLock()
+	defer s.initSyncBlocksLock.RUnlock()
+	_, ok := s.initSyncBlocks[r]
+	return ok
+}
+
+// This retrieves a beacon block from the initial sync blocks cache using the root of
+// the block.
+func (s *Service) getInitSyncBlock(r [32]byte) *ethpb.SignedBeaconBlock {
+	s.initSyncBlocksLock.RLock()
+	defer s.initSyncBlocksLock.RUnlock()
+	b := s.initSyncBlocks[r]
+	return b
+}
+
+// This retrieves all the beacon blocks from the initial sync blocks cache, the returned
+// blocks are unordered.
+func (s *Service) getInitSyncBlocks() []*ethpb.SignedBeaconBlock {
+	s.initSyncBlocksLock.RLock()
+	defer s.initSyncBlocksLock.RUnlock()
+
+	blks := make([]*ethpb.SignedBeaconBlock, 0, len(s.initSyncBlocks))
+	for _, b := range s.initSyncBlocks {
+		blks = append(blks, b)
+	}
+	return blks
+}
+
+// This clears out the initial sync blocks cache.
+func (s *Service) clearInitSyncBlocks() {
+	s.initSyncBlocksLock.Lock()
+	defer s.initSyncBlocksLock.Unlock()
+	s.initSyncBlocks = make(map[[32]byte]*ethpb.SignedBeaconBlock)
+}

beacon-chain/blockchain/init_sync_process_block_test.go

@@ -0,0 +1,314 @@
+package blockchain
+
+import (
+	"context"
+	"testing"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/go-ssz"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
+	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
+	messagediff "gopkg.in/d4l3k/messagediff.v1"
+)
+
+func TestFilterBoundaryCandidates_FilterCorrect(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	st, err := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := uint64(0); i < 500; i++ {
+		if err := st.SetSlot(i); err != nil {
+			t.Fatal(err)
+		}
+		root := [32]byte{}
+		copy(root[:], bytesutil.Bytes32(i))
+		service.initSyncState[root] = st.Copy()
+		if helpers.IsEpochStart(i) {
+			service.boundaryRoots = append(service.boundaryRoots, root)
+		}
+	}
+	lastIndex := len(service.boundaryRoots) - 1
+	for i := uint64(500); i < 2000; i++ {
+		if err := st.SetSlot(i); err != nil {
+			t.Fatal(err)
+		}
+		root := [32]byte{}
+		copy(root[:], bytesutil.Bytes32(i))
+		service.initSyncState[root] = st.Copy()
+	}
+	// Set current state.
+	latestSlot := helpers.RoundUpToNearestEpoch(2000)
+	if err := st.SetSlot(latestSlot); err != nil {
+		t.Fatal(err)
+	}
+	lastRoot := [32]byte{}
+	copy(lastRoot[:], bytesutil.Bytes32(latestSlot))
+
+	service.initSyncState[lastRoot] = st.Copy()
+	service.finalizedCheckpt = &ethpb.Checkpoint{
+		Epoch: 0,
+		Root:  []byte{},
+	}
+	service.filterBoundaryCandidates(context.Background(), lastRoot, st)
+	if len(service.boundaryRoots[lastIndex+1:]) == 0 {
+		t.Fatal("Wanted non zero added boundary roots")
+	}
+	for _, rt := range service.boundaryRoots[lastIndex+1:] {
+		st, ok := service.initSyncState[rt]
+		if !ok {
+			t.Error("Root doen't exist in cache map")
+			continue
+		}
+		if !(helpers.IsEpochStart(st.Slot()) || helpers.IsEpochStart(st.Slot()-1) || helpers.IsEpochStart(st.Slot()+1)) {
+			t.Errorf("boundary roots not validly stored. They have slot %d", st.Slot())
+		}
+	}
+}
+
+func TestFilterBoundaryCandidates_HandleSkippedSlots(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	st, err := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := uint64(0); i < 500; i++ {
+		if err := st.SetSlot(i); err != nil {
+			t.Fatal(err)
+		}
+		root := [32]byte{}
+		copy(root[:], bytesutil.Bytes32(i))
+		service.initSyncState[root] = st.Copy()
+		if helpers.IsEpochStart(i) {
+			service.boundaryRoots = append(service.boundaryRoots, root)
+		}
+	}
+	lastIndex := len(service.boundaryRoots) - 1
+	for i := uint64(500); i < 2000; i++ {
+		if err := st.SetSlot(i); err != nil {
+			t.Fatal(err)
+		}
+		root := [32]byte{}
+		copy(root[:], bytesutil.Bytes32(i))
+		// save only for offsetted slots
+		if helpers.IsEpochStart(i + 10) {
+			service.initSyncState[root] = st.Copy()
+		}
+	}
+	// Set current state.
+	latestSlot := helpers.RoundUpToNearestEpoch(2000)
+	if err := st.SetSlot(latestSlot); err != nil {
+		t.Fatal(err)
+	}
+	lastRoot := [32]byte{}
+	copy(lastRoot[:], bytesutil.Bytes32(latestSlot))
+
+	service.initSyncState[lastRoot] = st.Copy()
+	service.finalizedCheckpt = &ethpb.Checkpoint{
+		Epoch: 0,
+		Root:  []byte{},
+	}
+	service.filterBoundaryCandidates(context.Background(), lastRoot, st)
+	if len(service.boundaryRoots[lastIndex+1:]) == 0 {
+		t.Fatal("Wanted non zero added boundary roots")
+	}
+	for _, rt := range service.boundaryRoots[lastIndex+1:] {
+		st, ok := service.initSyncState[rt]
+		if !ok {
+			t.Error("Root doen't exist in cache map")
+			continue
+		}
+		if st.Slot() >= 500 {
+			// Ignore head boundary root.
+			if st.Slot() == 2016 {
+				continue
+			}
+			if !helpers.IsEpochStart(st.Slot() + 10) {
+				t.Errorf("boundary roots not validly stored. They have slot %d "+
+					"instead of the offset from epoch start", st.Slot())
+			}
+		}
+	}
+}
+
+func TestPruneOldStates_AlreadyFinalized(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	st, err := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := uint64(100); i < 200; i++ {
+		if err := st.SetSlot(i); err != nil {
+			t.Fatal(err)
+		}
+		root := [32]byte{}
+		copy(root[:], bytesutil.Bytes32(i))
+		service.initSyncState[root] = st.Copy()
+		service.boundaryRoots = append(service.boundaryRoots, root)
+	}
+	finalizedEpoch := uint64(5)
+	service.finalizedCheckpt = &ethpb.Checkpoint{Epoch: finalizedEpoch}
+	service.pruneOldStates()
+	for _, rt := range service.boundaryRoots {
+		st, ok := service.initSyncState[rt]
+		if !ok {
+			t.Error("Root doen't exist in cache map")
+			continue
+		}
+		if st.Slot() < helpers.StartSlot(finalizedEpoch) {
+			t.Errorf("State with slot %d still exists and not pruned", st.Slot())
+		}
+	}
+}
+
+func TestPruneNonBoundary_CanPrune(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	st, err := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := uint64(0); i < 2000; i++ {
+		if err := st.SetSlot(i); err != nil {
+			t.Fatal(err)
+		}
+		root := [32]byte{}
+		copy(root[:], bytesutil.Bytes32(i))
+		service.initSyncState[root] = st.Copy()
+		if helpers.IsEpochStart(i) {
+			service.boundaryRoots = append(service.boundaryRoots, root)
+		}
+	}
+	service.pruneNonBoundaryStates()
+	for _, rt := range service.boundaryRoots {
+		st, ok := service.initSyncState[rt]
+		if !ok {
+			t.Error("Root doesn't exist in cache map")
+			continue
+		}
+		if !helpers.IsEpochStart(st.Slot()) {
+			t.Errorf("Non boundary state with slot %d still exists and not pruned", st.Slot())
+		}
+	}
+}
+
+func TestGenerateState_CorrectlyGenerated(t *testing.T) {
+	db := testDB.SetupDB(t)
+	cfg := &Config{BeaconDB: db, StateGen: stategen.New(db, cache.NewStateSummaryCache())}
+	service, err := NewService(context.Background(), cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	beaconState, privs := testutil.DeterministicGenesisState(t, 32)
+	genesisBlock := blocks.NewGenesisBlock([]byte{})
+	bodyRoot, err := stateutil.BlockRoot(genesisBlock.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = beaconState.SetLatestBlockHeader(&ethpb.BeaconBlockHeader{
+		Slot:       genesisBlock.Block.Slot,
+		ParentRoot: genesisBlock.Block.ParentRoot,
+		StateRoot:  params.BeaconConfig().ZeroHash[:],
+		BodyRoot:   bodyRoot[:],
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := beaconState.SetSlashings(make([]uint64, params.BeaconConfig().EpochsPerSlashingsVector)); err != nil {
+		t.Fatal(err)
+	}
+	cp := beaconState.CurrentJustifiedCheckpoint()
+	mockRoot := [32]byte{}
+	copy(mockRoot[:], "hello-world")
+	cp.Root = mockRoot[:]
+	if err := beaconState.SetCurrentJustifiedCheckpoint(cp); err != nil {
+		t.Fatal(err)
+	}
+	if err := beaconState.SetCurrentEpochAttestations([]*pb.PendingAttestation{}); err != nil {
+		t.Fatal(err)
+	}
+	err = db.SaveBlock(context.Background(), genesisBlock)
+	if err != nil {
+		t.Fatal(err)
+	}
+	genRoot, err := stateutil.BlockRoot(genesisBlock.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = db.SaveState(context.Background(), beaconState, genRoot)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	lastBlock := &ethpb.SignedBeaconBlock{}
+	for i := uint64(1); i < 10; i++ {
+		block, err := testutil.GenerateFullBlock(beaconState, privs, testutil.DefaultBlockGenConfig(), i)
+		if err != nil {
+			t.Fatal(err)
+		}
+		beaconState, err = state.ExecuteStateTransition(context.Background(), beaconState, block)
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = db.SaveBlock(context.Background(), block)
+		if err != nil {
+			t.Fatal(err)
+		}
+		lastBlock = block
+	}
+	root, err := stateutil.BlockRoot(lastBlock.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	newState, err := service.generateState(context.Background(), genRoot, root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !ssz.DeepEqual(newState.InnerStateUnsafe(), beaconState.InnerStateUnsafe()) {
+		diff, _ := messagediff.PrettyDiff(newState.InnerStateUnsafe(), beaconState.InnerStateUnsafe())
+		t.Errorf("Generated state is different from what is expected: %s", diff)
+	}
+}

beacon-chain/blockchain/log.go

@@ -1,17 +1,62 @@
 package blockchain
 
 import (
+	"encoding/hex"
+	"fmt"
+
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/sirupsen/logrus"
 )
 
 var log = logrus.WithField("prefix", "blockchain")
 
 // logs state transition related data every slot.
-func logStateTransitionData(b *ethpb.BeaconBlock, r []byte) {
+func logStateTransitionData(b *ethpb.BeaconBlock) {
 	log.WithFields(logrus.Fields{
-		"slot":         b.Slot,
-		"attestations": len(b.Body.Attestations),
-		"deposits":     len(b.Body.Deposits),
+		"attestations":      len(b.Body.Attestations),
+		"deposits":          len(b.Body.Deposits),
+		"attesterSlashings": len(b.Body.AttesterSlashings),
+		"proposerSlashings": len(b.Body.ProposerSlashings),
+		"voluntaryExits":    len(b.Body.VoluntaryExits),
 	}).Info("Finished applying state transition")
 }
+
+func logBlockSyncStatus(block *ethpb.BeaconBlock, blockRoot [32]byte, finalized *ethpb.Checkpoint) {
+	log.WithFields(logrus.Fields{
+		"slot":           block.Slot,
+		"block":          fmt.Sprintf("0x%s...", hex.EncodeToString(blockRoot[:])[:8]),
+		"epoch":          helpers.SlotToEpoch(block.Slot),
+		"finalizedEpoch": finalized.Epoch,
+		"finalizedRoot":  fmt.Sprintf("0x%s...", hex.EncodeToString(finalized.Root[:])[:8]),
+	}).Info("Synced new block")
+}
+
+func logEpochData(beaconState *stateTrie.BeaconState) {
+	log.WithFields(logrus.Fields{
+		"epoch":                  helpers.CurrentEpoch(beaconState),
+		"finalizedEpoch":         beaconState.FinalizedCheckpointEpoch(),
+		"justifiedEpoch":         beaconState.CurrentJustifiedCheckpoint().Epoch,
+		"previousJustifiedEpoch": beaconState.PreviousJustifiedCheckpoint().Epoch,
+	}).Info("Starting next epoch")
+	activeVals, err := helpers.ActiveValidatorIndices(beaconState, helpers.CurrentEpoch(beaconState))
+	if err != nil {
+		log.WithError(err).Error("Could not get active validator indices")
+		return
+	}
+	log.WithFields(logrus.Fields{
+		"totalValidators":  len(beaconState.Validators()),
+		"activeValidators": len(activeVals),
+		"averageBalance":   fmt.Sprintf("%.5f ETH", averageBalance(beaconState.Balances())),
+	}).Info("Validator registry information")
+}
+
+func averageBalance(balances []uint64) float64 {
+	total := uint64(0)
+	for i := 0; i < len(balances); i++ {
+		total += balances[i]
+	}
+	return float64(total) / float64(len(balances)) / float64(params.BeaconConfig().GweiPerEth)
+}

beacon-chain/blockchain/metrics.go

@@ -1,9 +1,17 @@
 package blockchain
 
 import (
+	"time"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/roughtime"
 )
 
 var (
@@ -15,38 +23,14 @@ var (
 		Name: "beacon_head_slot",
 		Help: "Slot of the head block of the beacon chain",
 	})
-	beaconHeadRoot = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_head_root",
-		Help: "Root of the head block of the beacon chain, it returns the lowest 8 bytes interpreted as little endian",
-	})
-	competingAtts = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "competing_attestations",
-		Help: "The # of attestations received and processed from a competing chain",
+	clockTimeSlot = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_clock_time_slot",
+		Help: "The current slot based on the genesis time and current clock",
 	})
 	competingBlks = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "competing_blocks",
 		Help: "The # of blocks received and processed from a competing chain",
 	})
-	processedBlkNoPubsub = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "processed_no_pubsub_block_counter",
-		Help: "The # of processed block without pubsub, this usually means the blocks from sync",
-	})
-	processedBlkNoPubsubForkchoice = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "processed_no_pubsub_forkchoice_block_counter",
-		Help: "The # of processed block without pubsub and forkchoice, this means indicate blocks from initial sync",
-	})
-	processedBlk = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "processed_block_counter",
-		Help: "The # of total processed in block chain service, with fork choice and pubsub",
-	})
-	processedAttNoPubsub = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "processed_no_pubsub_attestation_counter",
-		Help: "The # of processed attestation without pubsub, this usually means the attestations from sync",
-	})
-	processedAtt = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "processed_attestation_counter",
-		Help: "The # of processed attestation with pubsub and fork choice, this ususally means attestations from rpc",
-	})
 	headFinalizedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
 		Name: "head_finalized_epoch",
 		Help: "Last finalized epoch of the head state",
@@ -55,14 +39,187 @@ var (
 		Name: "head_finalized_root",
 		Help: "Last finalized root of the head state",
 	})
+	beaconFinalizedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_finalized_epoch",
+		Help: "Last finalized epoch of the processed state",
+	})
+	beaconFinalizedRoot = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_finalized_root",
+		Help: "Last finalized root of the processed state",
+	})
+	beaconCurrentJustifiedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_current_justified_epoch",
+		Help: "Current justified epoch of the processed state",
+	})
+	beaconCurrentJustifiedRoot = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_current_justified_root",
+		Help: "Current justified root of the processed state",
+	})
+	beaconPrevJustifiedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_previous_justified_epoch",
+		Help: "Previous justified epoch of the processed state",
+	})
+	beaconPrevJustifiedRoot = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_previous_justified_root",
+		Help: "Previous justified root of the processed state",
+	})
+	validatorsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "validator_count",
+		Help: "The total number of validators",
+	}, []string{"state"})
+	validatorsBalance = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "validators_total_balance",
+		Help: "The total balance of validators, in GWei",
+	}, []string{"state"})
+	validatorsEffectiveBalance = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "validators_total_effective_balance",
+		Help: "The total effective balance of validators, in GWei",
+	}, []string{"state"})
+	currentEth1DataDepositCount = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "current_eth1_data_deposit_count",
+		Help: "The current eth1 deposit count in the last processed state eth1data field.",
+	})
+	totalEligibleBalances = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "total_eligible_balances",
+		Help: "The total amount of ether, in gwei, that has been used in voting attestation target of previous epoch",
+	})
+	totalVotedTargetBalances = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "total_voted_target_balances",
+		Help: "The total amount of ether, in gwei, that is eligible for voting of previous epoch",
+	})
+	reorgCount = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "beacon_reorg_total",
+		Help: "Count the number of times beacon chain has a reorg",
+	})
+	sentBlockPropagationHistogram = promauto.NewHistogram(
+		prometheus.HistogramOpts{
+			Name:    "block_sent_latency_milliseconds",
+			Help:    "Captures blocks broadcast time. Blocks sent in milliseconds distribution",
+			Buckets: []float64{1000, 2000, 3000, 4000, 5000, 6000},
+		},
+	)
+	attestationInclusionDelay = promauto.NewHistogram(
+		prometheus.HistogramOpts{
+			Name:    "attestation_inclusion_delay_slots",
+			Help:    "The number of slots between att.Slot and block.Slot",
+			Buckets: []float64{1, 2, 3, 4, 6, 32, 64},
+		},
+	)
 )
 
-func (s *Service) reportSlotMetrics(currentSlot uint64) {
-	beaconSlot.Set(float64(currentSlot))
-	beaconHeadSlot.Set(float64(s.HeadSlot()))
-	beaconHeadRoot.Set(float64(bytesutil.ToLowInt64(s.HeadRoot())))
-	if s.headState != nil {
-		headFinalizedEpoch.Set(float64(s.headState.FinalizedCheckpoint.Epoch))
-		headFinalizedRoot.Set(float64(bytesutil.ToLowInt64(s.headState.FinalizedCheckpoint.Root)))
+// reportSlotMetrics reports slot related metrics.
+func reportSlotMetrics(stateSlot uint64, headSlot uint64, clockSlot uint64, finalizedCheckpoint *ethpb.Checkpoint) {
+	clockTimeSlot.Set(float64(clockSlot))
+	beaconSlot.Set(float64(stateSlot))
+	beaconHeadSlot.Set(float64(headSlot))
+	if finalizedCheckpoint != nil {
+		headFinalizedEpoch.Set(float64(finalizedCheckpoint.Epoch))
+		headFinalizedRoot.Set(float64(bytesutil.ToLowInt64(finalizedCheckpoint.Root)))
+	}
+}
+
+// reportEpochMetrics reports epoch related metrics.
+func reportEpochMetrics(state *stateTrie.BeaconState) {
+	currentEpoch := state.Slot() / params.BeaconConfig().SlotsPerEpoch
+
+	// Validator instances
+	pendingInstances := 0
+	activeInstances := 0
+	slashingInstances := 0
+	slashedInstances := 0
+	exitingInstances := 0
+	exitedInstances := 0
+	// Validator balances
+	pendingBalance := uint64(0)
+	activeBalance := uint64(0)
+	activeEffectiveBalance := uint64(0)
+	exitingBalance := uint64(0)
+	exitingEffectiveBalance := uint64(0)
+	slashingBalance := uint64(0)
+	slashingEffectiveBalance := uint64(0)
+
+	for i, validator := range state.Validators() {
+		bal, err := state.BalanceAtIndex(uint64(i))
+		if err != nil {
+			continue
+		}
+		if validator.Slashed {
+			if currentEpoch < validator.ExitEpoch {
+				slashingInstances++
+				slashingBalance += bal
+				slashingEffectiveBalance += validator.EffectiveBalance
+			} else {
+				slashedInstances++
+			}
+			continue
+		}
+		if validator.ExitEpoch != params.BeaconConfig().FarFutureEpoch {
+			if currentEpoch < validator.ExitEpoch {
+				exitingInstances++
+				exitingBalance += bal
+				exitingEffectiveBalance += validator.EffectiveBalance
+			} else {
+				exitedInstances++
+			}
+			continue
+		}
+		if currentEpoch < validator.ActivationEpoch {
+			pendingInstances++
+			pendingBalance += bal
+			continue
+		}
+		activeInstances++
+		activeBalance += bal
+		activeEffectiveBalance += validator.EffectiveBalance
+	}
+	validatorsCount.WithLabelValues("Pending").Set(float64(pendingInstances))
+	validatorsCount.WithLabelValues("Active").Set(float64(activeInstances))
+	validatorsCount.WithLabelValues("Exiting").Set(float64(exitingInstances))
+	validatorsCount.WithLabelValues("Exited").Set(float64(exitedInstances))
+	validatorsCount.WithLabelValues("Slashing").Set(float64(slashingInstances))
+	validatorsCount.WithLabelValues("Slashed").Set(float64(slashedInstances))
+	validatorsBalance.WithLabelValues("Pending").Set(float64(pendingBalance))
+	validatorsBalance.WithLabelValues("Active").Set(float64(activeBalance))
+	validatorsBalance.WithLabelValues("Exiting").Set(float64(exitingBalance))
+	validatorsBalance.WithLabelValues("Slashing").Set(float64(slashingBalance))
+	validatorsEffectiveBalance.WithLabelValues("Active").Set(float64(activeEffectiveBalance))
+	validatorsEffectiveBalance.WithLabelValues("Exiting").Set(float64(exitingEffectiveBalance))
+	validatorsEffectiveBalance.WithLabelValues("Slashing").Set(float64(slashingEffectiveBalance))
+
+	// Last justified slot
+	beaconCurrentJustifiedEpoch.Set(float64(state.CurrentJustifiedCheckpoint().Epoch))
+	beaconCurrentJustifiedRoot.Set(float64(bytesutil.ToLowInt64(state.CurrentJustifiedCheckpoint().Root)))
+
+	// Last previous justified slot
+	beaconPrevJustifiedEpoch.Set(float64(state.PreviousJustifiedCheckpoint().Epoch))
+	beaconPrevJustifiedRoot.Set(float64(bytesutil.ToLowInt64(state.PreviousJustifiedCheckpoint().Root)))
+
+	// Last finalized slot
+	beaconFinalizedEpoch.Set(float64(state.FinalizedCheckpointEpoch()))
+	beaconFinalizedRoot.Set(float64(bytesutil.ToLowInt64(state.FinalizedCheckpoint().Root)))
+
+	currentEth1DataDepositCount.Set(float64(state.Eth1Data().DepositCount))
+
+	if precompute.Balances != nil {
+		totalEligibleBalances.Set(float64(precompute.Balances.ActivePrevEpoch))
+		totalVotedTargetBalances.Set(float64(precompute.Balances.PrevEpochTargetAttested))
+	}
+}
+
+// This captures metrics for block sent time by subtracts slot start time.
+func captureSentTimeMetric(genesisTime uint64, currentSlot uint64) error {
+	startTime, err := helpers.SlotToTime(genesisTime, currentSlot)
+	if err != nil {
+		return err
+	}
+	diffMs := roughtime.Now().Sub(startTime) / time.Millisecond
+	sentBlockPropagationHistogram.Observe(float64(diffMs))
+
+	return nil
+}
+
+func reportAttestationInclusion(blk *ethpb.BeaconBlock) {
+	for _, att := range blk.Body.Attestations {
+		attestationInclusionDelay.Observe(float64(blk.Slot - att.Data.Slot))
 	}
 }

beacon-chain/blockchain/process_attestation.go

@@ -0,0 +1,146 @@
+package blockchain
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/flags"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/roughtime"
+	"go.opencensus.io/trace"
+)
+
+// ErrTargetRootNotInDB returns when the target block root of an attestation cannot be found in the
+// beacon database.
+var ErrTargetRootNotInDB = errors.New("target root does not exist in db")
+
+// onAttestation is called whenever an attestation is received, verifies the attestation is valid and saves
+/// it to the DB.
+//
+// Spec pseudocode definition:
+//   def on_attestation(store: Service, attestation: Attestation) -> None:
+//    """
+//    Run ``on_attestation`` upon receiving a new ``attestation`` from either within a block or directly on the wire.
+//
+//    An ``attestation`` that is asserted as invalid may be valid at a later time,
+//    consider scheduling it for later processing in such case.
+//    """
+//    target = attestation.data.target
+//
+//    # Attestations must be from the current or previous epoch
+//    current_epoch = compute_epoch_at_slot(get_current_slot(store))
+//    # Use GENESIS_EPOCH for previous when genesis to avoid underflow
+//    previous_epoch = current_epoch - 1 if current_epoch > GENESIS_EPOCH else GENESIS_EPOCH
+//    assert target.epoch in [current_epoch, previous_epoch]
+//    assert target.epoch == compute_epoch_at_slot(attestation.data.slot)
+//
+//    # Attestations target be for a known block. If target block is unknown, delay consideration until the block is found
+//    assert target.root in store.blocks
+//    # Attestations cannot be from future epochs. If they are, delay consideration until the epoch arrives
+//    base_state = store.block_states[target.root].copy()
+//    assert store.time >= base_state.genesis_time + compute_start_slot_at_epoch(target.epoch) * SECONDS_PER_SLOT
+//
+//    # Attestations must be for a known block. If block is unknown, delay consideration until the block is found
+//    assert attestation.data.beacon_block_root in store.blocks
+//    # Attestations must not be for blocks in the future. If not, the attestation should not be considered
+//    assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot
+//
+//    # Service target checkpoint state if not yet seen
+//    if target not in store.checkpoint_states:
+//        process_slots(base_state, compute_start_slot_at_epoch(target.epoch))
+//        store.checkpoint_states[target] = base_state
+//    target_state = store.checkpoint_states[target]
+//
+//    # Attestations can only affect the fork choice of subsequent slots.
+//    # Delay consideration in the fork choice until their slot is in the past.
+//    assert store.time >= (attestation.data.slot + 1) * SECONDS_PER_SLOT
+//
+//    # Get state at the `target` to validate attestation and calculate the committees
+//    indexed_attestation = get_indexed_attestation(target_state, attestation)
+//    assert is_valid_indexed_attestation(target_state, indexed_attestation)
+//
+//    # Update latest messages
+//    for i in indexed_attestation.attesting_indices:
+//        if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch:
+//            store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=attestation.data.beacon_block_root)
+func (s *Service) onAttestation(ctx context.Context, a *ethpb.Attestation) ([]uint64, error) {
+	ctx, span := trace.StartSpan(ctx, "blockchain.onAttestation")
+	defer span.End()
+
+	if a == nil {
+		return nil, errors.New("nil attestation")
+	}
+	if a.Data == nil {
+		return nil, errors.New("nil attestation.Data field")
+	}
+	if a.Data.Target == nil {
+		return nil, errors.New("nil attestation.Data.Target field")
+	}
+
+	tgt := stateTrie.CopyCheckpoint(a.Data.Target)
+	tgtSlot := helpers.StartSlot(tgt.Epoch)
+
+	if helpers.SlotToEpoch(a.Data.Slot) != a.Data.Target.Epoch {
+		return nil, fmt.Errorf("data slot is not in the same epoch as target %d != %d", helpers.SlotToEpoch(a.Data.Slot), a.Data.Target.Epoch)
+	}
+
+	// Verify beacon node has seen the target block before.
+	if !s.hasBlock(ctx, bytesutil.ToBytes32(tgt.Root)) {
+		return nil, ErrTargetRootNotInDB
+	}
+
+	// Retrieve attestation's data beacon block pre state. Advance pre state to latest epoch if necessary and
+	// save it to the cache.
+	baseState, err := s.getAttPreState(ctx, tgt)
+	if err != nil {
+		return nil, err
+	}
+
+	genesisTime := baseState.GenesisTime()
+
+	// Verify attestation target is from current epoch or previous epoch.
+	if err := s.verifyAttTargetEpoch(ctx, genesisTime, uint64(roughtime.Now().Unix()), tgt); err != nil {
+		return nil, err
+	}
+
+	// Verify Attestations cannot be from future epochs.
+	if err := helpers.VerifySlotTime(genesisTime, tgtSlot, helpers.TimeShiftTolerance); err != nil {
+		return nil, errors.Wrap(err, "could not verify attestation target slot")
+	}
+
+	// Verify attestation beacon block is known and not from the future.
+	if err := s.verifyBeaconBlock(ctx, a.Data); err != nil {
+		return nil, errors.Wrap(err, "could not verify attestation beacon block")
+	}
+
+	// Verify attestations can only affect the fork choice of subsequent slots.
+	if err := helpers.VerifySlotTime(genesisTime, a.Data.Slot+1, helpers.TimeShiftTolerance); err != nil {
+		return nil, err
+	}
+
+	// Use the target state to to validate attestation and calculate the committees.
+	indexedAtt, err := s.verifyAttestation(ctx, baseState, a)
+	if err != nil {
+		return nil, err
+	}
+
+	// Only save attestation in DB for archival node.
+	if flags.Get().EnableArchive {
+		if err := s.beaconDB.SaveAttestation(ctx, a); err != nil {
+			return nil, err
+		}
+	}
+
+	if indexedAtt.AttestingIndices == nil {
+		return nil, errors.New("nil attesting indices")
+	}
+
+	// Update forkchoice store with the new attestation for updating weight.
+	s.forkChoiceStore.ProcessAttestation(ctx, indexedAtt.AttestingIndices, bytesutil.ToBytes32(a.Data.BeaconBlockRoot), a.Data.Target.Epoch)
+
+	return indexedAtt.AttestingIndices, nil
+}

beacon-chain/blockchain/process_attestation_helpers.go

@@ -0,0 +1,200 @@
+package blockchain
+
+import (
+	"bytes"
+	"context"
+	"encoding/hex"
+	"fmt"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/attestationutil"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+	"github.com/prysmaticlabs/prysm/shared/params"
+)
+
+// getAttPreState retrieves the att pre state by either from the cache or the DB.
+func (s *Service) getAttPreState(ctx context.Context, c *ethpb.Checkpoint) (*stateTrie.BeaconState, error) {
+	s.checkpointStateLock.Lock()
+	defer s.checkpointStateLock.Unlock()
+	cachedState, err := s.checkpointState.StateByCheckpoint(c)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get cached checkpoint state")
+	}
+	if cachedState != nil {
+		return cachedState, nil
+	}
+
+	if featureconfig.Get().NewStateMgmt {
+		if !s.stateGen.HasState(ctx, bytesutil.ToBytes32(c.Root)) {
+			if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+				return nil, errors.Wrap(err, "could not save initial sync blocks")
+			}
+			s.clearInitSyncBlocks()
+		}
+
+		baseState, err := s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(c.Root))
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not get pre state for slot %d", helpers.StartSlot(c.Epoch))
+		}
+
+		if helpers.StartSlot(c.Epoch) > baseState.Slot() {
+			baseState = baseState.Copy()
+			baseState, err = state.ProcessSlots(ctx, baseState, helpers.StartSlot(c.Epoch))
+			if err != nil {
+				return nil, errors.Wrapf(err, "could not process slots up to %d", helpers.StartSlot(c.Epoch))
+			}
+		}
+
+		if err := s.checkpointState.AddCheckpointState(&cache.CheckpointState{
+			Checkpoint: c,
+			State:      baseState,
+		}); err != nil {
+			return nil, errors.Wrap(err, "could not saved checkpoint state to cache")
+		}
+
+		return baseState, nil
+	}
+
+	if featureconfig.Get().CheckHeadState {
+		headRoot, err := s.HeadRoot(ctx)
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not get head root")
+		}
+		if bytes.Equal(headRoot, c.Root) {
+			st, err := s.HeadState(ctx)
+			if err != nil {
+				return nil, errors.Wrapf(err, "could not get head state")
+			}
+			if err := s.checkpointState.AddCheckpointState(&cache.CheckpointState{
+				Checkpoint: c,
+				State:      st.Copy(),
+			}); err != nil {
+				return nil, errors.Wrap(err, "could not saved checkpoint state to cache")
+			}
+			return st, nil
+		}
+	}
+
+	baseState, err := s.beaconDB.State(ctx, bytesutil.ToBytes32(c.Root))
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not get pre state for slot %d", helpers.StartSlot(c.Epoch))
+	}
+	if baseState == nil {
+		return nil, fmt.Errorf("pre state of target block %d does not exist", helpers.StartSlot(c.Epoch))
+	}
+
+	if helpers.StartSlot(c.Epoch) > baseState.Slot() {
+		savedState := baseState.Copy()
+		savedState, err = state.ProcessSlots(ctx, savedState, helpers.StartSlot(c.Epoch))
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not process slots up to %d", helpers.StartSlot(c.Epoch))
+		}
+		if err := s.checkpointState.AddCheckpointState(&cache.CheckpointState{
+			Checkpoint: c,
+			State:      savedState.Copy(),
+		}); err != nil {
+			return nil, errors.Wrap(err, "could not saved checkpoint state to cache")
+		}
+		return savedState, nil
+	}
+
+	if err := s.checkpointState.AddCheckpointState(&cache.CheckpointState{
+		Checkpoint: c,
+		State:      baseState.Copy(),
+	}); err != nil {
+		return nil, errors.Wrap(err, "could not saved checkpoint state to cache")
+	}
+
+	return baseState, nil
+}
+
+// verifyAttTargetEpoch validates attestation is from the current or previous epoch.
+func (s *Service) verifyAttTargetEpoch(ctx context.Context, genesisTime uint64, nowTime uint64, c *ethpb.Checkpoint) error {
+	currentSlot := (nowTime - genesisTime) / params.BeaconConfig().SecondsPerSlot
+	currentEpoch := helpers.SlotToEpoch(currentSlot)
+	var prevEpoch uint64
+	// Prevents previous epoch under flow
+	if currentEpoch > 1 {
+		prevEpoch = currentEpoch - 1
+	}
+	if c.Epoch != prevEpoch && c.Epoch != currentEpoch {
+		return fmt.Errorf("target epoch %d does not match current epoch %d or prev epoch %d", c.Epoch, currentEpoch, prevEpoch)
+	}
+	return nil
+}
+
+// verifyBeaconBlock verifies beacon head block is known and not from the future.
+func (s *Service) verifyBeaconBlock(ctx context.Context, data *ethpb.AttestationData) error {
+	b, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(data.BeaconBlockRoot))
+	if err != nil {
+		return err
+	}
+	if b == nil || b.Block == nil {
+		return fmt.Errorf("beacon block %#x does not exist", bytesutil.Trunc(data.BeaconBlockRoot))
+	}
+	if b.Block.Slot > data.Slot {
+		return fmt.Errorf("could not process attestation for future block, block.Slot=%d > attestation.Data.Slot=%d", b.Block.Slot, data.Slot)
+	}
+	return nil
+}
+
+// verifyAttestation validates input attestation is valid.
+func (s *Service) verifyAttestation(ctx context.Context, baseState *stateTrie.BeaconState, a *ethpb.Attestation) (*ethpb.IndexedAttestation, error) {
+	committee, err := helpers.BeaconCommitteeFromState(baseState, a.Data.Slot, a.Data.CommitteeIndex)
+	if err != nil {
+		return nil, err
+	}
+	indexedAtt := attestationutil.ConvertToIndexed(ctx, a, committee)
+	if err := blocks.VerifyIndexedAttestation(ctx, baseState, indexedAtt); err != nil {
+		if err == helpers.ErrSigFailedToVerify {
+			// When sig fails to verify, check if there's a differences in committees due to
+			// different seeds.
+			var aState *stateTrie.BeaconState
+			var err error
+			if featureconfig.Get().NewStateMgmt {
+				if !s.stateGen.HasState(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot)) {
+					if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+						return nil, errors.Wrap(err, "could not save initial sync blocks")
+					}
+					s.clearInitSyncBlocks()
+				}
+				aState, err = s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
+				if err != nil {
+					return nil, err
+				}
+			} else {
+				aState, err = s.beaconDB.State(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
+				if err != nil {
+					return nil, err
+				}
+			}
+			if aState == nil {
+				return nil, fmt.Errorf("nil state for block root %#x", a.Data.BeaconBlockRoot)
+			}
+			epoch := helpers.SlotToEpoch(a.Data.Slot)
+			origSeed, err := helpers.Seed(baseState, epoch, params.BeaconConfig().DomainBeaconAttester)
+			if err != nil {
+				return nil, errors.Wrap(err, "could not get original seed")
+			}
+
+			aSeed, err := helpers.Seed(aState, epoch, params.BeaconConfig().DomainBeaconAttester)
+			if err != nil {
+				return nil, errors.Wrap(err, "could not get attester's seed")
+			}
+			if origSeed != aSeed {
+				return nil, fmt.Errorf("could not verify indexed attestation due to differences in seeds: %v != %v",
+					hex.EncodeToString(bytesutil.Trunc(origSeed[:])), hex.EncodeToString(bytesutil.Trunc(aSeed[:])))
+			}
+		}
+		return nil, errors.Wrap(err, "could not verify indexed attestation")
+	}
+
+	return indexedAtt, nil
+}

beacon-chain/blockchain/process_attestation_test.go

@@ -0,0 +1,470 @@
+package blockchain
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/gogo/protobuf/proto"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
+	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
+)
+
+func TestStore_OnAttestation(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{
+		BeaconDB:        db,
+		ForkChoiceStore: protoarray.New(0, 0, [32]byte{}),
+		StateGen:        stategen.New(db, cache.NewStateSummaryCache()),
+	}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = blockTree1(db, []byte{'g'})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	BlkWithOutState := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 0}}
+	if err := db.SaveBlock(ctx, BlkWithOutState); err != nil {
+		t.Fatal(err)
+	}
+	BlkWithOutStateRoot, err := stateutil.BlockRoot(BlkWithOutState.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	BlkWithStateBadAtt := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1}}
+	if err := db.SaveBlock(ctx, BlkWithStateBadAtt); err != nil {
+		t.Fatal(err)
+	}
+	BlkWithStateBadAttRoot, err := stateutil.BlockRoot(BlkWithStateBadAtt.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s := testutil.NewBeaconState()
+	if err := s.SetSlot(100 * params.BeaconConfig().SlotsPerEpoch); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveState(ctx, s, BlkWithStateBadAttRoot); err != nil {
+		t.Fatal(err)
+	}
+
+	BlkWithValidState := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 2}}
+	if err := db.SaveBlock(ctx, BlkWithValidState); err != nil {
+		t.Fatal(err)
+	}
+
+	BlkWithValidStateRoot, err := stateutil.BlockRoot(BlkWithValidState.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+	s = testutil.NewBeaconState()
+	if err := s.SetFork(&pb.Fork{
+		Epoch:           0,
+		CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
+		PreviousVersion: params.BeaconConfig().GenesisForkVersion,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveState(ctx, s, BlkWithValidStateRoot); err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct {
+		name          string
+		a             *ethpb.Attestation
+		s             *pb.BeaconState
+		wantErr       bool
+		wantErrString string
+	}{
+		{
+			name:          "attestation's data slot not aligned with target vote",
+			a:             &ethpb.Attestation{Data: &ethpb.AttestationData{Slot: params.BeaconConfig().SlotsPerEpoch, Target: &ethpb.Checkpoint{}}},
+			s:             &pb.BeaconState{},
+			wantErr:       true,
+			wantErrString: "data slot is not in the same epoch as target 1 != 0",
+		},
+		{
+			name:          "attestation's target root not in db",
+			a:             &ethpb.Attestation{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{Root: []byte{'A'}}}},
+			s:             &pb.BeaconState{},
+			wantErr:       true,
+			wantErrString: "target root does not exist in db",
+		},
+		{
+			name:          "no pre state for attestations's target block",
+			a:             &ethpb.Attestation{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{Root: BlkWithOutStateRoot[:]}}},
+			s:             &pb.BeaconState{},
+			wantErr:       true,
+			wantErrString: "pre state of target block 0 does not exist",
+		},
+		{
+			name: "process attestation doesn't match current epoch",
+			a: &ethpb.Attestation{Data: &ethpb.AttestationData{Slot: 100 * params.BeaconConfig().SlotsPerEpoch, Target: &ethpb.Checkpoint{Epoch: 100,
+				Root: BlkWithStateBadAttRoot[:]}}},
+			s:             &pb.BeaconState{Slot: 100 * params.BeaconConfig().SlotsPerEpoch},
+			wantErr:       true,
+			wantErrString: "does not match current epoch",
+		},
+		{
+			name:          "process nil field (a.Target) in attestation",
+			a:             nil,
+			s:             &pb.BeaconState{},
+			wantErr:       true,
+			wantErrString: "nil attestation",
+		},
+		{
+			name:          "process nil field (a.Data) in attestation",
+			a:             &ethpb.Attestation{},
+			s:             &pb.BeaconState{},
+			wantErr:       true,
+			wantErrString: "nil attestation.Data field",
+		},
+		{
+			name:          "process nil field (a.Target) in attestation",
+			a:             &ethpb.Attestation{Data: &ethpb.AttestationData{}},
+			s:             &pb.BeaconState{},
+			wantErr:       true,
+			wantErrString: "nil attestation.Data.Target field",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, err := service.onAttestation(ctx, tt.a)
+			if tt.wantErr {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErrString) {
+					t.Errorf("Store.onAttestation() error = %v, wantErr = %v", err, tt.wantErrString)
+				}
+			} else {
+				t.Error(err)
+			}
+		})
+	}
+}
+
+func TestStore_SaveCheckpointState(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{
+		BeaconDB: db,
+		StateGen: stategen.New(db, cache.NewStateSummaryCache()),
+	}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Fork: &pb.Fork{
+			Epoch:           0,
+			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
+			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
+		},
+		RandaoMixes:         make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+		StateRoots:          make([][]byte, params.BeaconConfig().SlotsPerHistoricalRoot),
+		BlockRoots:          make([][]byte, params.BeaconConfig().SlotsPerHistoricalRoot),
+		LatestBlockHeader:   &ethpb.BeaconBlockHeader{},
+		JustificationBits:   []byte{0},
+		Slashings:           make([]uint64, params.BeaconConfig().EpochsPerSlashingsVector),
+		FinalizedCheckpoint: &ethpb.Checkpoint{Root: bytesutil.PadTo([]byte{'A'}, 32)},
+		Validators:          []*ethpb.Validator{{PublicKey: bytesutil.PadTo([]byte("foo"), 48)}},
+		Balances:            []uint64{0},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	r := [32]byte{'g'}
+	if err := service.beaconDB.SaveState(ctx, s, r); err != nil {
+		t.Fatal(err)
+	}
+
+	service.justifiedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	service.bestJustifiedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	service.finalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	service.prevFinalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+
+	r = bytesutil.ToBytes32([]byte{'A'})
+	cp1 := &ethpb.Checkpoint{Epoch: 1, Root: bytesutil.PadTo([]byte{'A'}, 32)}
+	if err := service.beaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'A'})); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Root: bytesutil.PadTo([]byte{'A'}, 32)}); err != nil {
+		t.Fatal(err)
+	}
+	s1, err := service.getAttPreState(ctx, cp1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s1.Slot() != 1*params.BeaconConfig().SlotsPerEpoch {
+		t.Errorf("Wanted state slot: %d, got: %d", 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot())
+	}
+
+	cp2 := &ethpb.Checkpoint{Epoch: 2, Root: bytesutil.PadTo([]byte{'B'}, 32)}
+	if err := service.beaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'B'})); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Root: bytesutil.PadTo([]byte{'B'}, 32)}); err != nil {
+		t.Fatal(err)
+	}
+	s2, err := service.getAttPreState(ctx, cp2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s2.Slot() != 2*params.BeaconConfig().SlotsPerEpoch {
+		t.Errorf("Wanted state slot: %d, got: %d", 2*params.BeaconConfig().SlotsPerEpoch, s2.Slot())
+	}
+
+	s1, err = service.getAttPreState(ctx, cp1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s1.Slot() != 1*params.BeaconConfig().SlotsPerEpoch {
+		t.Errorf("Wanted state slot: %d, got: %d", 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot())
+	}
+
+	s1, err = service.checkpointState.StateByCheckpoint(cp1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s1.Slot() != 1*params.BeaconConfig().SlotsPerEpoch {
+		t.Errorf("Wanted state slot: %d, got: %d", 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot())
+	}
+
+	s2, err = service.checkpointState.StateByCheckpoint(cp2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s2.Slot() != 2*params.BeaconConfig().SlotsPerEpoch {
+		t.Errorf("Wanted state slot: %d, got: %d", 2*params.BeaconConfig().SlotsPerEpoch, s2.Slot())
+	}
+
+	if err := s.SetSlot(params.BeaconConfig().SlotsPerEpoch + 1); err != nil {
+		t.Fatal(err)
+	}
+	service.justifiedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	service.bestJustifiedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	service.finalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	service.prevFinalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	cp3 := &ethpb.Checkpoint{Epoch: 1, Root: bytesutil.PadTo([]byte{'C'}, 32)}
+	if err := service.beaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'C'})); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Root: bytesutil.PadTo([]byte{'C'}, 32)}); err != nil {
+		t.Fatal(err)
+	}
+	s3, err := service.getAttPreState(ctx, cp3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s3.Slot() != s.Slot() {
+		t.Errorf("Wanted state slot: %d, got: %d", s.Slot(), s3.Slot())
+	}
+}
+
+func TestStore_UpdateCheckpointState(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{
+		BeaconDB: db,
+		StateGen: stategen.New(db, cache.NewStateSummaryCache()),
+	}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	epoch := uint64(1)
+	baseState, _ := testutil.DeterministicGenesisState(t, 1)
+	if err := baseState.SetSlot(epoch * params.BeaconConfig().SlotsPerEpoch); err != nil {
+		t.Fatal(err)
+	}
+	checkpoint := &ethpb.Checkpoint{Epoch: epoch}
+	if err := service.beaconDB.SaveState(ctx, baseState, bytesutil.ToBytes32(checkpoint.Root)); err != nil {
+		t.Fatal(err)
+	}
+	returned, err := service.getAttPreState(ctx, checkpoint)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if baseState.Slot() != returned.Slot() {
+		t.Error("Incorrectly returned base state")
+	}
+
+	cached, err := service.checkpointState.StateByCheckpoint(checkpoint)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if cached == nil {
+		t.Error("State should have been cached")
+	}
+
+	epoch = uint64(2)
+	newCheckpoint := &ethpb.Checkpoint{Epoch: epoch}
+	if err := service.beaconDB.SaveState(ctx, baseState, bytesutil.ToBytes32(newCheckpoint.Root)); err != nil {
+		t.Fatal(err)
+	}
+	returned, err = service.getAttPreState(ctx, newCheckpoint)
+	if err != nil {
+		t.Fatal(err)
+	}
+	baseState, err = state.ProcessSlots(ctx, baseState, helpers.StartSlot(newCheckpoint.Epoch))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if baseState.Slot() != returned.Slot() {
+		t.Error("Incorrectly returned base state")
+	}
+
+	cached, err = service.checkpointState.StateByCheckpoint(newCheckpoint)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !proto.Equal(returned.InnerStateUnsafe(), cached.InnerStateUnsafe()) {
+		t.Error("Incorrectly cached base state")
+	}
+}
+
+func TestAttEpoch_MatchPrevEpoch(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := service.verifyAttTargetEpoch(
+		ctx,
+		0,
+		params.BeaconConfig().SlotsPerEpoch*params.BeaconConfig().SecondsPerSlot,
+		&ethpb.Checkpoint{}); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestAttEpoch_MatchCurrentEpoch(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := service.verifyAttTargetEpoch(
+		ctx,
+		0,
+		params.BeaconConfig().SlotsPerEpoch*params.BeaconConfig().SecondsPerSlot,
+		&ethpb.Checkpoint{Epoch: 1}); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestAttEpoch_NotMatch(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := service.verifyAttTargetEpoch(
+		ctx,
+		0,
+		2*params.BeaconConfig().SlotsPerEpoch*params.BeaconConfig().SecondsPerSlot,
+		&ethpb.Checkpoint{}); !strings.Contains(err.Error(),
+		"target epoch 0 does not match current epoch 2 or prev epoch 1") {
+		t.Error("Did not receive wanted error")
+	}
+}
+
+func TestVerifyBeaconBlock_NoBlock(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	d := &ethpb.AttestationData{}
+	if err := service.verifyBeaconBlock(ctx, d); !strings.Contains(err.Error(), "beacon block  does not exist") {
+		t.Error("Did not receive the wanted error")
+	}
+}
+
+func TestVerifyBeaconBlock_futureBlock(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 2}}
+	if err := service.beaconDB.SaveBlock(ctx, b); err != nil {
+		t.Fatal(err)
+	}
+	r, err := stateutil.BlockRoot(b.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+	d := &ethpb.AttestationData{Slot: 1, BeaconBlockRoot: r[:]}
+
+	err = service.verifyBeaconBlock(ctx, d)
+	if err == nil || !strings.Contains(err.Error(), "could not process attestation for future block") {
+		t.Error("Did not receive the wanted error")
+	}
+}
+
+func TestVerifyBeaconBlock_OK(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 2}}
+	if err := service.beaconDB.SaveBlock(ctx, b); err != nil {
+		t.Fatal(err)
+	}
+	r, err := stateutil.BlockRoot(b.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+	d := &ethpb.AttestationData{Slot: 2, BeaconBlockRoot: r[:]}
+
+	if err := service.verifyBeaconBlock(ctx, d); err != nil {
+		t.Error("Did not receive the wanted error")
+	}
+}

beacon-chain/blockchain/process_block.go

@@ -0,0 +1,376 @@
+package blockchain
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/flags"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/attestationutil"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/sirupsen/logrus"
+	"go.opencensus.io/trace"
+)
+
+// This defines size of the upper bound for initial sync block cache.
+var initialSyncBlockCacheSize = 2 * params.BeaconConfig().SlotsPerEpoch
+
+// onBlock is called when a gossip block is received. It runs regular state transition on the block.
+// The block's signing root should be computed before calling this method to avoid redundant
+// computation in this method and methods it calls into.
+//
+// Spec pseudocode definition:
+//   def on_block(store: Store, block: BeaconBlock) -> None:
+//    # Make a copy of the state to avoid mutability issues
+//    assert block.parent_root in store.block_states
+//    pre_state = store.block_states[block.parent_root].copy()
+//    # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past.
+//    assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT
+//    # Add new block to the store
+//    store.blocks[signing_root(block)] = block
+//    # Check block is a descendant of the finalized block
+//    assert (
+//        get_ancestor(store, signing_root(block), store.blocks[store.finalized_checkpoint.root].slot) ==
+//        store.finalized_checkpoint.root
+//    )
+//    # Check that block is later than the finalized epoch slot
+//    assert block.slot > compute_start_slot_of_epoch(store.finalized_checkpoint.epoch)
+//    # Check the block is valid and compute the post-state
+//    state = state_transition(pre_state, block)
+//    # Add new state for this block to the store
+//    store.block_states[signing_root(block)] = state
+//
+//    # Update justified checkpoint
+//    if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch:
+//        if state.current_justified_checkpoint.epoch > store.best_justified_checkpoint.epoch:
+//            store.best_justified_checkpoint = state.current_justified_checkpoint
+//
+//    # Update finalized checkpoint
+//    if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch:
+//        store.finalized_checkpoint = state.finalized_checkpoint
+func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock, blockRoot [32]byte) (*stateTrie.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "blockchain.onBlock")
+	defer span.End()
+
+	if signed == nil || signed.Block == nil {
+		return nil, errors.New("nil block")
+	}
+
+	b := signed.Block
+
+	// Retrieve incoming block's pre state.
+	preState, err := s.getBlockPreState(ctx, b)
+	if err != nil {
+		return nil, err
+	}
+
+	log.WithFields(logrus.Fields{
+		"slot": b.Slot,
+		"root": fmt.Sprintf("0x%s...", hex.EncodeToString(blockRoot[:])[:8]),
+	}).Debug("Executing state transition on block")
+
+	postState, err := state.ExecuteStateTransition(ctx, preState, signed)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not execute state transition")
+	}
+
+	if err := s.beaconDB.SaveBlock(ctx, signed); err != nil {
+		return nil, errors.Wrapf(err, "could not save block from slot %d", b.Slot)
+	}
+
+	if err := s.insertBlockToForkChoiceStore(ctx, b, blockRoot, postState); err != nil {
+		return nil, errors.Wrapf(err, "could not insert block %d to fork choice store", b.Slot)
+	}
+
+	if featureconfig.Get().NewStateMgmt {
+		if err := s.stateGen.SaveState(ctx, blockRoot, postState); err != nil {
+			return nil, errors.Wrap(err, "could not save state")
+		}
+	} else {
+		if err := s.beaconDB.SaveState(ctx, postState, blockRoot); err != nil {
+			return nil, errors.Wrap(err, "could not save state")
+		}
+	}
+
+	// Update justified check point.
+	if postState.CurrentJustifiedCheckpoint().Epoch > s.justifiedCheckpt.Epoch {
+		if err := s.updateJustified(ctx, postState); err != nil {
+			return nil, err
+		}
+	}
+
+	// Update finalized check point. Prune the block cache and helper caches on every new finalized epoch.
+	if postState.FinalizedCheckpointEpoch() > s.finalizedCheckpt.Epoch {
+		if !featureconfig.Get().NoInitSyncBatchSaveBlocks {
+			if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+				return nil, err
+			}
+			s.clearInitSyncBlocks()
+		}
+
+		if err := s.beaconDB.SaveFinalizedCheckpoint(ctx, postState.FinalizedCheckpoint()); err != nil {
+			return nil, errors.Wrap(err, "could not save finalized checkpoint")
+		}
+
+		if !featureconfig.Get().NewStateMgmt {
+			startSlot := helpers.StartSlot(s.prevFinalizedCheckpt.Epoch)
+			endSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
+			if endSlot > startSlot {
+				if err := s.rmStatesOlderThanLastFinalized(ctx, startSlot, endSlot); err != nil {
+					return nil, errors.Wrapf(err, "could not delete states prior to finalized check point, range: %d, %d",
+						startSlot, endSlot)
+				}
+			}
+		}
+		fRoot := bytesutil.ToBytes32(postState.FinalizedCheckpoint().Root)
+
+		// Prune proto array fork choice nodes, all nodes before finalized check point will
+		// be pruned.
+		if err := s.forkChoiceStore.Prune(ctx, fRoot); err != nil {
+			return nil, errors.Wrap(err, "could not prune proto array fork choice nodes")
+		}
+
+		s.prevFinalizedCheckpt = s.finalizedCheckpt
+		s.finalizedCheckpt = postState.FinalizedCheckpoint()
+
+		if err := s.finalizedImpliesNewJustified(ctx, postState); err != nil {
+			return nil, errors.Wrap(err, "could not save new justified")
+		}
+
+		if featureconfig.Get().NewStateMgmt {
+			fRoot := bytesutil.ToBytes32(postState.FinalizedCheckpoint().Root)
+			fBlock, err := s.beaconDB.Block(ctx, fRoot)
+			if err != nil {
+				return nil, errors.Wrap(err, "could not get finalized block to migrate")
+			}
+			if err := s.stateGen.MigrateToCold(ctx, fBlock.Block.Slot, fRoot); err != nil {
+				return nil, errors.Wrap(err, "could not migrate to cold")
+			}
+		}
+	}
+
+	// Epoch boundary bookkeeping such as logging epoch summaries.
+	if postState.Slot() >= s.nextEpochBoundarySlot {
+		logEpochData(postState)
+		reportEpochMetrics(postState)
+
+		// Update committees cache at epoch boundary slot.
+		if err := helpers.UpdateCommitteeCache(postState, helpers.CurrentEpoch(postState)); err != nil {
+			return nil, err
+		}
+		if err := helpers.UpdateProposerIndicesInCache(postState, helpers.CurrentEpoch(postState)); err != nil {
+			return nil, err
+		}
+
+		s.nextEpochBoundarySlot = helpers.StartSlot(helpers.NextEpoch(postState))
+	}
+
+	// Delete the processed block attestations from attestation pool.
+	if err := s.deletePoolAtts(b.Body.Attestations); err != nil {
+		return nil, err
+	}
+
+	// Delete the processed block attester slashings from slashings pool.
+	for i := 0; i < len(b.Body.AttesterSlashings); i++ {
+		s.slashingPool.MarkIncludedAttesterSlashing(b.Body.AttesterSlashings[i])
+	}
+
+	defer reportAttestationInclusion(b)
+
+	return postState, nil
+}
+
+// onBlockInitialSyncStateTransition is called when an initial sync block is received.
+// It runs state transition on the block and without any BLS verification. The excluded BLS verification
+// includes attestation's aggregated signature. It also does not save attestations.
+// The block's signing root should be computed before calling this method to avoid redundant
+// computation in this method and methods it calls into.
+func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
+	ctx, span := trace.StartSpan(ctx, "blockchain.onBlock")
+	defer span.End()
+
+	if signed == nil || signed.Block == nil {
+		return errors.New("nil block")
+	}
+
+	b := signed.Block
+
+	// Retrieve incoming block's pre state.
+	preState, err := s.verifyBlkPreState(ctx, b)
+	if err != nil {
+		return err
+	}
+	// To invalidate cache for parent root because pre state will get mutated.
+	s.stateGen.DeleteHotStateInCache(bytesutil.ToBytes32(b.ParentRoot))
+
+	// Exit early if the pre state slot is higher than incoming block's slot.
+	if preState.Slot() >= signed.Block.Slot {
+		return nil
+	}
+
+	postState, err := state.ExecuteStateTransitionNoVerifyAttSigs(ctx, preState, signed)
+	if err != nil {
+		return errors.Wrap(err, "could not execute state transition")
+	}
+
+	if !featureconfig.Get().NoInitSyncBatchSaveBlocks {
+		s.saveInitSyncBlock(blockRoot, signed)
+	} else {
+		if err := s.beaconDB.SaveBlock(ctx, signed); err != nil {
+			return errors.Wrapf(err, "could not save block from slot %d", b.Slot)
+		}
+	}
+
+	if err := s.insertBlockToForkChoiceStore(ctx, b, blockRoot, postState); err != nil {
+		return errors.Wrapf(err, "could not insert block %d to fork choice store", b.Slot)
+	}
+
+	if featureconfig.Get().NewStateMgmt {
+		if err := s.stateGen.SaveState(ctx, blockRoot, postState); err != nil {
+			return errors.Wrap(err, "could not save state")
+		}
+	} else {
+		s.initSyncStateLock.Lock()
+		defer s.initSyncStateLock.Unlock()
+		s.initSyncState[blockRoot] = postState.Copy()
+		s.filterBoundaryCandidates(ctx, blockRoot, postState)
+	}
+
+	if flags.Get().EnableArchive {
+		atts := signed.Block.Body.Attestations
+		if err := s.beaconDB.SaveAttestations(ctx, atts); err != nil {
+			return errors.Wrapf(err, "could not save block attestations from slot %d", b.Slot)
+		}
+	}
+
+	// Update justified check point.
+	if postState.CurrentJustifiedCheckpoint().Epoch > s.justifiedCheckpt.Epoch {
+		if err := s.updateJustified(ctx, postState); err != nil {
+			return err
+		}
+	}
+
+	// Rate limit how many blocks (2 epochs worth of blocks) a node keeps in the memory.
+	if len(s.getInitSyncBlocks()) > int(initialSyncBlockCacheSize) {
+		if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+			return err
+		}
+		s.clearInitSyncBlocks()
+	}
+
+	// Update finalized check point. Prune the block cache and helper caches on every new finalized epoch.
+	if postState.FinalizedCheckpointEpoch() > s.finalizedCheckpt.Epoch {
+		if !featureconfig.Get().NewStateMgmt {
+			startSlot := helpers.StartSlot(s.prevFinalizedCheckpt.Epoch)
+			endSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
+			if endSlot > startSlot {
+				if err := s.rmStatesOlderThanLastFinalized(ctx, startSlot, endSlot); err != nil {
+					return errors.Wrapf(err, "could not delete states prior to finalized check point, range: %d, %d",
+						startSlot, endSlot)
+				}
+			}
+
+			if err := s.saveInitState(ctx, postState); err != nil {
+				return errors.Wrap(err, "could not save init sync finalized state")
+			}
+		}
+
+		if !featureconfig.Get().NoInitSyncBatchSaveBlocks {
+			if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+				return err
+			}
+			s.clearInitSyncBlocks()
+		}
+
+		if err := s.beaconDB.SaveFinalizedCheckpoint(ctx, postState.FinalizedCheckpoint()); err != nil {
+			return errors.Wrap(err, "could not save finalized checkpoint")
+		}
+
+		s.prevFinalizedCheckpt = s.finalizedCheckpt
+		s.finalizedCheckpt = postState.FinalizedCheckpoint()
+
+		if err := s.finalizedImpliesNewJustified(ctx, postState); err != nil {
+			return errors.Wrap(err, "could not save new justified")
+		}
+
+		if featureconfig.Get().NewStateMgmt {
+			fRoot := bytesutil.ToBytes32(postState.FinalizedCheckpoint().Root)
+			fBlock, err := s.beaconDB.Block(ctx, fRoot)
+			if err != nil {
+				return errors.Wrap(err, "could not get finalized block to migrate")
+			}
+			if err := s.stateGen.MigrateToCold(ctx, fBlock.Block.Slot, fRoot); err != nil {
+				return errors.Wrap(err, "could not migrate to cold")
+			}
+		}
+	}
+
+	if !featureconfig.Get().NewStateMgmt {
+		numOfStates := len(s.boundaryRoots)
+		if numOfStates > initialSyncCacheSize {
+			if err = s.persistCachedStates(ctx, numOfStates); err != nil {
+				return err
+			}
+		}
+		if len(s.initSyncState) > maxCacheSize {
+			s.pruneOldNonFinalizedStates()
+		}
+	}
+
+	// Epoch boundary bookkeeping such as logging epoch summaries.
+	if postState.Slot() >= s.nextEpochBoundarySlot {
+		reportEpochMetrics(postState)
+		s.nextEpochBoundarySlot = helpers.StartSlot(helpers.NextEpoch(postState))
+
+		// Update committees cache at epoch boundary slot.
+		if err := helpers.UpdateCommitteeCache(postState, helpers.CurrentEpoch(postState)); err != nil {
+			return err
+		}
+		if err := helpers.UpdateProposerIndicesInCache(postState, helpers.CurrentEpoch(postState)); err != nil {
+			return err
+		}
+
+		if !featureconfig.Get().NewStateMgmt && helpers.IsEpochStart(postState.Slot()) {
+			if err := s.beaconDB.SaveState(ctx, postState, blockRoot); err != nil {
+				return errors.Wrap(err, "could not save state")
+			}
+		}
+	}
+
+	return nil
+}
+
+// This feeds in the block and block's attestations to fork choice store. It's allows fork choice store
+// to gain information on the most current chain.
+func (s *Service) insertBlockToForkChoiceStore(ctx context.Context, blk *ethpb.BeaconBlock, root [32]byte, state *stateTrie.BeaconState) error {
+	if err := s.fillInForkChoiceMissingBlocks(ctx, blk, state); err != nil {
+		return err
+	}
+
+	// Feed in block to fork choice store.
+	if err := s.forkChoiceStore.ProcessBlock(ctx,
+		blk.Slot, root, bytesutil.ToBytes32(blk.ParentRoot), bytesutil.ToBytes32(blk.Body.Graffiti),
+		state.CurrentJustifiedCheckpoint().Epoch,
+		state.FinalizedCheckpointEpoch()); err != nil {
+		return errors.Wrap(err, "could not process block for proto array fork choice")
+	}
+
+	// Feed in block's attestations to fork choice store.
+	for _, a := range blk.Body.Attestations {
+		committee, err := helpers.BeaconCommitteeFromState(state, a.Data.Slot, a.Data.CommitteeIndex)
+		if err != nil {
+			return err
+		}
+		indices := attestationutil.AttestingIndices(a.AggregationBits, committee)
+		s.forkChoiceStore.ProcessAttestation(ctx, indices, bytesutil.ToBytes32(a.Data.BeaconBlockRoot), a.Data.Target.Epoch)
+	}
+
+	return nil
+}

beacon-chain/blockchain/process_block_helpers.go

@@ -0,0 +1,514 @@
+package blockchain
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/go-ssz"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/roughtime"
+	"github.com/prysmaticlabs/prysm/shared/traceutil"
+	"go.opencensus.io/trace"
+)
+
+// CurrentSlot returns the current slot based on time.
+func (s *Service) CurrentSlot() uint64 {
+	now := roughtime.Now().Unix()
+	genesis := s.genesisTime.Unix()
+	if now < genesis {
+		return 0
+	}
+	return uint64(now-genesis) / params.BeaconConfig().SecondsPerSlot
+}
+
+// getBlockPreState returns the pre state of an incoming block. It uses the parent root of the block
+// to retrieve the state in DB. It verifies the pre state's validity and the incoming block
+// is in the correct time window.
+func (s *Service) getBlockPreState(ctx context.Context, b *ethpb.BeaconBlock) (*stateTrie.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "forkchoice.getBlockPreState")
+	defer span.End()
+
+	// Verify incoming block has a valid pre state.
+	preState, err := s.verifyBlkPreState(ctx, b)
+	if err != nil {
+		return nil, err
+	}
+
+	//  For new state management, this ensures the state does not get mutated since initial syncing
+	//  uses verifyBlkPreState.
+	if featureconfig.Get().NewStateMgmt {
+		preState, err = s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(b.ParentRoot))
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not get pre state for slot %d", b.Slot)
+		}
+		if preState == nil {
+			return nil, errors.Wrapf(err, "nil pre state for slot %d", b.Slot)
+		}
+	}
+
+	// Verify block slot time is not from the feature.
+	if err := helpers.VerifySlotTime(preState.GenesisTime(), b.Slot, helpers.TimeShiftTolerance); err != nil {
+		return nil, err
+	}
+
+	// Verify block is a descendent of a finalized block.
+	if err := s.verifyBlkDescendant(ctx, bytesutil.ToBytes32(b.ParentRoot), b.Slot); err != nil {
+		return nil, err
+	}
+
+	// Verify block is later than the finalized epoch slot.
+	if err := s.verifyBlkFinalizedSlot(b); err != nil {
+		return nil, err
+	}
+
+	return preState, nil
+}
+
+// verifyBlkPreState validates input block has a valid pre-state.
+func (s *Service) verifyBlkPreState(ctx context.Context, b *ethpb.BeaconBlock) (*stateTrie.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "chainService.verifyBlkPreState")
+	defer span.End()
+
+	if featureconfig.Get().NewStateMgmt {
+		parentRoot := bytesutil.ToBytes32(b.ParentRoot)
+		// Loosen the check to HasBlock because state summary gets saved in batches
+		// during initial syncing. There's no risk given a state summary object is just a
+		// a subset of the block object.
+		if !s.stateGen.StateSummaryExists(ctx, parentRoot) && !s.beaconDB.HasBlock(ctx, parentRoot) {
+			return nil, errors.New("could not reconstruct parent state")
+		}
+		if !s.stateGen.HasState(ctx, parentRoot) {
+			if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+				return nil, errors.Wrap(err, "could not save initial sync blocks")
+			}
+			s.clearInitSyncBlocks()
+		}
+		preState, err := s.stateGen.StateByRootInitialSync(ctx, parentRoot)
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not get pre state for slot %d", b.Slot)
+		}
+		if preState == nil {
+			return nil, errors.Wrapf(err, "nil pre state for slot %d", b.Slot)
+		}
+
+		return preState, nil // No copy needed from newly hydrated state gen object.
+	}
+
+	preState := s.initSyncState[bytesutil.ToBytes32(b.ParentRoot)]
+	var err error
+	if preState == nil {
+		if featureconfig.Get().CheckHeadState {
+			headRoot, err := s.HeadRoot(ctx)
+			if err != nil {
+				return nil, errors.Wrapf(err, "could not get head root")
+			}
+			if bytes.Equal(headRoot, b.ParentRoot) {
+				return s.HeadState(ctx)
+			}
+		}
+		preState, err = s.beaconDB.State(ctx, bytesutil.ToBytes32(b.ParentRoot))
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not get pre state for slot %d", b.Slot)
+		}
+		if preState == nil {
+			if bytes.Equal(s.finalizedCheckpt.Root, b.ParentRoot) {
+				return nil, fmt.Errorf("pre state of slot %d does not exist", b.Slot)
+			}
+			preState, err = s.generateState(ctx, bytesutil.ToBytes32(s.finalizedCheckpt.Root), bytesutil.ToBytes32(b.ParentRoot))
+			if err != nil {
+				return nil, err
+			}
+		}
+		return preState, nil // No copy needed from newly hydrated DB object.
+	}
+	return preState.Copy(), nil
+}
+
+// verifyBlkDescendant validates input block root is a descendant of the
+// current finalized block root.
+func (s *Service) verifyBlkDescendant(ctx context.Context, root [32]byte, slot uint64) error {
+	ctx, span := trace.StartSpan(ctx, "forkchoice.verifyBlkDescendant")
+	defer span.End()
+
+	finalizedBlkSigned, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(s.finalizedCheckpt.Root))
+	if err != nil || finalizedBlkSigned == nil || finalizedBlkSigned.Block == nil {
+		return errors.Wrap(err, "could not get finalized block")
+	}
+	finalizedBlk := finalizedBlkSigned.Block
+
+	bFinalizedRoot, err := s.ancestor(ctx, root[:], finalizedBlk.Slot)
+	if err != nil {
+		return errors.Wrap(err, "could not get finalized block root")
+	}
+	if bFinalizedRoot == nil {
+		return fmt.Errorf("no finalized block known for block from slot %d", slot)
+	}
+
+	if !bytes.Equal(bFinalizedRoot, s.finalizedCheckpt.Root) {
+		err := fmt.Errorf("block from slot %d is not a descendent of the current finalized block slot %d, %#x != %#x",
+			slot, finalizedBlk.Slot, bytesutil.Trunc(bFinalizedRoot), bytesutil.Trunc(s.finalizedCheckpt.Root))
+		traceutil.AnnotateError(span, err)
+		return err
+	}
+	return nil
+}
+
+// verifyBlkFinalizedSlot validates input block is not less than or equal
+// to current finalized slot.
+func (s *Service) verifyBlkFinalizedSlot(b *ethpb.BeaconBlock) error {
+	finalizedSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
+	if finalizedSlot >= b.Slot {
+		return fmt.Errorf("block is equal or earlier than finalized block, slot %d < slot %d", b.Slot, finalizedSlot)
+	}
+	return nil
+}
+
+// rmStatesOlderThanLastFinalized deletes the states in db since last finalized check point.
+func (s *Service) rmStatesOlderThanLastFinalized(ctx context.Context, startSlot uint64, endSlot uint64) error {
+	ctx, span := trace.StartSpan(ctx, "forkchoice.rmStatesBySlots")
+	defer span.End()
+
+	// Make sure start slot is not a skipped slot
+	for i := startSlot; i > 0; i-- {
+		filter := filters.NewFilter().SetStartSlot(i).SetEndSlot(i)
+		b, err := s.beaconDB.Blocks(ctx, filter)
+		if err != nil {
+			return err
+		}
+		if len(b) > 0 {
+			startSlot = i
+			break
+		}
+	}
+
+	// Make sure finalized slot is not a skipped slot.
+	for i := endSlot; i > 0; i-- {
+		filter := filters.NewFilter().SetStartSlot(i).SetEndSlot(i)
+		b, err := s.beaconDB.Blocks(ctx, filter)
+		if err != nil {
+			return err
+		}
+		if len(b) > 0 {
+			endSlot = i - 1
+			break
+		}
+	}
+
+	// Do not remove genesis state
+	if startSlot == 0 {
+		startSlot++
+	}
+	// If end slot comes less than start slot
+	if endSlot < startSlot {
+		endSlot = startSlot
+	}
+
+	filter := filters.NewFilter().SetStartSlot(startSlot).SetEndSlot(endSlot)
+	roots, err := s.beaconDB.BlockRoots(ctx, filter)
+	if err != nil {
+		return err
+	}
+
+	roots, err = s.filterBlockRoots(ctx, roots)
+	if err != nil {
+		return err
+	}
+
+	if err := s.beaconDB.DeleteStates(ctx, roots); err != nil {
+		log.Warnf("Could not delete states: %v", err)
+	}
+
+	return nil
+}
+
+// shouldUpdateCurrentJustified prevents bouncing attack, by only update conflicting justified
+// checkpoints in the fork choice if in the early slots of the epoch.
+// Otherwise, delay incorporation of new justified checkpoint until next epoch boundary.
+// See https://ethresear.ch/t/prevention-of-bouncing-attack-on-ffg/6114 for more detailed analysis and discussion.
+func (s *Service) shouldUpdateCurrentJustified(ctx context.Context, newJustifiedCheckpt *ethpb.Checkpoint) (bool, error) {
+	if helpers.SlotsSinceEpochStarts(s.CurrentSlot()) < params.BeaconConfig().SafeSlotsToUpdateJustified {
+		return true, nil
+	}
+	var newJustifiedBlockSigned *ethpb.SignedBeaconBlock
+	justifiedRoot := bytesutil.ToBytes32(newJustifiedCheckpt.Root)
+	var err error
+	if !featureconfig.Get().NoInitSyncBatchSaveBlocks && s.hasInitSyncBlock(justifiedRoot) {
+		newJustifiedBlockSigned = s.getInitSyncBlock(justifiedRoot)
+	} else {
+		newJustifiedBlockSigned, err = s.beaconDB.Block(ctx, justifiedRoot)
+		if err != nil {
+			return false, err
+		}
+	}
+	if newJustifiedBlockSigned == nil || newJustifiedBlockSigned.Block == nil {
+		return false, errors.New("nil new justified block")
+	}
+
+	newJustifiedBlock := newJustifiedBlockSigned.Block
+	if newJustifiedBlock.Slot <= helpers.StartSlot(s.justifiedCheckpt.Epoch) {
+		return false, nil
+	}
+	var justifiedBlockSigned *ethpb.SignedBeaconBlock
+	cachedJustifiedRoot := bytesutil.ToBytes32(s.justifiedCheckpt.Root)
+	if !featureconfig.Get().NoInitSyncBatchSaveBlocks && s.hasInitSyncBlock(cachedJustifiedRoot) {
+		justifiedBlockSigned = s.getInitSyncBlock(cachedJustifiedRoot)
+	} else {
+		justifiedBlockSigned, err = s.beaconDB.Block(ctx, cachedJustifiedRoot)
+		if err != nil {
+			return false, err
+		}
+	}
+
+	if justifiedBlockSigned == nil || justifiedBlockSigned.Block == nil {
+		return false, errors.New("nil justified block")
+	}
+	justifiedBlock := justifiedBlockSigned.Block
+	b, err := s.ancestor(ctx, newJustifiedCheckpt.Root, justifiedBlock.Slot)
+	if err != nil {
+		return false, err
+	}
+	if !bytes.Equal(b, s.justifiedCheckpt.Root) {
+		return false, nil
+	}
+	return true, nil
+}
+
+func (s *Service) updateJustified(ctx context.Context, state *stateTrie.BeaconState) error {
+	cpt := state.CurrentJustifiedCheckpoint()
+	if cpt.Epoch > s.bestJustifiedCheckpt.Epoch {
+		s.bestJustifiedCheckpt = cpt
+	}
+	canUpdate, err := s.shouldUpdateCurrentJustified(ctx, cpt)
+	if err != nil {
+		return err
+	}
+
+	if canUpdate {
+		s.prevJustifiedCheckpt = s.justifiedCheckpt
+		s.justifiedCheckpt = cpt
+	}
+
+	if !featureconfig.Get().NewStateMgmt {
+		justifiedRoot := bytesutil.ToBytes32(cpt.Root)
+
+		justifiedState := s.initSyncState[justifiedRoot]
+		// If justified state is nil, resume back to normal syncing process and save
+		// justified check point.
+		var err error
+		if justifiedState == nil {
+			if s.beaconDB.HasState(ctx, justifiedRoot) {
+				return s.beaconDB.SaveJustifiedCheckpoint(ctx, cpt)
+			}
+			justifiedState, err = s.generateState(ctx, bytesutil.ToBytes32(s.finalizedCheckpt.Root), justifiedRoot)
+			if err != nil {
+				log.Error(err)
+				return s.beaconDB.SaveJustifiedCheckpoint(ctx, cpt)
+			}
+		}
+		if err := s.beaconDB.SaveState(ctx, justifiedState, justifiedRoot); err != nil {
+			return errors.Wrap(err, "could not save justified state")
+		}
+	}
+
+	return s.beaconDB.SaveJustifiedCheckpoint(ctx, cpt)
+}
+
+// This saves every finalized state in DB during initial sync, needed as part of optimization to
+// use cache state during initial sync in case of restart.
+func (s *Service) saveInitState(ctx context.Context, state *stateTrie.BeaconState) error {
+	cpt := state.FinalizedCheckpoint()
+	finalizedRoot := bytesutil.ToBytes32(cpt.Root)
+	fs := s.initSyncState[finalizedRoot]
+	if fs == nil {
+		var err error
+		fs, err = s.beaconDB.State(ctx, finalizedRoot)
+		if err != nil {
+			return err
+		}
+		if fs == nil {
+			fs, err = s.generateState(ctx, bytesutil.ToBytes32(s.prevFinalizedCheckpt.Root), finalizedRoot)
+			if err != nil {
+				// This might happen if the client was in sync and is now re-syncing for whatever reason.
+				log.Warn("Initial sync cache did not have finalized state root cached")
+				return err
+			}
+		}
+	}
+
+	if err := s.beaconDB.SaveState(ctx, fs, finalizedRoot); err != nil {
+		return errors.Wrap(err, "could not save state")
+	}
+	return nil
+}
+
+// This filters block roots that are not known as head root and finalized root in DB.
+// It serves as the last line of defence before we prune states.
+func (s *Service) filterBlockRoots(ctx context.Context, roots [][32]byte) ([][32]byte, error) {
+	f, err := s.beaconDB.FinalizedCheckpoint(ctx)
+	if err != nil {
+		return nil, err
+	}
+	fRoot := f.Root
+	h, err := s.beaconDB.HeadBlock(ctx)
+	if err != nil {
+		return nil, err
+	}
+	hRoot, err := stateutil.BlockRoot(h.Block)
+	if err != nil {
+		return nil, err
+	}
+
+	filtered := make([][32]byte, 0, len(roots))
+	for _, root := range roots {
+		if bytes.Equal(root[:], fRoot[:]) || bytes.Equal(root[:], hRoot[:]) {
+			continue
+		}
+		filtered = append(filtered, root)
+	}
+
+	return filtered, nil
+}
+
+// ancestor returns the block root of an ancestry block from the input block root.
+//
+// Spec pseudocode definition:
+//   def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash:
+//    block = store.blocks[root]
+//    if block.slot > slot:
+//      return get_ancestor(store, block.parent_root, slot)
+//    elif block.slot == slot:
+//      return root
+//    else:
+//      return Bytes32()  # root is older than queried slot: no results.
+func (s *Service) ancestor(ctx context.Context, root []byte, slot uint64) ([]byte, error) {
+	ctx, span := trace.StartSpan(ctx, "forkchoice.ancestor")
+	defer span.End()
+
+	// Stop recursive ancestry lookup if context is cancelled.
+	if ctx.Err() != nil {
+		return nil, ctx.Err()
+	}
+
+	signed, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(root))
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get ancestor block")
+	}
+
+	if !featureconfig.Get().NoInitSyncBatchSaveBlocks && s.hasInitSyncBlock(bytesutil.ToBytes32(root)) {
+		signed = s.getInitSyncBlock(bytesutil.ToBytes32(root))
+	}
+
+	if signed == nil || signed.Block == nil {
+		return nil, errors.New("nil block")
+	}
+	b := signed.Block
+
+	// If we dont have the ancestor in the DB, simply return nil so rest of fork choice
+	// operation can proceed. This is not an error condition.
+	if b == nil || b.Slot < slot {
+		return nil, nil
+	}
+
+	if b.Slot == slot {
+		return root, nil
+	}
+
+	return s.ancestor(ctx, b.ParentRoot, slot)
+}
+
+// This updates justified check point in store, if the new justified is later than stored justified or
+// the store's justified is not in chain with finalized check point.
+//
+// Spec definition:
+//   if (
+//            state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch
+//            or get_ancestor(store, store.justified_checkpoint.root, finalized_slot) != store.finalized_checkpoint.root
+//        ):
+//            store.justified_checkpoint = state.current_justified_checkpoint
+func (s *Service) finalizedImpliesNewJustified(ctx context.Context, state *stateTrie.BeaconState) error {
+	finalizedBlkSigned, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(s.finalizedCheckpt.Root))
+	if err != nil || finalizedBlkSigned == nil || finalizedBlkSigned.Block == nil {
+		return errors.Wrap(err, "could not get finalized block")
+	}
+	finalizedBlk := finalizedBlkSigned.Block
+
+	anc, err := s.ancestor(ctx, s.justifiedCheckpt.Root, finalizedBlk.Slot)
+	if err != nil {
+		return err
+	}
+
+	// Either the new justified is later than stored justified or not in chain with finalized check pint.
+	if cpt := state.CurrentJustifiedCheckpoint(); cpt != nil && cpt.Epoch > s.justifiedCheckpt.Epoch || !bytes.Equal(anc, s.finalizedCheckpt.Root) {
+		s.justifiedCheckpt = state.CurrentJustifiedCheckpoint()
+	}
+
+	return nil
+}
+
+// This retrieves missing blocks from DB (ie. the blocks that couldn't received over sync) and inserts them to fork choice store.
+// This is useful for block tree visualizer and additional vote accounting.
+func (s *Service) fillInForkChoiceMissingBlocks(ctx context.Context, blk *ethpb.BeaconBlock, state *stateTrie.BeaconState) error {
+	pendingNodes := make([]*ethpb.BeaconBlock, 0)
+
+	parentRoot := bytesutil.ToBytes32(blk.ParentRoot)
+	slot := blk.Slot
+	// Fork choice only matters from last finalized slot.
+	higherThanFinalized := slot > helpers.StartSlot(s.finalizedCheckpt.Epoch)
+	// As long as parent node is not in fork choice store, and parent node is in DB.
+	for !s.forkChoiceStore.HasNode(parentRoot) && s.beaconDB.HasBlock(ctx, parentRoot) && higherThanFinalized {
+		b, err := s.beaconDB.Block(ctx, parentRoot)
+		if err != nil {
+			return err
+		}
+
+		pendingNodes = append(pendingNodes, b.Block)
+		parentRoot = bytesutil.ToBytes32(b.Block.ParentRoot)
+		slot = b.Block.Slot
+		higherThanFinalized = slot > helpers.StartSlot(s.finalizedCheckpt.Epoch)
+	}
+
+	// Insert parent nodes to fork choice store in reverse order.
+	// Lower slots should be at the end of the list.
+	for i := len(pendingNodes) - 1; i >= 0; i-- {
+		b := pendingNodes[i]
+		r, err := ssz.HashTreeRoot(b)
+		if err != nil {
+			return err
+		}
+
+		if err := s.forkChoiceStore.ProcessBlock(ctx,
+			b.Slot, r, bytesutil.ToBytes32(b.ParentRoot), bytesutil.ToBytes32(b.Body.Graffiti),
+			state.CurrentJustifiedCheckpoint().Epoch,
+			state.FinalizedCheckpointEpoch()); err != nil {
+			return errors.Wrap(err, "could not process block for proto array fork choice")
+		}
+	}
+
+	return nil
+}
+
+// The deletes input attestations from the attestation pool, so proposers don't include them in a block for the future.
+func (s *Service) deletePoolAtts(atts []*ethpb.Attestation) error {
+	for _, att := range atts {
+		if helpers.IsAggregated(att) {
+			if err := s.attPool.DeleteAggregatedAttestation(att); err != nil {
+				return err
+			}
+		} else {
+			if err := s.attPool.DeleteUnaggregatedAttestation(att); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

beacon-chain/blockchain/process_block_test.go

@@ -0,0 +1,774 @@
+package blockchain
+
+import (
+	"context"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/go-ssz"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/roughtime"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
+)
+
+func TestStore_OnBlock(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{
+		BeaconDB: db,
+		StateGen: stategen.New(db, cache.NewStateSummaryCache()),
+	}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	genesisStateRoot := [32]byte{}
+	genesis := blocks.NewGenesisBlock(genesisStateRoot[:])
+	if err := db.SaveBlock(ctx, genesis); err != nil {
+		t.Error(err)
+	}
+	validGenesisRoot, err := stateutil.BlockRoot(genesis.Block)
+	if err != nil {
+		t.Error(err)
+	}
+	st := testutil.NewBeaconState()
+	if err := service.beaconDB.SaveState(ctx, st.Copy(), validGenesisRoot); err != nil {
+		t.Fatal(err)
+	}
+	roots, err := blockTree1(db, validGenesisRoot[:])
+	if err != nil {
+		t.Fatal(err)
+	}
+	random := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1, ParentRoot: validGenesisRoot[:]}}
+	if err := db.SaveBlock(ctx, random); err != nil {
+		t.Error(err)
+	}
+	randomParentRoot, err := stateutil.BlockRoot(random.Block)
+	if err != nil {
+		t.Error(err)
+	}
+	if err := service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Slot: st.Slot(), Root: randomParentRoot[:]}); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveState(ctx, st.Copy(), randomParentRoot); err != nil {
+		t.Fatal(err)
+	}
+	randomParentRoot2 := roots[1]
+	if err := service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Slot: st.Slot(), Root: randomParentRoot2[:]}); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveState(ctx, st.Copy(), bytesutil.ToBytes32(randomParentRoot2)); err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct {
+		name          string
+		blk           *ethpb.BeaconBlock
+		s             *stateTrie.BeaconState
+		time          uint64
+		wantErrString string
+	}{
+		{
+			name:          "parent block root does not have a state",
+			blk:           &ethpb.BeaconBlock{},
+			s:             st.Copy(),
+			wantErrString: "provided block root does not have block saved in the db",
+		},
+		{
+			name:          "block is from the future",
+			blk:           &ethpb.BeaconBlock{ParentRoot: randomParentRoot[:], Slot: params.BeaconConfig().FarFutureEpoch},
+			s:             st.Copy(),
+			wantErrString: "far distant future",
+		},
+		{
+			name:          "could not get finalized block",
+			blk:           &ethpb.BeaconBlock{ParentRoot: randomParentRoot[:]},
+			s:             st.Copy(),
+			wantErrString: "block from slot 0 is not a descendent of the current finalized block",
+		},
+		{
+			name:          "same slot as finalized block",
+			blk:           &ethpb.BeaconBlock{Slot: 0, ParentRoot: randomParentRoot2},
+			s:             st.Copy(),
+			wantErrString: "block is equal or earlier than finalized block, slot 0 < slot 0",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			service.justifiedCheckpt = &ethpb.Checkpoint{Root: validGenesisRoot[:]}
+			service.bestJustifiedCheckpt = &ethpb.Checkpoint{Root: validGenesisRoot[:]}
+			service.finalizedCheckpt = &ethpb.Checkpoint{Root: validGenesisRoot[:]}
+			service.prevFinalizedCheckpt = &ethpb.Checkpoint{Root: validGenesisRoot[:]}
+			service.finalizedCheckpt.Root = roots[0]
+
+			root, err := stateutil.BlockRoot(tt.blk)
+			if err != nil {
+				t.Error(err)
+			}
+			_, err = service.onBlock(ctx, &ethpb.SignedBeaconBlock{Block: tt.blk}, root)
+			if err == nil || !strings.Contains(err.Error(), tt.wantErrString) {
+				t.Errorf("Store.OnBlock() error = %v, wantErr = %v", err, tt.wantErrString)
+			}
+		})
+	}
+}
+
+func TestRemoveStateSinceLastFinalized(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	params.UseMinimalConfig()
+	defer params.UseMainnetConfig()
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Save 100 blocks in DB, each has a state.
+	numBlocks := 100
+	totalBlocks := make([]*ethpb.SignedBeaconBlock, numBlocks)
+	blockRoots := make([][32]byte, 0)
+	for i := 0; i < len(totalBlocks); i++ {
+		totalBlocks[i] = &ethpb.SignedBeaconBlock{
+			Block: &ethpb.BeaconBlock{
+				Slot: uint64(i),
+			},
+		}
+		r, err := stateutil.BlockRoot(totalBlocks[i].Block)
+		if err != nil {
+			t.Fatal(err)
+		}
+		s := testutil.NewBeaconState()
+		if err := s.SetSlot(uint64(i)); err != nil {
+			t.Fatal(err)
+		}
+		if err := service.beaconDB.SaveState(ctx, s, r); err != nil {
+			t.Fatal(err)
+		}
+		if err := service.beaconDB.SaveBlock(ctx, totalBlocks[i]); err != nil {
+			t.Fatal(err)
+		}
+		blockRoots = append(blockRoots, r)
+		if err := service.beaconDB.SaveHeadBlockRoot(ctx, r); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// New finalized epoch: 1
+	finalizedEpoch := uint64(1)
+	finalizedSlot := finalizedEpoch * params.BeaconConfig().SlotsPerEpoch
+	endSlot := helpers.StartSlot(finalizedEpoch+1) - 1 // Inclusive
+	if err := service.rmStatesOlderThanLastFinalized(ctx, 0, endSlot); err != nil {
+		t.Fatal(err)
+	}
+	for _, r := range blockRoots {
+		s, err := service.beaconDB.State(ctx, r)
+		if err != nil {
+			t.Fatal(err)
+		}
+		// Also verifies genesis state didnt get deleted
+		if s != nil && s.Slot() != finalizedSlot && s.Slot() != 0 && s.Slot() < endSlot {
+			t.Errorf("State with slot %d should not be in DB", s.Slot())
+		}
+	}
+
+	// New finalized epoch: 5
+	newFinalizedEpoch := uint64(5)
+	newFinalizedSlot := newFinalizedEpoch * params.BeaconConfig().SlotsPerEpoch
+	endSlot = helpers.StartSlot(newFinalizedEpoch+1) - 1 // Inclusive
+	if err := service.rmStatesOlderThanLastFinalized(ctx, helpers.StartSlot(finalizedEpoch+1)-1, endSlot); err != nil {
+		t.Fatal(err)
+	}
+	for _, r := range blockRoots {
+		s, err := service.beaconDB.State(ctx, r)
+		if err != nil {
+			t.Fatal(err)
+		}
+		// Also verifies genesis state didnt get deleted
+		if s != nil && s.Slot() != newFinalizedSlot && s.Slot() != finalizedSlot && s.Slot() != 0 && s.Slot() < endSlot {
+			t.Errorf("State with slot %d should not be in DB", s.Slot())
+		}
+	}
+}
+
+func TestRemoveStateSinceLastFinalized_EmptyStartSlot(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	params.UseMinimalConfig()
+	defer params.UseMainnetConfig()
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	service.genesisTime = time.Now()
+
+	update, err := service.shouldUpdateCurrentJustified(ctx, &ethpb.Checkpoint{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !update {
+		t.Error("Should be able to update justified, received false")
+	}
+
+	lastJustifiedBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{ParentRoot: []byte{'G'}}}
+	lastJustifiedRoot, err := stateutil.BlockRoot(lastJustifiedBlk.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+	newJustifiedBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1, ParentRoot: lastJustifiedRoot[:]}}
+	newJustifiedRoot, err := stateutil.BlockRoot(newJustifiedBlk.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveBlock(ctx, newJustifiedBlk); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveBlock(ctx, lastJustifiedBlk); err != nil {
+		t.Fatal(err)
+	}
+
+	diff := (params.BeaconConfig().SlotsPerEpoch - 1) * params.BeaconConfig().SecondsPerSlot
+	service.genesisTime = time.Unix(time.Now().Unix()-int64(diff), 0)
+	service.justifiedCheckpt = &ethpb.Checkpoint{Root: lastJustifiedRoot[:]}
+	update, err = service.shouldUpdateCurrentJustified(ctx, &ethpb.Checkpoint{Root: newJustifiedRoot[:]})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !update {
+		t.Error("Should be able to update justified, received false")
+	}
+}
+
+func TestShouldUpdateJustified_ReturnFalse(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	params.UseMinimalConfig()
+	defer params.UseMainnetConfig()
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	lastJustifiedBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{ParentRoot: []byte{'G'}}}
+	lastJustifiedRoot, err := stateutil.BlockRoot(lastJustifiedBlk.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+	newJustifiedBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{ParentRoot: lastJustifiedRoot[:]}}
+	newJustifiedRoot, err := stateutil.BlockRoot(newJustifiedBlk.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveBlock(ctx, newJustifiedBlk); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveBlock(ctx, lastJustifiedBlk); err != nil {
+		t.Fatal(err)
+	}
+
+	diff := (params.BeaconConfig().SlotsPerEpoch - 1) * params.BeaconConfig().SecondsPerSlot
+	service.genesisTime = time.Unix(time.Now().Unix()-int64(diff), 0)
+	service.justifiedCheckpt = &ethpb.Checkpoint{Root: lastJustifiedRoot[:]}
+
+	update, err := service.shouldUpdateCurrentJustified(ctx, &ethpb.Checkpoint{Root: newJustifiedRoot[:]})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if update {
+		t.Error("Should not be able to update justified, received true")
+	}
+}
+
+func TestCachedPreState_CanGetFromStateSummary(t *testing.T) {
+	resetCfg := featureconfig.InitWithReset(&featureconfig.Flags{NewStateMgmt: true})
+	defer resetCfg()
+
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{
+		BeaconDB: db,
+		StateGen: stategen.New(db, cache.NewStateSummaryCache()),
+	}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s, err := stateTrie.InitializeFromProto(&pb.BeaconState{Slot: 1, GenesisValidatorsRoot: params.BeaconConfig().ZeroHash[:]})
+	if err != nil {
+		t.Fatal(err)
+	}
+	r := [32]byte{'A'}
+	b := &ethpb.BeaconBlock{Slot: 1, ParentRoot: r[:]}
+	if err := service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Slot: 1, Root: r[:]}); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.stateGen.SaveState(ctx, r, s); err != nil {
+		t.Fatal(err)
+	}
+
+	received, err := service.verifyBlkPreState(ctx, b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(s.InnerStateUnsafe(), received.InnerStateUnsafe()) {
+		t.Error("cached state not the same")
+	}
+}
+
+func TestCachedPreState_CanGetFromDB(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	resetCfg := featureconfig.InitWithReset(&featureconfig.Flags{NewStateMgmt: true})
+	defer resetCfg()
+
+	cfg := &Config{
+		BeaconDB: db,
+		StateGen: stategen.New(db, cache.NewStateSummaryCache()),
+	}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	r := [32]byte{'A'}
+	b := &ethpb.BeaconBlock{Slot: 1, ParentRoot: r[:]}
+
+	service.finalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	_, err = service.verifyBlkPreState(ctx, b)
+	wanted := "could not reconstruct parent state"
+	if err.Error() != wanted {
+		t.Error("Did not get wanted error")
+	}
+
+	s, err := stateTrie.InitializeFromProto(&pb.BeaconState{Slot: 1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Slot: 1, Root: r[:]}); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.stateGen.SaveState(ctx, r, s); err != nil {
+		t.Fatal(err)
+	}
+
+	received, err := service.verifyBlkPreState(ctx, b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s.Slot() != received.Slot() {
+		t.Error("cached state not the same")
+	}
+}
+
+func TestSaveInitState_CanSaveDelete(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := uint64(0); i < 64; i++ {
+		b := &ethpb.BeaconBlock{Slot: i}
+		s := testutil.NewBeaconState()
+		if err := s.SetSlot(i); err != nil {
+			t.Fatal(err)
+		}
+		r, err := ssz.HashTreeRoot(b)
+		if err != nil {
+			t.Fatal(err)
+		}
+		service.initSyncState[r] = s
+	}
+
+	// Set finalized root as slot 32
+	finalizedRoot, err := ssz.HashTreeRoot(&ethpb.BeaconBlock{Slot: 32})
+	if err != nil {
+		t.Fatal(err)
+	}
+	s := testutil.NewBeaconState()
+	if err := s.SetFinalizedCheckpoint(&ethpb.Checkpoint{
+		Epoch: 1, Root: finalizedRoot[:]}); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.saveInitState(ctx, s); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify finalized state is saved in DB
+	finalizedState, err := service.beaconDB.State(ctx, finalizedRoot)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if finalizedState == nil {
+		t.Error("finalized state can't be nil")
+	}
+}
+
+func TestUpdateJustified_CouldUpdateBest(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	signedBlock := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{}}
+	if err := db.SaveBlock(ctx, signedBlock); err != nil {
+		t.Fatal(err)
+	}
+	r, err := stateutil.BlockRoot(signedBlock.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+	service.justifiedCheckpt = &ethpb.Checkpoint{Root: []byte{'A'}}
+	service.bestJustifiedCheckpt = &ethpb.Checkpoint{Root: []byte{'A'}}
+	st := testutil.NewBeaconState()
+	service.initSyncState[r] = st.Copy()
+	if err := db.SaveState(ctx, st.Copy(), r); err != nil {
+		t.Fatal(err)
+	}
+
+	// Could update
+	s := testutil.NewBeaconState()
+	if err := s.SetCurrentJustifiedCheckpoint(&ethpb.Checkpoint{Epoch: 1, Root: r[:]}); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.updateJustified(context.Background(), s); err != nil {
+		t.Fatal(err)
+	}
+
+	if service.bestJustifiedCheckpt.Epoch != s.CurrentJustifiedCheckpoint().Epoch {
+		t.Error("Incorrect justified epoch in service")
+	}
+
+	// Could not update
+	service.bestJustifiedCheckpt.Epoch = 2
+	if err := service.updateJustified(context.Background(), s); err != nil {
+		t.Fatal(err)
+	}
+
+	if service.bestJustifiedCheckpt.Epoch != 2 {
+		t.Error("Incorrect justified epoch in service")
+	}
+}
+
+func TestFilterBlockRoots_CanFilter(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fBlock := &ethpb.BeaconBlock{}
+	fRoot, err := stateutil.BlockRoot(fBlock)
+	if err != nil {
+		t.Fatal(err)
+	}
+	hBlock := &ethpb.BeaconBlock{Slot: 1}
+	headRoot, err := stateutil.BlockRoot(hBlock)
+	if err != nil {
+		t.Fatal(err)
+	}
+	st := testutil.NewBeaconState()
+	if err := service.beaconDB.SaveBlock(ctx, &ethpb.SignedBeaconBlock{Block: fBlock}); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveState(ctx, st.Copy(), fRoot); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveFinalizedCheckpoint(ctx, &ethpb.Checkpoint{Root: fRoot[:]}); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveBlock(ctx, &ethpb.SignedBeaconBlock{Block: hBlock}); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveState(ctx, st.Copy(), headRoot); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveHeadBlockRoot(ctx, headRoot); err != nil {
+		t.Fatal(err)
+	}
+
+	roots := [][32]byte{{'C'}, {'D'}, headRoot, {'E'}, fRoot, {'F'}}
+	wanted := [][32]byte{{'C'}, {'D'}, {'E'}, {'F'}}
+
+	received, err := service.filterBlockRoots(ctx, roots)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(wanted, received) {
+		t.Error("Did not filter correctly")
+	}
+}
+
+func TestPersistCache_CanSave(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	st := testutil.NewBeaconState()
+
+	for i := uint64(0); i < initialSyncCacheSize; i++ {
+		if err := st.SetSlot(i); err != nil {
+			t.Fatal(err)
+		}
+		root := [32]byte{}
+		copy(root[:], bytesutil.Bytes32(i))
+		service.initSyncState[root] = st.Copy()
+		service.boundaryRoots = append(service.boundaryRoots, root)
+	}
+
+	if err = service.persistCachedStates(ctx, initialSyncCacheSize); err != nil {
+		t.Fatal(err)
+	}
+
+	for i := uint64(0); i < initialSyncCacheSize-minimumCacheSize; i++ {
+		root := [32]byte{}
+		copy(root[:], bytesutil.Bytes32(i))
+		state, err := db.State(context.Background(), root)
+		if err != nil {
+			t.Errorf("State with root of %#x , could not be retrieved: %v", root, err)
+		}
+		if state == nil {
+			t.Errorf("State with root of %#x , does not exist", root)
+		}
+		if state.Slot() != i {
+			t.Errorf("Incorrect slot retrieved. Wanted %d but got %d", i, state.Slot())
+		}
+	}
+}
+
+func TestFillForkChoiceMissingBlocks_CanSave(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	service.forkChoiceStore = protoarray.New(0, 0, [32]byte{'A'})
+	service.finalizedCheckpt = &ethpb.Checkpoint{}
+
+	genesisStateRoot := [32]byte{}
+	genesis := blocks.NewGenesisBlock(genesisStateRoot[:])
+	if err := db.SaveBlock(ctx, genesis); err != nil {
+		t.Error(err)
+	}
+	validGenesisRoot, err := stateutil.BlockRoot(genesis.Block)
+	if err != nil {
+		t.Error(err)
+	}
+	st := testutil.NewBeaconState()
+
+	if err := service.beaconDB.SaveState(ctx, st.Copy(), validGenesisRoot); err != nil {
+		t.Fatal(err)
+	}
+	roots, err := blockTree1(db, validGenesisRoot[:])
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	beaconState, _ := testutil.DeterministicGenesisState(t, 32)
+	block := &ethpb.BeaconBlock{Slot: 9, ParentRoot: roots[8], Body: &ethpb.BeaconBlockBody{Graffiti: []byte{}}}
+	if err := service.fillInForkChoiceMissingBlocks(context.Background(), block, beaconState); err != nil {
+		t.Fatal(err)
+	}
+
+	// 5 nodes from the block tree 1. B0 - B3 - B4 - B6 - B8
+	if len(service.forkChoiceStore.Nodes()) != 5 {
+		t.Error("Miss match nodes")
+	}
+
+	if !service.forkChoiceStore.HasNode(bytesutil.ToBytes32(roots[4])) {
+		t.Error("Didn't save node")
+	}
+	if !service.forkChoiceStore.HasNode(bytesutil.ToBytes32(roots[6])) {
+		t.Error("Didn't save node")
+	}
+	if !service.forkChoiceStore.HasNode(bytesutil.ToBytes32(roots[8])) {
+		t.Error("Didn't save node")
+	}
+}
+
+func TestFillForkChoiceMissingBlocks_FilterFinalized(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	service.forkChoiceStore = protoarray.New(0, 0, [32]byte{'A'})
+	// Set finalized epoch to 1.
+	service.finalizedCheckpt = &ethpb.Checkpoint{Epoch: 1}
+
+	genesisStateRoot := [32]byte{}
+	genesis := blocks.NewGenesisBlock(genesisStateRoot[:])
+	if err := db.SaveBlock(ctx, genesis); err != nil {
+		t.Error(err)
+	}
+	validGenesisRoot, err := stateutil.BlockRoot(genesis.Block)
+	if err != nil {
+		t.Error(err)
+	}
+	st := testutil.NewBeaconState()
+
+	if err := service.beaconDB.SaveState(ctx, st.Copy(), validGenesisRoot); err != nil {
+		t.Fatal(err)
+	}
+
+	// Define a tree branch, slot 63 <- 64 <- 65
+	b63 := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 63, Body: &ethpb.BeaconBlockBody{}}}
+	if err := service.beaconDB.SaveBlock(ctx, b63); err != nil {
+		t.Fatal(err)
+	}
+	r63, err := stateutil.BlockRoot(b63.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+	b64 := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 64, ParentRoot: r63[:], Body: &ethpb.BeaconBlockBody{}}}
+	if err := service.beaconDB.SaveBlock(ctx, b64); err != nil {
+		t.Fatal(err)
+	}
+	r64, err := stateutil.BlockRoot(b64.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+	b65 := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 65, ParentRoot: r64[:], Body: &ethpb.BeaconBlockBody{}}}
+	if err := service.beaconDB.SaveBlock(ctx, b65); err != nil {
+		t.Fatal(err)
+	}
+
+	beaconState, _ := testutil.DeterministicGenesisState(t, 32)
+	if err := service.fillInForkChoiceMissingBlocks(context.Background(), b65.Block, beaconState); err != nil {
+		t.Fatal(err)
+	}
+
+	// There should be 2 nodes, block 65 and block 64.
+	if len(service.forkChoiceStore.Nodes()) != 2 {
+		t.Error("Miss match nodes")
+	}
+
+	// Block with slot 63 should be in fork choice because it's less than finalized epoch 1.
+	if !service.forkChoiceStore.HasNode(r63) {
+		t.Error("Didn't save node")
+	}
+}
+
+// blockTree1 constructs the following tree:
+//    /- B1
+// B0           /- B5 - B7
+//    \- B3 - B4 - B6 - B8
+// (B1, and B3 are all from the same slots)
+func blockTree1(db db.Database, genesisRoot []byte) ([][]byte, error) {
+	b0 := &ethpb.BeaconBlock{Slot: 0, ParentRoot: genesisRoot}
+	r0, err := ssz.HashTreeRoot(b0)
+	if err != nil {
+		return nil, err
+	}
+	b1 := &ethpb.BeaconBlock{Slot: 1, ParentRoot: r0[:]}
+	r1, err := ssz.HashTreeRoot(b1)
+	if err != nil {
+		return nil, err
+	}
+	b3 := &ethpb.BeaconBlock{Slot: 3, ParentRoot: r0[:]}
+	r3, err := ssz.HashTreeRoot(b3)
+	if err != nil {
+		return nil, err
+	}
+	b4 := &ethpb.BeaconBlock{Slot: 4, ParentRoot: r3[:]}
+	r4, err := ssz.HashTreeRoot(b4)
+	if err != nil {
+		return nil, err
+	}
+	b5 := &ethpb.BeaconBlock{Slot: 5, ParentRoot: r4[:]}
+	r5, err := ssz.HashTreeRoot(b5)
+	if err != nil {
+		return nil, err
+	}
+	b6 := &ethpb.BeaconBlock{Slot: 6, ParentRoot: r4[:]}
+	r6, err := ssz.HashTreeRoot(b6)
+	if err != nil {
+		return nil, err
+	}
+	b7 := &ethpb.BeaconBlock{Slot: 7, ParentRoot: r5[:]}
+	r7, err := ssz.HashTreeRoot(b7)
+	if err != nil {
+		return nil, err
+	}
+	b8 := &ethpb.BeaconBlock{Slot: 8, ParentRoot: r6[:]}
+	r8, err := ssz.HashTreeRoot(b8)
+	if err != nil {
+		return nil, err
+	}
+	st := testutil.NewBeaconState()
+
+	for _, b := range []*ethpb.BeaconBlock{b0, b1, b3, b4, b5, b6, b7, b8} {
+		b.Body = &ethpb.BeaconBlockBody{}
+		if err := db.SaveBlock(context.Background(), &ethpb.SignedBeaconBlock{Block: b}); err != nil {
+			return nil, err
+		}
+		if err := db.SaveState(context.Background(), st.Copy(), bytesutil.ToBytes32(b.ParentRoot)); err != nil {
+			return nil, err
+		}
+	}
+	if err := db.SaveState(context.Background(), st.Copy(), r1); err != nil {
+		return nil, err
+	}
+	if err := db.SaveState(context.Background(), st.Copy(), r7); err != nil {
+		return nil, err
+	}
+	if err := db.SaveState(context.Background(), st.Copy(), r8); err != nil {
+		return nil, err
+	}
+	return [][]byte{r0[:], r1[:], nil, r3[:], r4[:], r5[:], r6[:], r7[:], r8[:]}, nil
+}
+
+func TestCurrentSlot_HandlesOverflow(t *testing.T) {
+	svc := Service{genesisTime: roughtime.Now().Add(1 * time.Hour)}
+
+	slot := svc.CurrentSlot()
+
+	if slot != 0 {
+		t.Fatalf("Expected slot to be 0, got %d", slot)
+	}
+}

beacon-chain/blockchain/receive_attestation.go

@@ -1,15 +1,19 @@
 package blockchain
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/roughtime"
 	"github.com/prysmaticlabs/prysm/shared/slotutil"
 	"github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
@@ -18,6 +22,8 @@ import (
 // AttestationReceiver interface defines the methods of chain service receive and processing new attestations.
 type AttestationReceiver interface {
 	ReceiveAttestationNoPubsub(ctx context.Context, att *ethpb.Attestation) error
+	IsValidAttestation(ctx context.Context, att *ethpb.Attestation) bool
+	AttestationPreState(ctx context.Context, att *ethpb.Attestation) (*state.BeaconState, error)
 }
 
 // ReceiveAttestationNoPubsub is a function that defines the operations that are preformed on
@@ -29,39 +35,56 @@ func (s *Service) ReceiveAttestationNoPubsub(ctx context.Context, att *ethpb.Att
 	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveAttestationNoPubsub")
 	defer span.End()
 
-	// Update forkchoice store for the new attestation
-	if err := s.forkChoiceStore.OnAttestation(ctx, att); err != nil {
-		return errors.Wrap(err, "could not process attestation from fork choice service")
-	}
-
-	// Run fork choice for head block after updating fork choice store.
-	headRoot, err := s.forkChoiceStore.Head(ctx)
+	_, err := s.onAttestation(ctx, att)
 	if err != nil {
-		return errors.Wrap(err, "could not get head from fork choice service")
+		return errors.Wrap(err, "could not process attestation")
 	}
-	// Only save head if it's different than the current head.
-	if !bytes.Equal(headRoot, s.HeadRoot()) {
-		signed, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(headRoot))
+
+	if !featureconfig.Get().DisableUpdateHeadPerAttestation {
+		baseState, err := s.getAttPreState(ctx, att.Data.Target)
 		if err != nil {
-			return errors.Wrap(err, "could not compute state from block head")
+			return err
 		}
-		if signed == nil || signed.Block == nil {
-			return errors.New("nil head block")
-		}
-		if err := s.saveHead(ctx, signed, bytesutil.ToBytes32(headRoot)); err != nil {
-			return errors.Wrap(err, "could not save head")
+
+		// This updates fork choice head, if a new head could not be updated due to
+		// long range or intermediate forking. It simply logs a warning and returns nil
+		// as that's more appropriate than returning errors.
+		if err := s.updateHead(ctx, baseState.Balances()); err != nil {
+			log.Warnf("Resolving fork due to new attestation: %v", err)
+			return nil
 		}
 	}
 
-	processedAttNoPubsub.Inc()
 	return nil
 }
 
+// IsValidAttestation returns true if the attestation can be verified against its pre-state.
+func (s *Service) IsValidAttestation(ctx context.Context, att *ethpb.Attestation) bool {
+	baseState, err := s.AttestationPreState(ctx, att)
+	if err != nil {
+		log.WithError(err).Error("Failed to get attestation pre state")
+		return false
+	}
+
+	if err := blocks.VerifyAttestation(ctx, baseState, att); err != nil {
+		log.WithError(err).Error("Failed to validate attestation")
+		return false
+	}
+
+	return true
+}
+
+// AttestationPreState returns the pre state of attestation.
+func (s *Service) AttestationPreState(ctx context.Context, att *ethpb.Attestation) (*state.BeaconState, error) {
+	return s.getAttPreState(ctx, att.Data.Target)
+}
+
 // This processes attestations from the attestation pool to account for validator votes and fork choice.
-func (s *Service) processAttestation() {
+func (s *Service) processAttestation(subscribedToStateEvents chan struct{}) {
 	// Wait for state to be initialized.
 	stateChannel := make(chan *feed.Event, 1)
 	stateSub := s.stateNotifier.StateFeed().Subscribe(stateChannel)
+	subscribedToStateEvents <- struct{}{}
 	<-stateChannel
 	stateSub.Unsubscribe()
 
@@ -74,16 +97,64 @@ func (s *Service) processAttestation() {
 			ctx := context.Background()
 			atts := s.attPool.ForkchoiceAttestations()
 			for _, a := range atts {
+				// Based on the spec, don't process the attestation until the subsequent slot.
+				// This delays consideration in the fork choice until their slot is in the past.
+				// https://github.com/ethereum/eth2.0-specs/blob/dev/specs/phase0/fork-choice.md#validate_on_attestation
+				nextSlot := a.Data.Slot + 1
+				if err := helpers.VerifySlotTime(uint64(s.genesisTime.Unix()), nextSlot, helpers.TimeShiftTolerance); err != nil {
+					continue
+				}
+
+				var hasState bool
+				if featureconfig.Get().NewStateMgmt {
+					hasState = s.stateGen.StateSummaryExists(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
+				} else {
+					hasState = s.beaconDB.HasState(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot)) && s.beaconDB.HasState(ctx, bytesutil.ToBytes32(a.Data.Target.Root))
+				}
+
+				hasBlock := s.hasBlock(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
+				if !(hasState && hasBlock) {
+					continue
+				}
+
 				if err := s.attPool.DeleteForkchoiceAttestation(a); err != nil {
 					log.WithError(err).Error("Could not delete fork choice attestation in pool")
 				}
 
+				if !s.verifyCheckpointEpoch(a.Data.Target) {
+					continue
+				}
+
 				if err := s.ReceiveAttestationNoPubsub(ctx, a); err != nil {
 					log.WithFields(logrus.Fields{
-						"targetRoot": fmt.Sprintf("%#x", a.Data.Target.Root),
-					}).WithError(err).Error("Could not receive attestation in chain service")
+						"slot":             a.Data.Slot,
+						"committeeIndex":   a.Data.CommitteeIndex,
+						"beaconBlockRoot":  fmt.Sprintf("%#x", bytesutil.Trunc(a.Data.BeaconBlockRoot)),
+						"targetRoot":       fmt.Sprintf("%#x", bytesutil.Trunc(a.Data.Target.Root)),
+						"aggregationCount": a.AggregationBits.Count(),
+					}).WithError(err).Warn("Could not receive attestation in chain service")
 				}
 			}
 		}
 	}
 }
+
+// This verifies the epoch of input checkpoint is within current epoch and previous epoch
+// with respect to current time. Returns true if it's within, false if it's not.
+func (s *Service) verifyCheckpointEpoch(c *ethpb.Checkpoint) bool {
+	now := uint64(roughtime.Now().Unix())
+	genesisTime := uint64(s.genesisTime.Unix())
+	currentSlot := (now - genesisTime) / params.BeaconConfig().SecondsPerSlot
+	currentEpoch := helpers.SlotToEpoch(currentSlot)
+
+	var prevEpoch uint64
+	if currentEpoch > 1 {
+		prevEpoch = currentEpoch - 1
+	}
+
+	if c.Epoch != prevEpoch && c.Epoch != currentEpoch {
+		return false
+	}
+
+	return true
+}

beacon-chain/blockchain/receive_attestation_test.go

@@ -2,45 +2,23 @@ package blockchain
 
 import (
 	"testing"
+	"time"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/testutil"
-	logTest "github.com/sirupsen/logrus/hooks/test"
-	"golang.org/x/net/context"
 )
 
-func TestReceiveAttestationNoPubsub_ProcessCorrectly(t *testing.T) {
-	hook := logTest.NewGlobal()
+func TestVerifyCheckpointEpoch_Ok(t *testing.T) {
 	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	ctx := context.Background()
 
 	chainService := setupBeaconChain(t, db)
-	r, _ := ssz.HashTreeRoot(&ethpb.BeaconBlock{})
-	chainService.forkChoiceStore = &store{headRoot: r[:]}
+	chainService.genesisTime = time.Now()
 
-	b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{}}
-	if err := chainService.beaconDB.SaveBlock(ctx, b); err != nil {
-		t.Fatal(err)
-	}
-	root, err := ssz.HashTreeRoot(b.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := chainService.beaconDB.SaveState(ctx, &pb.BeaconState{}, root); err != nil {
-		t.Fatal(err)
+	if !chainService.verifyCheckpointEpoch(&ethpb.Checkpoint{}) {
+		t.Error("Wanted true, got false")
 	}
 
-	a := &ethpb.Attestation{Data: &ethpb.AttestationData{
-		Target: &ethpb.Checkpoint{Root: root[:]},
-	}}
-	if err := chainService.ReceiveAttestationNoPubsub(ctx, a); err != nil {
-		t.Fatal(err)
+	if chainService.verifyCheckpointEpoch(&ethpb.Checkpoint{Epoch: 1}) {
+		t.Error("Wanted false, got true")
 	}
-
-	testutil.AssertLogsContain(t, hook, "Saved new head info")
-	testutil.AssertLogsDoNotContain(t, hook, "Broadcasting attestation")
 }

beacon-chain/blockchain/receive_block.go

@@ -5,15 +5,13 @@ import (
 	"context"
 	"encoding/hex"
 
-	"github.com/gogo/protobuf/proto"
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/traceutil"
 	"github.com/sirupsen/logrus"
@@ -22,10 +20,11 @@ import (
 
 // BlockReceiver interface defines the methods of chain service receive and processing new blocks.
 type BlockReceiver interface {
-	ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock) error
-	ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedBeaconBlock) error
-	ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *ethpb.SignedBeaconBlock) error
-	ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedBeaconBlock) error
+	ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error
+	ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error
+	ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error
+	ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error
+	HasInitSyncBlock(root [32]byte) bool
 }
 
 // ReceiveBlock is a function that defines the operations that are preformed on
@@ -34,28 +33,27 @@ type BlockReceiver interface {
 //   2. Validate block, apply state transition and update check points
 //   3. Apply fork choice to the processed block
 //   4. Save latest head info
-func (s *Service) ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
+func (s *Service) ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
 	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlock")
 	defer span.End()
 
-	root, err := ssz.HashTreeRoot(block.Block)
-	if err != nil {
-		return errors.Wrap(err, "could not get signing root on received block")
-	}
-
 	// Broadcast the new block to the network.
 	if err := s.p2p.Broadcast(ctx, block); err != nil {
 		return errors.Wrap(err, "could not broadcast block")
 	}
 	log.WithFields(logrus.Fields{
-		"blockRoot": hex.EncodeToString(root[:]),
+		"blockRoot": hex.EncodeToString(blockRoot[:]),
 	}).Debug("Broadcasting block")
 
-	if err := s.ReceiveBlockNoPubsub(ctx, block); err != nil {
+	if err := captureSentTimeMetric(uint64(s.genesisTime.Unix()), block.Block.Slot); err != nil {
+		// If a node fails to capture metric, this shouldn't cause the block processing to fail.
+		log.Warnf("Could not capture block sent time metric: %v", err)
+	}
+
+	if err := s.ReceiveBlockNoPubsub(ctx, block, blockRoot); err != nil {
 		return err
 	}
 
-	processedBlk.Inc()
 	return nil
 }
 
@@ -64,38 +62,38 @@ func (s *Service) ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlo
 //   1. Validate block, apply state transition and update check points
 //   2. Apply fork choice to the processed block
 //   3. Save latest head info
-func (s *Service) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
+func (s *Service) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
 	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlockNoPubsub")
 	defer span.End()
-	blockCopy := proto.Clone(block).(*ethpb.SignedBeaconBlock)
+	blockCopy := stateTrie.CopySignedBeaconBlock(block)
 
 	// Apply state transition on the new block.
-	if err := s.forkChoiceStore.OnBlock(ctx, blockCopy); err != nil {
-		err := errors.Wrap(err, "could not process block from fork choice service")
+	postState, err := s.onBlock(ctx, blockCopy, blockRoot)
+	if err != nil {
+		err := errors.Wrap(err, "could not process block")
 		traceutil.AnnotateError(span, err)
 		return err
 	}
-	root, err := ssz.HashTreeRoot(blockCopy.Block)
-	if err != nil {
-		return errors.Wrap(err, "could not get signing root on received block")
+
+	// Add attestations from the block to the pool for fork choice.
+	if err := s.attPool.SaveBlockAttestations(blockCopy.Block.Body.Attestations); err != nil {
+		log.Errorf("Could not save attestation for fork choice: %v", err)
+		return nil
+	}
+	for _, exit := range block.Block.Body.VoluntaryExits {
+		s.exitPool.MarkIncluded(exit)
 	}
 
-	// Run fork choice after applying state transition on the new block.
-	headRoot, err := s.forkChoiceStore.Head(ctx)
-	if err != nil {
-		return errors.Wrap(err, "could not get head from fork choice service")
-	}
-	signedHeadBlock, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(headRoot))
-	if err != nil {
-		return errors.Wrap(err, "could not compute state from block head")
-	}
-	if signedHeadBlock == nil || signedHeadBlock.Block == nil {
-		return errors.New("nil head block")
-	}
+	s.epochParticipationLock.Lock()
+	defer s.epochParticipationLock.Unlock()
+	s.epochParticipation[helpers.SlotToEpoch(blockCopy.Block.Slot)] = precompute.Balances
 
-	// Only save head if it's different than the current head.
-	if !bytes.Equal(headRoot, s.HeadRoot()) {
-		if err := s.saveHead(ctx, signedHeadBlock, bytesutil.ToBytes32(headRoot)); err != nil {
+	if featureconfig.Get().DisableForkChoice && block.Block.Slot > s.headSlot() {
+		if err := s.saveHead(ctx, blockRoot); err != nil {
+			return errors.Wrap(err, "could not save head")
+		}
+	} else {
+		if err := s.updateHead(ctx, postState.Balances()); err != nil {
 			return errors.Wrap(err, "could not save head")
 		}
 	}
@@ -104,31 +102,20 @@ func (s *Service) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedB
 	s.stateNotifier.StateFeed().Send(&feed.Event{
 		Type: statefeed.BlockProcessed,
 		Data: &statefeed.BlockProcessedData{
-			BlockRoot: root,
+			Slot:      blockCopy.Block.Slot,
+			BlockRoot: blockRoot,
 			Verified:  true,
 		},
 	})
 
-	// Add attestations from the block to the pool for fork choice.
-	if err := s.attPool.SaveBlockAttestations(blockCopy.Block.Body.Attestations); err != nil {
-		log.Errorf("Could not save attestation for fork choice: %v", err)
-		return nil
-	}
-
 	// Reports on block and fork choice metrics.
-	s.reportSlotMetrics(blockCopy.Block.Slot)
+	reportSlotMetrics(blockCopy.Block.Slot, s.headSlot(), s.CurrentSlot(), s.finalizedCheckpt)
 
-	// Log if block is a competing block.
-	isCompetingBlock(root[:], blockCopy.Block.Slot, headRoot, signedHeadBlock.Block.Slot)
+	// Log block sync status.
+	logBlockSyncStatus(blockCopy.Block, blockRoot, s.finalizedCheckpt)
 
 	// Log state transition data.
-	logStateTransitionData(blockCopy.Block, root[:])
-
-	s.epochParticipationLock.Lock()
-	defer s.epochParticipationLock.Unlock()
-	s.epochParticipation[helpers.SlotToEpoch(blockCopy.Block.Slot)] = precompute.Balances
-
-	processedBlkNoPubsub.Inc()
+	logStateTransitionData(blockCopy.Block)
 
 	return nil
 }
@@ -137,24 +124,25 @@ func (s *Service) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedB
 // that are preformed blocks that is received from initial sync service. The operations consists of:
 //   1. Validate block, apply state transition and update check points
 //   2. Save latest head info
-func (s *Service) ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
+func (s *Service) ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
 	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlockNoForkchoice")
 	defer span.End()
-	blockCopy := proto.Clone(block).(*ethpb.SignedBeaconBlock)
+	blockCopy := stateTrie.CopySignedBeaconBlock(block)
 
-	// Apply state transition on the incoming newly received block.
-	if err := s.forkChoiceStore.OnBlock(ctx, blockCopy); err != nil {
-		err := errors.Wrap(err, "could not process block from fork choice service")
+	// Apply state transition on the new block.
+	_, err := s.onBlock(ctx, blockCopy, blockRoot)
+	if err != nil {
+		err := errors.Wrap(err, "could not process block")
 		traceutil.AnnotateError(span, err)
 		return err
 	}
-	root, err := ssz.HashTreeRoot(blockCopy.Block)
-	if err != nil {
-		return errors.Wrap(err, "could not get signing root on received block")
-	}
 
-	if !bytes.Equal(root[:], s.HeadRoot()) {
-		if err := s.saveHead(ctx, blockCopy, root); err != nil {
+	cachedHeadRoot, err := s.HeadRoot(ctx)
+	if err != nil {
+		return errors.Wrap(err, "could not get head root from cache")
+	}
+	if !bytes.Equal(blockRoot[:], cachedHeadRoot) {
+		if err := s.saveHead(ctx, blockRoot); err != nil {
 			return errors.Wrap(err, "could not save head")
 		}
 	}
@@ -163,57 +151,53 @@ func (s *Service) ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *eth
 	s.stateNotifier.StateFeed().Send(&feed.Event{
 		Type: statefeed.BlockProcessed,
 		Data: &statefeed.BlockProcessedData{
-			BlockRoot: root,
+			Slot:      blockCopy.Block.Slot,
+			BlockRoot: blockRoot,
 			Verified:  true,
 		},
 	})
 
 	// Reports on block and fork choice metrics.
-	s.reportSlotMetrics(blockCopy.Block.Slot)
+	reportSlotMetrics(blockCopy.Block.Slot, s.headSlot(), s.CurrentSlot(), s.finalizedCheckpt)
+
+	// Log block sync status.
+	logBlockSyncStatus(blockCopy.Block, blockRoot, s.finalizedCheckpt)
 
 	// Log state transition data.
-	logStateTransitionData(blockCopy.Block, root[:])
+	logStateTransitionData(blockCopy.Block)
 
 	s.epochParticipationLock.Lock()
 	defer s.epochParticipationLock.Unlock()
 	s.epochParticipation[helpers.SlotToEpoch(blockCopy.Block.Slot)] = precompute.Balances
 
-	processedBlkNoPubsubForkchoice.Inc()
 	return nil
 }
 
 // ReceiveBlockNoVerify runs state transition on a input block without verifying the block's BLS contents.
 // Depends on the security model, this is the "minimal" work a node can do to sync the chain.
 // It simulates light client behavior and assumes 100% trust with the syncing peer.
-func (s *Service) ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
+func (s *Service) ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
 	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlockNoVerify")
 	defer span.End()
-	blockCopy := proto.Clone(block).(*ethpb.SignedBeaconBlock)
+	blockCopy := stateTrie.CopySignedBeaconBlock(block)
 
 	// Apply state transition on the incoming newly received blockCopy without verifying its BLS contents.
-	if err := s.forkChoiceStore.OnBlockInitialSyncStateTransition(ctx, blockCopy); err != nil {
-		return errors.Wrap(err, "could not process blockCopy from fork choice service")
-	}
-	root, err := ssz.HashTreeRoot(blockCopy.Block)
-	if err != nil {
-		return errors.Wrap(err, "could not get signing root on received blockCopy")
+	if err := s.onBlockInitialSyncStateTransition(ctx, blockCopy, blockRoot); err != nil {
+		err := errors.Wrap(err, "could not process block")
+		traceutil.AnnotateError(span, err)
+		return err
 	}
 
-	if featureconfig.Get().InitSyncCacheState {
-		if !bytes.Equal(root[:], s.HeadRoot()) {
-			if err := s.saveHeadNoDB(ctx, blockCopy, root); err != nil {
-				err := errors.Wrap(err, "could not save head")
-				traceutil.AnnotateError(span, err)
-				return err
-			}
-		}
-	} else {
-		if !bytes.Equal(root[:], s.HeadRoot()) {
-			if err := s.saveHead(ctx, blockCopy, root); err != nil {
-				err := errors.Wrap(err, "could not save head")
-				traceutil.AnnotateError(span, err)
-				return err
-			}
+	cachedHeadRoot, err := s.HeadRoot(ctx)
+	if err != nil {
+		return errors.Wrap(err, "could not get head root from cache")
+	}
+
+	if !bytes.Equal(blockRoot[:], cachedHeadRoot) {
+		if err := s.saveHeadNoDB(ctx, blockCopy, blockRoot); err != nil {
+			err := errors.Wrap(err, "could not save head")
+			traceutil.AnnotateError(span, err)
+			return err
 		}
 	}
 
@@ -221,13 +205,14 @@ func (s *Service) ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedB
 	s.stateNotifier.StateFeed().Send(&feed.Event{
 		Type: statefeed.BlockProcessed,
 		Data: &statefeed.BlockProcessedData{
-			BlockRoot: root,
+			Slot:      blockCopy.Block.Slot,
+			BlockRoot: blockRoot,
 			Verified:  false,
 		},
 	})
 
 	// Reports on blockCopy and fork choice metrics.
-	s.reportSlotMetrics(blockCopy.Block.Slot)
+	reportSlotMetrics(blockCopy.Block.Slot, s.headSlot(), s.CurrentSlot(), s.finalizedCheckpt)
 
 	// Log state transition data.
 	log.WithFields(logrus.Fields{
@@ -243,15 +228,7 @@ func (s *Service) ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedB
 	return nil
 }
 
-// This checks if the block is from a competing chain, emits warning and updates metrics.
-func isCompetingBlock(root []byte, slot uint64, headRoot []byte, headSlot uint64) {
-	if !bytes.Equal(root[:], headRoot) {
-		log.WithFields(logrus.Fields{
-			"blkSlot":  slot,
-			"blkRoot":  hex.EncodeToString(root[:]),
-			"headSlot": headSlot,
-			"headRoot": hex.EncodeToString(headRoot),
-		}).Warn("Calculated head diffs from new block")
-		competingBlks.Inc()
-	}
+// HasInitSyncBlock returns true if the block of the input root exists in initial sync blocks cache.
+func (s *Service) HasInitSyncBlock(root [32]byte) bool {
+	return s.hasInitSyncBlock(root)
 }

beacon-chain/blockchain/receive_block_test.go

@@ -1,195 +0,0 @@
-package blockchain
-
-import (
-	"bytes"
-	"context"
-	"reflect"
-	"testing"
-
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	b "github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
-	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/stateutil"
-	"github.com/prysmaticlabs/prysm/shared/testutil"
-	logTest "github.com/sirupsen/logrus/hooks/test"
-)
-
-func TestReceiveBlock_ProcessCorrectly(t *testing.T) {
-	hook := logTest.NewGlobal()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	ctx := context.Background()
-
-	chainService := setupBeaconChain(t, db)
-
-	beaconState, privKeys := testutil.DeterministicGenesisState(t, 100)
-	genesis, _ := testutil.GenerateFullBlock(beaconState, privKeys, nil, beaconState.Slot+1)
-	beaconState, err := state.ExecuteStateTransition(ctx, beaconState, genesis)
-	if err != nil {
-		t.Fatal(err)
-	}
-	genesisBlkRoot, err := ssz.HashTreeRoot(genesis.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveState(ctx, beaconState, genesisBlkRoot); err != nil {
-		t.Fatal(err)
-	}
-	cp := &ethpb.Checkpoint{Root: genesisBlkRoot[:]}
-	if err := chainService.forkChoiceStore.GenesisStore(ctx, cp, cp); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := chainService.beaconDB.SaveBlock(ctx, genesis); err != nil {
-		t.Fatalf("Could not save block to db: %v", err)
-	}
-
-	if err := db.SaveState(ctx, beaconState, genesisBlkRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	slot := beaconState.Slot + 1
-	block, err := testutil.GenerateFullBlock(beaconState, privKeys, nil, slot)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := chainService.beaconDB.SaveBlock(ctx, block); err != nil {
-		t.Fatal(err)
-	}
-	if err := chainService.ReceiveBlock(context.Background(), block); err != nil {
-		t.Errorf("Block failed processing: %v", err)
-	}
-	testutil.AssertLogsContain(t, hook, "Finished applying state transition")
-}
-
-func TestReceiveReceiveBlockNoPubsub_CanSaveHeadInfo(t *testing.T) {
-	hook := logTest.NewGlobal()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	ctx := context.Background()
-
-	chainService := setupBeaconChain(t, db)
-
-	headBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 100}}
-	if err := db.SaveBlock(ctx, headBlk); err != nil {
-		t.Fatal(err)
-	}
-	r, err := ssz.HashTreeRoot(headBlk.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	head := &pb.BeaconState{Slot: 100, FinalizedCheckpoint: &ethpb.Checkpoint{Root: r[:]}}
-	if err := db.SaveState(ctx, head, r); err != nil {
-		t.Fatal(err)
-	}
-	chainService.forkChoiceStore = &store{headRoot: r[:]}
-
-	if err := chainService.ReceiveBlockNoPubsub(ctx, &ethpb.SignedBeaconBlock{
-		Block: &ethpb.BeaconBlock{
-			Slot: 1,
-			Body: &ethpb.BeaconBlockBody{},
-		},
-	}); err != nil {
-		t.Fatal(err)
-	}
-
-	if !bytes.Equal(r[:], chainService.HeadRoot()) {
-		t.Error("Incorrect head root saved")
-	}
-
-	if !reflect.DeepEqual(headBlk, chainService.HeadBlock()) {
-		t.Error("Incorrect head block saved")
-	}
-
-	testutil.AssertLogsContain(t, hook, "Saved new head info")
-}
-
-func TestReceiveReceiveBlockNoPubsub_SameHead(t *testing.T) {
-	hook := logTest.NewGlobal()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	ctx := context.Background()
-
-	chainService := setupBeaconChain(t, db)
-
-	headBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{}}
-	if err := db.SaveBlock(ctx, headBlk); err != nil {
-		t.Fatal(err)
-	}
-	newBlk := &ethpb.SignedBeaconBlock{
-		Block: &ethpb.BeaconBlock{
-			Slot: 1,
-			Body: &ethpb.BeaconBlockBody{},
-		},
-	}
-	newRoot, _ := ssz.HashTreeRoot(newBlk.Block)
-	if err := db.SaveBlock(ctx, newBlk); err != nil {
-		t.Fatal(err)
-	}
-
-	chainService.forkChoiceStore = &store{headRoot: newRoot[:]}
-	chainService.canonicalRoots[0] = newRoot[:]
-
-	if err := chainService.ReceiveBlockNoPubsub(ctx, newBlk); err != nil {
-		t.Fatal(err)
-	}
-
-	testutil.AssertLogsDoNotContain(t, hook, "Saved new head info")
-}
-
-func TestReceiveBlockNoPubsubForkchoice_ProcessCorrectly(t *testing.T) {
-	hook := logTest.NewGlobal()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	ctx := context.Background()
-
-	chainService := setupBeaconChain(t, db)
-	beaconState, privKeys := testutil.DeterministicGenesisState(t, 100)
-
-	block, err := testutil.GenerateFullBlock(beaconState, privKeys, nil, beaconState.Slot)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	stateRoot, err := stateutil.HashTreeRootState(beaconState)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	genesis := b.NewGenesisBlock(stateRoot[:])
-	parentRoot, err := ssz.HashTreeRoot(genesis.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveState(ctx, beaconState, parentRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := chainService.forkChoiceStore.GenesisStore(ctx, &ethpb.Checkpoint{Root: parentRoot[:]}, &ethpb.Checkpoint{Root: parentRoot[:]}); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := chainService.beaconDB.SaveBlock(ctx, block); err != nil {
-		t.Fatalf("Could not save block to db: %v", err)
-	}
-
-	block, err = testutil.GenerateFullBlock(beaconState, privKeys, nil, beaconState.Slot)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveState(ctx, beaconState, bytesutil.ToBytes32(block.Block.ParentRoot)); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := chainService.beaconDB.SaveBlock(ctx, block); err != nil {
-		t.Fatal(err)
-	}
-	if err := chainService.ReceiveBlockNoPubsubForkchoice(context.Background(), block); err != nil {
-		t.Errorf("Block failed processing: %v", err)
-	}
-	testutil.AssertLogsContain(t, hook, "Finished applying state transition")
-	testutil.AssertLogsDoNotContain(t, hook, "Finished fork choice")
-}

beacon-chain/blockchain/service.go

@@ -1,6 +1,6 @@
-// Package blockchain defines the life-cycle and status of the beacon chain
-// as well as the Ethereum Serenity beacon chain fork-choice rule based on
-// Casper Proof of Stake finality.
+// Package blockchain defines the life-cycle of the blockchain at the core of
+// eth2, including processing of new blocks and attestations using casper
+// proof of stake.
 package blockchain
 
 import (
@@ -12,8 +12,7 @@ import (
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/blockchain/forkchoice"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
@@ -22,70 +21,108 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
+	"github.com/prysmaticlabs/prysm/beacon-chain/flags"
+	f "github.com/prysmaticlabs/prysm/beacon-chain/forkchoice"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/attestations"
+	"github.com/prysmaticlabs/prysm/beacon-chain/operations/slashings"
+	"github.com/prysmaticlabs/prysm/beacon-chain/operations/voluntaryexits"
 	"github.com/prysmaticlabs/prysm/beacon-chain/p2p"
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"github.com/sirupsen/logrus"
+	"github.com/prysmaticlabs/prysm/shared/params"
 	"go.opencensus.io/trace"
 )
 
 // Service represents a service that handles the internal
 // logic of managing the full PoS beacon chain.
 type Service struct {
-	ctx                    context.Context
-	cancel                 context.CancelFunc
-	beaconDB               db.Database
-	depositCache           *depositcache.DepositCache
-	chainStartFetcher      powchain.ChainStartFetcher
-	attPool                attestations.Pool
-	forkChoiceStore        forkchoice.ForkChoicer
-	genesisTime            time.Time
-	p2p                    p2p.Broadcaster
-	maxRoutines            int64
-	headSlot               uint64
-	headBlock              *ethpb.SignedBeaconBlock
-	headState              *pb.BeaconState
-	canonicalRoots         map[uint64][]byte
-	headLock               sync.RWMutex
-	stateNotifier          statefeed.Notifier
-	genesisRoot            [32]byte
-	epochParticipation     map[uint64]*precompute.Balance
-	epochParticipationLock sync.RWMutex
+	ctx                       context.Context
+	cancel                    context.CancelFunc
+	beaconDB                  db.HeadAccessDatabase
+	depositCache              *depositcache.DepositCache
+	chainStartFetcher         powchain.ChainStartFetcher
+	attPool                   attestations.Pool
+	slashingPool              *slashings.Pool
+	exitPool                  *voluntaryexits.Pool
+	genesisTime               time.Time
+	p2p                       p2p.Broadcaster
+	maxRoutines               int64
+	head                      *head
+	headLock                  sync.RWMutex
+	stateNotifier             statefeed.Notifier
+	genesisRoot               [32]byte
+	epochParticipation        map[uint64]*precompute.Balance
+	epochParticipationLock    sync.RWMutex
+	forkChoiceStore           f.ForkChoicer
+	justifiedCheckpt          *ethpb.Checkpoint
+	prevJustifiedCheckpt      *ethpb.Checkpoint
+	bestJustifiedCheckpt      *ethpb.Checkpoint
+	finalizedCheckpt          *ethpb.Checkpoint
+	prevFinalizedCheckpt      *ethpb.Checkpoint
+	nextEpochBoundarySlot     uint64
+	voteLock                  sync.RWMutex
+	initSyncState             map[[32]byte]*stateTrie.BeaconState
+	boundaryRoots             [][32]byte
+	initSyncStateLock         sync.RWMutex
+	checkpointState           *cache.CheckpointStateCache
+	checkpointStateLock       sync.Mutex
+	stateGen                  *stategen.State
+	opsService                *attestations.Service
+	initSyncBlocks            map[[32]byte]*ethpb.SignedBeaconBlock
+	initSyncBlocksLock        sync.RWMutex
+	recentCanonicalBlocks     map[[32]byte]bool
+	recentCanonicalBlocksLock sync.RWMutex
 }
 
 // Config options for the service.
 type Config struct {
 	BeaconBlockBuf    int
 	ChainStartFetcher powchain.ChainStartFetcher
-	BeaconDB          db.Database
+	BeaconDB          db.HeadAccessDatabase
 	DepositCache      *depositcache.DepositCache
 	AttPool           attestations.Pool
+	ExitPool          *voluntaryexits.Pool
+	SlashingPool      *slashings.Pool
 	P2p               p2p.Broadcaster
 	MaxRoutines       int64
 	StateNotifier     statefeed.Notifier
+	ForkChoiceStore   f.ForkChoicer
+	OpsService        *attestations.Service
+	StateGen          *stategen.State
 }
 
 // NewService instantiates a new block service instance that will
 // be registered into a running beacon node.
 func NewService(ctx context.Context, cfg *Config) (*Service, error) {
 	ctx, cancel := context.WithCancel(ctx)
-	store := forkchoice.NewForkChoiceService(ctx, cfg.BeaconDB)
 	return &Service{
-		ctx:                ctx,
-		cancel:             cancel,
-		beaconDB:           cfg.BeaconDB,
-		depositCache:       cfg.DepositCache,
-		chainStartFetcher:  cfg.ChainStartFetcher,
-		attPool:            cfg.AttPool,
-		forkChoiceStore:    store,
-		p2p:                cfg.P2p,
-		canonicalRoots:     make(map[uint64][]byte),
-		maxRoutines:        cfg.MaxRoutines,
-		stateNotifier:      cfg.StateNotifier,
-		epochParticipation: make(map[uint64]*precompute.Balance),
+		ctx:                   ctx,
+		cancel:                cancel,
+		beaconDB:              cfg.BeaconDB,
+		depositCache:          cfg.DepositCache,
+		chainStartFetcher:     cfg.ChainStartFetcher,
+		attPool:               cfg.AttPool,
+		exitPool:              cfg.ExitPool,
+		slashingPool:          cfg.SlashingPool,
+		p2p:                   cfg.P2p,
+		maxRoutines:           cfg.MaxRoutines,
+		stateNotifier:         cfg.StateNotifier,
+		epochParticipation:    make(map[uint64]*precompute.Balance),
+		forkChoiceStore:       cfg.ForkChoiceStore,
+		initSyncState:         make(map[[32]byte]*stateTrie.BeaconState),
+		boundaryRoots:         [][32]byte{},
+		checkpointState:       cache.NewCheckpointStateCache(),
+		opsService:            cfg.OpsService,
+		stateGen:              cfg.StateGen,
+		initSyncBlocks:        make(map[[32]byte]*ethpb.SignedBeaconBlock),
+		recentCanonicalBlocks: make(map[[32]byte]bool),
 	}, nil
 }
 
@@ -100,12 +137,18 @@ func (s *Service) Start() {
 	// For running initial sync with state cache, in an event of restart, we use
 	// last finalized check point as start point to sync instead of head
 	// state. This is because we no longer save state every slot during sync.
-	if featureconfig.Get().InitSyncCacheState {
-		cp, err := s.beaconDB.FinalizedCheckpoint(ctx)
-		if err != nil {
-			log.Fatalf("Could not fetch finalized cp: %v", err)
-		}
-		if beaconState == nil {
+	cp, err := s.beaconDB.FinalizedCheckpoint(ctx)
+	if err != nil {
+		log.Fatalf("Could not fetch finalized cp: %v", err)
+	}
+
+	if beaconState == nil {
+		if featureconfig.Get().NewStateMgmt {
+			beaconState, err = s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(cp.Root))
+			if err != nil {
+				log.Fatalf("Could not fetch beacon state by root: %v", err)
+			}
+		} else {
 			beaconState, err = s.beaconDB.State(ctx, bytesutil.ToBytes32(cp.Root))
 			if err != nil {
 				log.Fatalf("Could not fetch beacon state: %v", err)
@@ -113,10 +156,14 @@ func (s *Service) Start() {
 		}
 	}
 
+	// Make sure that attestation processor is subscribed and ready for state initializing event.
+	attestationProcessorSubscribed := make(chan struct{}, 1)
+
 	// If the chain has already been initialized, simply start the block processing routine.
 	if beaconState != nil {
 		log.Info("Blockchain data already exists in DB, initializing...")
-		s.genesisTime = time.Unix(int64(beaconState.GenesisTime), 0)
+		s.genesisTime = time.Unix(int64(beaconState.GenesisTime()), 0)
+		s.opsService.SetGenesisTime(beaconState.GenesisTime())
 		if err := s.initializeChainInfo(ctx); err != nil {
 			log.Fatalf("Could not set up chain info: %v", err)
 		}
@@ -128,13 +175,28 @@ func (s *Service) Start() {
 		if err != nil {
 			log.Fatalf("Could not get finalized checkpoint: %v", err)
 		}
-		if err := s.forkChoiceStore.GenesisStore(ctx, justifiedCheckpoint, finalizedCheckpoint); err != nil {
-			log.Fatalf("Could not start fork choice service: %v", err)
+
+		// Resume fork choice.
+		s.justifiedCheckpt = stateTrie.CopyCheckpoint(justifiedCheckpoint)
+		s.prevJustifiedCheckpt = stateTrie.CopyCheckpoint(justifiedCheckpoint)
+		s.bestJustifiedCheckpt = stateTrie.CopyCheckpoint(justifiedCheckpoint)
+		s.finalizedCheckpt = stateTrie.CopyCheckpoint(finalizedCheckpoint)
+		s.prevFinalizedCheckpt = stateTrie.CopyCheckpoint(finalizedCheckpoint)
+		s.resumeForkChoice(justifiedCheckpoint, finalizedCheckpoint)
+
+		if !featureconfig.Get().NewStateMgmt {
+			if finalizedCheckpoint.Epoch > 1 {
+				if err := s.pruneGarbageState(ctx, helpers.StartSlot(finalizedCheckpoint.Epoch)-params.BeaconConfig().SlotsPerEpoch); err != nil {
+					log.WithError(err).Warn("Could not prune old states")
+				}
+			}
 		}
+
 		s.stateNotifier.StateFeed().Send(&feed.Event{
 			Type: statefeed.Initialized,
 			Data: &statefeed.InitializedData{
-				StartTime: s.genesisTime,
+				StartTime:             s.genesisTime,
+				GenesisValidatorsRoot: beaconState.GenesisValidatorRoot(),
 			},
 		})
 	} else {
@@ -147,11 +209,16 @@ func (s *Service) Start() {
 			stateChannel := make(chan *feed.Event, 1)
 			stateSub := s.stateNotifier.StateFeed().Subscribe(stateChannel)
 			defer stateSub.Unsubscribe()
+			<-attestationProcessorSubscribed
 			for {
 				select {
 				case event := <-stateChannel:
 					if event.Type == statefeed.ChainStarted {
-						data := event.Data.(*statefeed.ChainStartedData)
+						data, ok := event.Data.(*statefeed.ChainStartedData)
+						if !ok {
+							log.Error("event data is not type *statefeed.ChainStartedData")
+							return
+						}
 						log.WithField("starttime", data.StartTime).Debug("Received chain start event")
 						s.processChainStartTime(ctx, data.StartTime)
 						return
@@ -167,20 +234,22 @@ func (s *Service) Start() {
 		}()
 	}
 
-	go s.processAttestation()
+	go s.processAttestation(attestationProcessorSubscribed)
 }
 
 // processChainStartTime initializes a series of deposits from the ChainStart deposits in the eth1
 // deposit contract, initializes the beacon chain's state, and kicks off the beacon chain.
 func (s *Service) processChainStartTime(ctx context.Context, genesisTime time.Time) {
 	preGenesisState := s.chainStartFetcher.PreGenesisState()
-	if err := s.initializeBeaconChain(ctx, genesisTime, preGenesisState, s.chainStartFetcher.ChainStartEth1Data()); err != nil {
+	initializedState, err := s.initializeBeaconChain(ctx, genesisTime, preGenesisState, s.chainStartFetcher.ChainStartEth1Data())
+	if err != nil {
 		log.Fatalf("Could not initialize beacon chain: %v", err)
 	}
 	s.stateNotifier.StateFeed().Send(&feed.Event{
 		Type: statefeed.Initialized,
 		Data: &statefeed.InitializedData{
-			StartTime: genesisTime,
+			StartTime:             genesisTime,
+			GenesisValidatorsRoot: initializedState.GenesisValidatorRoot(),
 		},
 	})
 }
@@ -191,31 +260,38 @@ func (s *Service) processChainStartTime(ctx context.Context, genesisTime time.Ti
 func (s *Service) initializeBeaconChain(
 	ctx context.Context,
 	genesisTime time.Time,
-	preGenesisState *pb.BeaconState,
-	eth1data *ethpb.Eth1Data) error {
+	preGenesisState *stateTrie.BeaconState,
+	eth1data *ethpb.Eth1Data) (*stateTrie.BeaconState, error) {
 	_, span := trace.StartSpan(context.Background(), "beacon-chain.Service.initializeBeaconChain")
 	defer span.End()
-	log.Info("Genesis time reached, starting the beacon chain")
 	s.genesisTime = genesisTime
 	unixTime := uint64(genesisTime.Unix())
 
 	genesisState, err := state.OptimizedGenesisBeaconState(unixTime, preGenesisState, eth1data)
 	if err != nil {
-		return errors.Wrap(err, "could not initialize genesis state")
+		return nil, errors.Wrap(err, "could not initialize genesis state")
 	}
 
 	if err := s.saveGenesisData(ctx, genesisState); err != nil {
-		return errors.Wrap(err, "could not save genesis data")
+		return nil, errors.Wrap(err, "could not save genesis data")
 	}
 
+	log.Info("Initialized beacon chain genesis state")
+
+	// Clear out all pre-genesis data now that the state is initialized.
+	s.chainStartFetcher.ClearPreGenesisData()
+
 	// Update committee shuffled indices for genesis epoch.
-	if featureconfig.Get().EnableNewCache {
-		if err := helpers.UpdateCommitteeCache(genesisState, 0 /* genesis epoch */); err != nil {
-			return err
-		}
+	if err := helpers.UpdateCommitteeCache(genesisState, 0 /* genesis epoch */); err != nil {
+		return nil, err
+	}
+	if err := helpers.UpdateProposerIndicesInCache(genesisState, 0 /* genesis epoch */); err != nil {
+		return nil, err
 	}
 
-	return nil
+	s.opsService.SetGenesisTime(genesisState.GenesisTime())
+
+	return genesisState, nil
 }
 
 // Stop the blockchain service's main event loop and associated goroutines.
@@ -233,84 +309,20 @@ func (s *Service) Status() error {
 	return nil
 }
 
-// This gets called to update canonical root mapping.
-func (s *Service) saveHead(ctx context.Context, signed *ethpb.SignedBeaconBlock, r [32]byte) error {
-	s.headLock.Lock()
-	defer s.headLock.Unlock()
-
-	if signed == nil || signed.Block == nil {
-		return errors.New("cannot save nil head block")
-	}
-
-	s.headSlot = signed.Block.Slot
-
-	s.canonicalRoots[signed.Block.Slot] = r[:]
-
-	if err := s.beaconDB.SaveHeadBlockRoot(ctx, r); err != nil {
-		return errors.Wrap(err, "could not save head root in DB")
-	}
-	s.headBlock = signed
-
-	headState, err := s.beaconDB.State(ctx, r)
-	if err != nil {
-		return errors.Wrap(err, "could not retrieve head state in DB")
-	}
-	s.headState = headState
-
-	log.WithFields(logrus.Fields{
-		"slot":     signed.Block.Slot,
-		"headRoot": fmt.Sprintf("%#x", r),
-	}).Debug("Saved new head info")
-	return nil
+// ClearCachedStates removes all stored caches states. This is done after the node
+// is synced.
+func (s *Service) ClearCachedStates() {
+	s.initSyncState = map[[32]byte]*stateTrie.BeaconState{}
 }
 
-// This gets called to update canonical root mapping. It does not save head block
-// root in DB. With the inception of inital-sync-cache-state flag, it uses finalized
-// check point as anchors to resume sync therefore head is no longer needed to be saved on per slot basis.
-func (s *Service) saveHeadNoDB(ctx context.Context, b *ethpb.SignedBeaconBlock, r [32]byte) error {
-	s.headLock.Lock()
-	defer s.headLock.Unlock()
-
-	s.headSlot = b.Block.Slot
-
-	s.canonicalRoots[b.Block.Slot] = r[:]
-
-	s.headBlock = b
-
-	headState, err := s.beaconDB.State(ctx, r)
+// This gets called when beacon chain is first initialized to save genesis data (state, block, and more) in db.
+func (s *Service) saveGenesisData(ctx context.Context, genesisState *stateTrie.BeaconState) error {
+	stateRoot, err := genesisState.HashTreeRoot(ctx)
 	if err != nil {
-		return errors.Wrap(err, "could not retrieve head state in DB")
-	}
-	s.headState = headState
-
-	log.WithFields(logrus.Fields{
-		"slot":     b.Block.Slot,
-		"headRoot": fmt.Sprintf("%#x", r),
-	}).Debug("Saved new head info")
-	return nil
-}
-
-// This gets called when beacon chain is first initialized to save validator indices and pubkeys in db
-func (s *Service) saveGenesisValidators(ctx context.Context, state *pb.BeaconState) error {
-	for i, v := range state.Validators {
-		if err := s.beaconDB.SaveValidatorIndex(ctx, bytesutil.ToBytes48(v.PublicKey), uint64(i)); err != nil {
-			return errors.Wrapf(err, "could not save validator index: %d", i)
-		}
-	}
-	return nil
-}
-
-// This gets called when beacon chain is first initialized to save genesis data (state, block, and more) in db
-func (s *Service) saveGenesisData(ctx context.Context, genesisState *pb.BeaconState) error {
-	s.headLock.Lock()
-	defer s.headLock.Unlock()
-
-	stateRoot, err := ssz.HashTreeRoot(genesisState)
-	if err != nil {
-		return errors.Wrap(err, "could not tree hash genesis state")
+		return err
 	}
 	genesisBlk := blocks.NewGenesisBlock(stateRoot[:])
-	genesisBlkRoot, err := ssz.HashTreeRoot(genesisBlk.Block)
+	genesisBlkRoot, err := stateutil.BlockRoot(genesisBlk.Block)
 	if err != nil {
 		return errors.Wrap(err, "could not get genesis block root")
 	}
@@ -318,8 +330,20 @@ func (s *Service) saveGenesisData(ctx context.Context, genesisState *pb.BeaconSt
 	if err := s.beaconDB.SaveBlock(ctx, genesisBlk); err != nil {
 		return errors.Wrap(err, "could not save genesis block")
 	}
-	if err := s.beaconDB.SaveState(ctx, genesisState, genesisBlkRoot); err != nil {
-		return errors.Wrap(err, "could not save genesis state")
+	if featureconfig.Get().NewStateMgmt {
+		if err := s.stateGen.SaveState(ctx, genesisBlkRoot, genesisState); err != nil {
+			return errors.Wrap(err, "could not save genesis state")
+		}
+		if err := s.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{
+			Slot: 0,
+			Root: genesisBlkRoot[:],
+		}); err != nil {
+			return err
+		}
+	} else {
+		if err := s.beaconDB.SaveState(ctx, genesisState, genesisBlkRoot); err != nil {
+			return errors.Wrap(err, "could not save genesis state")
+		}
 	}
 	if err := s.beaconDB.SaveHeadBlockRoot(ctx, genesisBlkRoot); err != nil {
 		return errors.Wrap(err, "could not save head block root")
@@ -327,28 +351,33 @@ func (s *Service) saveGenesisData(ctx context.Context, genesisState *pb.BeaconSt
 	if err := s.beaconDB.SaveGenesisBlockRoot(ctx, genesisBlkRoot); err != nil {
 		return errors.Wrap(err, "could save genesis block root")
 	}
-	if err := s.saveGenesisValidators(ctx, genesisState); err != nil {
-		return errors.Wrap(err, "could not save genesis validators")
-	}
 
 	genesisCheckpoint := &ethpb.Checkpoint{Root: genesisBlkRoot[:]}
-	if err := s.forkChoiceStore.GenesisStore(ctx, genesisCheckpoint, genesisCheckpoint); err != nil {
-		return errors.Wrap(err, "Could not start fork choice service: %v")
+
+	// Add the genesis block to the fork choice store.
+	s.justifiedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
+	s.prevJustifiedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
+	s.bestJustifiedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
+	s.finalizedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
+	s.prevFinalizedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
+
+	if err := s.forkChoiceStore.ProcessBlock(ctx,
+		genesisBlk.Block.Slot,
+		genesisBlkRoot,
+		params.BeaconConfig().ZeroHash,
+		[32]byte{},
+		genesisCheckpoint.Epoch,
+		genesisCheckpoint.Epoch); err != nil {
+		log.Fatalf("Could not process genesis block for fork choice: %v", err)
 	}
 
-	s.genesisRoot = genesisBlkRoot
-	s.headBlock = genesisBlk
-	s.headState = genesisState
-	s.canonicalRoots[genesisState.Slot] = genesisBlkRoot[:]
+	s.setHead(genesisBlkRoot, genesisBlk, genesisState)
 
 	return nil
 }
 
 // This gets called to initialize chain info variables using the finalized checkpoint stored in DB
 func (s *Service) initializeChainInfo(ctx context.Context) error {
-	s.headLock.Lock()
-	defer s.headLock.Unlock()
-
 	genesisBlock, err := s.beaconDB.GenesisBlock(ctx)
 	if err != nil {
 		return errors.Wrap(err, "could not get genesis block from db")
@@ -356,12 +385,29 @@ func (s *Service) initializeChainInfo(ctx context.Context) error {
 	if genesisBlock == nil {
 		return errors.New("no genesis block in db")
 	}
-	genesisBlkRoot, err := ssz.HashTreeRoot(genesisBlock.Block)
+	genesisBlkRoot, err := stateutil.BlockRoot(genesisBlock.Block)
 	if err != nil {
 		return errors.Wrap(err, "could not get signing root of genesis block")
 	}
 	s.genesisRoot = genesisBlkRoot
 
+	if flags.Get().UnsafeSync {
+		headBlock, err := s.beaconDB.HeadBlock(ctx)
+		if err != nil {
+			return errors.Wrap(err, "could not retrieve head block")
+		}
+		headRoot, err := stateutil.BlockRoot(headBlock.Block)
+		if err != nil {
+			return errors.Wrap(err, "could not hash head block")
+		}
+		headState, err := s.beaconDB.HeadState(ctx)
+		if err != nil {
+			return errors.Wrap(err, "could not retrieve head state")
+		}
+		s.setHead(headRoot, headBlock, headState)
+		return nil
+	}
+
 	finalized, err := s.beaconDB.FinalizedCheckpoint(ctx)
 	if err != nil {
 		return errors.Wrap(err, "could not get finalized checkpoint from db")
@@ -371,19 +417,90 @@ func (s *Service) initializeChainInfo(ctx context.Context) error {
 		// would be the genesis state and block.
 		return errors.New("no finalized epoch in the database")
 	}
-	s.headState, err = s.beaconDB.State(ctx, bytesutil.ToBytes32(finalized.Root))
-	if err != nil {
-		return errors.Wrap(err, "could not get finalized state from db")
+	finalizedRoot := bytesutil.ToBytes32(finalized.Root)
+	var finalizedState *stateTrie.BeaconState
+
+	if featureconfig.Get().NewStateMgmt {
+		finalizedState, err = s.stateGen.Resume(ctx)
+		if err != nil {
+			return errors.Wrap(err, "could not get finalized state from db")
+		}
+		if !featureconfig.Get().SkipRegenHistoricalStates {
+			// Since historical states were skipped, the node should start from last finalized check point.
+			finalizedRoot = s.beaconDB.LastArchivedIndexRoot(ctx)
+			if finalizedRoot == params.BeaconConfig().ZeroHash {
+				finalizedRoot = bytesutil.ToBytes32(finalized.Root)
+			}
+		}
+	} else {
+		finalizedState, err = s.beaconDB.State(ctx, bytesutil.ToBytes32(finalized.Root))
+		if err != nil {
+			return errors.Wrap(err, "could not get finalized state from db")
+		}
 	}
-	s.headBlock, err = s.beaconDB.Block(ctx, bytesutil.ToBytes32(finalized.Root))
+	finalizedBlock, err := s.beaconDB.Block(ctx, finalizedRoot)
 	if err != nil {
 		return errors.Wrap(err, "could not get finalized block from db")
 	}
 
-	if s.headBlock != nil && s.headBlock.Block != nil {
-		s.headSlot = s.headBlock.Block.Slot
+	if featureconfig.Get().NewStateMgmt && featureconfig.Get().SkipRegenHistoricalStates {
+		// To skip the regeneration of historical state, the node has to generate the parent of the last finalized state.
+		parentRoot := bytesutil.ToBytes32(finalizedBlock.Block.ParentRoot)
+		parentState, err := s.generateState(ctx, finalizedRoot, parentRoot)
+		if err != nil {
+			return err
+		}
+		if s.beaconDB.SaveState(ctx, parentState, parentRoot) != nil {
+			return err
+		}
 	}
-	s.canonicalRoots[s.headSlot] = finalized.Root
+
+	if finalizedState == nil || finalizedBlock == nil {
+		return errors.New("finalized state and block can't be nil")
+	}
+	s.setHead(finalizedRoot, finalizedBlock, finalizedState)
 
 	return nil
 }
+
+// This is called when a client starts from a non-genesis slot. It deletes the states in DB
+// from slot 1 (avoid genesis state) to `slot`.
+func (s *Service) pruneGarbageState(ctx context.Context, slot uint64) error {
+	if featureconfig.Get().DontPruneStateStartUp {
+		return nil
+	}
+
+	filter := filters.NewFilter().SetStartSlot(1).SetEndSlot(slot)
+	roots, err := s.beaconDB.BlockRoots(ctx, filter)
+	if err != nil {
+		return err
+	}
+	if err := s.beaconDB.DeleteStates(ctx, roots); err != nil {
+		return err
+	}
+
+	if err := s.beaconDB.SaveLastArchivedIndex(ctx, 0); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// This is called when a client starts from non-genesis slot. This passes last justified and finalized
+// information to fork choice service to initializes fork choice store.
+func (s *Service) resumeForkChoice(justifiedCheckpoint *ethpb.Checkpoint, finalizedCheckpoint *ethpb.Checkpoint) {
+	store := protoarray.New(justifiedCheckpoint.Epoch, finalizedCheckpoint.Epoch, bytesutil.ToBytes32(finalizedCheckpoint.Root))
+	s.forkChoiceStore = store
+}
+
+// This returns true if block has been processed before. Two ways to verify the block has been processed:
+// 1.) Check fork choice store.
+// 2.) Check DB.
+// Checking 1.) is ten times faster than checking 2.)
+func (s *Service) hasBlock(ctx context.Context, root [32]byte) bool {
+	if s.forkChoiceStore.HasNode(root) {
+		return true
+	}
+
+	return s.beaconDB.HasBlock(ctx, root)
+}

beacon-chain/blockchain/service_norace_test.go

@@ -5,7 +5,6 @@ import (
 	"io/ioutil"
 	"testing"
 
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/sirupsen/logrus"
 )
@@ -17,21 +16,15 @@ func init() {
 
 func TestChainService_SaveHead_DataRace(t *testing.T) {
 	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
 	s := &Service{
-		beaconDB:       db,
-		canonicalRoots: make(map[uint64][]byte),
+		beaconDB: db,
 	}
 	go func() {
-		s.saveHead(
-			context.Background(),
-			&ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 777}},
-			[32]byte{},
-		)
+		if err := s.saveHead(context.Background(), [32]byte{}); err != nil {
+			t.Fatal(err)
+		}
 	}()
-	s.saveHead(
-		context.Background(),
-		&ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 888}},
-		[32]byte{},
-	)
+	if err := s.saveHead(context.Background(), [32]byte{}); err != nil {
+		t.Fatal(err)
+	}
 }

beacon-chain/blockchain/service_test.go

@@ -3,7 +3,6 @@ package blockchain
 import (
 	"bytes"
 	"context"
-	"encoding/hex"
 	"io/ioutil"
 	"reflect"
 	"testing"
@@ -13,6 +12,7 @@ import (
 	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	ssz "github.com/prysmaticlabs/go-ssz"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache"
 	b "github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
@@ -20,12 +20,17 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/attestations"
 	"github.com/prysmaticlabs/prysm/beacon-chain/p2p"
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
+	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
+	protodb "github.com/prysmaticlabs/prysm/proto/beacon/db"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/event"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
@@ -42,16 +47,20 @@ type store struct {
 	headRoot []byte
 }
 
-func (s *store) OnBlock(ctx context.Context, b *ethpb.SignedBeaconBlock) error {
-	return nil
+func (s *store) OnBlock(ctx context.Context, b *ethpb.SignedBeaconBlock) (*beaconstate.BeaconState, error) {
+	return nil, nil
 }
 
-func (s *store) OnBlockInitialSyncStateTransition(ctx context.Context, b *ethpb.SignedBeaconBlock) error {
-	return nil
+func (s *store) OnBlockCacheFilteredTree(ctx context.Context, b *ethpb.SignedBeaconBlock) (*beaconstate.BeaconState, error) {
+	return nil, nil
 }
 
-func (s *store) OnAttestation(ctx context.Context, a *ethpb.Attestation) error {
-	return nil
+func (s *store) OnBlockInitialSyncStateTransition(ctx context.Context, b *ethpb.SignedBeaconBlock) (*beaconstate.BeaconState, error) {
+	return nil, nil
+}
+
+func (s *store) OnAttestation(ctx context.Context, a *ethpb.Attestation) ([]uint64, error) {
+	return nil, nil
 }
 
 func (s *store) GenesisStore(ctx context.Context, justifiedCheckpoint *ethpb.Checkpoint, finalizedCheckpoint *ethpb.Checkpoint) error {
@@ -62,6 +71,10 @@ func (s *store) FinalizedCheckpt() *ethpb.Checkpoint {
 	return nil
 }
 
+func (s *store) JustifiedCheckpt() *ethpb.Checkpoint {
+	return nil
+}
+
 func (s *store) Head(ctx context.Context) ([]byte, error) {
 	return s.headRoot, nil
 }
@@ -94,6 +107,25 @@ func setupBeaconChain(t *testing.T, beaconDB db.Database) *Service {
 	ctx := context.Background()
 	var web3Service *powchain.Service
 	var err error
+	bState, _ := testutil.DeterministicGenesisState(t, 10)
+	err = beaconDB.SavePowchainData(ctx, &protodb.ETH1ChainData{
+		BeaconState: bState.InnerStateUnsafe(),
+		Trie:        &protodb.SparseMerkleTrie{},
+		CurrentEth1Data: &protodb.LatestETH1Data{
+			BlockHash: make([]byte, 32),
+		},
+		ChainstartData: &protodb.ChainStartData{
+			Eth1Data: &ethpb.Eth1Data{
+				DepositRoot:  make([]byte, 32),
+				DepositCount: 0,
+				BlockHash:    make([]byte, 32),
+			},
+		},
+		DepositContainers: []*protodb.DepositContainer{},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
 	web3Service, err = powchain.NewService(ctx, &powchain.Web3ServiceConfig{
 		BeaconDB:        beaconDB,
 		ETH1Endpoint:    endpoint,
@@ -103,6 +135,10 @@ func setupBeaconChain(t *testing.T, beaconDB db.Database) *Service {
 		t.Fatalf("unable to set up web3 service: %v", err)
 	}
 
+	opsService, err := attestations.NewService(ctx, &attestations.Config{Pool: attestations.NewPool()})
+	if err != nil {
+		t.Fatal(err)
+	}
 	cfg := &Config{
 		BeaconBlockBuf:    0,
 		BeaconDB:          beaconDB,
@@ -111,10 +147,11 @@ func setupBeaconChain(t *testing.T, beaconDB db.Database) *Service {
 		P2p:               &mockBroadcaster{},
 		StateNotifier:     &mockBeaconNode{},
 		AttPool:           attestations.NewPool(),
+		StateGen:          stategen.New(beaconDB, cache.NewStateSummaryCache()),
+		ForkChoiceStore:   protoarray.New(0, 0, params.BeaconConfig().ZeroHash),
+		OpsService:        opsService,
 	}
-	if err != nil {
-		t.Fatalf("could not register blockchain service: %v", err)
-	}
+
 	chainService, err := NewService(ctx, cfg)
 	if err != nil {
 		t.Fatalf("unable to setup chain service: %v", err)
@@ -127,7 +164,6 @@ func setupBeaconChain(t *testing.T, beaconDB db.Database) *Service {
 func TestChainStartStop_Uninitialized(t *testing.T) {
 	hook := logTest.NewGlobal()
 	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
 	chainService := setupBeaconChain(t, db)
 
 	// Listen for state events.
@@ -165,7 +201,7 @@ func TestChainStartStop_Uninitialized(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if beaconState == nil || beaconState.Slot != 0 {
+	if beaconState == nil || beaconState.Slot() != 0 {
 		t.Error("Expected canonical state feed to send a state with genesis block")
 	}
 	if err := chainService.Stop(); err != nil {
@@ -176,26 +212,29 @@ func TestChainStartStop_Uninitialized(t *testing.T) {
 		t.Error("Context was not canceled")
 	}
 	testutil.AssertLogsContain(t, hook, "Waiting")
-	testutil.AssertLogsContain(t, hook, "Genesis time reached")
+	testutil.AssertLogsContain(t, hook, "Initialized beacon chain genesis state")
 }
 
 func TestChainStartStop_Initialized(t *testing.T) {
 	hook := logTest.NewGlobal()
 	ctx := context.Background()
 	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
 
 	chainService := setupBeaconChain(t, db)
 
 	genesisBlk := b.NewGenesisBlock([]byte{})
-	blkRoot, err := ssz.HashTreeRoot(genesisBlk.Block)
+	blkRoot, err := stateutil.BlockRoot(genesisBlk.Block)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if err := db.SaveBlock(ctx, genesisBlk); err != nil {
 		t.Fatal(err)
 	}
-	if err := db.SaveState(ctx, &pb.BeaconState{Slot: 1}, blkRoot); err != nil {
+	s := testutil.NewBeaconState()
+	if err := s.SetSlot(1); err != nil {
+		t.Fatal(err)
+	}
+	if err := db.SaveState(ctx, s, blkRoot); err != nil {
 		t.Fatal(err)
 	}
 	if err := db.SaveHeadBlockRoot(ctx, blkRoot); err != nil {
@@ -224,7 +263,6 @@ func TestChainStartStop_Initialized(t *testing.T) {
 
 func TestChainService_InitializeBeaconChain(t *testing.T) {
 	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
 	ctx := context.Background()
 
 	bc := setupBeaconChain(t, db)
@@ -232,56 +270,57 @@ func TestChainService_InitializeBeaconChain(t *testing.T) {
 
 	// Set up 10 deposits pre chain start for validators to register
 	count := uint64(10)
-	deposits, _, _ := testutil.DeterministicDepositsAndKeys(count)
+	deposits, _, err := testutil.DeterministicDepositsAndKeys(count)
+	if err != nil {
+		t.Fatal(err)
+	}
 	trie, _, err := testutil.DepositTrieFromDeposits(deposits)
 	if err != nil {
 		t.Fatal(err)
 	}
 	hashTreeRoot := trie.HashTreeRoot()
-	genState := state.EmptyGenesisState()
-	genState.Eth1Data = &ethpb.Eth1Data{
-		DepositRoot:  hashTreeRoot[:],
-		DepositCount: uint64(len(deposits)),
-	}
-	genState, err = b.ProcessDeposits(ctx, genState, &ethpb.BeaconBlockBody{Deposits: deposits})
+	genState, err := state.EmptyGenesisState()
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := bc.initializeBeaconChain(ctx, time.Unix(0, 0), genState, &ethpb.Eth1Data{
+	err = genState.SetEth1Data(&ethpb.Eth1Data{
+		DepositRoot:  hashTreeRoot[:],
+		DepositCount: uint64(len(deposits)),
+	})
+	for _, deposit := range deposits {
+		genState, err = b.ProcessPreGenesisDeposit(ctx, genState, deposit)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if _, err := bc.initializeBeaconChain(ctx, time.Unix(0, 0), genState, &ethpb.Eth1Data{
 		DepositRoot: hashTreeRoot[:],
 	}); err != nil {
 		t.Fatal(err)
 	}
 
-	s, err := bc.beaconDB.State(ctx, bytesutil.ToBytes32(bc.canonicalRoots[0]))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for _, v := range s.Validators {
-		if !db.HasValidatorIndex(ctx, bytesutil.ToBytes48(v.PublicKey)) {
-			t.Errorf("Validator %s missing from db", hex.EncodeToString(v.PublicKey))
-		}
-	}
-
 	if _, err := bc.HeadState(ctx); err != nil {
 		t.Error(err)
 	}
-	if bc.HeadBlock() == nil {
+	headBlk, err := bc.HeadBlock(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if headBlk == nil {
 		t.Error("Head state can't be nil after initialize beacon chain")
 	}
-	if bc.CanonicalRoot(0) == nil {
-		t.Error("Canonical root for slot 0 can't be nil after initialize beacon chain")
+	if bc.headRoot() == params.BeaconConfig().ZeroHash {
+		t.Error("Canonical root for slot 0 can't be zeros after initialize beacon chain")
 	}
 }
 
 func TestChainService_InitializeChainInfo(t *testing.T) {
 	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
 	ctx := context.Background()
 
 	genesis := b.NewGenesisBlock([]byte{})
-	genesisRoot, err := ssz.HashTreeRoot(genesis.Block)
+	genesisRoot, err := stateutil.BlockRoot(genesis.Block)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -294,11 +333,23 @@ func TestChainService_InitializeChainInfo(t *testing.T) {
 
 	finalizedSlot := params.BeaconConfig().SlotsPerEpoch*2 + 1
 	headBlock := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: finalizedSlot, ParentRoot: genesisRoot[:]}}
-	headState := &pb.BeaconState{Slot: finalizedSlot}
-	headRoot, _ := ssz.HashTreeRoot(headBlock.Block)
+	headState := testutil.NewBeaconState()
+	if err := headState.SetSlot(finalizedSlot); err != nil {
+		t.Fatal(err)
+	}
+	if err := headState.SetGenesisValidatorRoot(params.BeaconConfig().ZeroHash[:]); err != nil {
+		t.Fatal(err)
+	}
+	headRoot, err := stateutil.BlockRoot(headBlock.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
 	if err := db.SaveState(ctx, headState, headRoot); err != nil {
 		t.Fatal(err)
 	}
+	if err := db.SaveState(ctx, headState, genesisRoot); err != nil {
+		t.Fatal(err)
+	}
 	if err := db.SaveBlock(ctx, headBlock); err != nil {
 		t.Fatal(err)
 	}
@@ -311,24 +362,32 @@ func TestChainService_InitializeChainInfo(t *testing.T) {
 	if err := db.SaveBlock(ctx, headBlock); err != nil {
 		t.Fatal(err)
 	}
-	c := &Service{beaconDB: db, canonicalRoots: make(map[uint64][]byte)}
+	c := &Service{beaconDB: db, stateGen: stategen.New(db, cache.NewStateSummaryCache())}
 	if err := c.initializeChainInfo(ctx); err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(c.HeadBlock(), headBlock) {
+	headBlk, err := c.HeadBlock(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(headBlk, headBlock) {
 		t.Error("head block incorrect")
 	}
 	s, err := c.HeadState(ctx)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(s, headState) {
+	if !reflect.DeepEqual(s.InnerStateUnsafe(), headState.InnerStateUnsafe()) {
 		t.Error("head state incorrect")
 	}
 	if headBlock.Block.Slot != c.HeadSlot() {
 		t.Error("head slot incorrect")
 	}
-	if !bytes.Equal(headRoot[:], c.HeadRoot()) {
+	r, err := c.HeadRoot(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(headRoot[:], r) {
 		t.Error("head slot incorrect")
 	}
 	if c.genesisRoot != genesisRoot {
@@ -338,14 +397,21 @@ func TestChainService_InitializeChainInfo(t *testing.T) {
 
 func TestChainService_SaveHeadNoDB(t *testing.T) {
 	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
 	ctx := context.Background()
 	s := &Service{
-		beaconDB:       db,
-		canonicalRoots: make(map[uint64][]byte),
+		beaconDB: db,
+		stateGen: stategen.New(db, cache.NewStateSummaryCache()),
 	}
 	b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1}}
-	r, _ := ssz.HashTreeRoot(b)
+	r, err := ssz.HashTreeRoot(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	newState := testutil.NewBeaconState()
+	if err := s.stateGen.SaveState(ctx, r, newState); err != nil {
+		t.Fatal(err)
+	}
+
 	if err := s.saveHeadNoDB(ctx, b, r); err != nil {
 		t.Fatal(err)
 	}
@@ -358,3 +424,134 @@ func TestChainService_SaveHeadNoDB(t *testing.T) {
 		t.Error("head block should not be equal")
 	}
 }
+
+func TestChainService_PruneOldStates(t *testing.T) {
+	db := testDB.SetupDB(t)
+	ctx := context.Background()
+	s := &Service{
+		beaconDB: db,
+	}
+
+	for i := 0; i < 100; i++ {
+		block := &ethpb.BeaconBlock{Slot: uint64(i)}
+		if err := s.beaconDB.SaveBlock(ctx, &ethpb.SignedBeaconBlock{Block: block}); err != nil {
+			t.Fatal(err)
+		}
+		r, err := stateutil.BlockRoot(block)
+		if err != nil {
+			t.Fatal(err)
+		}
+		newState := testutil.NewBeaconState()
+		if err := newState.SetSlot(uint64(i)); err != nil {
+			t.Fatal(err)
+		}
+		if err := s.beaconDB.SaveState(ctx, newState, r); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// Delete half of the states.
+	if err := s.pruneGarbageState(ctx, 50); err != nil {
+		t.Fatal(err)
+	}
+
+	filter := filters.NewFilter().SetStartSlot(1).SetEndSlot(100)
+	roots, err := s.beaconDB.BlockRoots(ctx, filter)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := 1; i < 50; i++ {
+		s, err := s.beaconDB.State(ctx, roots[i])
+		if err != nil {
+			t.Fatal(err)
+		}
+		if s != nil {
+			t.Errorf("wanted nil for slot %d", i)
+		}
+	}
+}
+
+func TestHasBlock_ForkChoiceAndDB(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	s := &Service{
+		forkChoiceStore:  protoarray.New(0, 0, [32]byte{}),
+		finalizedCheckpt: &ethpb.Checkpoint{},
+		beaconDB:         db,
+	}
+	block := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Body: &ethpb.BeaconBlockBody{}}}
+	r, err := stateutil.BlockRoot(block.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+	bs := &pb.BeaconState{FinalizedCheckpoint: &ethpb.Checkpoint{}, CurrentJustifiedCheckpoint: &ethpb.Checkpoint{}}
+	state, err := beaconstate.InitializeFromProto(bs)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := s.insertBlockToForkChoiceStore(ctx, block.Block, r, state); err != nil {
+		t.Fatal(err)
+	}
+
+	if s.hasBlock(ctx, [32]byte{}) {
+		t.Error("Should not have block")
+	}
+
+	if !s.hasBlock(ctx, r) {
+		t.Error("Should have block")
+	}
+}
+
+func BenchmarkHasBlockDB(b *testing.B) {
+	db := testDB.SetupDB(b)
+	ctx := context.Background()
+	s := &Service{
+		beaconDB: db,
+	}
+	block := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{}}
+	if err := s.beaconDB.SaveBlock(ctx, block); err != nil {
+		b.Fatal(err)
+	}
+	r, err := stateutil.BlockRoot(block.Block)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if !s.beaconDB.HasBlock(ctx, r) {
+			b.Fatal("Block is not in DB")
+		}
+	}
+}
+
+func BenchmarkHasBlockForkChoiceStore(b *testing.B) {
+	ctx := context.Background()
+	db := testDB.SetupDB(b)
+	s := &Service{
+		forkChoiceStore:  protoarray.New(0, 0, [32]byte{}),
+		finalizedCheckpt: &ethpb.Checkpoint{},
+		beaconDB:         db,
+	}
+	block := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Body: &ethpb.BeaconBlockBody{}}}
+	r, err := stateutil.BlockRoot(block.Block)
+	if err != nil {
+		b.Fatal(err)
+	}
+	bs := &pb.BeaconState{FinalizedCheckpoint: &ethpb.Checkpoint{}, CurrentJustifiedCheckpoint: &ethpb.Checkpoint{}}
+	state, err := beaconstate.InitializeFromProto(bs)
+	if err != nil {
+		b.Fatal(err)
+	}
+	if err := s.insertBlockToForkChoiceStore(ctx, block.Block, r, state); err != nil {
+		b.Fatal(err)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if !s.forkChoiceStore.HasNode(r) {
+			b.Fatal("Block is not in fork choice store")
+		}
+	}
+}

beacon-chain/blockchain/testing/BUILD.bazel

@@ -1,23 +1,29 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@prysm//tools/go:def.bzl", "go_library")
 
 go_library(
     name = "go_default_library",
     testonly = True,
     srcs = ["mock.go"],
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing",
-    visibility = ["//beacon-chain:__subpackages__"],
+    visibility = [
+        "//beacon-chain:__subpackages__",
+        "//fuzz:__pkg__",
+    ],
     deps = [
         "//beacon-chain/core/epoch/precompute:go_default_library",
+        "//beacon-chain/core/feed/block:go_default_library",
         "//beacon-chain/core/feed/operation:go_default_library",
         "//beacon-chain/core/feed/state:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/db:go_default_library",
+        "//beacon-chain/forkchoice/protoarray:go_default_library",
+        "//beacon-chain/state:go_default_library",
+        "//beacon-chain/state/stateutil:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/event:go_default_library",
         "//shared/params:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
-        "@com_github_prysmaticlabs_go_ssz//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
     ],
 )

beacon-chain/blockchain/testing/mock.go

@@ -1,3 +1,5 @@
+// Package testing includes useful mocks for writing unit
+// tests which depend on logic from the blockchain package.
 package testing
 
 import (
@@ -7,12 +9,15 @@ import (
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
+	blockfeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/block"
 	opfeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/operation"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/event"
 	"github.com/prysmaticlabs/prysm/shared/params"
@@ -21,7 +26,7 @@ import (
 
 // ChainService defines the mock interface for testing
 type ChainService struct {
-	State                       *pb.BeaconState
+	State                       *stateTrie.BeaconState
 	Root                        []byte
 	Block                       *ethpb.SignedBeaconBlock
 	FinalizedCheckPoint         *ethpb.Checkpoint
@@ -30,10 +35,14 @@ type ChainService struct {
 	BlocksReceived              []*ethpb.SignedBeaconBlock
 	Balance                     *precompute.Balance
 	Genesis                     time.Time
+	ValidatorsRoot              [32]byte
 	Fork                        *pb.Fork
 	DB                          db.Database
 	stateNotifier               statefeed.Notifier
+	blockNotifier               blockfeed.Notifier
 	opNotifier                  opfeed.Notifier
+	ValidAttestation            bool
+	ForkChoiceStore             *protoarray.Store
 }
 
 // StateNotifier mocks the same method in the chain service.
@@ -44,6 +53,27 @@ func (ms *ChainService) StateNotifier() statefeed.Notifier {
 	return ms.stateNotifier
 }
 
+// BlockNotifier mocks the same method in the chain service.
+func (ms *ChainService) BlockNotifier() blockfeed.Notifier {
+	if ms.blockNotifier == nil {
+		ms.blockNotifier = &MockBlockNotifier{}
+	}
+	return ms.blockNotifier
+}
+
+// MockBlockNotifier mocks the block notifier.
+type MockBlockNotifier struct {
+	feed *event.Feed
+}
+
+// BlockFeed returns a block feed.
+func (msn *MockBlockNotifier) BlockFeed() *event.Feed {
+	if msn.feed == nil {
+		msn.feed = new(event.Feed)
+	}
+	return msn.feed
+}
+
 // MockStateNotifier mocks the state notifier.
 type MockStateNotifier struct {
 	feed *event.Feed
@@ -79,31 +109,33 @@ func (mon *MockOperationNotifier) OperationFeed() *event.Feed {
 }
 
 // ReceiveBlock mocks ReceiveBlock method in chain service.
-func (ms *ChainService) ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
+func (ms *ChainService) ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
 	return nil
 }
 
 // ReceiveBlockNoVerify mocks ReceiveBlockNoVerify method in chain service.
-func (ms *ChainService) ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
+func (ms *ChainService) ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
 	return nil
 }
 
 // ReceiveBlockNoPubsub mocks ReceiveBlockNoPubsub method in chain service.
-func (ms *ChainService) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
+func (ms *ChainService) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
 	return nil
 }
 
 // ReceiveBlockNoPubsubForkchoice mocks ReceiveBlockNoPubsubForkchoice method in chain service.
-func (ms *ChainService) ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
+func (ms *ChainService) ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
 	if ms.State == nil {
-		ms.State = &pb.BeaconState{}
+		ms.State = &stateTrie.BeaconState{}
 	}
 	if !bytes.Equal(ms.Root, block.Block.ParentRoot) {
 		return errors.Errorf("wanted %#x but got %#x", ms.Root, block.Block.ParentRoot)
 	}
-	ms.State.Slot = block.Block.Slot
+	if err := ms.State.SetSlot(block.Block.Slot); err != nil {
+		return err
+	}
 	ms.BlocksReceived = append(ms.BlocksReceived, block)
-	signingRoot, err := ssz.HashTreeRoot(block.Block)
+	signingRoot, err := stateutil.BlockRoot(block.Block)
 	if err != nil {
 		return err
 	}
@@ -123,23 +155,22 @@ func (ms *ChainService) HeadSlot() uint64 {
 	if ms.State == nil {
 		return 0
 	}
-	return ms.State.Slot
-
+	return ms.State.Slot()
 }
 
 // HeadRoot mocks HeadRoot method in chain service.
-func (ms *ChainService) HeadRoot() []byte {
-	return ms.Root
+func (ms *ChainService) HeadRoot(ctx context.Context) ([]byte, error) {
+	return ms.Root, nil
 
 }
 
 // HeadBlock mocks HeadBlock method in chain service.
-func (ms *ChainService) HeadBlock() *ethpb.SignedBeaconBlock {
-	return ms.Block
+func (ms *ChainService) HeadBlock(context.Context) (*ethpb.SignedBeaconBlock, error) {
+	return ms.Block, nil
 }
 
 // HeadState mocks HeadState method in chain service.
-func (ms *ChainService) HeadState(context.Context) (*pb.BeaconState, error) {
+func (ms *ChainService) HeadState(context.Context) (*stateTrie.BeaconState, error) {
 	return ms.State, nil
 }
 
@@ -173,6 +204,11 @@ func (ms *ChainService) ReceiveAttestationNoPubsub(context.Context, *ethpb.Attes
 	return nil
 }
 
+// AttestationPreState mocks AttestationPreState method in chain service.
+func (ms *ChainService) AttestationPreState(ctx context.Context, att *ethpb.Attestation) (*stateTrie.BeaconState, error) {
+	return ms.State, nil
+}
+
 // HeadValidatorsIndices mocks the same method in the chain service.
 func (ms *ChainService) HeadValidatorsIndices(epoch uint64) ([]uint64, error) {
 	if ms.State == nil {
@@ -186,12 +222,45 @@ func (ms *ChainService) HeadSeed(epoch uint64) ([32]byte, error) {
 	return helpers.Seed(ms.State, epoch, params.BeaconConfig().DomainBeaconAttester)
 }
 
+// ProtoArrayStore mocks the same method in the chain service.
+func (ms *ChainService) ProtoArrayStore() *protoarray.Store {
+	return ms.ForkChoiceStore
+}
+
 // GenesisTime mocks the same method in the chain service.
 func (ms *ChainService) GenesisTime() time.Time {
 	return ms.Genesis
 }
 
+// GenesisValidatorRoot mocks the same method in the chain service.
+func (ms *ChainService) GenesisValidatorRoot() [32]byte {
+	return ms.ValidatorsRoot
+}
+
+// CurrentSlot mocks the same method in the chain service.
+func (ms *ChainService) CurrentSlot() uint64 {
+	return uint64(time.Now().Unix()-ms.Genesis.Unix()) / params.BeaconConfig().SecondsPerSlot
+}
+
 // Participation mocks the same method in the chain service.
 func (ms *ChainService) Participation(epoch uint64) *precompute.Balance {
 	return ms.Balance
 }
+
+// IsValidAttestation always returns true.
+func (ms *ChainService) IsValidAttestation(ctx context.Context, att *ethpb.Attestation) bool {
+	return ms.ValidAttestation
+}
+
+// ClearCachedStates does nothing.
+func (ms *ChainService) ClearCachedStates() {}
+
+// HasInitSyncBlock mocks the same method in the chain service.
+func (ms *ChainService) HasInitSyncBlock(root [32]byte) bool {
+	return false
+}
+
+// HeadGenesisValidatorRoot mocks HeadGenesisValidatorRoot method in chain service.
+func (ms *ChainService) HeadGenesisValidatorRoot() [32]byte {
+	return [32]byte{}
+}

beacon-chain/cache/BUILD.bazel

@@ -1,4 +1,5 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -6,22 +7,33 @@ go_library(
         "attestation_data.go",
         "checkpoint_state.go",
         "committee.go",
+        "committee_ids.go",
         "common.go",
-        "eth1_data.go",
+        "doc.go",
+        "hot_state_cache.go",
+        "skip_slot_cache.go",
+        "state_summary.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/cache",
-    visibility = ["//beacon-chain:__subpackages__"],
+    visibility = [
+        "//beacon-chain:__subpackages__",
+        "//fuzz:__pkg__",
+        "//tools:__subpackages__",
+    ],
     deps = [
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/featureconfig:go_default_library",
         "//shared/hashutil:go_default_library",
         "//shared/params:go_default_library",
         "//shared/sliceutil:go_default_library",
-        "@com_github_gogo_protobuf//proto:go_default_library",
+        "@com_github_hashicorp_golang_lru//:go_default_library",
+        "@com_github_patrickmn_go_cache//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
         "@io_k8s_client_go//tools/cache:go_default_library",
+        "@io_opencensus_go//trace:go_default_library",
     ],
 )
 
@@ -31,19 +43,23 @@ go_test(
     srcs = [
         "attestation_data_test.go",
         "checkpoint_state_test.go",
+        "committee_fuzz_test.go",
+        "committee_ids_test.go",
         "committee_test.go",
-        "eth1_data_test.go",
         "feature_flag_test.go",
+        "hot_state_cache_test.go",
+        "skip_slot_cache_test.go",
     ],
     embed = [":go_default_library"],
-    race = "on",
     deps = [
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/bytesutil:go_default_library",
         "//shared/featureconfig:go_default_library",
         "//shared/hashutil:go_default_library",
         "//shared/params:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
+        "@com_github_google_gofuzz//:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
     ],
 )

beacon-chain/cache/attestation_data.go

@@ -11,6 +11,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"k8s.io/client-go/tools/cache"
 )
@@ -59,12 +60,6 @@ func NewAttestationCache() *AttestationCache {
 // Get waits for any in progress calculation to complete before returning a
 // cached response, if any.
 func (c *AttestationCache) Get(ctx context.Context, req *ethpb.AttestationDataRequest) (*ethpb.AttestationData, error) {
-	if !featureconfig.Get().EnableAttestationCache {
-		// Return a miss result if cache is not enabled.
-		attestationCacheMiss.Inc()
-		return nil, nil
-	}
-
 	if req == nil {
 		return nil, errors.New("nil attestation data request")
 	}
@@ -104,6 +99,9 @@ func (c *AttestationCache) Get(ctx context.Context, req *ethpb.AttestationDataRe
 
 	if exists && item != nil && item.(*attestationReqResWrapper).res != nil {
 		attestationCacheHit.Inc()
+		if featureconfig.Get().ReduceAttesterStateCopy {
+			return state.CopyAttestationData(item.(*attestationReqResWrapper).res), nil
+		}
 		return item.(*attestationReqResWrapper).res, nil
 	}
 	attestationCacheMiss.Inc()
@@ -113,10 +111,6 @@ func (c *AttestationCache) Get(ctx context.Context, req *ethpb.AttestationDataRe
 // MarkInProgress a request so that any other similar requests will block on
 // Get until MarkNotInProgress is called.
 func (c *AttestationCache) MarkInProgress(req *ethpb.AttestationDataRequest) error {
-	if !featureconfig.Get().EnableAttestationCache {
-		return nil
-	}
-
 	c.lock.Lock()
 	defer c.lock.Unlock()
 	s, e := reqToKey(req)
@@ -126,19 +120,13 @@ func (c *AttestationCache) MarkInProgress(req *ethpb.AttestationDataRequest) err
 	if c.inProgress[s] {
 		return ErrAlreadyInProgress
 	}
-	if featureconfig.Get().EnableAttestationCache {
-		c.inProgress[s] = true
-	}
+	c.inProgress[s] = true
 	return nil
 }
 
 // MarkNotInProgress will release the lock on a given request. This should be
 // called after put.
 func (c *AttestationCache) MarkNotInProgress(req *ethpb.AttestationDataRequest) error {
-	if !featureconfig.Get().EnableAttestationCache {
-		return nil
-	}
-
 	c.lock.Lock()
 	defer c.lock.Unlock()
 	s, e := reqToKey(req)
@@ -151,10 +139,6 @@ func (c *AttestationCache) MarkNotInProgress(req *ethpb.AttestationDataRequest)
 
 // Put the response in the cache.
 func (c *AttestationCache) Put(ctx context.Context, req *ethpb.AttestationDataRequest, res *ethpb.AttestationData) error {
-	if !featureconfig.Get().EnableAttestationCache {
-		return nil
-	}
-
 	data := &attestationReqResWrapper{
 		req,
 		res,
@@ -169,7 +153,10 @@ func (c *AttestationCache) Put(ctx context.Context, req *ethpb.AttestationDataRe
 }
 
 func wrapperToKey(i interface{}) (string, error) {
-	w := i.(*attestationReqResWrapper)
+	w, ok := i.(*attestationReqResWrapper)
+	if !ok {
+		return "", errors.New("key is not of type *attestationReqResWrapper")
+	}
 	if w == nil {
 		return "", errors.New("nil wrapper")
 	}
@@ -180,6 +167,9 @@ func wrapperToKey(i interface{}) (string, error) {
 }
 
 func reqToKey(req *ethpb.AttestationDataRequest) (string, error) {
+	if featureconfig.Get().ReduceAttesterStateCopy {
+		return fmt.Sprintf("%d", req.Slot), nil
+	}
 	return fmt.Sprintf("%d-%d", req.CommitteeIndex, req.Slot), nil
 }
 

beacon-chain/cache/checkpoint_state.go

@@ -4,11 +4,10 @@ import (
 	"errors"
 	"sync"
 
-	"github.com/gogo/protobuf/proto"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/hashutil"
 	"k8s.io/client-go/tools/cache"
 )
@@ -19,7 +18,9 @@ var (
 	ErrNotCheckpointState = errors.New("object is not a state by check point struct")
 
 	// maxCheckpointStateSize defines the max number of entries check point to state cache can contain.
-	maxCheckpointStateSize = 4
+	// Choosing 10 to account for multiple forks, this allows 5 forks per epoch boundary with 2 epochs
+	// window to accept attestation based on latest spec.
+	maxCheckpointStateSize = 10
 
 	// Metrics.
 	checkpointStateMiss = promauto.NewCounter(prometheus.CounterOpts{
@@ -35,7 +36,7 @@ var (
 // CheckpointState defines the active validator indices per epoch.
 type CheckpointState struct {
 	Checkpoint *ethpb.Checkpoint
-	State      *pb.BeaconState
+	State      *stateTrie.BeaconState
 }
 
 // CheckpointStateCache is a struct with 1 queue for looking up state by checkpoint.
@@ -67,7 +68,7 @@ func NewCheckpointStateCache() *CheckpointStateCache {
 
 // StateByCheckpoint fetches state by checkpoint. Returns true with a
 // reference to the CheckpointState info, if exists. Otherwise returns false, nil.
-func (c *CheckpointStateCache) StateByCheckpoint(cp *ethpb.Checkpoint) (*pb.BeaconState, error) {
+func (c *CheckpointStateCache) StateByCheckpoint(cp *ethpb.Checkpoint) (*stateTrie.BeaconState, error) {
 	c.lock.RLock()
 	defer c.lock.RUnlock()
 	h, err := hashutil.HashProto(cp)
@@ -92,7 +93,8 @@ func (c *CheckpointStateCache) StateByCheckpoint(cp *ethpb.Checkpoint) (*pb.Beac
 		return nil, ErrNotCheckpointState
 	}
 
-	return proto.Clone(info.State).(*pb.BeaconState), nil
+	// Copy here is unnecessary since the return will only be used to verify attestation signature.
+	return info.State, nil
 }
 
 // AddCheckpointState adds CheckpointState object to the cache. This method also trims the least
@@ -100,7 +102,10 @@ func (c *CheckpointStateCache) StateByCheckpoint(cp *ethpb.Checkpoint) (*pb.Beac
 func (c *CheckpointStateCache) AddCheckpointState(cp *CheckpointState) error {
 	c.lock.Lock()
 	defer c.lock.Unlock()
-	if err := c.cache.AddIfNotPresent(cp); err != nil {
+	if err := c.cache.AddIfNotPresent(&CheckpointState{
+		Checkpoint: stateTrie.CopyCheckpoint(cp.Checkpoint),
+		State:      cp.State.Copy(),
+	}); err != nil {
 		return err
 	}
 

beacon-chain/cache/checkpoint_state_test.go

@@ -4,16 +4,26 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/hashutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
 func TestCheckpointStateCacheKeyFn_OK(t *testing.T) {
-	cp := &ethpb.Checkpoint{Epoch: 1, Root: []byte{'A'}}
+	cp := &ethpb.Checkpoint{Epoch: 1, Root: bytesutil.PadTo([]byte{'A'}, 32)}
+	st, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot: 64,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
 	info := &CheckpointState{
 		Checkpoint: cp,
-		State:      &pb.BeaconState{Slot: 64},
+		State:      st,
 	}
 	key, err := checkpointState(info)
 	if err != nil {
@@ -38,10 +48,17 @@ func TestCheckpointStateCacheKeyFn_InvalidObj(t *testing.T) {
 func TestCheckpointStateCache_StateByCheckpoint(t *testing.T) {
 	cache := NewCheckpointStateCache()
 
-	cp1 := &ethpb.Checkpoint{Epoch: 1, Root: []byte{'A'}}
+	cp1 := &ethpb.Checkpoint{Epoch: 1, Root: bytesutil.PadTo([]byte{'A'}, 32)}
+	st, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		GenesisValidatorsRoot: params.BeaconConfig().ZeroHash[:],
+		Slot:                  64,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
 	info1 := &CheckpointState{
 		Checkpoint: cp1,
-		State:      &pb.BeaconState{Slot: 64},
+		State:      st,
 	}
 	state, err := cache.StateByCheckpoint(cp1)
 	if err != nil {
@@ -58,14 +75,20 @@ func TestCheckpointStateCache_StateByCheckpoint(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(state, info1.State) {
+	if !proto.Equal(state.InnerStateUnsafe(), info1.State.InnerStateUnsafe()) {
 		t.Error("incorrectly cached state")
 	}
 
-	cp2 := &ethpb.Checkpoint{Epoch: 2, Root: []byte{'B'}}
+	cp2 := &ethpb.Checkpoint{Epoch: 2, Root: bytesutil.PadTo([]byte{'B'}, 32)}
+	st2, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot: 128,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
 	info2 := &CheckpointState{
 		Checkpoint: cp2,
-		State:      &pb.BeaconState{Slot: 128},
+		State:      st2,
 	}
 	if err := cache.AddCheckpointState(info2); err != nil {
 		t.Fatal(err)
@@ -74,7 +97,7 @@ func TestCheckpointStateCache_StateByCheckpoint(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(state, info2.State) {
+	if !reflect.DeepEqual(state.CloneInnerState(), info2.State.CloneInnerState()) {
 		t.Error("incorrectly cached state")
 	}
 
@@ -82,18 +105,26 @@ func TestCheckpointStateCache_StateByCheckpoint(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(state, info1.State) {
+	if !reflect.DeepEqual(state.CloneInnerState(), info1.State.CloneInnerState()) {
 		t.Error("incorrectly cached state")
 	}
 }
 
-func TestCheckpointStateCache__MaxSize(t *testing.T) {
+func TestCheckpointStateCache_MaxSize(t *testing.T) {
 	c := NewCheckpointStateCache()
-
+	st, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot: 0,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
 	for i := 0; i < maxCheckpointStateSize+100; i++ {
+		if err := st.SetSlot(uint64(i)); err != nil {
+			t.Fatal(err)
+		}
 		info := &CheckpointState{
 			Checkpoint: &ethpb.Checkpoint{Epoch: uint64(i)},
-			State:      &pb.BeaconState{Slot: uint64(i)},
+			State:      st,
 		}
 		if err := c.AddCheckpointState(info); err != nil {
 			t.Fatal(err)

beacon-chain/cache/committee.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/sliceutil"
 	"k8s.io/client-go/tools/cache"
@@ -40,6 +39,7 @@ type Committees struct {
 	Seed            [32]byte
 	ShuffledIndices []uint64
 	SortedIndices   []uint64
+	ProposerIndices []uint64
 }
 
 // CommitteeCache is a struct with 1 queue for looking up shuffled indices list by seed.
@@ -68,9 +68,6 @@ func NewCommitteesCache() *CommitteeCache {
 // Committee fetches the shuffled indices by slot and committee index. Every list of indices
 // represent one committee. Returns true if the list exists with slot and committee index. Otherwise returns false, nil.
 func (c *CommitteeCache) Committee(slot uint64, seed [32]byte, index uint64) ([]uint64, error) {
-	if !featureconfig.Get().EnableShuffledIndexCache && !featureconfig.Get().EnableNewCache {
-		return nil, nil
-	}
 	c.lock.RLock()
 	defer c.lock.RUnlock()
 
@@ -99,21 +96,50 @@ func (c *CommitteeCache) Committee(slot uint64, seed [32]byte, index uint64) ([]
 	indexOffSet := index + (slot%params.BeaconConfig().SlotsPerEpoch)*committeeCountPerSlot
 	start, end := startEndIndices(item, indexOffSet)
 
+	if int(end) > len(item.ShuffledIndices) || end < start {
+		return nil, errors.New("requested index out of bound")
+	}
+
 	return item.ShuffledIndices[start:end], nil
 }
 
 // AddCommitteeShuffledList adds Committee shuffled list object to the cache. T
 // his method also trims the least recently list if the cache size has ready the max cache size limit.
 func (c *CommitteeCache) AddCommitteeShuffledList(committees *Committees) error {
-	if !featureconfig.Get().EnableShuffledIndexCache && !featureconfig.Get().EnableNewCache {
-		return nil
-	}
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
 	if err := c.CommitteeCache.AddIfNotPresent(committees); err != nil {
 		return err
 	}
+	trim(c.CommitteeCache, maxCommitteesCacheSize)
+	return nil
+}
+
+// AddProposerIndicesList updates the committee shuffled list with proposer indices.
+func (c *CommitteeCache) AddProposerIndicesList(seed [32]byte, indices []uint64) error {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	obj, exists, err := c.CommitteeCache.GetByKey(key(seed))
+	if err != nil {
+		return err
+	}
+	if !exists {
+		committees := &Committees{ProposerIndices: indices}
+		if err := c.CommitteeCache.Add(committees); err != nil {
+			return err
+		}
+	} else {
+		committees, ok := obj.(*Committees)
+		if !ok {
+			return ErrNotCommittee
+		}
+		committees.ProposerIndices = indices
+		if err := c.CommitteeCache.Add(committees); err != nil {
+			return err
+		}
+	}
 
 	trim(c.CommitteeCache, maxCommitteesCacheSize)
 	return nil
@@ -121,10 +147,6 @@ func (c *CommitteeCache) AddCommitteeShuffledList(committees *Committees) error
 
 // ActiveIndices returns the active indices of a given seed stored in cache.
 func (c *CommitteeCache) ActiveIndices(seed [32]byte) ([]uint64, error) {
-	if !featureconfig.Get().EnableShuffledIndexCache && !featureconfig.Get().EnableNewCache {
-		return nil, nil
-	}
-
 	c.lock.RLock()
 	defer c.lock.RUnlock()
 	obj, exists, err := c.CommitteeCache.GetByKey(key(seed))
@@ -147,6 +169,30 @@ func (c *CommitteeCache) ActiveIndices(seed [32]byte) ([]uint64, error) {
 	return item.SortedIndices, nil
 }
 
+// ProposerIndices returns the proposer indices of a given seed.
+func (c *CommitteeCache) ProposerIndices(seed [32]byte) ([]uint64, error) {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	obj, exists, err := c.CommitteeCache.GetByKey(key(seed))
+	if err != nil {
+		return nil, err
+	}
+
+	if exists {
+		CommitteeCacheHit.Inc()
+	} else {
+		CommitteeCacheMiss.Inc()
+		return nil, nil
+	}
+
+	item, ok := obj.(*Committees)
+	if !ok {
+		return nil, ErrNotCommittee
+	}
+
+	return item.ProposerIndices, nil
+}
+
 func startEndIndices(c *Committees, index uint64) (uint64, uint64) {
 	validatorCount := uint64(len(c.ShuffledIndices))
 	start := sliceutil.SplitOffset(validatorCount, c.CommitteeCount, index)
@@ -157,7 +203,7 @@ func startEndIndices(c *Committees, index uint64) (uint64, uint64) {
 // Using seed as source for key to handle reorgs in the same epoch.
 // The seed is derived from state's array of randao mixes and epoch value
 // hashed together. This avoids collisions on different validator set. Spec definition:
-// https://github.com/ethereum/eth2.0-specs/blob/v0.9.2/specs/core/0_beacon-chain.md#get_seed
+// https://github.com/ethereum/eth2.0-specs/blob/v0.9.3/specs/core/0_beacon-chain.md#get_seed
 func key(seed [32]byte) string {
 	return string(seed[:])
 }

beacon-chain/cache/committee_fuzz_test.go

@@ -0,0 +1,68 @@
+package cache
+
+import (
+	"reflect"
+	"testing"
+
+	fuzz "github.com/google/gofuzz"
+)
+
+func TestCommitteeKeyFuzz_OK(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	c := &Committees{}
+
+	for i := 0; i < 100000; i++ {
+		fuzzer.Fuzz(c)
+		k, err := committeeKeyFn(c)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if k != key(c.Seed) {
+			t.Errorf("Incorrect hash k: %s, expected %s", k, key(c.Seed))
+		}
+	}
+}
+
+func TestCommitteeCache_FuzzCommitteesByEpoch(t *testing.T) {
+	cache := NewCommitteesCache()
+	fuzzer := fuzz.NewWithSeed(0)
+	c := &Committees{}
+
+	for i := 0; i < 100000; i++ {
+		fuzzer.Fuzz(c)
+		if err := cache.AddCommitteeShuffledList(c); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := cache.Committee(0, c.Seed, 0); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if len(cache.CommitteeCache.ListKeys()) != maxCommitteesCacheSize {
+		t.Error("Incorrect key size")
+	}
+}
+
+func TestCommitteeCache_FuzzActiveIndices(t *testing.T) {
+	cache := NewCommitteesCache()
+	fuzzer := fuzz.NewWithSeed(0)
+	c := &Committees{}
+
+	for i := 0; i < 100000; i++ {
+		fuzzer.Fuzz(c)
+		if err := cache.AddCommitteeShuffledList(c); err != nil {
+			t.Fatal(err)
+		}
+		indices, err := cache.ActiveIndices(c.Seed)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !reflect.DeepEqual(indices, c.SortedIndices) {
+			t.Error("Saved indices not the same")
+		}
+	}
+
+	if len(cache.CommitteeCache.ListKeys()) != maxCommitteesCacheSize {
+		t.Error("Incorrect key size")
+	}
+}

beacon-chain/cache/committee_ids.go

@@ -0,0 +1,134 @@
+package cache
+
+import (
+	"sync"
+	"time"
+
+	lru "github.com/hashicorp/golang-lru"
+	"github.com/patrickmn/go-cache"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/sliceutil"
+)
+
+type committeeIDs struct {
+	attester          *lru.Cache
+	attesterLock      sync.RWMutex
+	aggregator        *lru.Cache
+	aggregatorLock    sync.RWMutex
+	persistentSubnets *cache.Cache
+	subnetsLock       sync.RWMutex
+}
+
+// CommitteeIDs for attester and aggregator.
+var CommitteeIDs = newCommitteeIDs()
+
+func newCommitteeIDs() *committeeIDs {
+	// Given a node can calculate committee assignments of current epoch and next epoch.
+	// Max size is set to 2 epoch length.
+	cacheSize := int(params.BeaconConfig().MaxCommitteesPerSlot * params.BeaconConfig().SlotsPerEpoch * 2)
+	attesterCache, err := lru.New(cacheSize)
+	if err != nil {
+		panic(err)
+	}
+	aggregatorCache, err := lru.New(cacheSize)
+	if err != nil {
+		panic(err)
+	}
+	epochDuration := time.Duration(params.BeaconConfig().SlotsPerEpoch * params.BeaconConfig().SecondsPerSlot)
+	subLength := epochDuration * time.Duration(params.BeaconNetworkConfig().EpochsPerRandomSubnetSubscription)
+	persistentCache := cache.New(subLength*time.Second, epochDuration*time.Second)
+	return &committeeIDs{attester: attesterCache, aggregator: aggregatorCache, persistentSubnets: persistentCache}
+}
+
+// AddAttesterCommiteeID adds committee ID for subscribing subnet for the attester of a given slot.
+func (c *committeeIDs) AddAttesterCommiteeID(slot uint64, committeeID uint64) {
+	c.attesterLock.Lock()
+	defer c.attesterLock.Unlock()
+
+	ids := []uint64{committeeID}
+	val, exists := c.attester.Get(slot)
+	if exists {
+		ids = sliceutil.UnionUint64(append(val.([]uint64), ids...))
+	}
+	c.attester.Add(slot, ids)
+}
+
+// GetAttesterCommitteeIDs gets the committee ID for subscribing subnet for attester of the slot.
+func (c *committeeIDs) GetAttesterCommitteeIDs(slot uint64) []uint64 {
+	c.attesterLock.RLock()
+	defer c.attesterLock.RUnlock()
+
+	val, exists := c.attester.Get(slot)
+	if !exists {
+		return nil
+	}
+	if v, ok := val.([]uint64); ok {
+		return v
+	}
+	return nil
+}
+
+// AddAggregatorCommiteeID adds committee ID for subscribing subnet for the aggregator of a given slot.
+func (c *committeeIDs) AddAggregatorCommiteeID(slot uint64, committeeID uint64) {
+	c.aggregatorLock.Lock()
+	defer c.aggregatorLock.Unlock()
+
+	ids := []uint64{committeeID}
+	val, exists := c.aggregator.Get(slot)
+	if exists {
+		ids = sliceutil.UnionUint64(append(val.([]uint64), ids...))
+	}
+	c.aggregator.Add(slot, ids)
+}
+
+// GetAggregatorCommitteeIDs gets the committee ID for subscribing subnet for aggregator of the slot.
+func (c *committeeIDs) GetAggregatorCommitteeIDs(slot uint64) []uint64 {
+	c.aggregatorLock.RLock()
+	defer c.aggregatorLock.RUnlock()
+
+	val, exists := c.aggregator.Get(slot)
+	if !exists {
+		return []uint64{}
+	}
+	return val.([]uint64)
+}
+
+// GetPersistentCommittees retrieves the persistent committee and expiration time of that validator's
+// subscription.
+func (c *committeeIDs) GetPersistentCommittees(pubkey []byte) ([]uint64, bool, time.Time) {
+	c.subnetsLock.RLock()
+	defer c.subnetsLock.RUnlock()
+
+	id, duration, ok := c.persistentSubnets.GetWithExpiration(string(pubkey))
+	if !ok {
+		return []uint64{}, ok, time.Time{}
+	}
+	return id.([]uint64), ok, duration
+}
+
+// GetAllCommittees retrieves all the non-expired subscribed committees of all the validators
+// in the cache.
+func (c *committeeIDs) GetAllCommittees() []uint64 {
+	c.subnetsLock.RLock()
+	defer c.subnetsLock.RUnlock()
+
+	itemsMap := c.persistentSubnets.Items()
+	committees := []uint64{}
+
+	for _, v := range itemsMap {
+		if v.Expired() {
+			continue
+		}
+		committees = append(committees, v.Object.([]uint64)...)
+	}
+	return sliceutil.SetUint64(committees)
+}
+
+// AddPersistentCommittee adds the relevant committee for that particular validator along with its
+// expiration period.
+func (c *committeeIDs) AddPersistentCommittee(pubkey []byte, comIndex []uint64, duration time.Duration) {
+	c.subnetsLock.Lock()
+	defer c.subnetsLock.Unlock()
+
+	c.persistentSubnets.Set(string(pubkey), comIndex, duration)
+}

beacon-chain/cache/committee_ids_test.go

@@ -0,0 +1,84 @@
+package cache
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestCommitteeIDCache_RoundTrip(t *testing.T) {
+	c := newCommitteeIDs()
+	slot := uint64(100)
+	committeeIDs := c.GetAggregatorCommitteeIDs(slot)
+	if len(committeeIDs) != 0 {
+		t.Errorf("Empty cache returned an object: %v", committeeIDs)
+	}
+
+	c.AddAggregatorCommiteeID(slot, 1)
+	res := c.GetAggregatorCommitteeIDs(slot)
+	if !reflect.DeepEqual(res, []uint64{1}) {
+		t.Error("Expected equal value to return from cache")
+	}
+
+	c.AddAggregatorCommiteeID(slot, 2)
+	res = c.GetAggregatorCommitteeIDs(slot)
+	if !reflect.DeepEqual(res, []uint64{1, 2}) {
+		t.Error("Expected equal value to return from cache")
+	}
+
+	c.AddAggregatorCommiteeID(slot, 3)
+	res = c.GetAggregatorCommitteeIDs(slot)
+	if !reflect.DeepEqual(res, []uint64{1, 2, 3}) {
+		t.Error("Expected equal value to return from cache")
+	}
+
+	committeeIDs = c.GetAttesterCommitteeIDs(slot)
+	if len(committeeIDs) != 0 {
+		t.Errorf("Empty cache returned an object: %v", committeeIDs)
+	}
+
+	c.AddAttesterCommiteeID(slot, 11)
+	res = c.GetAttesterCommitteeIDs(slot)
+	if !reflect.DeepEqual(res, []uint64{11}) {
+		t.Error("Expected equal value to return from cache")
+	}
+
+	c.AddAttesterCommiteeID(slot, 22)
+	res = c.GetAttesterCommitteeIDs(slot)
+	if !reflect.DeepEqual(res, []uint64{11, 22}) {
+		t.Error("Expected equal value to return from cache")
+	}
+
+	c.AddAttesterCommiteeID(slot, 33)
+	res = c.GetAttesterCommitteeIDs(slot)
+	if !reflect.DeepEqual(res, []uint64{11, 22, 33}) {
+		t.Error("Expected equal value to return from cache")
+	}
+}
+
+func TestCommitteeIDs_PersistentCommitteeRoundtrip(t *testing.T) {
+	pubkeySet := [][48]byte{}
+	c := newCommitteeIDs()
+
+	for i := 0; i < 20; i++ {
+		pubkey := [48]byte{byte(i)}
+		pubkeySet = append(pubkeySet, pubkey)
+		c.AddPersistentCommittee(pubkey[:], []uint64{uint64(i)}, 0)
+	}
+
+	for i := 0; i < 20; i++ {
+		pubkey := [48]byte{byte(i)}
+
+		idxs, ok, _ := c.GetPersistentCommittees(pubkey[:])
+		if !ok {
+			t.Errorf("Couldn't find entry in cache for pubkey %#x", pubkey)
+			continue
+		}
+		if int(idxs[0]) != i {
+			t.Fatalf("Wanted index of %d but got %d", i, idxs[0])
+		}
+	}
+	coms := c.GetAllCommittees()
+	if len(coms) != 20 {
+		t.Errorf("Number of committees is not %d but is %d", 20, len(coms))
+	}
+}

beacon-chain/cache/committee_test.go

@@ -1,6 +1,7 @@
 package cache
 
 import (
+	"math"
 	"reflect"
 	"sort"
 	"strconv"
@@ -96,6 +97,53 @@ func TestCommitteeCache_ActiveIndices(t *testing.T) {
 	}
 }
 
+func TestCommitteeCache_AddProposerIndicesList(t *testing.T) {
+	cache := NewCommitteesCache()
+
+	seed := [32]byte{'A'}
+	indices := []uint64{1, 2, 3, 4, 5}
+	indices, err := cache.ProposerIndices(seed)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if indices != nil {
+		t.Error("Expected committee count not to exist in empty cache")
+	}
+	if err := cache.AddProposerIndicesList(seed, indices); err != nil {
+		t.Fatal(err)
+	}
+	received, err := cache.ProposerIndices(seed)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(indices, received) {
+		t.Error("Did not receive correct proposer indices from cache")
+	}
+
+	item := &Committees{Seed: [32]byte{'B'}, SortedIndices: []uint64{1, 2, 3, 4, 5, 6}}
+	if err := cache.AddCommitteeShuffledList(item); err != nil {
+		t.Fatal(err)
+	}
+	indices, err = cache.ProposerIndices(item.Seed)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if indices != nil {
+		t.Error("Expected committee count not to exist in empty cache")
+	}
+	if err := cache.AddProposerIndicesList(item.Seed, indices); err != nil {
+		t.Fatal(err)
+	}
+	received, err = cache.ProposerIndices(item.Seed)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(indices, received) {
+		t.Error("Did not receive correct proposer indices from cache")
+	}
+
+}
+
 func TestCommitteeCache_CanRotate(t *testing.T) {
 	cache := NewCommitteesCache()
 
@@ -125,3 +173,22 @@ func TestCommitteeCache_CanRotate(t *testing.T) {
 		t.Error("incorrect key received for slot 199")
 	}
 }
+
+func TestCommitteeCacheOutOfRange(t *testing.T) {
+	cache := NewCommitteesCache()
+	seed := bytesutil.ToBytes32([]byte("foo"))
+	err := cache.CommitteeCache.Add(&Committees{
+		CommitteeCount:  1,
+		Seed:            seed,
+		ShuffledIndices: []uint64{0},
+		SortedIndices:   []uint64{},
+		ProposerIndices: []uint64{},
+	})
+	if err != nil {
+		t.Error(err)
+	}
+	_, err = cache.Committee(0, seed, math.MaxUint64) // Overflow!
+	if err == nil {
+		t.Fatal("Did not fail as expected")
+	}
+}

beacon-chain/cache/common.go

@@ -14,8 +14,12 @@ var (
 // trim the FIFO queue to the maxSize.
 func trim(queue *cache.FIFO, maxSize int) {
 	for s := len(queue.ListKeys()); s > maxSize; s-- {
-		// #nosec G104 popProcessNoopFunc never returns an error
-		_, _ = queue.Pop(popProcessNoopFunc)
+		_, err := queue.Pop(popProcessNoopFunc)
+		if err != nil {
+			// popProcessNoopFunc never returns an error, but we handle this anyway to make linter
+			// happy.
+			return
+		}
 	}
 }
 

beacon-chain/cache/depositcache/BUILD.bazel

@@ -1,4 +1,5 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_library(
     name = "go_default_library",

beacon-chain/cache/depositcache/deposits_cache.go

@@ -1,3 +1,7 @@
+// Package depositcache is the source of validator deposits maintained
+// in-memory by the beacon node – deposits processed from the
+// eth1 powchain are then stored in this cache to be accessed by
+// any other service during a beacon node's runtime.
 package depositcache
 
 import (
@@ -35,12 +39,11 @@ type DepositFetcher interface {
 // stores all the deposit related data that is required by the beacon-node.
 type DepositCache struct {
 	// Beacon chain deposits in memory.
-	pendingDeposits       []*dbpb.DepositContainer
-	deposits              []*dbpb.DepositContainer
-	depositsLock          sync.RWMutex
-	chainStartDeposits    []*ethpb.Deposit
-	chainstartPubkeys     map[string]bool
-	chainstartPubkeysLock sync.RWMutex
+	pendingDeposits    []*dbpb.DepositContainer
+	deposits           []*dbpb.DepositContainer
+	depositsLock       sync.RWMutex
+	chainStartDeposits []*ethpb.Deposit
+	chainStartPubkeys  map[string]bool
 }
 
 // NewDepositCache instantiates a new deposit cache
@@ -48,7 +51,7 @@ func NewDepositCache() *DepositCache {
 	return &DepositCache{
 		pendingDeposits:    []*dbpb.DepositContainer{},
 		deposits:           []*dbpb.DepositContainer{},
-		chainstartPubkeys:  make(map[string]bool),
+		chainStartPubkeys:  make(map[string]bool),
 		chainStartDeposits: make([]*ethpb.Deposit, 0),
 	}
 }
@@ -69,7 +72,7 @@ func (dc *DepositCache) InsertDeposit(ctx context.Context, d *ethpb.Deposit, blo
 	}
 	dc.depositsLock.Lock()
 	defer dc.depositsLock.Unlock()
-	// keep the slice sorted on insertion in order to avoid costly sorting on retrival.
+	// Keep the slice sorted on insertion in order to avoid costly sorting on retrieval.
 	heightIdx := sort.Search(len(dc.deposits), func(i int) bool { return dc.deposits[i].Index >= index })
 	newDeposits := append([]*dbpb.DepositContainer{{Deposit: d, Eth1BlockHeight: blockNum, DepositRoot: depositRoot[:], Index: index}}, dc.deposits[heightIdx:]...)
 	dc.deposits = append(dc.deposits[:heightIdx], newDeposits...)
@@ -102,21 +105,17 @@ func (dc *DepositCache) AllDepositContainers(ctx context.Context) []*dbpb.Deposi
 func (dc *DepositCache) MarkPubkeyForChainstart(ctx context.Context, pubkey string) {
 	ctx, span := trace.StartSpan(ctx, "DepositsCache.MarkPubkeyForChainstart")
 	defer span.End()
-	dc.chainstartPubkeysLock.Lock()
-	defer dc.chainstartPubkeysLock.Unlock()
-	dc.chainstartPubkeys[pubkey] = true
+	dc.chainStartPubkeys[pubkey] = true
 }
 
 // PubkeyInChainstart returns bool for whether the pubkey passed in has deposited.
 func (dc *DepositCache) PubkeyInChainstart(ctx context.Context, pubkey string) bool {
 	ctx, span := trace.StartSpan(ctx, "DepositsCache.PubkeyInChainstart")
 	defer span.End()
-	dc.chainstartPubkeysLock.Lock()
-	defer dc.chainstartPubkeysLock.Unlock()
-	if dc.chainstartPubkeys != nil {
-		return dc.chainstartPubkeys[pubkey]
+	if dc.chainStartPubkeys != nil {
+		return dc.chainStartPubkeys[pubkey]
 	}
-	dc.chainstartPubkeys = make(map[string]bool)
+	dc.chainStartPubkeys = make(map[string]bool)
 	return false
 }
 

beacon-chain/cache/depositcache/pending_deposits_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	dbpb "github.com/prysmaticlabs/prysm/proto/beacon/db"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 )
 
 var _ = PendingDepositsFetcher(&DepositCache{})
@@ -33,8 +34,12 @@ func TestInsertPendingDeposit_ignoresNilDeposit(t *testing.T) {
 
 func TestRemovePendingDeposit_OK(t *testing.T) {
 	db := DepositCache{}
-	depToRemove := &ethpb.Deposit{Proof: [][]byte{[]byte("A")}}
-	otherDep := &ethpb.Deposit{Proof: [][]byte{[]byte("B")}}
+	proof1 := make([][]byte, 33)
+	proof1[0] = bytesutil.PadTo([]byte{'A'}, 32)
+	proof2 := make([][]byte, 33)
+	proof2[0] = bytesutil.PadTo([]byte{'A'}, 32)
+	depToRemove := &ethpb.Deposit{Proof: proof1}
+	otherDep := &ethpb.Deposit{Proof: proof2}
 	db.pendingDeposits = []*dbpb.DepositContainer{
 		{Deposit: depToRemove, Index: 1},
 		{Deposit: otherDep, Index: 5},
@@ -57,7 +62,9 @@ func TestRemovePendingDeposit_IgnoresNilDeposit(t *testing.T) {
 
 func TestPendingDeposit_RoundTrip(t *testing.T) {
 	dc := DepositCache{}
-	dep := &ethpb.Deposit{Proof: [][]byte{[]byte("A")}}
+	proof := make([][]byte, 33)
+	proof[0] = bytesutil.PadTo([]byte{'A'}, 32)
+	dep := &ethpb.Deposit{Proof: proof}
 	dc.InsertPendingDeposit(context.Background(), dep, 111, 100, [32]byte{})
 	dc.RemovePendingDeposit(context.Background(), dep)
 	if len(dc.pendingDeposits) != 0 {

beacon-chain/cache/doc.go

@@ -0,0 +1,5 @@
+// Package cache includes all important caches for the runtime
+// of an eth2 beacon node, ensuring the node does not spend
+// resources computing duplicate operations such as committee
+// calculations for validators during the same epoch, etc.
+package cache

beacon-chain/cache/eth1_data.go

@@ -1,136 +0,0 @@
-package cache
-
-import (
-	"errors"
-	"sync"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"k8s.io/client-go/tools/cache"
-)
-
-var (
-	// ErrNotEth1DataVote will be returned when a cache object is not a pointer to
-	// a Eth1DataVote struct.
-	ErrNotEth1DataVote = errors.New("object is not a eth1 data vote obj")
-
-	// maxEth1DataVoteSize defines the max number of eth1 data votes can cache.
-	maxEth1DataVoteSize = 1000
-
-	// Metrics.
-	eth1DataVoteCacheMiss = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "eth1_data_vote_cache_miss",
-		Help: "The number of eth1 data vote count requests that aren't present in the cache.",
-	})
-	eth1DataVoteCacheHit = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "eth1_data_vote_cache_hit",
-		Help: "The number of eth1 data vote count requests that are present in the cache.",
-	})
-)
-
-// Eth1DataVote defines the struct which keeps track of the vote count of individual deposit root.
-type Eth1DataVote struct {
-	Eth1DataHash [32]byte
-	VoteCount    uint64
-}
-
-// Eth1DataVoteCache is a struct with 1 queue for looking up eth1 data vote count by deposit root.
-type Eth1DataVoteCache struct {
-	eth1DataVoteCache *cache.FIFO
-	lock              sync.RWMutex
-}
-
-// eth1DataVoteKeyFn takes the eth1data hash as the key for the eth1 data vote count of a given eth1data object.
-func eth1DataVoteKeyFn(obj interface{}) (string, error) {
-	eInfo, ok := obj.(*Eth1DataVote)
-	if !ok {
-		return "", ErrNotEth1DataVote
-	}
-
-	return string(eInfo.Eth1DataHash[:]), nil
-}
-
-// NewEth1DataVoteCache creates a new eth1 data vote count cache for storing/accessing Eth1DataVote.
-func NewEth1DataVoteCache() *Eth1DataVoteCache {
-	return &Eth1DataVoteCache{
-		eth1DataVoteCache: cache.NewFIFO(eth1DataVoteKeyFn),
-	}
-}
-
-// Eth1DataVote fetches eth1 data vote count by the eth1data hash. Returns vote count,
-// if exists. Otherwise returns false, nil.
-func (c *Eth1DataVoteCache) Eth1DataVote(eth1DataHash [32]byte) (uint64, error) {
-	if !featureconfig.Get().EnableEth1DataVoteCache {
-		// Return a miss result if cache is not enabled.
-		eth1DataVoteCacheMiss.Inc()
-		return 0, nil
-	}
-
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-	obj, exists, err := c.eth1DataVoteCache.GetByKey(string(eth1DataHash[:]))
-	if err != nil {
-		return 0, err
-	}
-
-	if exists {
-		eth1DataVoteCacheHit.Inc()
-	} else {
-		eth1DataVoteCacheMiss.Inc()
-		return 0, nil
-	}
-
-	eInfo, ok := obj.(*Eth1DataVote)
-	if !ok {
-		return 0, ErrNotEth1DataVote
-	}
-
-	return eInfo.VoteCount, nil
-}
-
-// AddEth1DataVote adds eth1 data vote object to the cache. This method also trims the least
-// recently added Eth1DataVoteByEpoch object if the cache size has ready the max cache size limit.
-func (c *Eth1DataVoteCache) AddEth1DataVote(eth1DataVote *Eth1DataVote) error {
-	if !featureconfig.Get().EnableEth1DataVoteCache {
-		return nil
-	}
-
-	c.lock.Lock()
-	defer c.lock.Unlock()
-	if err := c.eth1DataVoteCache.Add(eth1DataVote); err != nil {
-		return err
-	}
-
-	trim(c.eth1DataVoteCache, maxEth1DataVoteSize)
-	return nil
-}
-
-// IncrementEth1DataVote increments the existing eth1 data object's vote count by 1,
-// and returns the vote count.
-func (c *Eth1DataVoteCache) IncrementEth1DataVote(eth1DataHash [32]byte) (uint64, error) {
-	if !featureconfig.Get().EnableEth1DataVoteCache {
-		return 0, nil
-	}
-
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-	obj, exists, err := c.eth1DataVoteCache.GetByKey(string(eth1DataHash[:]))
-	if err != nil {
-		return 0, err
-	}
-	if !exists {
-		return 0, errors.New("eth1 data vote object does not exist")
-	}
-
-	eth1DataVoteCacheHit.Inc()
-
-	eInfo, _ := obj.(*Eth1DataVote)
-	eInfo.VoteCount++
-
-	if err := c.eth1DataVoteCache.Add(eInfo); err != nil {
-		return 0, err
-	}
-
-	return eInfo.VoteCount, nil
-}

beacon-chain/cache/eth1_data_test.go

@@ -1,110 +0,0 @@
-package cache
-
-import (
-	"strconv"
-	"testing"
-)
-
-func TestEth1DataVoteKeyFn_OK(t *testing.T) {
-	eInfo := &Eth1DataVote{
-		VoteCount:    44,
-		Eth1DataHash: [32]byte{'A'},
-	}
-
-	key, err := eth1DataVoteKeyFn(eInfo)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if key != string(eInfo.Eth1DataHash[:]) {
-		t.Errorf("Incorrect hash key: %s, expected %s", key, string(eInfo.Eth1DataHash[:]))
-	}
-}
-
-func TestEth1DataVoteKeyFn_InvalidObj(t *testing.T) {
-	_, err := eth1DataVoteKeyFn("bad")
-	if err != ErrNotEth1DataVote {
-		t.Errorf("Expected error %v, got %v", ErrNotEth1DataVote, err)
-	}
-}
-
-func TestEth1DataVoteCache_CanAdd(t *testing.T) {
-	cache := NewEth1DataVoteCache()
-
-	eInfo := &Eth1DataVote{
-		VoteCount:    55,
-		Eth1DataHash: [32]byte{'B'},
-	}
-	count, err := cache.Eth1DataVote(eInfo.Eth1DataHash)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if count != 0 {
-		t.Error("Expected seed not to exist in empty cache")
-	}
-
-	if err := cache.AddEth1DataVote(eInfo); err != nil {
-		t.Fatal(err)
-	}
-	count, err = cache.Eth1DataVote(eInfo.Eth1DataHash)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if count != eInfo.VoteCount {
-		t.Errorf(
-			"Expected vote count to be %d, got %d",
-			eInfo.VoteCount,
-			count,
-		)
-	}
-}
-
-func TestEth1DataVoteCache_CanIncrement(t *testing.T) {
-	cache := NewEth1DataVoteCache()
-
-	eInfo := &Eth1DataVote{
-		VoteCount:    55,
-		Eth1DataHash: [32]byte{'B'},
-	}
-
-	if err := cache.AddEth1DataVote(eInfo); err != nil {
-		t.Fatal(err)
-	}
-
-	_, err := cache.IncrementEth1DataVote(eInfo.Eth1DataHash)
-	if err != nil {
-		t.Fatal(err)
-	}
-	_, _ = cache.IncrementEth1DataVote(eInfo.Eth1DataHash)
-	count, _ := cache.IncrementEth1DataVote(eInfo.Eth1DataHash)
-
-	if count != 58 {
-		t.Errorf(
-			"Expected vote count to be %d, got %d",
-			58,
-			count,
-		)
-	}
-}
-
-func TestEth1Data_MaxSize(t *testing.T) {
-	cache := NewEth1DataVoteCache()
-
-	for i := 0; i < maxEth1DataVoteSize+1; i++ {
-		var hash [32]byte
-		copy(hash[:], []byte(strconv.Itoa(i)))
-		eInfo := &Eth1DataVote{
-			Eth1DataHash: hash,
-		}
-		if err := cache.AddEth1DataVote(eInfo); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	if len(cache.eth1DataVoteCache.ListKeys()) != maxEth1DataVoteSize {
-		t.Errorf(
-			"Expected hash cache key size to be %d, got %d",
-			maxEth1DataVoteSize,
-			len(cache.eth1DataVoteCache.ListKeys()),
-		)
-	}
-}

beacon-chain/cache/feature_flag_test.go

@@ -1,11 +1,14 @@
 package cache
 
-import "github.com/prysmaticlabs/prysm/shared/featureconfig"
+import (
+	"os"
+	"testing"
 
-func init() {
-	featureconfig.Init(&featureconfig.Flags{
-		EnableAttestationCache:   true,
-		EnableEth1DataVoteCache:  true,
-		EnableShuffledIndexCache: true,
-	})
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+)
+
+func TestMain(m *testing.M) {
+	resetCfg := featureconfig.InitWithReset(&featureconfig.Flags{EnableEth1DataVoteCache: true})
+	defer resetCfg()
+	os.Exit(m.Run())
 }

beacon-chain/cache/hot_state_cache.go

@@ -0,0 +1,77 @@
+package cache
+
+import (
+	lru "github.com/hashicorp/golang-lru"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+)
+
+var (
+	// hotStateCacheSize defines the max number of hot state this can cache.
+	hotStateCacheSize = 16
+	// Metrics
+	hotStateCacheHit = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "hot_state_cache_hit",
+		Help: "The total number of cache hits on the hot state cache.",
+	})
+	hotStateCacheMiss = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "hot_state_cache_miss",
+		Help: "The total number of cache misses on the hot state cache.",
+	})
+)
+
+// HotStateCache is used to store the processed beacon state after finalized check point..
+type HotStateCache struct {
+	cache *lru.Cache
+}
+
+// NewHotStateCache initializes the map and underlying cache.
+func NewHotStateCache() *HotStateCache {
+	cache, err := lru.New(hotStateCacheSize)
+	if err != nil {
+		panic(err)
+	}
+	return &HotStateCache{
+		cache: cache,
+	}
+}
+
+// Get returns a cached response via input block root, if any.
+// The response is copied by default.
+func (c *HotStateCache) Get(root [32]byte) *stateTrie.BeaconState {
+	item, exists := c.cache.Get(root)
+
+	if exists && item != nil {
+		hotStateCacheHit.Inc()
+		return item.(*stateTrie.BeaconState).Copy()
+	}
+	hotStateCacheMiss.Inc()
+	return nil
+}
+
+// GetWithoutCopy returns a non-copied cached response via input block root.
+func (c *HotStateCache) GetWithoutCopy(root [32]byte) *stateTrie.BeaconState {
+	item, exists := c.cache.Get(root)
+	if exists && item != nil {
+		hotStateCacheHit.Inc()
+		return item.(*stateTrie.BeaconState)
+	}
+	hotStateCacheMiss.Inc()
+	return nil
+}
+
+// Put the response in the cache.
+func (c *HotStateCache) Put(root [32]byte, state *stateTrie.BeaconState) {
+	c.cache.Add(root, state)
+}
+
+// Has returns true if the key exists in the cache.
+func (c *HotStateCache) Has(root [32]byte) bool {
+	return c.cache.Contains(root)
+}
+
+// Delete deletes the key exists in the cache.
+func (c *HotStateCache) Delete(root [32]byte) bool {
+	return c.cache.Remove(root)
+}

beacon-chain/cache/hot_state_cache_test.go

@@ -0,0 +1,46 @@
+package cache_test
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+)
+
+func TestHotStateCache_RoundTrip(t *testing.T) {
+	c := cache.NewHotStateCache()
+	root := [32]byte{'A'}
+	state := c.Get(root)
+	if state != nil {
+		t.Errorf("Empty cache returned an object: %v", state)
+	}
+	if c.Has(root) {
+		t.Error("Empty cache has an object")
+	}
+
+	state, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot: 10,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	c.Put(root, state)
+
+	if !c.Has(root) {
+		t.Error("Empty cache does not have an object")
+	}
+	res := c.Get(root)
+	if state == nil {
+		t.Errorf("Empty cache returned an object: %v", state)
+	}
+	if !reflect.DeepEqual(state.CloneInnerState(), res.CloneInnerState()) {
+		t.Error("Expected equal protos to return from cache")
+	}
+
+	c.Delete(root)
+	if c.Has(root) {
+		t.Error("Cache not suppose to have the object")
+	}
+}

beacon-chain/cache/skip_slot_cache.go

@@ -0,0 +1,148 @@
+package cache
+
+import (
+	"context"
+	"math"
+	"sync"
+	"time"
+
+	lru "github.com/hashicorp/golang-lru"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"go.opencensus.io/trace"
+)
+
+var (
+	// Metrics
+	skipSlotCacheHit = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "skip_slot_cache_hit",
+		Help: "The total number of cache hits on the skip slot cache.",
+	})
+	skipSlotCacheMiss = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "skip_slot_cache_miss",
+		Help: "The total number of cache misses on the skip slot cache.",
+	})
+)
+
+// SkipSlotCache is used to store the cached results of processing skip slots in state.ProcessSlots.
+type SkipSlotCache struct {
+	cache      *lru.Cache
+	lock       sync.RWMutex
+	disabled   bool // Allow for programmatic toggling of the cache, useful during initial sync.
+	inProgress map[uint64]bool
+}
+
+// NewSkipSlotCache initializes the map and underlying cache.
+func NewSkipSlotCache() *SkipSlotCache {
+	cache, err := lru.New(8)
+	if err != nil {
+		panic(err)
+	}
+	return &SkipSlotCache{
+		cache:      cache,
+		inProgress: make(map[uint64]bool),
+	}
+}
+
+// Enable the skip slot cache.
+func (c *SkipSlotCache) Enable() {
+	c.disabled = false
+}
+
+// Disable the skip slot cache.
+func (c *SkipSlotCache) Disable() {
+	c.disabled = true
+}
+
+// Get waits for any in progress calculation to complete before returning a
+// cached response, if any.
+func (c *SkipSlotCache) Get(ctx context.Context, slot uint64) (*stateTrie.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "skipSlotCache.Get")
+	defer span.End()
+	if c.disabled {
+		// Return a miss result if cache is not enabled.
+		skipSlotCacheMiss.Inc()
+		return nil, nil
+	}
+
+	delay := minDelay
+
+	// Another identical request may be in progress already. Let's wait until
+	// any in progress request resolves or our timeout is exceeded.
+	inProgress := false
+	for {
+		if ctx.Err() != nil {
+			return nil, ctx.Err()
+		}
+
+		c.lock.RLock()
+		if !c.inProgress[slot] {
+			c.lock.RUnlock()
+			break
+		}
+		inProgress = true
+		c.lock.RUnlock()
+
+		// This increasing backoff is to decrease the CPU cycles while waiting
+		// for the in progress boolean to flip to false.
+		time.Sleep(time.Duration(delay) * time.Nanosecond)
+		delay *= delayFactor
+		delay = math.Min(delay, maxDelay)
+	}
+	span.AddAttributes(trace.BoolAttribute("inProgress", inProgress))
+
+	item, exists := c.cache.Get(slot)
+
+	if exists && item != nil {
+		skipSlotCacheHit.Inc()
+		span.AddAttributes(trace.BoolAttribute("hit", true))
+		return item.(*stateTrie.BeaconState).Copy(), nil
+	}
+	skipSlotCacheMiss.Inc()
+	span.AddAttributes(trace.BoolAttribute("hit", false))
+	return nil, nil
+}
+
+// MarkInProgress a request so that any other similar requests will block on
+// Get until MarkNotInProgress is called.
+func (c *SkipSlotCache) MarkInProgress(slot uint64) error {
+	if c.disabled {
+		return nil
+	}
+
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	if c.inProgress[slot] {
+		return ErrAlreadyInProgress
+	}
+	c.inProgress[slot] = true
+	return nil
+}
+
+// MarkNotInProgress will release the lock on a given request. This should be
+// called after put.
+func (c *SkipSlotCache) MarkNotInProgress(slot uint64) error {
+	if c.disabled {
+		return nil
+	}
+
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	delete(c.inProgress, slot)
+	return nil
+}
+
+// Put the response in the cache.
+func (c *SkipSlotCache) Put(ctx context.Context, slot uint64, state *stateTrie.BeaconState) error {
+	if c.disabled {
+		return nil
+	}
+
+	// Copy state so cached value is not mutated.
+	c.cache.Add(slot, state.Copy())
+
+	return nil
+}

beacon-chain/cache/skip_slot_cache_test.go

@@ -0,0 +1,53 @@
+package cache_test
+
+import (
+	"context"
+	"reflect"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+)
+
+func TestSkipSlotCache_RoundTrip(t *testing.T) {
+	ctx := context.Background()
+	c := cache.NewSkipSlotCache()
+
+	state, err := c.Get(ctx, 5)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if state != nil {
+		t.Errorf("Empty cache returned an object: %v", state)
+	}
+
+	if err := c.MarkInProgress(5); err != nil {
+		t.Error(err)
+	}
+
+	state, err = stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot: 10,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err = c.Put(ctx, 5, state); err != nil {
+		t.Error(err)
+	}
+
+	if err := c.MarkNotInProgress(5); err != nil {
+		t.Error(err)
+	}
+
+	res, err := c.Get(ctx, 5)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !reflect.DeepEqual(state.CloneInnerState(), res.CloneInnerState()) {
+		t.Error("Expected equal protos to return from cache")
+	}
+}

beacon-chain/cache/state_summary.go

@@ -0,0 +1,65 @@
+package cache
+
+import (
+	"sync"
+
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+)
+
+// StateSummaryCache caches state summary object.
+type StateSummaryCache struct {
+	initSyncStateSummaries     map[[32]byte]*pb.StateSummary
+	initSyncStateSummariesLock sync.RWMutex
+}
+
+// NewStateSummaryCache creates a new state summary cache.
+func NewStateSummaryCache() *StateSummaryCache {
+	return &StateSummaryCache{
+		initSyncStateSummaries: make(map[[32]byte]*pb.StateSummary),
+	}
+}
+
+// Put saves a state summary to the initial sync state summaries cache.
+func (s *StateSummaryCache) Put(r [32]byte, b *pb.StateSummary) {
+	s.initSyncStateSummariesLock.Lock()
+	defer s.initSyncStateSummariesLock.Unlock()
+	s.initSyncStateSummaries[r] = b
+}
+
+// Has checks if a state summary exists in the initial sync state summaries cache using the root
+// of the block.
+func (s *StateSummaryCache) Has(r [32]byte) bool {
+	s.initSyncStateSummariesLock.RLock()
+	defer s.initSyncStateSummariesLock.RUnlock()
+	_, ok := s.initSyncStateSummaries[r]
+	return ok
+}
+
+// Get retrieves a state summary from the initial sync state summaries cache using the root of
+// the block.
+func (s *StateSummaryCache) Get(r [32]byte) *pb.StateSummary {
+	s.initSyncStateSummariesLock.RLock()
+	defer s.initSyncStateSummariesLock.RUnlock()
+	b := s.initSyncStateSummaries[r]
+	return b
+}
+
+// GetAll retrieves all the beacon state summaries from the initial sync state summaries cache, the returned
+// state summaries are unordered.
+func (s *StateSummaryCache) GetAll() []*pb.StateSummary {
+	s.initSyncStateSummariesLock.RLock()
+	defer s.initSyncStateSummariesLock.RUnlock()
+
+	blks := make([]*pb.StateSummary, 0, len(s.initSyncStateSummaries))
+	for _, b := range s.initSyncStateSummaries {
+		blks = append(blks, b)
+	}
+	return blks
+}
+
+// Clear clears out the initial sync state summaries cache.
+func (s *StateSummaryCache) Clear() {
+	s.initSyncStateSummariesLock.Lock()
+	defer s.initSyncStateSummariesLock.Unlock()
+	s.initSyncStateSummaries = make(map[[32]byte]*pb.StateSummary)
+}

beacon-chain/core/blocks/BUILD.bazel

@@ -1,4 +1,5 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -9,17 +10,18 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/core/blocks",
     visibility = [
         "//beacon-chain:__subpackages__",
+        "//fuzz:__pkg__",
         "//shared/testutil:__pkg__",
     ],
     deps = [
-        "//beacon-chain/cache:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
-        "//beacon-chain/core/state/stateutils:go_default_library",
         "//beacon-chain/core/validators:go_default_library",
+        "//beacon-chain/state:go_default_library",
+        "//beacon-chain/state/stateutil:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
+        "//shared/attestationutil:go_default_library",
         "//shared/bls:go_default_library",
         "//shared/bytesutil:go_default_library",
-        "//shared/featureconfig:go_default_library",
         "//shared/hashutil:go_default_library",
         "//shared/mathutil:go_default_library",
         "//shared/params:go_default_library",
@@ -40,15 +42,19 @@ go_test(
     srcs = [
         "block_operations_fuzz_test.go",
         "block_operations_test.go",
+        "block_regression_test.go",
         "block_test.go",
         "eth1_data_test.go",
     ],
     embed = [":go_default_library"],
     deps = [
         "//beacon-chain/core/helpers:go_default_library",
-        "//beacon-chain/core/state/stateutils:go_default_library",
+        "//beacon-chain/state:go_default_library",
+        "//beacon-chain/state/stateutil:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
+        "//shared/attestationutil:go_default_library",
         "//shared/bls:go_default_library",
+        "//shared/bytesutil:go_default_library",
         "//shared/params:go_default_library",
         "//shared/testutil:go_default_library",
         "//shared/trieutil:go_default_library",

beacon-chain/core/blocks/block.go

@@ -1,6 +1,5 @@
-// Package blocks contains block processing libraries. These libraries
-// process and verify block specific messages such as PoW receipt root,
-// RANDAO, validator deposits, exits and slashing proofs.
+// Package blocks contains block processing libraries according to
+// the eth2spec.
 package blocks
 
 import (

beacon-chain/core/blocks/block_operations_fuzz_test.go

@@ -1,4 +1,4 @@
-package blocks_test
+package blocks
 
 import (
 	"context"
@@ -6,11 +6,16 @@ import (
 
 	fuzz "github.com/google/gofuzz"
 	eth "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/params"
+
+	//"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	ethereum_beacon_p2p_v1 "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 )
 
-func TestFuzzProcessAttestation_10000(t *testing.T) {
+func TestFuzzProcessAttestationNoVerify_10000(t *testing.T) {
 	fuzzer := fuzz.NewWithSeed(0)
 	ctx := context.Background()
 	state := &ethereum_beacon_p2p_v1.BeaconState{}
@@ -19,7 +24,9 @@ func TestFuzzProcessAttestation_10000(t *testing.T) {
 	for i := 0; i < 10000; i++ {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(att)
-		_, _ = blocks.ProcessAttestationNoVerify(ctx, state, att)
+		s, err := beaconstate.InitializeFromProtoUnsafe(state)
+		_, err = ProcessAttestationNoVerify(ctx, s, att)
+		_ = err
 	}
 }
 
@@ -31,6 +38,388 @@ func TestFuzzProcessBlockHeader_10000(t *testing.T) {
 	for i := 0; i < 10000; i++ {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(block)
-		_, _ = blocks.ProcessBlockHeader(state, block)
+
+		s, err := beaconstate.InitializeFromProtoUnsafe(state)
+		_, err = ProcessBlockHeader(s, block)
+		_ = err
+	}
+}
+
+func TestFuzzverifyDepositDataSigningRoot_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	ba := []byte{}
+	pubkey := [48]byte{}
+	sig := [96]byte{}
+	domain := [4]byte{}
+	p := []byte{}
+	s := []byte{}
+	d := []byte{}
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(&ba)
+		fuzzer.Fuzz(&pubkey)
+		fuzzer.Fuzz(&sig)
+		fuzzer.Fuzz(&domain)
+		fuzzer.Fuzz(&p)
+		fuzzer.Fuzz(&s)
+		fuzzer.Fuzz(&d)
+		err := verifySignature(ba, pubkey[:], sig[:], domain[:])
+		err = verifySignature(ba, p, s, d)
+		_ = err
+	}
+}
+
+func TestFuzzProcessEth1DataInBlock_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	block := &eth.BeaconBlock{}
+	state := &stateTrie.BeaconState{}
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(block)
+		s, err := ProcessEth1DataInBlock(state, block)
+		if err != nil && s != nil {
+			t.Fatalf("state should be nil on err. found: %v on error: %v for state: %v and block: %v", s, err, state, block)
+		}
+	}
+}
+
+func TestFuzzareEth1DataEqual_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	eth1data := &eth.Eth1Data{}
+	eth1data2 := &eth.Eth1Data{}
+
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(eth1data)
+		fuzzer.Fuzz(eth1data2)
+		areEth1DataEqual(eth1data, eth1data2)
+		areEth1DataEqual(eth1data, eth1data)
+	}
+}
+
+func TestFuzzEth1DataHasEnoughSupport_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	eth1data := &eth.Eth1Data{}
+	stateVotes := []*eth.Eth1Data{}
+	for i := 0; i < 100000; i++ {
+		fuzzer.Fuzz(eth1data)
+		fuzzer.Fuzz(&stateVotes)
+		s, err := beaconstate.InitializeFromProto(&ethereum_beacon_p2p_v1.BeaconState{
+			Eth1DataVotes: stateVotes,
+		})
+		_, err = Eth1DataHasEnoughSupport(s, eth1data)
+		_ = err
+	}
+
+}
+
+func TestFuzzProcessBlockHeaderNoVerify_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	block := &eth.BeaconBlock{}
+
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(block)
+		s, err := beaconstate.InitializeFromProtoUnsafe(state)
+		_, err = ProcessBlockHeaderNoVerify(s, block)
+		_ = err
+	}
+}
+
+func TestFuzzProcessRandao_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	blockBody := &eth.BeaconBlockBody{}
+
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(blockBody)
+		s, err := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessRandao(s, blockBody)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
+		}
+	}
+}
+
+func TestFuzzProcessRandaoNoVerify_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	blockBody := &eth.BeaconBlockBody{}
+
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(blockBody)
+		s, err := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessRandaoNoVerify(s, blockBody)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
+		}
+	}
+}
+
+func TestFuzzProcessProposerSlashings_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	blockBody := &eth.BeaconBlockBody{}
+	ctx := context.Background()
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(blockBody)
+		s, err := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessProposerSlashings(ctx, s, blockBody)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
+		}
+	}
+}
+
+func TestFuzzVerifyProposerSlashing_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	proposerSlashing := &eth.ProposerSlashing{}
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(proposerSlashing)
+		s, err := beaconstate.InitializeFromProtoUnsafe(state)
+		err = VerifyProposerSlashing(s, proposerSlashing)
+		_ = err
+	}
+}
+
+func TestFuzzProcessAttesterSlashings_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	blockBody := &eth.BeaconBlockBody{}
+	ctx := context.Background()
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(blockBody)
+		s, err := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessAttesterSlashings(ctx, s, blockBody)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
+		}
+	}
+}
+
+func TestFuzzVerifyAttesterSlashing_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	attesterSlashing := &eth.AttesterSlashing{}
+	ctx := context.Background()
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(attesterSlashing)
+		s, err := beaconstate.InitializeFromProtoUnsafe(state)
+		err = VerifyAttesterSlashing(ctx, s, attesterSlashing)
+		_ = err
+	}
+}
+
+func TestFuzzIsSlashableAttestationData_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	attestationData := &eth.AttestationData{}
+	attestationData2 := &eth.AttestationData{}
+
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(attestationData)
+		fuzzer.Fuzz(attestationData2)
+		IsSlashableAttestationData(attestationData, attestationData2)
+	}
+}
+
+func TestFuzzslashableAttesterIndices_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	attesterSlashing := &eth.AttesterSlashing{}
+
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(attesterSlashing)
+		slashableAttesterIndices(attesterSlashing)
+	}
+}
+
+func TestFuzzProcessAttestations_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	blockBody := &eth.BeaconBlockBody{}
+	ctx := context.Background()
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(blockBody)
+		s, err := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessAttestations(ctx, s, blockBody)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
+		}
+	}
+}
+
+func TestFuzzProcessAttestationsNoVerify_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	blockBody := &eth.BeaconBlockBody{}
+	ctx := context.Background()
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(blockBody)
+		s, err := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessAttestationsNoVerify(ctx, s, blockBody)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
+		}
+	}
+}
+
+func TestFuzzProcessAttestation_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	attestation := &eth.Attestation{}
+	ctx := context.Background()
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(attestation)
+		s, err := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessAttestation(ctx, s, attestation)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, attestation)
+		}
+	}
+}
+
+func TestFuzzVerifyIndexedAttestationn_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	idxAttestation := &eth.IndexedAttestation{}
+	ctx := context.Background()
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(idxAttestation)
+		s, err := beaconstate.InitializeFromProtoUnsafe(state)
+		err = VerifyIndexedAttestation(ctx, s, idxAttestation)
+		_ = err
+	}
+}
+
+func TestFuzzVerifyAttestation_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	attestation := &eth.Attestation{}
+	ctx := context.Background()
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(attestation)
+		s, err := beaconstate.InitializeFromProtoUnsafe(state)
+		err = VerifyAttestation(ctx, s, attestation)
+		_ = err
+	}
+}
+
+func TestFuzzProcessDeposits_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	blockBody := &eth.BeaconBlockBody{}
+	ctx := context.Background()
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(blockBody)
+		s, err := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessDeposits(ctx, s, blockBody)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
+		}
+	}
+}
+
+func TestFuzzProcessPreGenesisDeposit_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	deposit := &eth.Deposit{}
+	ctx := context.Background()
+
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(deposit)
+		s, err := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessPreGenesisDeposit(ctx, s, deposit)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, deposit)
+		}
+	}
+}
+
+func TestFuzzProcessDeposit_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	deposit := &eth.Deposit{}
+
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(deposit)
+		s, err := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessDeposit(s, deposit)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, deposit)
+		}
+	}
+}
+
+func TestFuzzverifyDeposit_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	deposit := &eth.Deposit{}
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(deposit)
+		s, err := beaconstate.InitializeFromProtoUnsafe(state)
+		err = verifyDeposit(s, deposit)
+		_ = err
+	}
+}
+
+func TestFuzzProcessVoluntaryExits_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	blockBody := &eth.BeaconBlockBody{}
+	ctx := context.Background()
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(blockBody)
+		s, err := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessVoluntaryExits(ctx, s, blockBody)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
+		}
+	}
+}
+
+func TestFuzzProcessVoluntaryExitsNoVerify_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	blockBody := &eth.BeaconBlockBody{}
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(blockBody)
+		s, err := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessVoluntaryExitsNoVerify(s, blockBody)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
+		}
+	}
+}
+
+func TestFuzzVerifyExit_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	ve := &eth.SignedVoluntaryExit{}
+	val := &beaconstate.ReadOnlyValidator{}
+	fork := &pb.Fork{}
+	var slot uint64
+
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(ve)
+		fuzzer.Fuzz(val)
+		fuzzer.Fuzz(fork)
+		fuzzer.Fuzz(&slot)
+		err := VerifyExit(val, slot, fork, ve, params.BeaconConfig().ZeroHash[:])
+		_ = err
 	}
 }

beacon-chain/core/blocks/block_regression_test.go

@@ -0,0 +1,113 @@
+package blocks_test
+
+import (
+	"context"
+	"testing"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/shared/bls"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
+)
+
+func TestProcessAttesterSlashings_RegressionSlashableIndices(t *testing.T) {
+	testutil.ResetCache()
+	beaconState, privKeys := testutil.DeterministicGenesisState(t, 5500)
+	for _, vv := range beaconState.Validators() {
+		vv.WithdrawableEpoch = 1 * params.BeaconConfig().SlotsPerEpoch
+	}
+	// This set of indices is very similar to the one from our sapphire testnet
+	// when close to 100 validators were incorrectly slashed. The set is from 0 -5500,
+	// instead of 55000 as it would take too long to generate a state.
+	setA := []uint64{21, 92, 236, 244, 281, 321, 510, 524,
+		538, 682, 828, 858, 913, 920, 922, 959, 1176, 1207,
+		1222, 1229, 1354, 1394, 1436, 1454, 1510, 1550,
+		1552, 1576, 1645, 1704, 1842, 1967, 2076, 2111, 2134, 2307,
+		2343, 2354, 2417, 2524, 2532, 2555, 2740, 2749, 2759, 2762,
+		2800, 2809, 2824, 2987, 3110, 3125, 3559, 3583, 3599, 3608,
+		3657, 3685, 3723, 3756, 3759, 3761, 3820, 3826, 3979, 4030,
+		4141, 4170, 4205, 4247, 4257, 4479, 4492, 4569, 5091,
+	}
+	// Only 2800 is the slashable index.
+	setB := []uint64{1361, 1438, 2383, 2800}
+	expectedSlashedVal := 2800
+
+	root1 := [32]byte{'d', 'o', 'u', 'b', 'l', 'e', '1'}
+	att1 := &ethpb.IndexedAttestation{
+		Data: &ethpb.AttestationData{
+			Source: &ethpb.Checkpoint{Epoch: 0},
+			Target: &ethpb.Checkpoint{Epoch: 0, Root: root1[:]},
+		},
+		AttestingIndices: setA,
+	}
+	domain, err := helpers.Domain(beaconState.Fork(), 0, params.BeaconConfig().DomainBeaconAttester, beaconState.GenesisValidatorRoot())
+	if err != nil {
+		t.Fatal(err)
+	}
+	signingRoot, err := helpers.ComputeSigningRoot(att1.Data, domain)
+	if err != nil {
+		t.Errorf("Could not get signing root of beacon block header: %v", err)
+	}
+	aggSigs := []*bls.Signature{}
+	for _, index := range setA {
+		sig := privKeys[index].Sign(signingRoot[:])
+		aggSigs = append(aggSigs, sig)
+	}
+	aggregateSig := bls.AggregateSignatures(aggSigs)
+	att1.Signature = aggregateSig.Marshal()[:]
+
+	root2 := [32]byte{'d', 'o', 'u', 'b', 'l', 'e', '2'}
+	att2 := &ethpb.IndexedAttestation{
+		Data: &ethpb.AttestationData{
+			Source: &ethpb.Checkpoint{Epoch: 0},
+			Target: &ethpb.Checkpoint{Epoch: 0, Root: root2[:]},
+		},
+		AttestingIndices: setB,
+	}
+	signingRoot, err = helpers.ComputeSigningRoot(att2.Data, domain)
+	if err != nil {
+		t.Errorf("Could not get signing root of beacon block header: %v", err)
+	}
+	aggSigs = []*bls.Signature{}
+	for _, index := range setB {
+		sig := privKeys[index].Sign(signingRoot[:])
+		aggSigs = append(aggSigs, sig)
+	}
+	aggregateSig = bls.AggregateSignatures(aggSigs)
+	att2.Signature = aggregateSig.Marshal()[:]
+
+	slashings := []*ethpb.AttesterSlashing{
+		{
+			Attestation_1: att1,
+			Attestation_2: att2,
+		},
+	}
+
+	currentSlot := 2 * params.BeaconConfig().SlotsPerEpoch
+	if err := beaconState.SetSlot(currentSlot); err != nil {
+		t.Fatal(err)
+	}
+
+	block := &ethpb.BeaconBlock{
+		Body: &ethpb.BeaconBlockBody{
+			AttesterSlashings: slashings,
+		},
+	}
+
+	newState, err := blocks.ProcessAttesterSlashings(context.Background(), beaconState, block.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	newRegistry := newState.Validators()
+	if !newRegistry[expectedSlashedVal].Slashed {
+		t.Errorf("Validator with index %d was not slashed despite performing a double vote", expectedSlashedVal)
+	}
+
+	for idx, val := range newRegistry {
+		if val.Slashed && idx != expectedSlashedVal {
+			t.Errorf("validator with index: %d was unintentionally slashed", idx)
+		}
+	}
+}

beacon-chain/core/blocks/eth1_data_test.go

@@ -6,10 +6,22 @@ import (
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
+func FakeDeposits(n int) []*ethpb.Eth1Data {
+	deposits := make([]*ethpb.Eth1Data, n)
+	for i := 0; i < n; i++ {
+		deposits[i] = &ethpb.Eth1Data{
+			DepositCount: 1,
+			DepositRoot:  []byte("root"),
+		}
+	}
+	return deposits
+}
+
 func TestEth1DataHasEnoughSupport(t *testing.T) {
 	tests := []struct {
 		stateVotes         []*ethpb.Eth1Data
@@ -18,21 +30,7 @@ func TestEth1DataHasEnoughSupport(t *testing.T) {
 		votingPeriodLength uint64
 	}{
 		{
-			stateVotes: []*ethpb.Eth1Data{
-				{
-					DepositCount: 1,
-					DepositRoot:  []byte("root"),
-				}, {
-					DepositCount: 1,
-					DepositRoot:  []byte("root"),
-				}, {
-					DepositCount: 1,
-					DepositRoot:  []byte("root"),
-				}, {
-					DepositCount: 1,
-					DepositRoot:  []byte("root"),
-				},
-			},
+			stateVotes: FakeDeposits(4 * int(params.BeaconConfig().SlotsPerEpoch)),
 			data: &ethpb.Eth1Data{
 				DepositCount: 1,
 				DepositRoot:  []byte("root"),
@@ -40,21 +38,7 @@ func TestEth1DataHasEnoughSupport(t *testing.T) {
 			hasSupport:         true,
 			votingPeriodLength: 7,
 		}, {
-			stateVotes: []*ethpb.Eth1Data{
-				{
-					DepositCount: 1,
-					DepositRoot:  []byte("root"),
-				}, {
-					DepositCount: 1,
-					DepositRoot:  []byte("root"),
-				}, {
-					DepositCount: 1,
-					DepositRoot:  []byte("root"),
-				}, {
-					DepositCount: 1,
-					DepositRoot:  []byte("root"),
-				},
-			},
+			stateVotes: FakeDeposits(4 * int(params.BeaconConfig().SlotsPerEpoch)),
 			data: &ethpb.Eth1Data{
 				DepositCount: 1,
 				DepositRoot:  []byte("root"),
@@ -62,21 +46,7 @@ func TestEth1DataHasEnoughSupport(t *testing.T) {
 			hasSupport:         false,
 			votingPeriodLength: 8,
 		}, {
-			stateVotes: []*ethpb.Eth1Data{
-				{
-					DepositCount: 1,
-					DepositRoot:  []byte("root"),
-				}, {
-					DepositCount: 1,
-					DepositRoot:  []byte("root"),
-				}, {
-					DepositCount: 1,
-					DepositRoot:  []byte("root"),
-				}, {
-					DepositCount: 1,
-					DepositRoot:  []byte("root"),
-				},
-			},
+			stateVotes: FakeDeposits(4 * int(params.BeaconConfig().SlotsPerEpoch)),
 			data: &ethpb.Eth1Data{
 				DepositCount: 1,
 				DepositRoot:  []byte("root"),
@@ -86,14 +56,18 @@ func TestEth1DataHasEnoughSupport(t *testing.T) {
 		},
 	}
 
+	params.SetupTestConfigCleanup(t)
 	for i, tt := range tests {
 		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
 			c := params.BeaconConfig()
-			c.SlotsPerEth1VotingPeriod = tt.votingPeriodLength
+			c.EpochsPerEth1VotingPeriod = tt.votingPeriodLength
 			params.OverrideBeaconConfig(c)
 
-			s := &pb.BeaconState{
+			s, err := beaconstate.InitializeFromProto(&pb.BeaconState{
 				Eth1DataVotes: tt.stateVotes,
+			})
+			if err != nil {
+				t.Fatal(err)
 			}
 			result, err := blocks.Eth1DataHasEnoughSupport(s, tt.data)
 			if err != nil {
@@ -102,8 +76,7 @@ func TestEth1DataHasEnoughSupport(t *testing.T) {
 
 			if result != tt.hasSupport {
 				t.Errorf(
-					"blocks.Eth1DataHasEnoughSupport(%+v, %+v) = %t, wanted %t",
-					s,
+					"blocks.Eth1DataHasEnoughSupport(%+v) = %t, wanted %t",
 					tt.data,
 					result,
 					tt.hasSupport,

beacon-chain/core/blocks/spectest/BUILD.bazel

@@ -1,4 +1,5 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -35,6 +36,7 @@ go_test(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/state:go_default_library",
         "//beacon-chain/core/state/stateutils:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/params/spectest:go_default_library",
         "//shared/testutil:go_default_library",
@@ -68,6 +70,7 @@ go_test(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/state:go_default_library",
         "//beacon-chain/core/state/stateutils:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/params/spectest:go_default_library",
         "//shared/testutil:go_default_library",

beacon-chain/core/blocks/spectest/attestation_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func runAttestationTest(t *testing.T, config string) {
-	if err := spectest.SetConfig(config); err != nil {
+	if err := spectest.SetConfig(t, config); err != nil {
 		t.Fatal(err)
 	}
 

beacon-chain/core/blocks/spectest/attester_slashing_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func runAttesterSlashingTest(t *testing.T, config string) {
-	if err := spectest.SetConfig(config); err != nil {
+	if err := spectest.SetConfig(t, config); err != nil {
 		t.Fatal(err)
 	}
 

beacon-chain/core/blocks/spectest/block_header_test.go

@@ -11,6 +11,7 @@ import (
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params/spectest"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
@@ -18,7 +19,7 @@ import (
 )
 
 func runBlockHeaderTest(t *testing.T, config string) {
-	if err := spectest.SetConfig(config); err != nil {
+	if err := spectest.SetConfig(t, config); err != nil {
 		t.Fatal(err)
 	}
 
@@ -38,10 +39,14 @@ func runBlockHeaderTest(t *testing.T, config string) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			preBeaconState := &pb.BeaconState{}
-			if err := ssz.Unmarshal(preBeaconStateFile, preBeaconState); err != nil {
+			preBeaconStateBase := &pb.BeaconState{}
+			if err := ssz.Unmarshal(preBeaconStateFile, preBeaconStateBase); err != nil {
 				t.Fatalf("Failed to unmarshal: %v", err)
 			}
+			preBeaconState, err := stateTrie.InitializeFromProto(preBeaconStateBase)
+			if err != nil {
+				t.Fatal(err)
+			}
 
 			// If the post.ssz is not present, it means the test should fail on our end.
 			postSSZFilepath, err := bazel.Runfile(path.Join(testsFolderPath, folder.Name(), "post.ssz"))
@@ -68,9 +73,8 @@ func runBlockHeaderTest(t *testing.T, config string) {
 				if err := ssz.Unmarshal(postBeaconStateFile, postBeaconState); err != nil {
 					t.Fatalf("Failed to unmarshal: %v", err)
 				}
-
-				if !proto.Equal(beaconState, postBeaconState) {
-					diff, _ := messagediff.PrettyDiff(beaconState, postBeaconState)
+				if !proto.Equal(beaconState.CloneInnerState(), postBeaconState) {
+					diff, _ := messagediff.PrettyDiff(beaconState.CloneInnerState(), postBeaconState)
 					t.Log(diff)
 					t.Fatal("Post state does not match expected")
 				}

beacon-chain/core/blocks/spectest/block_processing_test.go

@@ -1,3 +1,5 @@
+// Package spectest contains all comformity specification tests
+// for block processing according to the eth2 spec.
 package spectest
 
 import (
@@ -12,29 +14,40 @@ import (
 	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/go-ssz"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params/spectest"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 	"gopkg.in/d4l3k/messagediff.v1"
 )
 
+func init() {
+	state.SkipSlotCache.Disable()
+}
+
 func runBlockProcessingTest(t *testing.T, config string) {
-	if err := spectest.SetConfig(config); err != nil {
+	if err := spectest.SetConfig(t, config); err != nil {
 		t.Fatal(err)
 	}
 
 	testFolders, testsFolderPath := testutil.TestFolders(t, config, "sanity/blocks/pyspec_tests")
 	for _, folder := range testFolders {
 		t.Run(folder.Name(), func(t *testing.T) {
+			helpers.ClearCache()
 			preBeaconStateFile, err := testutil.BazelFileBytes(testsFolderPath, folder.Name(), "pre.ssz")
 			if err != nil {
 				t.Fatal(err)
 			}
-			beaconState := &pb.BeaconState{}
-			if err := ssz.Unmarshal(preBeaconStateFile, beaconState); err != nil {
+			beaconStateBase := &pb.BeaconState{}
+			if err := ssz.Unmarshal(preBeaconStateFile, beaconStateBase); err != nil {
 				t.Fatalf("Failed to unmarshal: %v", err)
 			}
+			beaconState, err := stateTrie.InitializeFromProto(beaconStateBase)
+			if err != nil {
+				t.Fatal(err)
+			}
 
 			file, err := testutil.BazelFileBytes(testsFolderPath, folder.Name(), "meta.yaml")
 			if err != nil {
@@ -87,8 +100,8 @@ func runBlockProcessingTest(t *testing.T, config string) {
 					t.Fatalf("Failed to unmarshal: %v", err)
 				}
 
-				if !proto.Equal(beaconState, postBeaconState) {
-					diff, _ := messagediff.PrettyDiff(beaconState, postBeaconState)
+				if !proto.Equal(beaconState.InnerStateUnsafe(), postBeaconState) {
+					diff, _ := messagediff.PrettyDiff(beaconState.InnerStateUnsafe(), postBeaconState)
 					t.Log(diff)
 					t.Fatal("Post state does not match expected")
 				}

beacon-chain/core/blocks/spectest/deposit_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func runDepositTest(t *testing.T, config string) {
-	if err := spectest.SetConfig(config); err != nil {
+	if err := spectest.SetConfig(t, config); err != nil {
 		t.Fatal(err)
 	}
 

beacon-chain/core/blocks/spectest/proposer_slashing_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func runProposerSlashingTest(t *testing.T, config string) {
-	if err := spectest.SetConfig(config); err != nil {
+	if err := spectest.SetConfig(t, config); err != nil {
 		t.Fatal(err)
 	}
 

beacon-chain/core/blocks/spectest/voluntary_exit_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func runVoluntaryExitTest(t *testing.T, config string) {
-	if err := spectest.SetConfig(config); err != nil {
+	if err := spectest.SetConfig(t, config); err != nil {
 		t.Fatal(err)
 	}