gaz

* connect to dv5 bootnodes

* fix test

* change polling period

* ignore

* Update beacon-chain/p2p/service.go

* Update beacon-chain/p2p/service_test.go

* fix test

Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com>
Co-authored-by: terence tsao <terence@prysmaticlabs.com>

.bazelrc

@@ -15,7 +15,7 @@ run --host_force_python=PY2
 # Network sandboxing only works on linux.
 --experimental_sandbox_default_allow_network=false
 
-# Use minimal protobufs at runtime
+# Use mainnet protobufs at runtime
 run --define ssz=mainnet
 test --define ssz=mainnet
 build --define ssz=mainnet
@@ -24,3 +24,161 @@ build --define ssz=mainnet
 build --incompatible_strict_action_env
 test --incompatible_strict_action_env
 run --incompatible_strict_action_env
+
+# Disable kafka by default, it takes a long time to build...
+build --define kafka_enabled=false
+test --define kafka_enabled=false
+run --define kafka_enabled=false
+
+# Release flags
+build:release --workspace_status_command=./scripts/workspace_status.sh
+build:release --stamp
+build:release --compilation_mode=opt
+
+# LLVM compiler for building C/C++ dependencies.
+build:llvm --crosstool_top=@llvm_toolchain//:toolchain
+build:llvm --define compiler=llvm
+
+# multi-arch cross-compiling toolchain configs:
+-----------------------------------------------
+build:cross --crosstool_top=@prysm_toolchains//:multiarch_toolchain
+build:cross --host_platform=@io_bazel_rules_go//go/toolchain:linux_amd64
+build:cross --host_crosstool_top=@prysm_toolchains//:hostonly_toolchain
+
+# linux_amd64 config for cross compiler toolchain, not strictly necessary since host/exec env is amd64
+build:linux_amd64 --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64_cgo
+
+# osx_amd64 config for cross compiler toolchain
+build:osx_amd64 --config=cross
+build:osx_amd64 --platforms=@io_bazel_rules_go//go/toolchain:darwin_amd64_cgo
+build:osx_amd64 --compiler=osxcross
+
+# windows
+build:windows_amd64 --config=cross
+build:windows_amd64  --platforms=@io_bazel_rules_go//go/toolchain:windows_amd64_cgo
+build:windows_amd64 --compiler=mingw-w64
+
+# linux_arm64 conifg for cross compiler toolchain
+build:linux_arm64 --config=cross
+build:linux_arm64 --platforms=@io_bazel_rules_go//go/toolchain:linux_arm64_cgo
+build:linux_arm64 --copt=-funsafe-math-optimizations
+build:linux_arm64 --copt=-ftree-vectorize
+build:linux_arm64 --copt=-fomit-frame-pointer
+build:linux_arm64 --cpu=aarch64
+build:linux_arm64 --compiler=clang
+build:linux_arm64 --copt=-march=armv8-a
+
+
+# toolchain build debug configs
+#------------------------------
+build:debug --sandbox_debug
+build:debug --toolchain_resolution_debug
+build:debug --verbose_failures
+build:debug -s
+
+# windows debug
+build:windows_amd64_debug --config=windows_amd64
+build:windows_amd64_debug --config=debug
+
+# osx_amd64 debug config
+build:osx_amd64_debug --config=debug
+build:osx_amd64_debug --config=osx_amd64
+
+# linux_arm64_debug
+build:linux_arm64_debug --config=linux_arm64
+build:linux_arm64_debug --config=debug
+
+# linux_amd64_debug
+build:linux_amd64_debug --config=linux_amd64
+build:linux_amd64_debug --config=debug
+
+
+# Docker Sandbox Configs
+#-----------------------
+# Note all docker sandbox configs must run from a linux x86_64 host
+# build:docker-sandbox --experimental_docker_image=gcr.io/prysmaticlabs/rbe-worker:latest
+build:docker-sandbox --spawn_strategy=docker --strategy=Javac=docker --genrule_strategy=docker
+build:docker-sandbox --define=EXECUTOR=remote
+build:docker-sandbox --experimental_docker_verbose
+build:docker-sandbox --experimental_enable_docker_sandbox
+build:docker-sandbox --crosstool_top=@rbe_ubuntu_clang//cc:toolchain 
+build:docker-sandbox --host_javabase=@rbe_ubuntu_clang//java:jdk 
+build:docker-sandbox --javabase=@rbe_ubuntu_clang//java:jdk 
+build:docker-sandbox --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 
+build:docker-sandbox --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 
+build:docker-sandbox --extra_execution_platforms=@rbe_ubuntu_clang//config:platform 
+build:docker-sandbox --host_platform=@rbe_ubuntu_clang//config:platform 
+build:docker-sandbox --platforms=@rbe_ubuntu_clang//config:platform
+build:docker-sandbox --extra_toolchains=@prysm_toolchains//:cc-toolchain-multiarch
+
+# windows_amd64 docker sandbox build config
+build:windows_amd64_docker --config=docker-sandbox --config=windows_amd64
+build:windows_amd64_docker_debug --config=windows_amd64_docker --config=debug
+
+# osx_amd64 docker sandbox build config
+build:osx_amd64_docker --config=docker-sandbox --config=osx_amd64
+build:osx_amd64_docker_debug --config=osx_amd64_docker --config=debug
+
+# linux_arm64 docker sandbox build config
+build:linux_arm64_docker --config=docker-sandbox --config=linux_arm64
+build:linux_arm64_docker_debug --config=linux_arm64_docker --config=debug
+
+# linux_amd64 docker sandbox build config
+build:linux_amd64_docker --config=docker-sandbox --config=linux_amd64
+build:linux_amd64_docker_debug --config=linux_amd64_docker --config=debug
+
+
+# Remote Build Execution
+#-----------------------
+# Originally from https://github.com/bazelbuild/bazel-toolchains/blob/master/bazelrc/bazel-2.0.0.bazelrc
+#
+# Depending on how many machines are in the remote execution instance, setting
+# this higher can make builds faster by allowing more jobs to run in parallel.
+# Setting it too high can result in jobs that timeout, however, while waiting
+# for a remote machine to execute them.
+build:remote --jobs=50
+
+# Set several flags related to specifying the platform, toolchain and java
+# properties.
+# These flags should only be used as is for the rbe-ubuntu16-04 container
+# and need to be adapted to work with other toolchain containers.
+build:remote --host_javabase=@rbe_ubuntu_clang//java:jdk
+build:remote --javabase=@rbe_ubuntu_clang//java:jdk
+build:remote --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
+build:remote --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
+build:remote --crosstool_top=@rbe_ubuntu_clang//cc:toolchain
+build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
+# Platform flags:
+# The toolchain container used for execution is defined in the target indicated
+# by "extra_execution_platforms", "host_platform" and "platforms".
+# More about platforms: https://docs.bazel.build/versions/master/platforms.html
+build:remote --extra_toolchains=@rbe_ubuntu_clang//config:cc-toolchain
+build:remote --extra_execution_platforms=@rbe_ubuntu_clang//config:platform
+build:remote --host_platform=@rbe_ubuntu_clang//config:platform
+build:remote --platforms=@rbe_ubuntu_clang//config:platform
+
+# Starting with Bazel 0.27.0 strategies do not need to be explicitly
+# defined. See https://github.com/bazelbuild/bazel/issues/7480
+build:remote --define=EXECUTOR=remote
+
+# Enable remote execution so actions are performed on the remote systems.
+# build:remote --remote_executor=grpcs://remotebuildexecution.googleapis.com
+
+# Enforce stricter environment rules, which eliminates some non-hermetic
+# behavior and therefore improves both the remote cache hit rate and the
+# correctness and repeatability of the build.
+build:remote --incompatible_strict_action_env=true
+
+# Set a higher timeout value, just in case.
+build:remote --remote_timeout=3600
+
+# Enable authentication. This will pick up application default credentials by
+# default. You can use --google_credentials=some_file.json to use a service
+# account credential instead.
+# build:remote --google_default_credentials=true
+
+# Enable build without the bytes
+# See: https://github.com/bazelbuild/bazel/issues/6862
+build:remote --experimental_remote_download_outputs=toplevel --experimental_inmemory_jdeps_files --experimental_inmemory_dotd_files
+
+build:remote --remote_local_fallback

.bazelversion

@@ -0,0 +1 @@
+2.1.1

.buildkite-bazelrc

@@ -45,3 +45,6 @@ build --stamp
 test --local_test_jobs=2
 # Disabled race detection due to unstable test results under constrained environment build kite
 # build --features=race
+
+# Enable kafka for CI tests only.
+test --define kafka_enabled=true

.gitignore

@@ -4,6 +4,9 @@ bazel-*
 .DS_Store
 .swp
 
+# Ignore VI/Vim swapfiles
+.*.sw?
+
 # IntelliJ
 .idea
 .ijwb
@@ -26,3 +29,6 @@ password.txt
 # go dependancy
 /go.mod
 /go.sum
+
+# Dist files
+dist

INTEROP.md

@@ -57,7 +57,7 @@ the system with 64 validators and the genesis time set to the current unix times
 Wait a bit until your beacon chain starts, and in the other window:
 
 ```
-bazel run //validator -- --interop-num-validators 64
+bazel run //validator -- --keymanager=interop --keymanageropts='{"keys":64}'
 ```
 
 This will launch and kickstart the system with your 64 validators performing their duties accordingly.
@@ -78,13 +78,7 @@ Assuming you generated a `genesis.ssz` file with 64 validators, open up two term
 Wait a bit until your beacon chain starts, and in the other window:
 
 ```
-bazel run //validator -- --interop-num-validators 64
+bazel run //validator -- --keymanager=interop --keymanageropts='{"keys":64}'
 ```
 
 This will launch and kickstart the system with your 64 validators performing their duties accordingly.
-
-
-
-
-
-

README.md

@@ -8,7 +8,7 @@
 This is the core repository for Prysm, a [Golang](https://golang.org/) implementation of the Ethereum 2.0 client specifications developed by [Prysmatic Labs](https://prysmaticlabs.com).
 
 ### Need assistance?
-A more detailed set of installation and usage instructions as well as breakdowns of each individual component are available in the [official documentation portal](https://prysmaticlabs.gitbook.io/prysm/). If you still have questions, feel free to stop by either our [Discord](https://discord.gg/KSA7rPr) or [Gitter](https://gitter.im/prysmaticlabs/geth-sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) and a member of the team or our community will be happy to assist you.
+A more detailed set of installation and usage instructions as well as breakdowns of each individual component are available in the [official documentation portal](https://docs.prylabs.network). If you still have questions, feel free to stop by either our [Discord](https://discord.gg/KSA7rPr) or [Gitter](https://gitter.im/prysmaticlabs/geth-sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) and a member of the team or our community will be happy to assist you.
 
 ### Come join the testnet!
 Participation is now open to the public for our Ethereum 2.0 phase 0 testnet release. Visit [prylabs.net](https://prylabs.net) for more information on the project or to sign up as a validator on the network.
@@ -23,7 +23,7 @@ Participation is now open to the public for our Ethereum 2.0 phase 0 testnet rel
     - [Running via Docker](#running-via-docker)
     - [Running via Bazel](#running-via-bazel)
 - [Staking ETH: running a validator client](#staking-eth-running-a-validator-client)
-    - [Activating your validator: depositing 3.2 Goerli ETH](#activating-your-validator-depositing-32-göerli-eth)
+    - [Activating your validator: depositing 3.2 Goerli ETH](#activating-your-validator-depositing-32-gerli-eth)
     - [Starting the validator with Bazel](#starting-the-validator-with-bazel)
 - [Setting up a local ETH2 development chain](#setting-up-a-local-eth2-development-chain)
     - [Installation and dependencies](#installation-and-dependencies) 
@@ -92,7 +92,7 @@ Bazel will automatically pull and install any dependencies as well, including Go
 
 ## Connecting to the testnet: running a beacon node
 
-Below are instructions for initialising a beacon node and connecting to the public testnet. To further understand the role that the beacon node plays in Prysm, see [this section of the documentation.](https://prysmaticlabs.gitbook.io/prysm/how-prysm-works/overview-technical)
+Below are instructions for initialising a beacon node and connecting to the public testnet. To further understand the role that the beacon node plays in Prysm, see [this section of the documentation.](https://docs.prylabs.network/docs/how-prysm-works/architecture-overview/)
 
 
 **NOTE:** It is recommended to open up port 13000 on your local router to improve connectivity and receive more peers from the network. To do so, navigate to `192.168.0.1` in your browser and login if required. Follow along with the interface to modify your routers firewall settings. When this task is completed, append the parameter`--p2p-host-ip=$(curl -s ident.me)` to your selected beacon startup command presented in this section to use the newly opened port.
@@ -104,10 +104,9 @@ Below are instructions for initialising a beacon node and connecting to the publ
 To start your beacon node, issue the following command:
 
 ```text
-docker run -it -v $HOME/prysm:/data -p 4000:4000 --name beacon-node \
+docker run -it -v $HOME/prysm:/data -p 4000:4000 -p 13000:13000 --name beacon-node \
   gcr.io/prysmaticlabs/prysm/beacon-chain:latest \
-  --datadir=/data \
-  --init-sync-no-verify
+  --datadir=/data
 ```
 
 The beacon node can be halted by either using `Ctrl+c` or with the command:
@@ -131,7 +130,7 @@ docker rm beacon-node
 To recreate a deleted container and refresh the chain database, issue the start command with an additional `--clear-db` parameter:
 
 ```text
-docker run -it -v $HOME/prysm:/data -p 4000:4000 --name beacon-node \
+docker run -it -v $HOME/prysm:/data -p 4000:4000 -p 13000:13000 --name beacon-node \
   gcr.io/prysmaticlabs/prysm/beacon-chain:latest \
   --datadir=/data \
   --clear-db
@@ -149,7 +148,7 @@ docker run -it -v $HOME/prysm:/data -p 4000:4000 --name beacon-node \
 3. To run the beacon node, issue the following command:
 
 ```text
-docker run -it -v c:/prysm/:/data -p 4000:4000 gcr.io/prysmaticlabs/prysm/beacon-chain:latest --datadir=/data --init-sync-no-verify --clear-db
+docker run -it -v c:/prysm/:/data -p 4000:4000 -p 13000:13000 --name beacon-node gcr.io/prysmaticlabs/prysm/beacon-chain:latest --datadir=/data --clear-db
 ```
 
 ### Running via Bazel
@@ -168,13 +167,13 @@ This will sync up the beacon node with the latest head block in the network.
 
 ## Staking ETH: Running a validator client
 
-Once your beacon node is up, the chain will be waiting for you to deposit 3.2 Goerli ETH into a [validator deposit contract](how-prysm-works/validator-deposit-contract.md) in order to activate your validator \(discussed in the section below\). First though, you will need to create this validator and connect to this node to participate in consensus.
+Once your beacon node is up, the chain will be waiting for you to deposit 3.2 Goerli ETH into a [validator deposit contract](https://docs.prylabs.network/docs/how-prysm-works/validator-deposit-contract) in order to activate your validator \(discussed in the section below\). First though, you will need to create this validator and connect to this node to participate in consensus.
 
 Each validator represents 3.2 Goerli ETH being staked in the system, and it is possible to spin up as many as you desire in order to have more stake in the network.
 
 ### Activating your validator: depositing 3.2 Göerli ETH
 
-To begin setting up a validator, follow the instructions found on [prylabs.net](https://prylabs.net) to use the Göerli ETH faucet and make a deposit. For step-by-step assistance with the deposit page, see the [Activating a Validator ](activating-a-validator.md)section of this documentation.
+To begin setting up a validator, follow the instructions found on [prylabs.net](https://prylabs.net) to use the Göerli ETH faucet and make a deposit. For step-by-step assistance with the deposit page, see the [Activating a Validator ](https://docs.prylabs.network/docs/activating-a-validator)section of this documentation.
 
 It will take a while for the nodes in the network to process a deposit. Once the node is active, the validator will immediately begin performing its responsibilities.
 
@@ -198,7 +197,7 @@ Open up two terminal windows. In the first, issue the command:
 
 ```text
 bazel run //beacon-chain -- \
---no-genesis-delay \
+--custom-genesis-delay=0 \
 --bootstrap-node= \
 --deposit-contract $(curl https://prylabs.net/contract) \
 --clear-db \
@@ -209,7 +208,7 @@ bazel run //beacon-chain -- \
 Wait a moment for the beacon chain to start. In the other terminal, issue the command:
 
 ```text
-bazel run //validator -- --interop-num-validators 64
+bazel run //validator -- --keymanager=interop --keymanageropts='{"keys":64}'
 ```
 
 This command will kickstart the system with your 64 validators performing their duties accordingly.
@@ -230,7 +229,7 @@ golangci-lint run
 
 
 ## Contributing
-Want to get involved? Check out our [Contribution Guide](https://prysmaticlabs.gitbook.io/prysm/getting-involved/contribution-guidelines) to learn more!
+Want to get involved? Check out our [Contribution Guide](https://docs.prylabs.network/docs/contribute/contribution-guidelines/) to learn more!
 
 ## License
 [GNU General Public License v3.0](https://www.gnu.org/licenses/gpl-3.0.en.html)

WORKSPACE

@@ -3,6 +3,46 @@ workspace(name = "prysm")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 
+http_archive(
+    name = "bazel_toolchains",
+    sha256 = "b5a8039df7119d618402472f3adff8a1bd0ae9d5e253f53fcc4c47122e91a3d2",
+    strip_prefix = "bazel-toolchains-2.1.1",
+    urls = [
+        "https://github.com/bazelbuild/bazel-toolchains/releases/download/2.1.1/bazel-toolchains-2.1.1.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/2.1.1.tar.gz",
+    ],
+)
+
+http_archive(
+    name = "com_grail_bazel_toolchain",
+    sha256 = "0bec89e35d8a141c87f28cfc506d6d344785c8eb2ff3a453140a1fe972ada79d",
+    strip_prefix = "bazel-toolchain-77a87103145f86f03f90475d19c2c8854398a444",
+    urls = ["https://github.com/grailbio/bazel-toolchain/archive/77a87103145f86f03f90475d19c2c8854398a444.tar.gz"],
+)
+
+load("@com_grail_bazel_toolchain//toolchain:deps.bzl", "bazel_toolchain_dependencies")
+
+bazel_toolchain_dependencies()
+
+load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "llvm_toolchain")
+
+llvm_toolchain(
+    name = "llvm_toolchain",
+    llvm_version = "9.0.0",
+)
+
+load("@llvm_toolchain//:toolchains.bzl", "llvm_register_toolchains")
+
+llvm_register_toolchains()
+
+load("@prysm//tools/cross-toolchain:prysm_toolchains.bzl", "configure_prysm_toolchains")
+
+configure_prysm_toolchains()
+
+load("@prysm//tools/cross-toolchain:rbe_toolchains_config.bzl", "rbe_toolchains_config")
+
+rbe_toolchains_config()
+
 http_archive(
     name = "bazel_skylib",
     sha256 = "2ea8a5ed2b448baf4a6855d3ce049c4c452a6470b1efd1504fdb7c1c134d220a",
@@ -28,9 +68,9 @@ http_archive(
 
 http_archive(
     name = "io_bazel_rules_docker",
-    #    sha256 = "9ff889216e28c918811b77999257d4ac001c26c1f7c7fb17a79bc28abf74182e",
-    strip_prefix = "rules_docker-0.12.1",
-    url = "https://github.com/bazelbuild/rules_docker/archive/v0.12.1.tar.gz",
+    sha256 = "dc97fccceacd4c6be14e800b2a00693d5e8d07f69ee187babfd04a80a9f8e250",
+    strip_prefix = "rules_docker-0.14.1",
+    url = "https://github.com/bazelbuild/rules_docker/archive/v0.14.1.tar.gz",
 )
 
 http_archive(
@@ -52,22 +92,21 @@ git_repository(
     name = "graknlabs_bazel_distribution",
     commit = "962f3a7e56942430c0ec120c24f9e9f2a9c2ce1a",
     remote = "https://github.com/graknlabs/bazel-distribution",
-    shallow_since = "1563544980 +0300",
+    shallow_since = "1569509514 +0300",
 )
 
 # Override default import in rules_go with special patch until
 # https://github.com/gogo/protobuf/pull/582 is merged.
 git_repository(
     name = "com_github_gogo_protobuf",
-    # v1.3.0 (latest) as of 2019-10-05
-    commit = "0ca988a254f991240804bf9821f3450d87ccbb1b",
+    commit = "5628607bb4c51c3157aacc3a50f0ab707582b805",
     patch_args = ["-p1"],
     patches = [
         "@io_bazel_rules_go//third_party:com_github_gogo_protobuf-gazelle.patch",
         "//third_party:com_github_gogo_protobuf-equal.patch",
     ],
     remote = "https://github.com/gogo/protobuf",
-    shallow_since = "1567336231 +0200",
+    shallow_since = "1571033717 +0200",
     # gazelle args: -go_prefix github.com/gogo/protobuf -proto legacy
 )
 
@@ -78,6 +117,10 @@ load(
 
 container_repositories()
 
+load("@prysm//third_party/herumi:herumi.bzl", "bls_dependencies")
+
+bls_dependencies()
+
 load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
 
 go_rules_dependencies()
@@ -97,15 +140,8 @@ load(
     _go_image_repos = "repositories",
 )
 
-_go_image_repos()
-
 # Golang images
 # This is using gcr.io/distroless/base
-load(
-    "@io_bazel_rules_docker//go:image.bzl",
-    _go_image_repos = "repositories",
-)
-
 _go_image_repos()
 
 # CC images
@@ -125,16 +161,16 @@ proto_library(
   srcs = ["src/proto/faucet.proto"],
   visibility = ["//visibility:public"],
 )""",
-    sha256 = "1184e44a7a9b8b172e68e82c02cc3b15a80122340e05a92bd1edeafe5e68debe",
-    strip_prefix = "prysm-testnet-site-ec6a4a4e421bf4445845969167d06e93ee8d7acc",
-    url = "https://github.com/prestonvanloon/prysm-testnet-site/archive/ec6a4a4e421bf4445845969167d06e93ee8d7acc.tar.gz",
+    sha256 = "29742136ff9faf47343073c4569a7cf21b8ed138f726929e09e3c38ab83544f7",
+    strip_prefix = "prysm-testnet-site-5c711600f0a77fc553b18cf37b880eaffef4afdb",
+    url = "https://github.com/prestonvanloon/prysm-testnet-site/archive/5c711600f0a77fc553b18cf37b880eaffef4afdb.tar.gz",
 )
 
 http_archive(
     name = "io_kubernetes_build",
-    sha256 = "5ab110312cd7665a1940ba0523b67b9fbb6053beb9dd4e147643867bebd7e809",
-    strip_prefix = "repo-infra-db6ceb5f992254db76af7c25db2edc5469b5ea82",
-    url = "https://github.com/kubernetes/repo-infra/archive/db6ceb5f992254db76af7c25db2edc5469b5ea82.tar.gz",
+    sha256 = "b84fbd1173acee9d02a7d3698ad269fdf4f7aa081e9cecd40e012ad0ad8cfa2a",
+    strip_prefix = "repo-infra-6537f2101fb432b679f3d103ee729dd8ac5d30a0",
+    url = "https://github.com/kubernetes/repo-infra/archive/6537f2101fb432b679f3d103ee729dd8ac5d30a0.tar.gz",
 )
 
 http_archive(
@@ -149,8 +185,8 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "72c6ee3c20d19736b1203f364a6eb0ddee2c173073e20bee2beccd288fdc42be",
-    url = "https://github.com/ethereum/eth2.0-spec-tests/releases/download/v0.9.4/general.tar.gz",
+    sha256 = "abba615d62ff895774fb354e99ea7fb11bd442760b3d1d2cd57e605a58be566c",
+    url = "https://github.com/ethereum/eth2.0-spec-tests/releases/download/v0.10.1/general.tar.gz",
 )
 
 http_archive(
@@ -165,8 +201,8 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "a3cc860a3679f6f62ee57b65677a9b48a65fdebb151cdcbf50f23852632845ef",
-    url = "https://github.com/ethereum/eth2.0-spec-tests/releases/download/v0.9.4/minimal.tar.gz",
+    sha256 = "f5fc250f1b73f14d9ebdb7f47371cae2ef9aa2175e37a69ab2e44ffa4baa16c4",
+    url = "https://github.com/ethereum/eth2.0-spec-tests/releases/download/v0.10.1/minimal.tar.gz",
 )
 
 http_archive(
@@ -181,8 +217,8 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "8fc1b6220973ca30fa4ddc4ed24d66b1719abadca8bedb5e06c3bd9bc0df28e9",
-    url = "https://github.com/ethereum/eth2.0-spec-tests/releases/download/v0.9.4/mainnet.tar.gz",
+    sha256 = "dbafe15d00ff6d75b7cb32a2ca7dc45ded786db8db7100e027e443f75469cfcf",
+    url = "https://github.com/ethereum/eth2.0-spec-tests/releases/download/v0.10.1/mainnet.tar.gz",
 )
 
 http_archive(
@@ -192,13 +228,6 @@ http_archive(
     url = "https://github.com/bazelbuild/buildtools/archive/bf564b4925ab5876a3f64d8b90fab7f769013d42.zip",
 )
 
-http_archive(
-    name = "com_github_herumi_bls_eth_go_binary",
-    sha256 = "15a41ddb0bf7d142ebffae68337f19c16e747676cb56794c5d80dbe388ce004c",
-    strip_prefix = "bls-go-binary-ac038c7cb6d3185c4a46f3bca0c99ebf7b191e16",
-    url = "https://github.com/nisdas/bls-go-binary/archive/ac038c7cb6d3185c4a46f3bca0c99ebf7b191e16.zip",
-)
-
 load("@com_github_bazelbuild_buildtools//buildifier:deps.bzl", "buildifier_dependencies")
 
 buildifier_dependencies()
@@ -211,7 +240,7 @@ go_repository(
 
 git_repository(
     name = "com_google_protobuf",
-    commit = "d09d649aea36f02c03f8396ba39a8d4db8a607e4",
+    commit = "4cf5bfee9546101d98754d23ff378ff718ba8438",
     remote = "https://github.com/protocolbuffers/protobuf",
     shallow_since = "1558721209 -0700",
 )
@@ -225,8 +254,9 @@ all_content = """filegroup(name = "all", srcs = glob(["**"]), visibility = ["//v
 
 http_archive(
     name = "rules_foreign_cc",
-    strip_prefix = "rules_foreign_cc-master",
-    url = "https://github.com/bazelbuild/rules_foreign_cc/archive/master.zip",
+    sha256 = "450563dc2938f38566a59596bb30a3e905fbbcc35b3fff5a1791b122bc140465",
+    strip_prefix = "rules_foreign_cc-456425521973736ef346d93d3d6ba07d807047df",
+    url = "https://github.com/bazelbuild/rules_foreign_cc/archive/456425521973736ef346d93d3d6ba07d807047df.zip",
 )
 
 load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
@@ -238,10 +268,18 @@ rules_foreign_cc_dependencies([
 http_archive(
     name = "librdkafka",
     build_file_content = all_content,
+    sha256 = "f6be27772babfdacbbf2e4c5432ea46c57ef5b7d82e52a81b885e7b804781fd6",
     strip_prefix = "librdkafka-1.2.1",
     urls = ["https://github.com/edenhill/librdkafka/archive/v1.2.1.tar.gz"],
 )
 
+http_archive(
+    name = "com_github_herumi_bls_eth_go_binary",
+    sha256 = "3b155ff597c307b5b0875c3b1ffc3beaa0a6634ee1cfad8768041b61c47eac39",
+    strip_prefix = "bls-eth-go-binary-946ee977cd0f585757741dda65073cbd84a160ac",
+    url = "https://github.com/prysmaticlabs/bls-eth-go-binary/archive/946ee977cd0f585757741dda65073cbd84a160ac.zip",
+)
+
 # External dependencies
 
 go_repository(
@@ -295,7 +333,7 @@ go_repository(
 
 go_repository(
     name = "com_github_libp2p_go_libp2p",
-    commit = "c1687281a5c19b61ee5e0dc07fad15697c3bde94",  # v0.4.0
+    commit = "76944c4fc848530530f6be36fb22b70431ca506c",  # v0.5.1
     importpath = "github.com/libp2p/go-libp2p",
 )
 
@@ -314,7 +352,7 @@ go_repository(
 
 go_repository(
     name = "com_github_multiformats_go_multiaddr",
-    commit = "f96df18bf0c217c77f6cc0f9e810a178cea12f38",  # v0.1.1
+    commit = "8c6cee15b340d7210c30a82a19231ee333b69b1d",  # v0.2.0
     importpath = "github.com/multiformats/go-multiaddr",
 )
 
@@ -326,7 +364,7 @@ go_repository(
 
 go_repository(
     name = "com_github_multiformats_go_multihash",
-    commit = "249ead2008065c476a2ee45e8e75e8b85d846a72",  # v0.0.8
+    commit = "6b39927dce4869bc1726861b65ada415ee1f7fc7",  # v0.0.13
     importpath = "github.com/multiformats/go-multihash",
 )
 
@@ -344,13 +382,13 @@ go_repository(
 
 go_repository(
     name = "com_github_libp2p_go_libp2p_peerstore",
-    commit = "f4c9af195c69379f1cf284dba31985482a56f78e",  # v0.1.3
+    commit = "dee88d7532302c001604811fa3fbb5a7f83225e7",  # v0.1.4
     importpath = "github.com/libp2p/go-libp2p-peerstore",
 )
 
 go_repository(
     name = "com_github_libp2p_go_libp2p_circuit",
-    commit = "0305622f3f146485f0ff6df0ae6c010787331ca7",  # v0.1.3
+    commit = "61af9db0dd78e01e53b9fb044be44dcc7255667e",  # v0.1.4
     importpath = "github.com/libp2p/go-libp2p-circuit",
 )
 
@@ -429,7 +467,7 @@ go_repository(
 go_repository(
     name = "com_github_libp2p_go_libp2p_secio",
     build_file_proto_mode = "disable_global",
-    commit = "7c3f577d99debb69c3b68be35fe14d9445a6569c",  # v0.2.0
+    commit = "6f83420d5715a8b1c4082aaf9c5c7785923e702e",  # v0.2.1
     importpath = "github.com/libp2p/go-libp2p-secio",
 )
 
@@ -447,7 +485,7 @@ go_repository(
 
 go_repository(
     name = "com_github_jbenet_goprocess",
-    commit = "1dc239722b2ba3784472fb5301f62640fa5a8bc3",  # v0.1.3
+    commit = "7f9d9ed286badffcf2122cfeb383ec37daf92508",
     importpath = "github.com/jbenet/goprocess",
 )
 
@@ -465,7 +503,7 @@ go_repository(
 
 go_repository(
     name = "com_github_libp2p_go_libp2p_nat",
-    commit = "c50c291a61bceccb914366d93eb24f58594e9134",  # v0.0.4
+    commit = "873ef75f6ab6273821d77197660c1fb3af4cc02e",  # v0.0.5
     importpath = "github.com/libp2p/go-libp2p-nat",
 )
 
@@ -483,7 +521,7 @@ go_repository(
 
 go_repository(
     name = "com_github_mattn_go_isatty",
-    commit = "e1f7b56ace729e4a73a29a6b4fac6cd5fcda7ab3",  # v0.0.9
+    commit = "7b513a986450394f7bbf1476909911b3aa3a55ce",
     importpath = "github.com/mattn/go-isatty",
 )
 
@@ -567,7 +605,7 @@ go_repository(
 
 go_repository(
     name = "com_github_libp2p_go_flow_metrics",
-    commit = "1f5b3acc846b2c8ce4c4e713296af74f5c24df55",  # v0.0.1
+    commit = "e5a6a4db89199d99b2a74b8da198277a826241d8",  # v0.0.3
     importpath = "github.com/libp2p/go-flow-metrics",
 )
 
@@ -591,7 +629,7 @@ go_repository(
 
 go_repository(
     name = "com_github_libp2p_go_ws_transport",
-    commit = "8cca0dbc7f3533b122bd2cbeaa4a9b07c2913b9d",  # v0.1.2
+    commit = "370d1a3a7420e27423417c37630cad3754ad5702",  # v0.2.0
     importpath = "github.com/libp2p/go-ws-transport",
 )
 
@@ -631,6 +669,12 @@ go_repository(
     importpath = "github.com/syndtr/goleveldb",
 )
 
+go_repository(
+    name = "com_github_emicklei_dot",
+    commit = "5810de2f2ab7aac98cd7bcbd59147a7ca6071768",
+    importpath = "github.com/emicklei/dot",
+)
+
 go_repository(
     name = "com_github_libp2p_go_libp2p_blankhost",
     commit = "da3b45205dfce3ef3926054ffd5dee76f5903382",  # v0.1.4
@@ -690,19 +734,19 @@ go_repository(
 
 go_repository(
     name = "com_github_prometheus_client_model",
-    commit = "fd36f4220a901265f90734c3183c5f0c91daa0b8",
+    commit = "7bc5445566f0fe75b15de23e6b93886e982d7bf9",
     importpath = "github.com/prometheus/client_model",
 )
 
 go_repository(
     name = "com_github_prometheus_common",
-    commit = "287d3e634a1e550c9e463dd7e5a75a422c614505",  # v0.7.0
+    commit = "d978bcb1309602d68bb4ba69cf3f8ed900e07308",
     importpath = "github.com/prometheus/common",
 )
 
 go_repository(
     name = "com_github_prometheus_procfs",
-    commit = "499c85531f756d1129edd26485a5f73871eeb308",  # v0.0.5
+    commit = "6d489fc7f1d9cd890a250f3ea3431b1744b9623f",
     importpath = "github.com/prometheus/procfs",
 )
 
@@ -771,7 +815,7 @@ go_repository(
 
 go_repository(
     name = "com_github_ipfs_go_datastore",
-    commit = "d0ca9bc39f9d5b77bd602abe1a897473e105be7f",  # v0.1.1
+    commit = "e7a498916ccca1b0b40fb08630659cd4d68a01e8",  # v0.3.1
     importpath = "github.com/ipfs/go-datastore",
 )
 
@@ -783,14 +827,14 @@ go_repository(
 
 go_repository(
     name = "com_github_ipfs_go_cid",
-    commit = "9bb7ea69202c6c9553479eb355ab8a8a97d43a2e",  # v0.0.3
+    commit = "3da5bbbe45260437a44f777e6b2e5effa2606901",  # v0.0.4
     importpath = "github.com/ipfs/go-cid",
 )
 
 go_repository(
     name = "com_github_libp2p_go_libp2p_record",
     build_file_proto_mode = "disable_global",
-    commit = "3f535b1abcdf698e11ac16f618c2e64c4e5a114a",  # v0.1.1
+    commit = "8ccbca30634f70a8f03d133ac64cbf245d079e1e",  # v0.1.2
     importpath = "github.com/libp2p/go-libp2p-record",
 )
 
@@ -802,7 +846,7 @@ go_repository(
 
 go_repository(
     name = "com_github_libp2p_go_libp2p_kbucket",
-    commit = "8b77351e0f784a5f71749d23000897c8aee71a76",  # v0.2.1
+    commit = "a0cac6f63c491504b18eeba24be2ac0bbbfa0e5c",  # v0.2.3
     importpath = "github.com/libp2p/go-libp2p-kbucket",
 )
 
@@ -826,7 +870,7 @@ go_repository(
 
 go_repository(
     name = "com_github_hashicorp_golang_lru",
-    commit = "7f827b33c0f158ec5dfbba01bb0b14a4541fd81d",  # v0.5.3
+    commit = "14eae340515388ca95aa8e7b86f0de668e981f54",  # v0.5.4
     importpath = "github.com/hashicorp/golang-lru",
 )
 
@@ -845,7 +889,7 @@ go_repository(
 
 go_repository(
     name = "com_github_libp2p_go_libp2p_autonat",
-    commit = "3464f9b4f7bfbd7bb008813eacb626c7ab7fb9a3",  # v0.1.0
+    commit = "60bf479cf6bc73c939f4db97ad711756e949e522",  # v0.1.1
     importpath = "github.com/libp2p/go-libp2p-autonat",
 )
 
@@ -871,7 +915,7 @@ go_repository(
 go_repository(
     name = "io_k8s_apimachinery",
     build_file_proto_mode = "disable_global",
-    commit = "bfcf53abc9f82bad3e534fcb1c36599d3c989ebf",
+    commit = "79c2a76c473a20cdc4ce59cae4b72529b5d9d16b",  # v0.17.2
     importpath = "k8s.io/apimachinery",
 )
 
@@ -953,7 +997,7 @@ go_repository(
 
 go_repository(
     name = "com_google_cloud_go",
-    commit = "264def2dd949cdb8a803bb9f50fa29a67b798a6a",  # v0.46.3
+    commit = "6daa679260d92196ffca2362d652c924fdcb7a22",  # v0.52.0
     importpath = "cloud.google.com/go",
 )
 
@@ -995,7 +1039,7 @@ go_repository(
 
 go_repository(
     name = "com_github_pkg_errors",
-    commit = "ba968bfe8b2f7e042a574c888954fccecfa385b4",  # v0.8.1
+    commit = "614d223910a179a466c1767a985424175c39b465",  # v0.9.1
     importpath = "github.com/pkg/errors",
 )
 
@@ -1037,7 +1081,7 @@ go_repository(
 
 go_repository(
     name = "com_github_apache_thrift",
-    commit = "384647d290e2e4a55a14b1b7ef1b7e66293a2c33",  # v0.12.0
+    commit = "cecee50308fc7e6f77f55b3fd906c1c6c471fa2f",  # v0.13.0
     importpath = "github.com/apache/thrift",
 )
 
@@ -1049,7 +1093,7 @@ go_repository(
 
 go_repository(
     name = "com_github_libp2p_go_libp2p_connmgr",
-    commit = "b46e9bdbcd8436b4fe4b30a53ec913c07e5e09c9",  # v0.1.1
+    commit = "273839464339f1885413b385feee35301c5cb76f",  # v0.2.1
     importpath = "github.com/libp2p/go-libp2p-connmgr",
 )
 
@@ -1080,13 +1124,13 @@ go_repository(
 go_repository(
     name = "com_github_libp2p_go_libp2p_core",
     build_file_proto_mode = "disable_global",
-    commit = "26b960839df84e2783f8f6125fa822a9978c2b8f",  # v0.2.3
+    commit = "f7f724862d85ec9f9ee7c58b0f79836abdee8cd9",  # v0.3.0
     importpath = "github.com/libp2p/go-libp2p-core",
 )
 
 go_repository(
     name = "com_github_libp2p_go_libp2p_testing",
-    commit = "1fa303da162dc57872d8fc553497f7602aa11c10",  # v0.1.0
+    commit = "82713a62880a5fe72d438bd58d737f0d3c4b7f36",  # v0.1.1
     importpath = "github.com/libp2p/go-libp2p-testing",
 )
 
@@ -1114,6 +1158,12 @@ go_repository(
     importpath = "github.com/multiformats/go-multiaddr-fmt",
 )
 
+go_repository(
+    name = "com_github_multiformats_go_varint",
+    commit = "0aa688902217dff2cba0f678c7e4a0f547b4983e",
+    importpath = "github.com/multiformats/go-varint",
+)
+
 go_repository(
     name = "com_github_libp2p_go_yamux",
     commit = "663972181d409e7263040f0b668462f87c85e1bd",  # v1.2.3
@@ -1122,7 +1172,7 @@ go_repository(
 
 go_repository(
     name = "com_github_libp2p_go_nat",
-    commit = "d13fdefb3bbb2fde2c6fc090a7ea992cec8b26df",  # v0.0.3
+    commit = "4b355d438085545df006ad9349686f30d8d37a27",  # v0.0.4
     importpath = "github.com/libp2p/go-nat",
 )
 
@@ -1187,7 +1237,7 @@ go_repository(
 
 go_repository(
     name = "com_github_googleapis_gnostic",
-    commit = "ab0dd09aa10e2952b28e12ecd35681b20463ebab",  # v0.3.1
+    commit = "896953e6749863beec38e27029c804e88c3144b8",  # v0.4.1
     importpath = "github.com/googleapis/gnostic",
 )
 
@@ -1211,7 +1261,7 @@ go_repository(
 
 go_repository(
     name = "com_github_google_go_cmp",
-    commit = "2d0692c2e9617365a95b295612ac0d4415ba4627",  # v0.3.1
+    commit = "5a6f75716e1203a923a78c9efb94089d857df0f6",  # v0.4.0
     importpath = "github.com/google/go-cmp",
 )
 
@@ -1241,12 +1291,6 @@ go_repository(
     importpath = "k8s.io/utils",
 )
 
-go_repository(
-    name = "com_github_googleapis_gnostic",
-    commit = "25d8b0b6698593f520d9d8dc5a88e6b16ca9ecc0",
-    importpath = "github.com/googleapis/gnostic",
-)
-
 go_repository(
     name = "com_github_patrickmn_go_cache",
     commit = "46f407853014144407b6c2ec7ccc76bf67958d93",
@@ -1255,7 +1299,7 @@ go_repository(
 
 go_repository(
     name = "com_github_prysmaticlabs_ethereumapis",
-    commit = "87118fb893cc6f32b25793d819790fd3bcce3221",
+    commit = "36966ba5fad8447f8ea7fedffa893112146fd362",
     importpath = "github.com/prysmaticlabs/ethereumapis",
     patch_args = ["-p1"],
     patches = [
@@ -1265,8 +1309,9 @@ go_repository(
 
 go_repository(
     name = "com_github_cloudflare_roughtime",
-    commit = "d41fdcee702eb3e5c3296288a453b9340184d37e",
     importpath = "github.com/cloudflare/roughtime",
+    sum = "h1:jeSxE3fepJdhASERvBHI6RFkMhISv6Ir2JUybYLIVXs=",
+    version = "v0.0.0-20200205191924-a69ef1dab727",
 )
 
 go_repository(
@@ -1417,9 +1462,9 @@ go_repository(
 )
 
 go_repository(
-    name = "com_github_emicklei_dot",
-    commit = "f4a04130244d60cef56086d2f649b4b55e9624aa",
-    importpath = "github.com/emicklei/dot",
+    name = "com_github_googleapis_gnostic",
+    commit = "25d8b0b6698593f520d9d8dc5a88e6b16ca9ecc0",
+    importpath = "github.com/googleapis/gnostic",
 )
 
 go_repository(
@@ -1445,7 +1490,7 @@ go_repository(
 
 go_repository(
     name = "com_github_dgraph_io_ristretto",
-    commit = "99d1bbbf28e64530eb246be0568fc7709a35ebdd",
+    commit = "99d1bbbf28e64530eb246be0568fc7709a35ebdd",  # v0.0.1
     importpath = "github.com/dgraph-io/ristretto",
 )
 
@@ -1463,13 +1508,105 @@ go_repository(
 )
 
 go_repository(
-    name = "com_github_dgraph_io_ristretto",
-    commit = "99d1bbbf28e64530eb246be0568fc7709a35ebdd",
-    importpath = "github.com/dgraph-io/ristretto",
+    name = "com_github_kevinms_leakybucket_go",
+    importpath = "github.com/kevinms/leakybucket-go",
+    sum = "h1:oq6BiN7v0MfWCRcJAxSV+hesVMAAV8COrQbTjYNnso4=",
+    version = "v0.0.0-20190611015032-8a3d0352aa79",
 )
 
 go_repository(
-    name = "com_github_cespare_xxhash",
-    commit = "d7df74196a9e781ede915320c11c378c1b2f3a1f",
-    importpath = "github.com/cespare/xxhash",
+    name = "com_github_wealdtech_go_eth2_wallet",
+    importpath = "github.com/wealdtech/go-eth2-wallet",
+    sum = "h1:H/T1n0SNd0jTsbf4rA4YxigsBPFWRUWgobsTOjzW4Hw=",
+    version = "v1.9.2",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_eth2_wallet_hd_v2",
+    importpath = "github.com/wealdtech/go-eth2-wallet-hd/v2",
+    sum = "h1:oqE/+zFOKteklEemecMWGlyNmPv+5OBaHmAo1LKG6LE=",
+    version = "v2.0.0",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_eth2_wallet_nd_v2",
+    importpath = "github.com/wealdtech/go-eth2-wallet-nd/v2",
+    sum = "h1:nWsbiaSVa1kwRdwPX5NfXsrowlRBjqoRpDv37i8ZecE=",
+    version = "v2.0.0",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_eth2_wallet_store_filesystem",
+    importpath = "github.com/wealdtech/go-eth2-wallet-store-filesystem",
+    sum = "h1:px7vV01opCUeeHjvdiBdkPbdnr60Ygq01Ddjy4dIbfg=",
+    version = "v1.7.1",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_eth2_wallet_store_s3",
+    importpath = "github.com/wealdtech/go-eth2-wallet-store-s3",
+    sum = "h1:f86TIVHqYkmDYc8VLsiIJ/KbGtNMeCGhkefqpXUVmYE=",
+    version = "v1.6.1",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_eth2_wallet_encryptor_keystorev4",
+    importpath = "github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4",
+    sum = "h1:IcpS4VpXhYz+TVupB5n6C6IQzaKwG+Rc8nvgCa/da4c=",
+    version = "v1.0.0",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_eth2_wallet_types_v2",
+    importpath = "github.com/wealdtech/go-eth2-wallet-types/v2",
+    sum = "h1:EyTwHO7zXtYkf62h3MqSB3OWc8pv0dnFl41yykUJY3s=",
+    version = "v2.0.0",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_eth2_types_v2",
+    importpath = "github.com/wealdtech/go-eth2-types/v2",
+    sum = "h1:L1Eg55aArRpUR2H8dnpSevHlSGRDuRQbQwA4IyYh0Js=",
+    version = "v2.0.2",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_eth2_util",
+    importpath = "github.com/wealdtech/go-eth2-util",
+    sum = "h1:m56HKJgWSuNy53Gt5GN7HcoFaGRCl1uE3OGWhIhWh1M=",
+    version = "v1.1.2",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_ecodec",
+    importpath = "github.com/wealdtech/go-ecodec",
+    sum = "h1:yggrTSckcPJRaxxOxQF7FPm21kgE8WA6+f5jdq5Kr8o=",
+    version = "v1.1.0",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_bytesutil",
+    importpath = "github.com/wealdtech/go-bytesutil",
+    sum = "h1:6XrN7OIQhhBjQy/PZ1HZ3ySE8v8UDyxzERkOgmsIc1g=",
+    version = "v1.1.0",
+)
+
+go_repository(
+    name = "com_github_wealdtech_go_indexer",
+    importpath = "github.com/wealdtech/go-indexer",
+    sum = "h1:/S4rfWQbSOnnYmwnvuTVatDibZ8o1s9bmTCHO16XINg=",
+    version = "v1.0.0",
+)
+
+go_repository(
+    name = "com_github_shibukawa_configdir",
+    commit = "e180dbdc8da04c4fa04272e875ce64949f38bd3e",
+    importpath = "github.com/shibukawa/configdir",
+)
+
+go_repository(
+    name = "com_github_libp2p_go_libp2p_noise",
+    importpath = "github.com/libp2p/go-libp2p-noise",
+    sum = "h1:J1gHJRNFEk7NdiaPQQqAvxEy+7hhCsVv3uzduWybmqY=",
+    version = "v0.0.0-20200302201340-8c54356e12c9",
 )

bazel.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# This script serves as a wrapper around bazel to limit the scope of environment variables that
+# may change the action output. Using this script should result in a higher cache hit ratio for
+# cached actions with a more heremtic build.
+
+env -i \
+ PATH=/usr/bin:/bin \
+ HOME=$HOME \
+ GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS \
+ bazel "$@"

beacon-chain/BUILD.bazel

@@ -1,7 +1,7 @@
 load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
 load("@io_bazel_rules_docker//go:image.bzl", "go_image")
 load("@io_bazel_rules_docker//container:container.bzl", "container_bundle")
-load("//tools:binary_targets.bzl", "binary_targets")
+load("//tools:binary_targets.bzl", "binary_targets", "go_image_debug")
 load("@io_bazel_rules_docker//contrib:push-all.bzl", "docker_push")
 
 go_library(
@@ -71,12 +71,32 @@ container_bundle(
     tags = ["manual"],
 )
 
+go_image_debug(
+    name = "image_debug",
+    image = ":image",
+)
+
+container_bundle(
+    name = "image_bundle_debug",
+    images = {
+        "gcr.io/prysmaticlabs/prysm/beacon-chain:latest-debug": ":image_debug",
+        "gcr.io/prysmaticlabs/prysm/beacon-chain:{DOCKER_TAG}-debug": ":image_debug",
+    },
+    tags = ["manual"],
+)
+
 docker_push(
     name = "push_images",
     bundle = ":image_bundle",
     tags = ["manual"],
 )
 
+docker_push(
+    name = "push_images_debug",
+    bundle = ":image_bundle_debug",
+    tags = ["manual"],
+)
+
 go_binary(
     name = "beacon-chain",
     embed = [":go_default_library"],

beacon-chain/README.md

@@ -5,6 +5,6 @@ This is the main project folder for the beacon chain implementation of Ethereum
 [![Discord](https://user-images.githubusercontent.com/7288322/34471967-1df7808a-efbb-11e7-9088-ed0b04151291.png)](https://discord.gg/KSA7rPr)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/prysmaticlabs/prysm?badge&utm_medium=badge&utm_campaign=pr-badge)
 
-Also, read the latest beacon chain [design spec](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md), this design spec serves as a source of truth for the beacon chain implementation we follow at prysmatic labs.
+Also, read the latest beacon chain [design spec](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/phase0/beacon-chain.md), this design spec serves as a source of truth for the beacon chain implementation we follow at prysmatic labs.
 Check out the [FAQs](https://notes.ethereum.org/9MMuzWeFTTSg-3Tz_YeiBA?view). Refer this page on [why](http://email.mg2.substack.com/c/eJwlj9GOhCAMRb9G3jRQQPGBh5mM8xsbhKrsDGIAM9m_X9xN2qZtbpt7rCm4xvSjj5gLOTOmL-809CMbKXFaOKakIl4DZYr2AGyQIGjHOnWH22OiYnoIxmDijaBhhS6fcy7GvjobA9m0mSXOcnZq5GBqLkilXBZhBsus5ZK89VbKkRt-a-BZI6DzZ7iur1lQ953KJ9bemnxgahuQU9XJu6pFPdu8meT8vragzEjpMCwMGLlgLo6h5z1JumQTu4IJd4v15xqMf_8ZLP_Y1bSLdbnrD-LL71i2Kj7DLxaWWF4)
 we are combining sharding and casper together.

beacon-chain/archiver/BUILD.bazel

@@ -12,6 +12,7 @@ go_library(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/validators:go_default_library",
         "//beacon-chain/db:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/params:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
@@ -32,6 +33,7 @@ go_test(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/db:go_default_library",
         "//beacon-chain/db/testing:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/params:go_default_library",
         "//shared/testutil:go_default_library",

beacon-chain/archiver/service.go

@@ -12,6 +12,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/validators"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/sirupsen/logrus"
@@ -24,7 +25,7 @@ var log = logrus.WithField("prefix", "archiver")
 type Service struct {
 	ctx                  context.Context
 	cancel               context.CancelFunc
-	beaconDB             db.Database
+	beaconDB             db.NoHeadAccessDatabase
 	headFetcher          blockchain.HeadFetcher
 	participationFetcher blockchain.ParticipationFetcher
 	stateNotifier        statefeed.Notifier
@@ -33,7 +34,7 @@ type Service struct {
 
 // Config options for the archiver service.
 type Config struct {
-	BeaconDB             db.Database
+	BeaconDB             db.NoHeadAccessDatabase
 	HeadFetcher          blockchain.HeadFetcher
 	ParticipationFetcher blockchain.ParticipationFetcher
 	StateNotifier        statefeed.Notifier
@@ -70,7 +71,7 @@ func (s *Service) Status() error {
 }
 
 // We archive committee information pertaining to the head state's epoch.
-func (s *Service) archiveCommitteeInfo(ctx context.Context, headState *pb.BeaconState, epoch uint64) error {
+func (s *Service) archiveCommitteeInfo(ctx context.Context, headState *state.BeaconState, epoch uint64) error {
 	proposerSeed, err := helpers.Seed(headState, epoch, params.BeaconConfig().DomainBeaconProposer)
 	if err != nil {
 		return errors.Wrap(err, "could not generate seed")
@@ -91,15 +92,16 @@ func (s *Service) archiveCommitteeInfo(ctx context.Context, headState *pb.Beacon
 }
 
 // We archive active validator set changes that happened during the previous epoch.
-func (s *Service) archiveActiveSetChanges(ctx context.Context, headState *pb.BeaconState, epoch uint64) error {
+func (s *Service) archiveActiveSetChanges(ctx context.Context, headState *state.BeaconState, epoch uint64) error {
 	prevEpoch := epoch - 1
-	activations := validators.ActivatedValidatorIndices(prevEpoch, headState.Validators)
-	slashings := validators.SlashedValidatorIndices(prevEpoch, headState.Validators)
+	vals := headState.Validators()
+	activations := validators.ActivatedValidatorIndices(prevEpoch, vals)
+	slashings := validators.SlashedValidatorIndices(prevEpoch, vals)
 	activeValidatorCount, err := helpers.ActiveValidatorCount(headState, prevEpoch)
 	if err != nil {
 		return errors.Wrap(err, "could not get active validator count")
 	}
-	exited, err := validators.ExitedValidatorIndices(prevEpoch, headState.Validators, activeValidatorCount)
+	exited, err := validators.ExitedValidatorIndices(prevEpoch, vals, activeValidatorCount)
 	if err != nil {
 		return errors.Wrap(err, "could not determine exited validator indices")
 	}
@@ -130,8 +132,7 @@ func (s *Service) archiveParticipation(ctx context.Context, epoch uint64) error
 }
 
 // We archive validator balances and active indices.
-func (s *Service) archiveBalances(ctx context.Context, headState *pb.BeaconState, epoch uint64) error {
-	balances := headState.Balances
+func (s *Service) archiveBalances(ctx context.Context, balances []uint64, epoch uint64) error {
 	if err := s.beaconDB.SaveArchivedBalances(ctx, epoch, balances); err != nil {
 		return errors.Wrap(err, "could not archive balances")
 	}
@@ -153,12 +154,13 @@ func (s *Service) run(ctx context.Context) {
 					log.WithError(err).Error("Head state is not available")
 					continue
 				}
-				currentEpoch := helpers.CurrentEpoch(headState)
-				if !helpers.IsEpochEnd(headState.Slot) && currentEpoch <= s.lastArchivedEpoch {
+				slot := headState.Slot()
+				currentEpoch := helpers.SlotToEpoch(slot)
+				if !helpers.IsEpochEnd(slot) && currentEpoch <= s.lastArchivedEpoch {
 					continue
 				}
 				epochToArchive := currentEpoch
-				if !helpers.IsEpochEnd(headState.Slot) {
+				if !helpers.IsEpochEnd(slot) {
 					epochToArchive--
 				}
 				if err := s.archiveCommitteeInfo(ctx, headState, epochToArchive); err != nil {
@@ -173,7 +175,7 @@ func (s *Service) run(ctx context.Context) {
 					log.WithError(err).Error("Could not archive validator participation")
 					continue
 				}
-				if err := s.archiveBalances(ctx, headState, epochToArchive); err != nil {
+				if err := s.archiveBalances(ctx, headState.Balances(), epochToArchive); err != nil {
 					log.WithError(err).Error("Could not archive validator balances and active indices")
 					continue
 				}

beacon-chain/archiver/service_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
 	dbutil "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
@@ -27,15 +28,20 @@ import (
 func init() {
 	logrus.SetLevel(logrus.DebugLevel)
 	logrus.SetOutput(ioutil.Discard)
-	params.OverrideBeaconConfig(params.MinimalSpecConfig())
 }
 
 func TestArchiverService_ReceivesBlockProcessedEvent(t *testing.T) {
 	hook := logTest.NewGlobal()
 	svc, beaconDB := setupService(t)
 	defer dbutil.TeardownDB(t, beaconDB)
+	st, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot: 1,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
 	svc.headFetcher = &mock.ChainService{
-		State: &pb.BeaconState{Slot: 1},
+		State: st,
 	}
 
 	event := &feed.Event{
@@ -55,8 +61,14 @@ func TestArchiverService_OnlyArchiveAtEpochEnd(t *testing.T) {
 	svc, beaconDB := setupService(t)
 	defer dbutil.TeardownDB(t, beaconDB)
 	// The head state is NOT an epoch end.
+	st, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot: params.BeaconConfig().SlotsPerEpoch - 2,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
 	svc.headFetcher = &mock.ChainService{
-		State: &pb.BeaconState{Slot: params.BeaconConfig().SlotsPerEpoch - 2},
+		State: st,
 	}
 	event := &feed.Event{
 		Type: statefeed.BlockProcessed,
@@ -81,7 +93,10 @@ func TestArchiverService_ArchivesEvenThroughSkipSlot(t *testing.T) {
 	hook := logTest.NewGlobal()
 	svc, beaconDB := setupService(t)
 	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
+	headState, err := setupState(validatorCount)
+	if err != nil {
+		t.Fatal(err)
+	}
 	defer dbutil.TeardownDB(t, beaconDB)
 	event := &feed.Event{
 		Type: statefeed.BlockProcessed,
@@ -99,7 +114,9 @@ func TestArchiverService_ArchivesEvenThroughSkipSlot(t *testing.T) {
 
 	// Send out an event every slot, skipping the end slot of the epoch.
 	for i := uint64(0); i < params.BeaconConfig().SlotsPerEpoch+1; i++ {
-		headState.Slot = i
+		if err := headState.SetSlot(i); err != nil {
+			t.Fatal(err)
+		}
 		svc.headFetcher = &mock.ChainService{
 			State: headState,
 		}
@@ -130,7 +147,10 @@ func TestArchiverService_ArchivesEvenThroughSkipSlot(t *testing.T) {
 func TestArchiverService_ComputesAndSavesParticipation(t *testing.T) {
 	hook := logTest.NewGlobal()
 	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
+	headState, err := setupState(validatorCount)
+	if err != nil {
+		t.Fatal(err)
+	}
 	svc, beaconDB := setupService(t)
 	defer dbutil.TeardownDB(t, beaconDB)
 	svc.headFetcher = &mock.ChainService{
@@ -168,7 +188,10 @@ func TestArchiverService_ComputesAndSavesParticipation(t *testing.T) {
 func TestArchiverService_SavesIndicesAndBalances(t *testing.T) {
 	hook := logTest.NewGlobal()
 	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
+	headState, err := setupState(validatorCount)
+	if err != nil {
+		t.Fatal(err)
+	}
 	svc, beaconDB := setupService(t)
 	defer dbutil.TeardownDB(t, beaconDB)
 	svc.headFetcher = &mock.ChainService{
@@ -187,11 +210,11 @@ func TestArchiverService_SavesIndicesAndBalances(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(headState.Balances, retrieved) {
+	if !reflect.DeepEqual(headState.Balances(), retrieved) {
 		t.Errorf(
 			"Wanted balances for epoch %d %v, retrieved %v",
 			helpers.CurrentEpoch(headState),
-			headState.Balances,
+			headState.Balances(),
 			retrieved,
 		)
 	}
@@ -201,7 +224,10 @@ func TestArchiverService_SavesIndicesAndBalances(t *testing.T) {
 func TestArchiverService_SavesCommitteeInfo(t *testing.T) {
 	hook := logTest.NewGlobal()
 	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
+	headState, err := setupState(validatorCount)
+	if err != nil {
+		t.Fatal(err)
+	}
 	svc, beaconDB := setupService(t)
 	defer dbutil.TeardownDB(t, beaconDB)
 	svc.headFetcher = &mock.ChainService{
@@ -248,16 +274,33 @@ func TestArchiverService_SavesCommitteeInfo(t *testing.T) {
 func TestArchiverService_SavesActivatedValidatorChanges(t *testing.T) {
 	hook := logTest.NewGlobal()
 	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
+	headState, err := setupState(validatorCount)
+	if err != nil {
+		t.Fatal(err)
+	}
 	svc, beaconDB := setupService(t)
 	defer dbutil.TeardownDB(t, beaconDB)
 	svc.headFetcher = &mock.ChainService{
 		State: headState,
 	}
 	prevEpoch := helpers.PrevEpoch(headState)
-	delayedActEpoch := helpers.DelayedActivationExitEpoch(prevEpoch)
-	headState.Validators[4].ActivationEpoch = delayedActEpoch
-	headState.Validators[5].ActivationEpoch = delayedActEpoch
+	delayedActEpoch := helpers.ActivationExitEpoch(prevEpoch)
+	val1, err := headState.ValidatorAtIndex(4)
+	if err != nil {
+		t.Fatal(err)
+	}
+	val1.ActivationEpoch = delayedActEpoch
+	val2, err := headState.ValidatorAtIndex(5)
+	if err != nil {
+		t.Fatal(err)
+	}
+	val2.ActivationEpoch = delayedActEpoch
+	if err := headState.UpdateValidatorAtIndex(4, val1); err != nil {
+		t.Fatal(err)
+	}
+	if err := headState.UpdateValidatorAtIndex(5, val1); err != nil {
+		t.Fatal(err)
+	}
 	event := &feed.Event{
 		Type: statefeed.BlockProcessed,
 		Data: &statefeed.BlockProcessedData{
@@ -283,15 +326,32 @@ func TestArchiverService_SavesActivatedValidatorChanges(t *testing.T) {
 func TestArchiverService_SavesSlashedValidatorChanges(t *testing.T) {
 	hook := logTest.NewGlobal()
 	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
+	headState, err := setupState(validatorCount)
+	if err != nil {
+		t.Fatal(err)
+	}
 	svc, beaconDB := setupService(t)
 	defer dbutil.TeardownDB(t, beaconDB)
 	svc.headFetcher = &mock.ChainService{
 		State: headState,
 	}
 	prevEpoch := helpers.PrevEpoch(headState)
-	headState.Validators[95].Slashed = true
-	headState.Validators[96].Slashed = true
+	val1, err := headState.ValidatorAtIndex(95)
+	if err != nil {
+		t.Fatal(err)
+	}
+	val1.Slashed = true
+	val2, err := headState.ValidatorAtIndex(96)
+	if err != nil {
+		t.Fatal(err)
+	}
+	val2.Slashed = true
+	if err := headState.UpdateValidatorAtIndex(95, val1); err != nil {
+		t.Fatal(err)
+	}
+	if err := headState.UpdateValidatorAtIndex(96, val1); err != nil {
+		t.Fatal(err)
+	}
 	event := &feed.Event{
 		Type: statefeed.BlockProcessed,
 		Data: &statefeed.BlockProcessedData{
@@ -317,15 +377,25 @@ func TestArchiverService_SavesSlashedValidatorChanges(t *testing.T) {
 func TestArchiverService_SavesExitedValidatorChanges(t *testing.T) {
 	hook := logTest.NewGlobal()
 	validatorCount := uint64(100)
-	headState := setupState(t, validatorCount)
+	headState, err := setupState(validatorCount)
+	if err != nil {
+		t.Fatal(err)
+	}
 	svc, beaconDB := setupService(t)
 	defer dbutil.TeardownDB(t, beaconDB)
 	svc.headFetcher = &mock.ChainService{
 		State: headState,
 	}
 	prevEpoch := helpers.PrevEpoch(headState)
-	headState.Validators[95].ExitEpoch = prevEpoch
-	headState.Validators[95].WithdrawableEpoch = prevEpoch + params.BeaconConfig().MinValidatorWithdrawabilityDelay
+	val, err := headState.ValidatorAtIndex(95)
+	if err != nil {
+		t.Fatal(err)
+	}
+	val.ExitEpoch = prevEpoch
+	val.WithdrawableEpoch = prevEpoch + params.BeaconConfig().MinValidatorWithdrawabilityDelay
+	if err := headState.UpdateValidatorAtIndex(95, val); err != nil {
+		t.Fatal(err)
+	}
 	event := &feed.Event{
 		Type: statefeed.BlockProcessed,
 		Data: &statefeed.BlockProcessedData{
@@ -347,7 +417,7 @@ func TestArchiverService_SavesExitedValidatorChanges(t *testing.T) {
 	}
 }
 
-func setupState(t *testing.T, validatorCount uint64) *pb.BeaconState {
+func setupState(validatorCount uint64) (*stateTrie.BeaconState, error) {
 	validators := make([]*ethpb.Validator, validatorCount)
 	balances := make([]uint64, validatorCount)
 	for i := 0; i < len(validators); i++ {
@@ -363,18 +433,18 @@ func setupState(t *testing.T, validatorCount uint64) *pb.BeaconState {
 
 	// We initialize a head state that has attestations from participated
 	// validators in a simulated fashion.
-	return &pb.BeaconState{
+	return stateTrie.InitializeFromProto(&pb.BeaconState{
 		Slot:                       (2 * params.BeaconConfig().SlotsPerEpoch) - 1,
 		Validators:                 validators,
 		Balances:                   balances,
-		BlockRoots:                 make([][]byte, 128),
+		BlockRoots:                 make([][]byte, params.BeaconConfig().SlotsPerHistoricalRoot),
 		Slashings:                  []uint64{0, 1e9, 1e9},
 		RandaoMixes:                make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
 		CurrentEpochAttestations:   atts,
 		FinalizedCheckpoint:        &ethpb.Checkpoint{},
 		JustificationBits:          bitfield.Bitvector4{0x00},
 		CurrentJustifiedCheckpoint: &ethpb.Checkpoint{},
-	}
+	})
 }
 
 func setupService(t *testing.T) (*Service, db.Database) {

beacon-chain/blockchain/BUILD.bazel

@@ -4,9 +4,15 @@ go_library(
     name = "go_default_library",
     srcs = [
         "chain_info.go",
+        "head.go",
         "info.go",
+        "init_sync_process_block.go",
         "log.go",
         "metrics.go",
+        "process_attestation.go",
+        "process_attestation_helpers.go",
+        "process_block.go",
+        "process_block_helpers.go",
         "receive_attestation.go",
         "receive_block.go",
         "service.go",
@@ -14,7 +20,7 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/blockchain",
     visibility = ["//beacon-chain:__subpackages__"],
     deps = [
-        "//beacon-chain/blockchain/forkchoice:go_default_library",
+        "//beacon-chain/cache:go_default_library",
         "//beacon-chain/cache/depositcache:go_default_library",
         "//beacon-chain/core/blocks:go_default_library",
         "//beacon-chain/core/epoch/precompute:go_default_library",
@@ -23,16 +29,25 @@ go_library(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/state:go_default_library",
         "//beacon-chain/db:go_default_library",
+        "//beacon-chain/db/filters:go_default_library",
+        "//beacon-chain/flags:go_default_library",
+        "//beacon-chain/forkchoice:go_default_library",
+        "//beacon-chain/forkchoice/protoarray:go_default_library",
         "//beacon-chain/operations/attestations:go_default_library",
+        "//beacon-chain/operations/slashings:go_default_library",
+        "//beacon-chain/operations/voluntaryexits:go_default_library",
         "//beacon-chain/p2p:go_default_library",
         "//beacon-chain/powchain:go_default_library",
+        "//beacon-chain/state:go_default_library",
+        "//beacon-chain/state/stategen:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
+        "//shared/attestationutil:go_default_library",
         "//shared/bytesutil:go_default_library",
         "//shared/featureconfig:go_default_library",
         "//shared/params:go_default_library",
         "//shared/slotutil:go_default_library",
         "//shared/traceutil:go_default_library",
-        "@com_github_gogo_protobuf//proto:go_default_library",
+        "@com_github_emicklei_dot//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
@@ -56,8 +71,11 @@ go_test(
     size = "medium",
     srcs = [
         "chain_info_test.go",
+        "head_test.go",
+        "init_sync_process_block_test.go",
+        "process_attestation_test.go",
+        "process_block_test.go",
         "receive_attestation_test.go",
-        "receive_block_test.go",
         "service_test.go",
     ],
     embed = [":go_default_library"],
@@ -70,11 +88,12 @@ go_test(
         "//beacon-chain/db/testing:go_default_library",
         "//beacon-chain/p2p:go_default_library",
         "//beacon-chain/powchain:go_default_library",
+        "//beacon-chain/state/stateutil:go_default_library",
+        "//proto/beacon/db:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/bytesutil:go_default_library",
         "//shared/event:go_default_library",
         "//shared/params:go_default_library",
-        "//shared/stateutil:go_default_library",
         "//shared/testutil:go_default_library",
         "@com_github_ethereum_go_ethereum//:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
@@ -84,6 +103,7 @@ go_test(
         "@com_github_prysmaticlabs_go_ssz//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",
+        "@in_gopkg_d4l3k_messagediff_v1//:go_default_library",
         "@org_golang_x_net//context:go_default_library",
     ],
 )

beacon-chain/blockchain/chain_info.go

@@ -5,10 +5,11 @@ import (
 	"context"
 	"time"
 
-	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
@@ -17,32 +18,26 @@ import (
 // directly retrieves chain info related data.
 type ChainInfoFetcher interface {
 	HeadFetcher
-	CanonicalRootFetcher
 	FinalizationFetcher
 }
 
-// GenesisTimeFetcher retrieves the Eth2 genesis timestamp.
-type GenesisTimeFetcher interface {
+// TimeFetcher retrieves the Eth2 data that's related to time.
+type TimeFetcher interface {
 	GenesisTime() time.Time
+	CurrentSlot() uint64
 }
 
 // HeadFetcher defines a common interface for methods in blockchain service which
 // directly retrieves head related data.
 type HeadFetcher interface {
 	HeadSlot() uint64
-	HeadRoot() []byte
-	HeadBlock() *ethpb.SignedBeaconBlock
-	HeadState(ctx context.Context) (*pb.BeaconState, error)
+	HeadRoot(ctx context.Context) ([]byte, error)
+	HeadBlock(ctx context.Context) (*ethpb.SignedBeaconBlock, error)
+	HeadState(ctx context.Context) (*state.BeaconState, error)
 	HeadValidatorsIndices(epoch uint64) ([]uint64, error)
 	HeadSeed(epoch uint64) ([32]byte, error)
 }
 
-// CanonicalRootFetcher defines a common interface for methods in blockchain service which
-// directly retrieves canonical roots related data.
-type CanonicalRootFetcher interface {
-	CanonicalRoot(slot uint64) []byte
-}
-
 // ForkFetcher retrieves the current fork information of the Ethereum beacon chain.
 type ForkFetcher interface {
 	CurrentFork() *pb.Fork
@@ -64,115 +59,118 @@ type ParticipationFetcher interface {
 
 // FinalizedCheckpt returns the latest finalized checkpoint from head state.
 func (s *Service) FinalizedCheckpt() *ethpb.Checkpoint {
-	if s.headState == nil || s.headState.FinalizedCheckpoint == nil {
+	if s.finalizedCheckpt == nil {
 		return &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
 	}
 
 	// If head state exists but there hasn't been a finalized check point,
 	// the check point's root should refer to genesis block root.
-	if bytes.Equal(s.headState.FinalizedCheckpoint.Root, params.BeaconConfig().ZeroHash[:]) {
+	if bytes.Equal(s.finalizedCheckpt.Root, params.BeaconConfig().ZeroHash[:]) {
 		return &ethpb.Checkpoint{Root: s.genesisRoot[:]}
 	}
 
-	return s.headState.FinalizedCheckpoint
+	return state.CopyCheckpoint(s.finalizedCheckpt)
 }
 
 // CurrentJustifiedCheckpt returns the current justified checkpoint from head state.
 func (s *Service) CurrentJustifiedCheckpt() *ethpb.Checkpoint {
-	if s.headState == nil || s.headState.CurrentJustifiedCheckpoint == nil {
+	if s.justifiedCheckpt == nil {
 		return &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
 	}
 
 	// If head state exists but there hasn't been a justified check point,
 	// the check point root should refer to genesis block root.
-	if bytes.Equal(s.headState.CurrentJustifiedCheckpoint.Root, params.BeaconConfig().ZeroHash[:]) {
+	if bytes.Equal(s.justifiedCheckpt.Root, params.BeaconConfig().ZeroHash[:]) {
 		return &ethpb.Checkpoint{Root: s.genesisRoot[:]}
 	}
 
-	return s.headState.CurrentJustifiedCheckpoint
+	return state.CopyCheckpoint(s.justifiedCheckpt)
 }
 
 // PreviousJustifiedCheckpt returns the previous justified checkpoint from head state.
 func (s *Service) PreviousJustifiedCheckpt() *ethpb.Checkpoint {
-	if s.headState == nil || s.headState.PreviousJustifiedCheckpoint == nil {
+	if s.prevJustifiedCheckpt == nil {
 		return &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
 	}
 
 	// If head state exists but there hasn't been a justified check point,
 	// the check point root should refer to genesis block root.
-	if bytes.Equal(s.headState.PreviousJustifiedCheckpoint.Root, params.BeaconConfig().ZeroHash[:]) {
+	if bytes.Equal(s.prevJustifiedCheckpt.Root, params.BeaconConfig().ZeroHash[:]) {
 		return &ethpb.Checkpoint{Root: s.genesisRoot[:]}
 	}
 
-	return s.headState.PreviousJustifiedCheckpoint
+	return state.CopyCheckpoint(s.prevJustifiedCheckpt)
 }
 
 // HeadSlot returns the slot of the head of the chain.
 func (s *Service) HeadSlot() uint64 {
-	s.headLock.RLock()
-	defer s.headLock.RUnlock()
+	if !s.hasHeadState() {
+		return 0
+	}
 
-	return s.headSlot
+	return s.headSlot()
 }
 
 // HeadRoot returns the root of the head of the chain.
-func (s *Service) HeadRoot() []byte {
-	s.headLock.RLock()
-	defer s.headLock.RUnlock()
-
-	root := s.canonicalRoots[s.headSlot]
-	if len(root) != 0 {
-		return root
+func (s *Service) HeadRoot(ctx context.Context) ([]byte, error) {
+	if s.headRoot() != params.BeaconConfig().ZeroHash {
+		r := s.headRoot()
+		return r[:], nil
 	}
 
-	return params.BeaconConfig().ZeroHash[:]
+	b, err := s.beaconDB.HeadBlock(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if b == nil {
+		return params.BeaconConfig().ZeroHash[:], nil
+	}
+
+	r, err := ssz.HashTreeRoot(b.Block)
+	if err != nil {
+		return nil, err
+	}
+
+	return r[:], nil
 }
 
 // HeadBlock returns the head block of the chain.
-func (s *Service) HeadBlock() *ethpb.SignedBeaconBlock {
-	s.headLock.RLock()
-	defer s.headLock.RUnlock()
+// If the head state is nil from service struct,
+// it will attempt to get the head block from DB.
+func (s *Service) HeadBlock(ctx context.Context) (*ethpb.SignedBeaconBlock, error) {
+	if s.hasHeadState() {
+		return s.headBlock(), nil
+	}
 
-	return proto.Clone(s.headBlock).(*ethpb.SignedBeaconBlock)
+	return s.beaconDB.HeadBlock(ctx)
 }
 
 // HeadState returns the head state of the chain.
 // If the head state is nil from service struct,
-// it will attempt to get from DB and error if nil again.
-func (s *Service) HeadState(ctx context.Context) (*pb.BeaconState, error) {
-	s.headLock.RLock()
-	defer s.headLock.RUnlock()
-
-	if s.headState == nil {
-		return s.beaconDB.HeadState(ctx)
+// it will attempt to get the head state from DB.
+func (s *Service) HeadState(ctx context.Context) (*state.BeaconState, error) {
+	if s.hasHeadState() {
+		return s.headState(), nil
 	}
 
-	return proto.Clone(s.headState).(*pb.BeaconState), nil
+	return s.beaconDB.HeadState(ctx)
 }
 
 // HeadValidatorsIndices returns a list of active validator indices from the head view of a given epoch.
 func (s *Service) HeadValidatorsIndices(epoch uint64) ([]uint64, error) {
-	if s.headState == nil {
+	if !s.hasHeadState() {
 		return []uint64{}, nil
 	}
-	return helpers.ActiveValidatorIndices(s.headState, epoch)
+	return helpers.ActiveValidatorIndices(s.headState(), epoch)
 }
 
 // HeadSeed returns the seed from the head view of a given epoch.
 func (s *Service) HeadSeed(epoch uint64) ([32]byte, error) {
-	if s.headState == nil {
+	if !s.hasHeadState() {
 		return [32]byte{}, nil
 	}
 
-	return helpers.Seed(s.headState, epoch, params.BeaconConfig().DomainBeaconAttester)
-}
-
-// CanonicalRoot returns the canonical root of a given slot.
-func (s *Service) CanonicalRoot(slot uint64) []byte {
-	s.headLock.RLock()
-	defer s.headLock.RUnlock()
-
-	return s.canonicalRoots[slot]
+	return helpers.Seed(s.headState(), epoch, params.BeaconConfig().DomainBeaconAttester)
 }
 
 // GenesisTime returns the genesis time of beacon chain.
@@ -182,13 +180,13 @@ func (s *Service) GenesisTime() time.Time {
 
 // CurrentFork retrieves the latest fork information of the beacon chain.
 func (s *Service) CurrentFork() *pb.Fork {
-	if s.headState == nil {
+	if !s.hasHeadState() {
 		return &pb.Fork{
 			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
 			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
 		}
 	}
-	return proto.Clone(s.headState.Fork).(*pb.Fork)
+	return s.headState().Fork()
 }
 
 // Participation returns the participation stats of a given epoch.

beacon-chain/blockchain/chain_info_norace_test.go

@@ -12,13 +12,11 @@ func TestHeadSlot_DataRace(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 	s := &Service{
-		beaconDB:       db,
-		canonicalRoots: make(map[uint64][]byte),
+		beaconDB: db,
 	}
 	go func() {
 		s.saveHead(
 			context.Background(),
-			&ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 777}},
 			[32]byte{},
 		)
 	}()
@@ -29,47 +27,45 @@ func TestHeadRoot_DataRace(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 	s := &Service{
-		beaconDB:       db,
-		canonicalRoots: make(map[uint64][]byte),
+		beaconDB: db,
+		head:     &head{root: [32]byte{'A'}},
 	}
 	go func() {
 		s.saveHead(
 			context.Background(),
-			&ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 777}},
 			[32]byte{},
 		)
 	}()
-	s.HeadRoot()
+	if _, err := s.HeadRoot(context.Background()); err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestHeadBlock_DataRace(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 	s := &Service{
-		beaconDB:       db,
-		canonicalRoots: make(map[uint64][]byte),
+		beaconDB: db,
+		head:     &head{block: &ethpb.SignedBeaconBlock{}},
 	}
 	go func() {
 		s.saveHead(
 			context.Background(),
-			&ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 777}},
 			[32]byte{},
 		)
 	}()
-	s.HeadBlock()
+	s.HeadBlock(context.Background())
 }
 
 func TestHeadState_DataRace(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 	s := &Service{
-		beaconDB:       db,
-		canonicalRoots: make(map[uint64][]byte),
+		beaconDB: db,
 	}
 	go func() {
 		s.saveHead(
 			context.Background(),
-			&ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 777}},
 			[32]byte{},
 		)
 	}()

beacon-chain/blockchain/chain_info_test.go

@@ -7,23 +7,23 @@ import (
 	"testing"
 	"time"
 
+	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
 
 // Ensure Service implements chain info interface.
 var _ = ChainInfoFetcher(&Service{})
-var _ = GenesisTimeFetcher(&Service{})
+var _ = TimeFetcher(&Service{})
 var _ = ForkFetcher(&Service{})
 
 func TestFinalizedCheckpt_Nil(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 	c := setupBeaconChain(t, db)
-	c.headState, _ = testutil.DeterministicGenesisState(t, 1)
 	if !bytes.Equal(c.FinalizedCheckpt().Root, params.BeaconConfig().ZeroHash[:]) {
 		t.Error("Incorrect pre chain start value")
 	}
@@ -33,7 +33,11 @@ func TestHeadRoot_Nil(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 	c := setupBeaconChain(t, db)
-	if !bytes.Equal(c.HeadRoot(), params.BeaconConfig().ZeroHash[:]) {
+	headRoot, err := c.HeadRoot(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(headRoot, params.BeaconConfig().ZeroHash[:]) {
 		t.Error("Incorrect pre chain start value")
 	}
 }
@@ -42,9 +46,9 @@ func TestFinalizedCheckpt_CanRetrieve(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 
-	cp := &ethpb.Checkpoint{Epoch: 5}
+	cp := &ethpb.Checkpoint{Epoch: 5, Root: []byte("foo")}
 	c := setupBeaconChain(t, db)
-	c.headState = &pb.BeaconState{FinalizedCheckpoint: cp}
+	c.finalizedCheckpt = cp
 
 	if c.FinalizedCheckpt().Epoch != cp.Epoch {
 		t.Errorf("Finalized epoch at genesis should be %d, got: %d", cp.Epoch, c.FinalizedCheckpt().Epoch)
@@ -55,10 +59,11 @@ func TestFinalizedCheckpt_GenesisRootOk(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 
-	cp := &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	genesisRoot := [32]byte{'A'}
+	cp := &ethpb.Checkpoint{Root: genesisRoot[:]}
 	c := setupBeaconChain(t, db)
-	c.headState = &pb.BeaconState{FinalizedCheckpoint: cp}
-	c.genesisRoot = [32]byte{'A'}
+	c.finalizedCheckpt = cp
+	c.genesisRoot = genesisRoot
 
 	if !bytes.Equal(c.FinalizedCheckpt().Root, c.genesisRoot[:]) {
 		t.Errorf("Got: %v, wanted: %v", c.FinalizedCheckpt().Root, c.genesisRoot[:])
@@ -69,9 +74,9 @@ func TestCurrentJustifiedCheckpt_CanRetrieve(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 
-	cp := &ethpb.Checkpoint{Epoch: 6}
+	cp := &ethpb.Checkpoint{Epoch: 6, Root: []byte("foo")}
 	c := setupBeaconChain(t, db)
-	c.headState = &pb.BeaconState{CurrentJustifiedCheckpoint: cp}
+	c.justifiedCheckpt = cp
 
 	if c.CurrentJustifiedCheckpt().Epoch != cp.Epoch {
 		t.Errorf("Current Justifiied epoch at genesis should be %d, got: %d", cp.Epoch, c.CurrentJustifiedCheckpt().Epoch)
@@ -82,10 +87,11 @@ func TestJustifiedCheckpt_GenesisRootOk(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 
-	cp := &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	genesisRoot := [32]byte{'B'}
+	cp := &ethpb.Checkpoint{Root: genesisRoot[:]}
 	c := setupBeaconChain(t, db)
-	c.headState = &pb.BeaconState{CurrentJustifiedCheckpoint: cp}
-	c.genesisRoot = [32]byte{'B'}
+	c.justifiedCheckpt = cp
+	c.genesisRoot = genesisRoot
 
 	if !bytes.Equal(c.CurrentJustifiedCheckpt().Root, c.genesisRoot[:]) {
 		t.Errorf("Got: %v, wanted: %v", c.CurrentJustifiedCheckpt().Root, c.genesisRoot[:])
@@ -96,9 +102,9 @@ func TestPreviousJustifiedCheckpt_CanRetrieve(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 
-	cp := &ethpb.Checkpoint{Epoch: 7}
+	cp := &ethpb.Checkpoint{Epoch: 7, Root: []byte("foo")}
 	c := setupBeaconChain(t, db)
-	c.headState = &pb.BeaconState{PreviousJustifiedCheckpoint: cp}
+	c.prevJustifiedCheckpt = cp
 
 	if c.PreviousJustifiedCheckpt().Epoch != cp.Epoch {
 		t.Errorf("Previous Justifiied epoch at genesis should be %d, got: %d", cp.Epoch, c.PreviousJustifiedCheckpt().Epoch)
@@ -109,10 +115,11 @@ func TestPrevJustifiedCheckpt_GenesisRootOk(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 
-	cp := &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	genesisRoot := [32]byte{'C'}
+	cp := &ethpb.Checkpoint{Root: genesisRoot[:]}
 	c := setupBeaconChain(t, db)
-	c.headState = &pb.BeaconState{PreviousJustifiedCheckpoint: cp}
-	c.genesisRoot = [32]byte{'C'}
+	c.prevJustifiedCheckpt = cp
+	c.genesisRoot = genesisRoot
 
 	if !bytes.Equal(c.PreviousJustifiedCheckpt().Root, c.genesisRoot[:]) {
 		t.Errorf("Got: %v, wanted: %v", c.PreviousJustifiedCheckpt().Root, c.genesisRoot[:])
@@ -121,37 +128,49 @@ func TestPrevJustifiedCheckpt_GenesisRootOk(t *testing.T) {
 
 func TestHeadSlot_CanRetrieve(t *testing.T) {
 	c := &Service{}
-	c.headSlot = 100
+	s, _ := state.InitializeFromProto(&pb.BeaconState{})
+	c.head = &head{slot: 100, state: s}
 	if c.HeadSlot() != 100 {
 		t.Errorf("Wanted head slot: %d, got: %d", 100, c.HeadSlot())
 	}
 }
 
 func TestHeadRoot_CanRetrieve(t *testing.T) {
-	c := &Service{canonicalRoots: make(map[uint64][]byte)}
-	c.headSlot = 100
-	c.canonicalRoots[c.headSlot] = []byte{'A'}
-	if !bytes.Equal([]byte{'A'}, c.HeadRoot()) {
-		t.Errorf("Wanted head root: %v, got: %d", []byte{'A'}, c.HeadRoot())
+	c := &Service{}
+	c.head = &head{root: [32]byte{'A'}}
+	if [32]byte{'A'} != c.headRoot() {
+		t.Errorf("Wanted head root: %v, got: %d", []byte{'A'}, c.headRoot())
 	}
 }
 
 func TestHeadBlock_CanRetrieve(t *testing.T) {
 	b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1}}
-	c := &Service{headBlock: b}
-	if !reflect.DeepEqual(b, c.HeadBlock()) {
+	s, _ := state.InitializeFromProto(&pb.BeaconState{})
+	c := &Service{}
+	c.head = &head{block: b, state: s}
+
+	recevied, err := c.HeadBlock(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(b, recevied) {
 		t.Error("incorrect head block received")
 	}
 }
 
 func TestHeadState_CanRetrieve(t *testing.T) {
-	s := &pb.BeaconState{Slot: 2}
-	c := &Service{headState: s}
+	s, err := state.InitializeFromProto(&pb.BeaconState{Slot: 2})
+	if err != nil {
+		t.Fatal(err)
+	}
+	c := &Service{}
+	c.head = &head{state: s}
 	headState, err := c.HeadState(context.Background())
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(s, headState) {
+	if !reflect.DeepEqual(s.InnerStateUnsafe(), headState.InnerStateUnsafe()) {
 		t.Error("incorrect head state received")
 	}
 }
@@ -166,19 +185,13 @@ func TestGenesisTime_CanRetrieve(t *testing.T) {
 
 func TestCurrentFork_CanRetrieve(t *testing.T) {
 	f := &pb.Fork{Epoch: 999}
-	s := &pb.BeaconState{Fork: f}
-	c := &Service{headState: s}
-	if !reflect.DeepEqual(c.CurrentFork(), f) {
-		t.Error("Recieved incorrect fork version")
-	}
-}
-
-func TestCanonicalRoot_CanRetrieve(t *testing.T) {
-	c := &Service{canonicalRoots: make(map[uint64][]byte)}
-	slot := uint64(123)
-	r := []byte{'B'}
-	c.canonicalRoots[slot] = r
-	if !bytes.Equal(r, c.CanonicalRoot(slot)) {
-		t.Errorf("Wanted head root: %v, got: %d", []byte{'A'}, c.CanonicalRoot(slot))
+	s, err := state.InitializeFromProto(&pb.BeaconState{Fork: f})
+	if err != nil {
+		t.Fatal(err)
+	}
+	c := &Service{}
+	c.head = &head{state: s}
+	if !proto.Equal(c.CurrentFork(), f) {
+		t.Error("Received incorrect fork version")
 	}
 }

beacon-chain/blockchain/forkchoice/benchmark_test.go

@@ -1,168 +0,0 @@
-package forkchoice
-
-import (
-	"context"
-	"testing"
-
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-)
-
-func BenchmarkForkChoiceTree1(b *testing.B) {
-	ctx := context.Background()
-	db := testDB.SetupDB(b)
-	defer testDB.TeardownDB(b, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db, []byte{'g'})
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	// Benchmark fork choice with 1024 validators
-	validators := make([]*ethpb.Validator, 1024)
-	for i := 0; i < len(validators); i++ {
-		validators[i] = &ethpb.Validator{ExitEpoch: 2, EffectiveBalance: 1e9}
-	}
-	s := &pb.BeaconState{Validators: validators}
-
-	if err := store.GenesisStore(ctx, &ethpb.Checkpoint{}, &ethpb.Checkpoint{}); err != nil {
-		b.Fatal(err)
-	}
-
-	store.justifiedCheckpt.Root = roots[0]
-	if err := store.db.SaveState(ctx, s, bytesutil.ToBytes32(roots[0])); err != nil {
-		b.Fatal(err)
-	}
-
-	if err := store.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: store.justifiedCheckpt,
-		State:      s,
-	}); err != nil {
-		b.Fatal(err)
-	}
-
-	// Spread out the votes evenly for all 3 leaf nodes
-	for i := 0; i < len(validators); i++ {
-		switch {
-		case i < 256:
-			store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[1]}
-		case i > 768:
-			store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[7]}
-		default:
-			store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[8]}
-		}
-	}
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := store.Head(ctx)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func BenchmarkForkChoiceTree2(b *testing.B) {
-	ctx := context.Background()
-	db := testDB.SetupDB(b)
-	defer testDB.TeardownDB(b, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree2(db)
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	// Benchmark fork choice with 1024 validators
-	validators := make([]*ethpb.Validator, 1024)
-	for i := 0; i < len(validators); i++ {
-		validators[i] = &ethpb.Validator{ExitEpoch: 2, EffectiveBalance: 1e9}
-	}
-	s := &pb.BeaconState{Validators: validators}
-	if err := store.GenesisStore(ctx, &ethpb.Checkpoint{}, &ethpb.Checkpoint{}); err != nil {
-		b.Fatal(err)
-	}
-
-	store.justifiedCheckpt.Root = roots[0]
-	if err := store.db.SaveState(ctx, s, bytesutil.ToBytes32(roots[0])); err != nil {
-		b.Fatal(err)
-	}
-
-	if err := store.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: store.justifiedCheckpt,
-		State:      s,
-	}); err != nil {
-		b.Fatal(err)
-	}
-
-	// Spread out the votes evenly for all the leaf nodes. 8 to 15
-	nodeIndex := 8
-	for i := 0; i < len(validators); i++ {
-		store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[nodeIndex]}
-		if i%155 == 0 {
-			nodeIndex++
-		}
-	}
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := store.Head(ctx)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func BenchmarkForkChoiceTree3(b *testing.B) {
-	ctx := context.Background()
-	db := testDB.SetupDB(b)
-	defer testDB.TeardownDB(b, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree3(db)
-	if err != nil {
-		b.Fatal(err)
-	}
-
-	// Benchmark fork choice with 1024 validators
-	validators := make([]*ethpb.Validator, 1024)
-	for i := 0; i < len(validators); i++ {
-		validators[i] = &ethpb.Validator{ExitEpoch: 2, EffectiveBalance: 1e9}
-	}
-	s := &pb.BeaconState{Validators: validators}
-	if err := store.GenesisStore(ctx, &ethpb.Checkpoint{}, &ethpb.Checkpoint{}); err != nil {
-		b.Fatal(err)
-	}
-
-	store.justifiedCheckpt.Root = roots[0]
-	if err := store.db.SaveState(ctx, s, bytesutil.ToBytes32(roots[0])); err != nil {
-		b.Fatal(err)
-	}
-
-	if err := store.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: store.justifiedCheckpt,
-		State:      s,
-	}); err != nil {
-		b.Fatal(err)
-	}
-
-	// All validators vote on the same head
-	for i := 0; i < len(validators); i++ {
-		store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[len(roots)-1]}
-	}
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := store.Head(ctx)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}

beacon-chain/blockchain/forkchoice/doc.go

@@ -1,9 +0,0 @@
-/*
-Package forkchoice implements the Latest Message Driven GHOST (Greediest Heaviest Observed
-Sub-Tree) algorithm as the Ethereum Serenity beacon chain fork choice rule. This algorithm is designed to
-properly detect the canonical chain based on validator votes even in the presence of high network
-latency, network partitions, and many conflicting blocks. To read more about fork choice, read the
-official accompanying document:
-https://github.com/ethereum/eth2.0-specs/blob/v0.9.0/specs/core/0_fork-choice.md
-*/
-package forkchoice

beacon-chain/blockchain/forkchoice/lmd_ghost_test.yaml

@@ -1,59 +0,0 @@
-test_cases:
-  # GHOST chooses b3 with the heaviest weight
-  - blocks:
-      - id: 'b0'
-        parent: 'b0'
-      - id: 'b1'
-        parent: 'b0'
-      - id: 'b2'
-        parent: 'b1'
-      - id: 'b3'
-        parent: 'b1'
-    weights:
-      b0: 0
-      b1: 0
-      b2: 5
-      b3: 10
-    head: 'b3'
-  # GHOST chooses b1 with the heaviest weight
-  - blocks:
-      - id: 'b0'
-        parent: 'b0'
-      - id: 'b1'
-        parent: 'b0'
-      - id: 'b2'
-        parent: 'b0'
-      - id: 'b3'
-        parent: 'b0'
-    weights:
-      b1: 5
-      b2: 4
-      b3: 3
-    head: 'b1'
-  # Equal weights children, GHOST chooses b2 because it is higher lexicographically than b3
-  - blocks:
-      - id: 'b0'
-        parent: 'b0'
-      - id: 'b1'
-        parent: 'b0'
-      - id: 'b2'
-        parent: 'b0'
-      - id: 'b3'
-        parent: 'b0'
-    weights:
-      b1: 5
-      b2: 6
-      b3: 6
-    head: 'b3'
-  # Equal weights children, GHOST chooses b2 because it is higher lexicographically than b1
-  - blocks:
-      - id: 'b0'
-        parent: 'b0'
-      - id: 'b1'
-        parent: 'b0'
-      - id: 'b2'
-        parent: 'b0'
-    weights:
-      b1: 0
-      b2: 0
-    head: 'b2'

beacon-chain/blockchain/forkchoice/lmd_ghost_yaml_test.go

@@ -1,140 +0,0 @@
-package forkchoice
-
-import (
-	"bytes"
-	"context"
-	"io/ioutil"
-	"path/filepath"
-	"strconv"
-	"testing"
-
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"gopkg.in/yaml.v2"
-)
-
-type Config struct {
-	TestCases []struct {
-		Blocks []struct {
-			ID     string `yaml:"id"`
-			Parent string `yaml:"parent"`
-		} `yaml:"blocks"`
-		Weights map[string]int `yaml:"weights"`
-		Head    string         `yaml:"head"`
-	} `yaml:"test_cases"`
-}
-
-func TestGetHeadFromYaml(t *testing.T) {
-	ctx := context.Background()
-	filename, _ := filepath.Abs("./lmd_ghost_test.yaml")
-	yamlFile, err := ioutil.ReadFile(filename)
-	if err != nil {
-		t.Fatal(err)
-	}
-	var c *Config
-	err = yaml.Unmarshal(yamlFile, &c)
-
-	params.UseMainnetConfig()
-
-	for _, test := range c.TestCases {
-		db := testDB.SetupDB(t)
-		defer testDB.TeardownDB(t, db)
-
-		blksRoot := make(map[int][]byte)
-		// Construct block tree from yaml.
-		for _, blk := range test.Blocks {
-			// genesis block condition
-			if blk.ID == blk.Parent {
-				b := &ethpb.BeaconBlock{Slot: 0, ParentRoot: []byte{'g'}}
-				if err := db.SaveBlock(ctx, &ethpb.SignedBeaconBlock{Block: b}); err != nil {
-					t.Fatal(err)
-				}
-				root, err := ssz.HashTreeRoot(b)
-				if err != nil {
-					t.Fatal(err)
-				}
-				blksRoot[0] = root[:]
-			} else {
-				slot, err := strconv.Atoi(blk.ID[1:])
-				if err != nil {
-					t.Fatal(err)
-				}
-				parentSlot, err := strconv.Atoi(blk.Parent[1:])
-				if err != nil {
-					t.Fatal(err)
-				}
-				b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: uint64(slot), ParentRoot: blksRoot[parentSlot]}}
-				if err := db.SaveBlock(ctx, b); err != nil {
-					t.Fatal(err)
-				}
-				root, err := ssz.HashTreeRoot(b.Block)
-				if err != nil {
-					t.Fatal(err)
-				}
-				blksRoot[slot] = root[:]
-				if err := db.SaveState(ctx, &pb.BeaconState{}, root); err != nil {
-					t.Fatal(err)
-				}
-			}
-		}
-
-		store := NewForkChoiceService(ctx, db)
-
-		// Assign validator votes to the blocks as weights.
-		count := 0
-		for blk, votes := range test.Weights {
-			slot, err := strconv.Atoi(blk[1:])
-			if err != nil {
-				t.Fatal(err)
-			}
-			max := count + votes
-			for i := count; i < max; i++ {
-				store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: blksRoot[slot]}
-				count++
-			}
-		}
-
-		validators := make([]*ethpb.Validator, count)
-		for i := 0; i < len(validators); i++ {
-			validators[i] = &ethpb.Validator{ExitEpoch: 2, EffectiveBalance: 1e9}
-		}
-
-		s := &pb.BeaconState{Validators: validators}
-
-		if err := store.db.SaveState(ctx, s, bytesutil.ToBytes32(blksRoot[0])); err != nil {
-			t.Fatal(err)
-		}
-		if err := store.GenesisStore(ctx, &ethpb.Checkpoint{Root: blksRoot[0]}, &ethpb.Checkpoint{Root: blksRoot[0]}); err != nil {
-			t.Fatal(err)
-		}
-
-		if err := store.checkpointState.AddCheckpointState(&cache.CheckpointState{
-			Checkpoint: store.justifiedCheckpt,
-			State:      s,
-		}); err != nil {
-			t.Fatal(err)
-		}
-
-		head, err := store.Head(ctx)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		headSlot, err := strconv.Atoi(test.Head[1:])
-		if err != nil {
-			t.Fatal(err)
-		}
-		wantedHead := blksRoot[headSlot]
-
-		if !bytes.Equal(head, wantedHead) {
-			t.Errorf("wanted root %#x, got root %#x", wantedHead, head)
-		}
-
-		testDB.TeardownDB(t, db)
-	}
-}

beacon-chain/blockchain/forkchoice/log.go

@@ -1,40 +0,0 @@
-package forkchoice
-
-import (
-	"fmt"
-
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/sirupsen/logrus"
-)
-
-var log = logrus.WithField("prefix", "forkchoice")
-
-// logs epoch related data during epoch boundary.
-func logEpochData(beaconState *pb.BeaconState) {
-	log.WithFields(logrus.Fields{
-		"epoch":                  helpers.CurrentEpoch(beaconState),
-		"finalizedEpoch":         beaconState.FinalizedCheckpoint.Epoch,
-		"justifiedEpoch":         beaconState.CurrentJustifiedCheckpoint.Epoch,
-		"previousJustifiedEpoch": beaconState.PreviousJustifiedCheckpoint.Epoch,
-	}).Info("Starting next epoch")
-	activeVals, err := helpers.ActiveValidatorIndices(beaconState, helpers.CurrentEpoch(beaconState))
-	if err != nil {
-		log.WithError(err).Error("Could not get active validator indices")
-		return
-	}
-	log.WithFields(logrus.Fields{
-		"totalValidators":  len(beaconState.Validators),
-		"activeValidators": len(activeVals),
-		"averageBalance":   fmt.Sprintf("%.5f ETH", averageBalance(beaconState.Balances)),
-	}).Info("Validator registry information")
-}
-
-func averageBalance(balances []uint64) float64 {
-	total := uint64(0)
-	for i := 0; i < len(balances); i++ {
-		total += balances[i]
-	}
-	return float64(total) / float64(len(balances)) / float64(params.BeaconConfig().GweiPerEth)
-}

beacon-chain/blockchain/forkchoice/metrics.go

@@ -1,161 +0,0 @@
-package forkchoice
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/params"
-)
-
-var (
-	beaconFinalizedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_finalized_epoch",
-		Help: "Last finalized epoch of the processed state",
-	})
-	beaconFinalizedRoot = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_finalized_root",
-		Help: "Last finalized root of the processed state",
-	})
-	cacheFinalizedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "cache_finalized_epoch",
-		Help: "Last cached finalized epoch",
-	})
-	cacheFinalizedRoot = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "cache_finalized_root",
-		Help: "Last cached finalized root",
-	})
-	beaconCurrentJustifiedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_current_justified_epoch",
-		Help: "Current justified epoch of the processed state",
-	})
-	beaconCurrentJustifiedRoot = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_current_justified_root",
-		Help: "Current justified root of the processed state",
-	})
-	beaconPrevJustifiedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_previous_justified_epoch",
-		Help: "Previous justified epoch of the processed state",
-	})
-	beaconPrevJustifiedRoot = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_previous_justified_root",
-		Help: "Previous justified root of the processed state",
-	})
-	sigFailsToVerify = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "att_signature_failed_to_verify_with_cache",
-		Help: "Number of attestation signatures that failed to verify with cache on, but succeeded without cache",
-	})
-	validatorsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Name: "validator_count",
-		Help: "The total number of validators, in GWei",
-	}, []string{"state"})
-	validatorsBalance = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Name: "validators_total_balance",
-		Help: "The total balance of validators, in GWei",
-	}, []string{"state"})
-	validatorsEffectiveBalance = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Name: "validators_total_effective_balance",
-		Help: "The total effective balance of validators, in GWei",
-	}, []string{"state"})
-	currentEth1DataDepositCount = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "current_eth1_data_deposit_count",
-		Help: "The current eth1 deposit count in the last processed state eth1data field.",
-	})
-	totalEligibleBalances = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "total_eligible_balances",
-		Help: "The total amount of ether, in gwei, that has been used in voting attestation target of previous epoch",
-	})
-	totalVotedTargetBalances = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "total_voted_target_balances",
-		Help: "The total amount of ether, in gwei, that is eligible for voting of previous epoch",
-	})
-)
-
-func reportEpochMetrics(state *pb.BeaconState) {
-	currentEpoch := state.Slot / params.BeaconConfig().SlotsPerEpoch
-
-	// Validator instances
-	pendingInstances := 0
-	activeInstances := 0
-	slashingInstances := 0
-	slashedInstances := 0
-	exitingInstances := 0
-	exitedInstances := 0
-	// Validator balances
-	pendingBalance := uint64(0)
-	activeBalance := uint64(0)
-	activeEffectiveBalance := uint64(0)
-	exitingBalance := uint64(0)
-	exitingEffectiveBalance := uint64(0)
-	slashingBalance := uint64(0)
-	slashingEffectiveBalance := uint64(0)
-
-	for i, validator := range state.Validators {
-		if validator.Slashed {
-			if currentEpoch < validator.ExitEpoch {
-				slashingInstances++
-				slashingBalance += state.Balances[i]
-				slashingEffectiveBalance += validator.EffectiveBalance
-			} else {
-				slashedInstances++
-			}
-			continue
-		}
-		if validator.ExitEpoch != params.BeaconConfig().FarFutureEpoch {
-			if currentEpoch < validator.ExitEpoch {
-				exitingInstances++
-				exitingBalance += state.Balances[i]
-				exitingEffectiveBalance += validator.EffectiveBalance
-			} else {
-				exitedInstances++
-			}
-			continue
-		}
-		if currentEpoch < validator.ActivationEpoch {
-			pendingInstances++
-			pendingBalance += state.Balances[i]
-			continue
-		}
-		activeInstances++
-		activeBalance += state.Balances[i]
-		activeEffectiveBalance += validator.EffectiveBalance
-	}
-	validatorsCount.WithLabelValues("Pending").Set(float64(pendingInstances))
-	validatorsCount.WithLabelValues("Active").Set(float64(activeInstances))
-	validatorsCount.WithLabelValues("Exiting").Set(float64(exitingInstances))
-	validatorsCount.WithLabelValues("Exited").Set(float64(exitedInstances))
-	validatorsCount.WithLabelValues("Slashing").Set(float64(slashingInstances))
-	validatorsCount.WithLabelValues("Slashed").Set(float64(slashedInstances))
-	validatorsBalance.WithLabelValues("Pending").Set(float64(pendingBalance))
-	validatorsBalance.WithLabelValues("Active").Set(float64(activeBalance))
-	validatorsBalance.WithLabelValues("Exiting").Set(float64(exitingBalance))
-	validatorsBalance.WithLabelValues("Slashing").Set(float64(slashingBalance))
-	validatorsEffectiveBalance.WithLabelValues("Active").Set(float64(activeEffectiveBalance))
-	validatorsEffectiveBalance.WithLabelValues("Exiting").Set(float64(exitingEffectiveBalance))
-	validatorsEffectiveBalance.WithLabelValues("Slashing").Set(float64(slashingEffectiveBalance))
-
-	// Last justified slot
-	if state.CurrentJustifiedCheckpoint != nil {
-		beaconCurrentJustifiedEpoch.Set(float64(state.CurrentJustifiedCheckpoint.Epoch))
-		beaconCurrentJustifiedRoot.Set(float64(bytesutil.ToLowInt64(state.CurrentJustifiedCheckpoint.Root)))
-	}
-	// Last previous justified slot
-	if state.PreviousJustifiedCheckpoint != nil {
-		beaconPrevJustifiedEpoch.Set(float64(state.PreviousJustifiedCheckpoint.Epoch))
-		beaconPrevJustifiedRoot.Set(float64(bytesutil.ToLowInt64(state.PreviousJustifiedCheckpoint.Root)))
-	}
-	// Last finalized slot
-	if state.FinalizedCheckpoint != nil {
-		beaconFinalizedEpoch.Set(float64(state.FinalizedCheckpoint.Epoch))
-		beaconFinalizedRoot.Set(float64(bytesutil.ToLowInt64(state.FinalizedCheckpoint.Root)))
-	}
-	if state.Eth1Data != nil {
-		currentEth1DataDepositCount.Set(float64(state.Eth1Data.DepositCount))
-	}
-
-	if precompute.Balances != nil {
-		totalEligibleBalances.Set(float64(precompute.Balances.PrevEpoch))
-		totalVotedTargetBalances.Set(float64(precompute.Balances.PrevEpochTargetAttesters))
-	}
-}

beacon-chain/blockchain/forkchoice/process_attestation.go

@@ -1,309 +0,0 @@
-package forkchoice
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"github.com/gogo/protobuf/proto"
-	"github.com/pkg/errors"
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/sirupsen/logrus"
-	"go.opencensus.io/trace"
-)
-
-// ErrTargetRootNotInDB returns when the target block root of an attestation cannot be found in the
-// beacon database.
-var ErrTargetRootNotInDB = errors.New("target root does not exist in db")
-
-// OnAttestation is called whenever an attestation is received, it updates validators latest vote,
-// as well as the fork choice store struct.
-//
-// Spec pseudocode definition:
-//   def on_attestation(store: Store, attestation: Attestation) -> None:
-//    """
-//    Run ``on_attestation`` upon receiving a new ``attestation`` from either within a block or directly on the wire.
-//
-//    An ``attestation`` that is asserted as invalid may be valid at a later time,
-//    consider scheduling it for later processing in such case.
-//    """
-//    target = attestation.data.target
-//
-//    # Attestations must be from the current or previous epoch
-//    current_epoch = compute_epoch_at_slot(get_current_slot(store))
-//    # Use GENESIS_EPOCH for previous when genesis to avoid underflow
-//    previous_epoch = current_epoch - 1 if current_epoch > GENESIS_EPOCH else GENESIS_EPOCH
-//    assert target.epoch in [current_epoch, previous_epoch]
-//    assert target.epoch == compute_epoch_at_slot(attestation.data.slot)
-//
-//    # Attestations target be for a known block. If target block is unknown, delay consideration until the block is found
-//    assert target.root in store.blocks
-//    # Attestations cannot be from future epochs. If they are, delay consideration until the epoch arrives
-//    base_state = store.block_states[target.root].copy()
-//    assert store.time >= base_state.genesis_time + compute_start_slot_at_epoch(target.epoch) * SECONDS_PER_SLOT
-//
-//    # Attestations must be for a known block. If block is unknown, delay consideration until the block is found
-//    assert attestation.data.beacon_block_root in store.blocks
-//    # Attestations must not be for blocks in the future. If not, the attestation should not be considered
-//    assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot
-//
-//    # Store target checkpoint state if not yet seen
-//    if target not in store.checkpoint_states:
-//        process_slots(base_state, compute_start_slot_at_epoch(target.epoch))
-//        store.checkpoint_states[target] = base_state
-//    target_state = store.checkpoint_states[target]
-//
-//    # Attestations can only affect the fork choice of subsequent slots.
-//    # Delay consideration in the fork choice until their slot is in the past.
-//    assert store.time >= (attestation.data.slot + 1) * SECONDS_PER_SLOT
-//
-//    # Get state at the `target` to validate attestation and calculate the committees
-//    indexed_attestation = get_indexed_attestation(target_state, attestation)
-//    assert is_valid_indexed_attestation(target_state, indexed_attestation)
-//
-//    # Update latest messages
-//    for i in indexed_attestation.attesting_indices:
-//        if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch:
-//            store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=attestation.data.beacon_block_root)
-func (s *Store) OnAttestation(ctx context.Context, a *ethpb.Attestation) error {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.onAttestation")
-	defer span.End()
-
-	tgt := proto.Clone(a.Data.Target).(*ethpb.Checkpoint)
-	tgtSlot := helpers.StartSlot(tgt.Epoch)
-
-	if helpers.SlotToEpoch(a.Data.Slot) != a.Data.Target.Epoch {
-		return fmt.Errorf("data slot is not in the same epoch as target %d != %d", helpers.SlotToEpoch(a.Data.Slot), a.Data.Target.Epoch)
-	}
-
-	// Verify beacon node has seen the target block before.
-	if !s.db.HasBlock(ctx, bytesutil.ToBytes32(tgt.Root)) {
-		return ErrTargetRootNotInDB
-	}
-
-	// Verify attestation target has had a valid pre state produced by the target block.
-	baseState, err := s.verifyAttPreState(ctx, tgt)
-	if err != nil {
-		return err
-	}
-
-	// Verify attestation target is from current epoch or previous epoch.
-	if err := s.verifyAttTargetEpoch(ctx, baseState.GenesisTime, uint64(time.Now().Unix()), tgt); err != nil {
-		return err
-	}
-
-	// Verify Attestations cannot be from future epochs.
-	if err := helpers.VerifySlotTime(baseState.GenesisTime, tgtSlot); err != nil {
-		return errors.Wrap(err, "could not verify attestation target slot")
-	}
-
-	// Verify attestation beacon block is known and not from the future.
-	if err := s.verifyBeaconBlock(ctx, a.Data); err != nil {
-		return errors.Wrap(err, "could not verify attestation beacon block")
-	}
-
-	// Store target checkpoint state if not yet seen.
-	baseState, err = s.saveCheckpointState(ctx, baseState, tgt)
-	if err != nil {
-		return err
-	}
-
-	// Verify attestations can only affect the fork choice of subsequent slots.
-	if err := helpers.VerifySlotTime(baseState.GenesisTime, a.Data.Slot+1); err != nil {
-		return err
-	}
-
-	// Use the target state to to validate attestation and calculate the committees.
-	indexedAtt, err := s.verifyAttestation(ctx, baseState, a)
-	if err != nil {
-		return err
-	}
-
-	// Update every validator's latest vote.
-	if err := s.updateAttVotes(ctx, indexedAtt, tgt.Root, tgt.Epoch); err != nil {
-		return err
-	}
-
-	if err := s.db.SaveAttestation(ctx, a); err != nil {
-		return err
-	}
-
-	log := log.WithFields(logrus.Fields{
-		"Slot":               a.Data.Slot,
-		"Index":              a.Data.CommitteeIndex,
-		"AggregatedBitfield": fmt.Sprintf("%08b", a.AggregationBits),
-		"BeaconBlockRoot":    fmt.Sprintf("%#x", bytesutil.Trunc(a.Data.BeaconBlockRoot)),
-	})
-	log.Debug("Updated latest votes")
-
-	return nil
-}
-
-// verifyAttPreState validates input attested check point has a valid pre-state.
-func (s *Store) verifyAttPreState(ctx context.Context, c *ethpb.Checkpoint) (*pb.BeaconState, error) {
-	baseState, err := s.db.State(ctx, bytesutil.ToBytes32(c.Root))
-	if err != nil {
-		return nil, errors.Wrapf(err, "could not get pre state for slot %d", helpers.StartSlot(c.Epoch))
-	}
-	if baseState == nil {
-		return nil, fmt.Errorf("pre state of target block %d does not exist", helpers.StartSlot(c.Epoch))
-	}
-	return baseState, nil
-}
-
-// verifyAttTargetEpoch validates attestation is from the current or previous epoch.
-func (s *Store) verifyAttTargetEpoch(ctx context.Context, genesisTime uint64, nowTime uint64, c *ethpb.Checkpoint) error {
-	currentSlot := (nowTime - genesisTime) / params.BeaconConfig().SecondsPerSlot
-	currentEpoch := helpers.SlotToEpoch(currentSlot)
-	var prevEpoch uint64
-	// Prevents previous epoch under flow
-	if currentEpoch > 1 {
-		prevEpoch = currentEpoch - 1
-	}
-	if c.Epoch != prevEpoch && c.Epoch != currentEpoch {
-		return fmt.Errorf("target epoch %d does not match current epoch %d or prev epoch %d", c.Epoch, currentEpoch, prevEpoch)
-	}
-	return nil
-}
-
-// verifyBeaconBlock verifies beacon head block is known and not from the future.
-func (s *Store) verifyBeaconBlock(ctx context.Context, data *ethpb.AttestationData) error {
-	b, err := s.db.Block(ctx, bytesutil.ToBytes32(data.BeaconBlockRoot))
-	if err != nil {
-		return err
-	}
-	if b == nil || b.Block == nil {
-		return fmt.Errorf("beacon block %#x does not exist", bytesutil.Trunc(data.BeaconBlockRoot))
-	}
-	if b.Block.Slot > data.Slot {
-		return fmt.Errorf("could not process attestation for future block, %d > %d", b.Block.Slot, data.Slot)
-	}
-	return nil
-}
-
-// saveCheckpointState saves and returns the processed state with the associated check point.
-func (s *Store) saveCheckpointState(ctx context.Context, baseState *pb.BeaconState, c *ethpb.Checkpoint) (*pb.BeaconState, error) {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.saveCheckpointState")
-	defer span.End()
-
-	s.checkpointStateLock.Lock()
-	defer s.checkpointStateLock.Unlock()
-	cachedState, err := s.checkpointState.StateByCheckpoint(c)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not get cached checkpoint state")
-	}
-	if cachedState != nil {
-		return cachedState, nil
-	}
-
-	// Advance slots only when it's higher than current state slot.
-	if helpers.StartSlot(c.Epoch) > baseState.Slot {
-		stateCopy := proto.Clone(baseState).(*pb.BeaconState)
-		stateCopy, err = state.ProcessSlots(ctx, stateCopy, helpers.StartSlot(c.Epoch))
-		if err != nil {
-			return nil, errors.Wrapf(err, "could not process slots up to %d", helpers.StartSlot(c.Epoch))
-		}
-
-		if err := s.checkpointState.AddCheckpointState(&cache.CheckpointState{
-			Checkpoint: c,
-			State:      stateCopy,
-		}); err != nil {
-			return nil, errors.Wrap(err, "could not saved checkpoint state to cache")
-		}
-
-		return stateCopy, nil
-	}
-
-	return baseState, nil
-}
-
-// verifyAttestation validates input attestation is valid.
-func (s *Store) verifyAttestation(ctx context.Context, baseState *pb.BeaconState, a *ethpb.Attestation) (*ethpb.IndexedAttestation, error) {
-	committee, err := helpers.BeaconCommitteeFromState(baseState, a.Data.Slot, a.Data.CommitteeIndex)
-	if err != nil {
-		return nil, err
-	}
-	indexedAtt, err := blocks.ConvertToIndexed(ctx, a, committee)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not convert attestation to indexed attestation")
-	}
-
-	if err := blocks.VerifyIndexedAttestation(ctx, baseState, indexedAtt); err != nil {
-
-		// TODO(3603): Delete the following signature verify fallback when issue 3603 closes.
-		// When signature fails to verify with committee cache enabled at run time,
-		// the following re-runs the same signature verify routine without cache in play.
-		// This provides extra assurance that committee cache can't break run time.
-		if err == blocks.ErrSigFailedToVerify {
-			committee, err = helpers.BeaconCommitteeWithoutCache(baseState, a.Data.Slot, a.Data.CommitteeIndex)
-			if err != nil {
-				return nil, errors.Wrap(err, "could not convert attestation to indexed attestation without cache")
-			}
-			indexedAtt, err = blocks.ConvertToIndexed(ctx, a, committee)
-			if err != nil {
-				return nil, errors.Wrap(err, "could not convert attestation to indexed attestation")
-			}
-			if err := blocks.VerifyIndexedAttestation(ctx, baseState, indexedAtt); err != nil {
-				return nil, errors.Wrap(err, "could not verify indexed attestation without cache")
-			}
-			sigFailsToVerify.Inc()
-			return indexedAtt, nil
-		}
-
-		return nil, errors.Wrap(err, "could not verify indexed attestation")
-	}
-	return indexedAtt, nil
-}
-
-// updateAttVotes updates validator's latest votes based on the incoming attestation.
-func (s *Store) updateAttVotes(
-	ctx context.Context,
-	indexedAtt *ethpb.IndexedAttestation,
-	tgtRoot []byte,
-	tgtEpoch uint64) error {
-
-	indices := indexedAtt.AttestingIndices
-	s.voteLock.Lock()
-	defer s.voteLock.Unlock()
-	for _, i := range indices {
-		vote, ok := s.latestVoteMap[i]
-		if !ok || tgtEpoch > vote.Epoch {
-			s.latestVoteMap[i] = &pb.ValidatorLatestVote{
-				Epoch: tgtEpoch,
-				Root:  tgtRoot,
-			}
-		}
-	}
-	return nil
-}
-
-// aggregatedAttestation returns the aggregated attestation after checking saved one in db.
-func (s *Store) aggregatedAttestations(ctx context.Context, att *ethpb.Attestation) ([]*ethpb.Attestation, error) {
-	r, err := ssz.HashTreeRoot(att.Data)
-	if err != nil {
-		return nil, err
-	}
-	saved, err := s.db.AttestationsByDataRoot(ctx, r)
-	if err != nil {
-		return nil, err
-	}
-
-	if saved == nil {
-		return []*ethpb.Attestation{att}, nil
-	}
-
-	aggregated, err := helpers.AggregateAttestations(append(saved, att))
-	if err != nil {
-		return nil, err
-	}
-
-	return aggregated, nil
-}

beacon-chain/blockchain/forkchoice/process_block.go

@@ -1,572 +0,0 @@
-package forkchoice
-
-import (
-	"bytes"
-	"context"
-	"encoding/hex"
-	"fmt"
-	"time"
-
-	"github.com/gogo/protobuf/proto"
-	"github.com/pkg/errors"
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
-	"github.com/prysmaticlabs/prysm/beacon-chain/flags"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/prysmaticlabs/prysm/shared/traceutil"
-	"github.com/sirupsen/logrus"
-	"go.opencensus.io/trace"
-)
-
-// OnBlock is called when a gossip block is received. It runs regular state transition on the block and
-// update fork choice store.
-//
-// Spec pseudocode definition:
-//   def on_block(store: Store, block: BeaconBlock) -> None:
-//    # Make a copy of the state to avoid mutability issues
-//    assert block.parent_root in store.block_states
-//    pre_state = store.block_states[block.parent_root].copy()
-//    # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past.
-//    assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT
-//    # Add new block to the store
-//    store.blocks[signing_root(block)] = block
-//    # Check block is a descendant of the finalized block
-//    assert (
-//        get_ancestor(store, signing_root(block), store.blocks[store.finalized_checkpoint.root].slot) ==
-//        store.finalized_checkpoint.root
-//    )
-//    # Check that block is later than the finalized epoch slot
-//    assert block.slot > compute_start_slot_of_epoch(store.finalized_checkpoint.epoch)
-//    # Check the block is valid and compute the post-state
-//    state = state_transition(pre_state, block)
-//    # Add new state for this block to the store
-//    store.block_states[signing_root(block)] = state
-//
-//    # Update justified checkpoint
-//    if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch:
-//        if state.current_justified_checkpoint.epoch > store.best_justified_checkpoint.epoch:
-//            store.best_justified_checkpoint = state.current_justified_checkpoint
-//
-//    # Update finalized checkpoint
-//    if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch:
-//        store.finalized_checkpoint = state.finalized_checkpoint
-func (s *Store) OnBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock) error {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.onBlock")
-	defer span.End()
-
-	if signed == nil || signed.Block == nil {
-		return errors.New("nil block")
-	}
-
-	b := signed.Block
-
-	// Retrieve incoming block's pre state.
-	preState, err := s.getBlockPreState(ctx, b)
-	if err != nil {
-		return err
-	}
-	preStateValidatorCount := len(preState.Validators)
-
-	root, err := ssz.HashTreeRoot(b)
-	if err != nil {
-		return errors.Wrapf(err, "could not get signing root of block %d", b.Slot)
-	}
-	log.WithFields(logrus.Fields{
-		"slot": b.Slot,
-		"root": fmt.Sprintf("0x%s...", hex.EncodeToString(root[:])[:8]),
-	}).Info("Executing state transition on block")
-	postState, err := state.ExecuteStateTransition(ctx, preState, signed)
-	if err != nil {
-		return errors.Wrap(err, "could not execute state transition")
-	}
-
-	if err := s.db.SaveBlock(ctx, signed); err != nil {
-		return errors.Wrapf(err, "could not save block from slot %d", b.Slot)
-	}
-	if err := s.db.SaveState(ctx, postState, root); err != nil {
-		return errors.Wrap(err, "could not save state")
-	}
-
-	// Update justified check point.
-	if postState.CurrentJustifiedCheckpoint.Epoch > s.justifiedCheckpt.Epoch {
-		if err := s.updateJustified(ctx, postState); err != nil {
-			return err
-		}
-	}
-
-	// Update finalized check point.
-	// Prune the block cache and helper caches on every new finalized epoch.
-	if postState.FinalizedCheckpoint.Epoch > s.finalizedCheckpt.Epoch {
-		if err := s.db.SaveFinalizedCheckpoint(ctx, postState.FinalizedCheckpoint); err != nil {
-			return errors.Wrap(err, "could not save finalized checkpoint")
-		}
-
-		startSlot := helpers.StartSlot(s.prevFinalizedCheckpt.Epoch)
-		endSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
-		if endSlot > startSlot {
-			if err := s.rmStatesOlderThanLastFinalized(ctx, startSlot, endSlot); err != nil {
-				return errors.Wrapf(err, "could not delete states prior to finalized check point, range: %d, %d",
-					startSlot, endSlot)
-			}
-		}
-
-		s.prevFinalizedCheckpt = s.finalizedCheckpt
-		s.finalizedCheckpt = postState.FinalizedCheckpoint
-	}
-
-	// Update validator indices in database as needed.
-	if err := s.saveNewValidators(ctx, preStateValidatorCount, postState); err != nil {
-		return errors.Wrap(err, "could not save finalized checkpoint")
-	}
-	// Save the unseen attestations from block to db.
-	if err := s.saveNewBlockAttestations(ctx, b.Body.Attestations); err != nil {
-		return errors.Wrap(err, "could not save attestations")
-	}
-
-	// Epoch boundary bookkeeping such as logging epoch summaries.
-	if postState.Slot >= s.nextEpochBoundarySlot {
-		logEpochData(postState)
-		reportEpochMetrics(postState)
-
-		// Update committees cache at epoch boundary slot.
-		if featureconfig.Get().EnableNewCache {
-			if err := helpers.UpdateCommitteeCache(postState, helpers.CurrentEpoch(postState)); err != nil {
-				return err
-			}
-		}
-
-		s.nextEpochBoundarySlot = helpers.StartSlot(helpers.NextEpoch(postState))
-	}
-
-	return nil
-}
-
-// OnBlockInitialSyncStateTransition is called when an initial sync block is received.
-// It runs state transition on the block and without any BLS verification. The BLS verification
-// includes proposer signature, randao and attestation's aggregated signature. It also does not save
-// attestations.
-func (s *Store) OnBlockInitialSyncStateTransition(ctx context.Context, signed *ethpb.SignedBeaconBlock) error {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.onBlock")
-	defer span.End()
-
-	if signed == nil || signed.Block == nil {
-		return errors.New("nil block")
-	}
-
-	b := signed.Block
-
-	s.initSyncStateLock.Lock()
-	defer s.initSyncStateLock.Unlock()
-
-	// Retrieve incoming block's pre state.
-	preState, err := s.cachedPreState(ctx, b)
-	if err != nil {
-		return err
-	}
-	preStateValidatorCount := len(preState.Validators)
-
-	log.WithField("slot", b.Slot).Debug("Executing state transition on block")
-
-	postState, err := state.ExecuteStateTransitionNoVerify(ctx, preState, signed)
-	if err != nil {
-		return errors.Wrap(err, "could not execute state transition")
-	}
-
-	if err := s.db.SaveBlock(ctx, signed); err != nil {
-		return errors.Wrapf(err, "could not save block from slot %d", b.Slot)
-	}
-	root, err := ssz.HashTreeRoot(b)
-	if err != nil {
-		return errors.Wrapf(err, "could not get signing root of block %d", b.Slot)
-	}
-
-	if featureconfig.Get().InitSyncCacheState {
-		s.initSyncState[root] = postState
-	} else {
-		if err := s.db.SaveState(ctx, postState, root); err != nil {
-			return errors.Wrap(err, "could not save state")
-		}
-	}
-
-	// Update justified check point.
-	if postState.CurrentJustifiedCheckpoint.Epoch > s.justifiedCheckpt.Epoch {
-		if err := s.updateJustified(ctx, postState); err != nil {
-			return err
-		}
-	}
-
-	// Update finalized check point.
-	// Prune the block cache and helper caches on every new finalized epoch.
-	if postState.FinalizedCheckpoint.Epoch > s.finalizedCheckpt.Epoch {
-		startSlot := helpers.StartSlot(s.prevFinalizedCheckpt.Epoch)
-		endSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
-		if endSlot > startSlot {
-			if err := s.rmStatesOlderThanLastFinalized(ctx, startSlot, endSlot); err != nil {
-				return errors.Wrapf(err, "could not delete states prior to finalized check point, range: %d, %d",
-					startSlot, endSlot)
-			}
-		}
-
-		if err := s.saveInitState(ctx, postState); err != nil {
-			return errors.Wrap(err, "could not save init sync finalized state")
-		}
-
-		if err := s.db.SaveFinalizedCheckpoint(ctx, postState.FinalizedCheckpoint); err != nil {
-			return errors.Wrap(err, "could not save finalized checkpoint")
-		}
-
-		s.prevFinalizedCheckpt = s.finalizedCheckpt
-		s.finalizedCheckpt = postState.FinalizedCheckpoint
-	}
-
-	// Update validator indices in database as needed.
-	if err := s.saveNewValidators(ctx, preStateValidatorCount, postState); err != nil {
-		return errors.Wrap(err, "could not save finalized checkpoint")
-	}
-
-	if flags.Get().EnableArchive {
-		// Save the unseen attestations from block to db.
-		if err := s.saveNewBlockAttestations(ctx, b.Body.Attestations); err != nil {
-			return errors.Wrap(err, "could not save attestations")
-		}
-	}
-
-	// Epoch boundary bookkeeping such as logging epoch summaries.
-	if postState.Slot >= s.nextEpochBoundarySlot {
-		reportEpochMetrics(postState)
-
-		s.nextEpochBoundarySlot = helpers.StartSlot(helpers.NextEpoch(postState))
-	}
-
-	return nil
-}
-
-// getBlockPreState returns the pre state of an incoming block. It uses the parent root of the block
-// to retrieve the state in DB. It verifies the pre state's validity and the incoming block
-// is in the correct time window.
-func (s *Store) getBlockPreState(ctx context.Context, b *ethpb.BeaconBlock) (*pb.BeaconState, error) {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.getBlockPreState")
-	defer span.End()
-
-	// Verify incoming block has a valid pre state.
-	preState, err := s.verifyBlkPreState(ctx, b)
-	if err != nil {
-		return nil, err
-	}
-
-	// Verify block slot time is not from the feature.
-	if err := helpers.VerifySlotTime(preState.GenesisTime, b.Slot); err != nil {
-		return nil, err
-	}
-
-	// Verify block is a descendent of a finalized block.
-	if err := s.verifyBlkDescendant(ctx, bytesutil.ToBytes32(b.ParentRoot), b.Slot); err != nil {
-		return nil, err
-	}
-
-	// Verify block is later than the finalized epoch slot.
-	if err := s.verifyBlkFinalizedSlot(b); err != nil {
-		return nil, err
-	}
-
-	return preState, nil
-}
-
-// verifyBlkPreState validates input block has a valid pre-state.
-func (s *Store) verifyBlkPreState(ctx context.Context, b *ethpb.BeaconBlock) (*pb.BeaconState, error) {
-	preState, err := s.db.State(ctx, bytesutil.ToBytes32(b.ParentRoot))
-	if err != nil {
-		return nil, errors.Wrapf(err, "could not get pre state for slot %d", b.Slot)
-	}
-	if preState == nil {
-		return nil, fmt.Errorf("pre state of slot %d does not exist", b.Slot)
-	}
-	return preState, nil
-}
-
-// verifyBlkDescendant validates input block root is a descendant of the
-// current finalized block root.
-func (s *Store) verifyBlkDescendant(ctx context.Context, root [32]byte, slot uint64) error {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.verifyBlkDescendant")
-	defer span.End()
-
-	finalizedBlkSigned, err := s.db.Block(ctx, bytesutil.ToBytes32(s.finalizedCheckpt.Root))
-	if err != nil || finalizedBlkSigned == nil || finalizedBlkSigned.Block == nil {
-		return errors.Wrap(err, "could not get finalized block")
-	}
-	finalizedBlk := finalizedBlkSigned.Block
-
-	bFinalizedRoot, err := s.ancestor(ctx, root[:], finalizedBlk.Slot)
-	if err != nil {
-		return errors.Wrap(err, "could not get finalized block root")
-	}
-	if !bytes.Equal(bFinalizedRoot, s.finalizedCheckpt.Root) {
-		err := fmt.Errorf("block from slot %d is not a descendent of the current finalized block slot %d, %#x != %#x",
-			slot, finalizedBlk.Slot, bytesutil.Trunc(bFinalizedRoot), bytesutil.Trunc(s.finalizedCheckpt.Root))
-		traceutil.AnnotateError(span, err)
-		return err
-	}
-	return nil
-}
-
-// verifyBlkFinalizedSlot validates input block is not less than or equal
-// to current finalized slot.
-func (s *Store) verifyBlkFinalizedSlot(b *ethpb.BeaconBlock) error {
-	finalizedSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
-	if finalizedSlot >= b.Slot {
-		return fmt.Errorf("block is equal or earlier than finalized block, slot %d < slot %d", b.Slot, finalizedSlot)
-	}
-	return nil
-}
-
-// saveNewValidators saves newly added validator index from state to db. Does nothing if validator count has not
-// changed.
-func (s *Store) saveNewValidators(ctx context.Context, preStateValidatorCount int, postState *pb.BeaconState) error {
-	postStateValidatorCount := len(postState.Validators)
-	if preStateValidatorCount != postStateValidatorCount {
-		for i := preStateValidatorCount; i < postStateValidatorCount; i++ {
-			pubKey := postState.Validators[i].PublicKey
-			if err := s.db.SaveValidatorIndex(ctx, bytesutil.ToBytes48(pubKey), uint64(i)); err != nil {
-				return errors.Wrapf(err, "could not save activated validator: %d", i)
-			}
-			log.WithFields(logrus.Fields{
-				"index":               i,
-				"pubKey":              hex.EncodeToString(bytesutil.Trunc(pubKey)),
-				"totalValidatorCount": i + 1,
-			}).Info("New validator index saved in DB")
-		}
-	}
-	return nil
-}
-
-// saveNewBlockAttestations saves the new attestations in block to DB.
-func (s *Store) saveNewBlockAttestations(ctx context.Context, atts []*ethpb.Attestation) error {
-	attestations := make([]*ethpb.Attestation, 0, len(atts))
-	for _, att := range atts {
-		aggregated, err := s.aggregatedAttestations(ctx, att)
-		if err != nil {
-			continue
-		}
-		attestations = append(attestations, aggregated...)
-	}
-	if err := s.db.SaveAttestations(ctx, atts); err != nil {
-		return err
-	}
-	return nil
-}
-
-// rmStatesOlderThanLastFinalized deletes the states in db since last finalized check point.
-func (s *Store) rmStatesOlderThanLastFinalized(ctx context.Context, startSlot uint64, endSlot uint64) error {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.rmStatesBySlots")
-	defer span.End()
-
-	// Make sure start slot is not a skipped slot
-	for i := startSlot; i > 0; i-- {
-		filter := filters.NewFilter().SetStartSlot(i).SetEndSlot(i)
-		b, err := s.db.Blocks(ctx, filter)
-		if err != nil {
-			return err
-		}
-		if len(b) > 0 {
-			startSlot = i
-			break
-		}
-	}
-
-	// Make sure finalized slot is not a skipped slot.
-	for i := endSlot; i > 0; i-- {
-		filter := filters.NewFilter().SetStartSlot(i).SetEndSlot(i)
-		b, err := s.db.Blocks(ctx, filter)
-		if err != nil {
-			return err
-		}
-		if len(b) > 0 {
-			endSlot = i - 1
-			break
-		}
-	}
-
-	// Do not remove genesis state
-	if startSlot == 0 {
-		startSlot++
-	}
-	// If end slot comes less than start slot
-	if endSlot < startSlot {
-		endSlot = startSlot
-	}
-
-	filter := filters.NewFilter().SetStartSlot(startSlot).SetEndSlot(endSlot)
-	roots, err := s.db.BlockRoots(ctx, filter)
-	if err != nil {
-		return err
-	}
-
-	roots, err = s.filterBlockRoots(ctx, roots)
-	if err != nil {
-		return err
-	}
-
-	if err := s.db.DeleteStates(ctx, roots); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// shouldUpdateCurrentJustified prevents bouncing attack, by only update conflicting justified
-// checkpoints in the fork choice if in the early slots of the epoch.
-// Otherwise, delay incorporation of new justified checkpoint until next epoch boundary.
-// See https://ethresear.ch/t/prevention-of-bouncing-attack-on-ffg/6114 for more detailed analysis and discussion.
-func (s *Store) shouldUpdateCurrentJustified(ctx context.Context, newJustifiedCheckpt *ethpb.Checkpoint) (bool, error) {
-	if helpers.SlotsSinceEpochStarts(s.currentSlot()) < params.BeaconConfig().SafeSlotsToUpdateJustified {
-		return true, nil
-	}
-	newJustifiedBlockSigned, err := s.db.Block(ctx, bytesutil.ToBytes32(newJustifiedCheckpt.Root))
-	if err != nil {
-		return false, err
-	}
-	if newJustifiedBlockSigned == nil || newJustifiedBlockSigned.Block == nil {
-		return false, errors.New("nil new justified block")
-	}
-	newJustifiedBlock := newJustifiedBlockSigned.Block
-	if newJustifiedBlock.Slot <= helpers.StartSlot(s.justifiedCheckpt.Epoch) {
-		return false, nil
-	}
-	justifiedBlockSigned, err := s.db.Block(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root))
-	if err != nil {
-		return false, err
-	}
-	if justifiedBlockSigned == nil || justifiedBlockSigned.Block == nil {
-		return false, errors.New("nil justified block")
-	}
-	justifiedBlock := justifiedBlockSigned.Block
-	b, err := s.ancestor(ctx, newJustifiedCheckpt.Root, justifiedBlock.Slot)
-	if err != nil {
-		return false, err
-	}
-	if !bytes.Equal(b, s.justifiedCheckpt.Root) {
-		return false, nil
-	}
-	return true, nil
-}
-
-func (s *Store) updateJustified(ctx context.Context, state *pb.BeaconState) error {
-	if state.CurrentJustifiedCheckpoint.Epoch > s.bestJustifiedCheckpt.Epoch {
-		s.bestJustifiedCheckpt = state.CurrentJustifiedCheckpoint
-	}
-	canUpdate, err := s.shouldUpdateCurrentJustified(ctx, state.CurrentJustifiedCheckpoint)
-	if err != nil {
-		return err
-	}
-	if canUpdate {
-		s.justifiedCheckpt = state.CurrentJustifiedCheckpoint
-	}
-
-	if featureconfig.Get().InitSyncCacheState {
-		justifiedRoot := bytesutil.ToBytes32(state.CurrentJustifiedCheckpoint.Root)
-		justifiedState := s.initSyncState[justifiedRoot]
-		if err := s.db.SaveState(ctx, justifiedState, justifiedRoot); err != nil {
-			return errors.Wrap(err, "could not save justified state")
-		}
-	}
-
-	return s.db.SaveJustifiedCheckpoint(ctx, state.CurrentJustifiedCheckpoint)
-}
-
-// currentSlot returns the current slot based on time.
-func (s *Store) currentSlot() uint64 {
-	return (uint64(time.Now().Unix()) - s.genesisTime) / params.BeaconConfig().SecondsPerSlot
-}
-
-// updates justified check point in store if a better check point is known
-func (s *Store) updateJustifiedCheckpoint() {
-	// Update at epoch boundary slot only
-	if !helpers.IsEpochStart(s.currentSlot()) {
-		return
-	}
-	if s.bestJustifiedCheckpt.Epoch > s.justifiedCheckpt.Epoch {
-		s.justifiedCheckpt = s.bestJustifiedCheckpt
-	}
-}
-
-// This receives cached state in memory for initial sync only during initial sync.
-func (s *Store) cachedPreState(ctx context.Context, b *ethpb.BeaconBlock) (*pb.BeaconState, error) {
-	if featureconfig.Get().InitSyncCacheState {
-		preState := s.initSyncState[bytesutil.ToBytes32(b.ParentRoot)]
-		var err error
-		if preState == nil {
-			preState, err = s.db.State(ctx, bytesutil.ToBytes32(b.ParentRoot))
-			if err != nil {
-				return nil, errors.Wrapf(err, "could not get pre state for slot %d", b.Slot)
-			}
-			if preState == nil {
-				return nil, fmt.Errorf("pre state of slot %d does not exist", b.Slot)
-			}
-		}
-		return proto.Clone(preState).(*pb.BeaconState), nil
-	}
-
-	preState, err := s.db.State(ctx, bytesutil.ToBytes32(b.ParentRoot))
-	if err != nil {
-		return nil, errors.Wrapf(err, "could not get pre state for slot %d", b.Slot)
-	}
-	if preState == nil {
-		return nil, fmt.Errorf("pre state of slot %d does not exist", b.Slot)
-	}
-
-	return preState, nil
-}
-
-// This saves every finalized state in DB during initial sync, needed as part of optimization to
-// use cache state during initial sync in case of restart.
-func (s *Store) saveInitState(ctx context.Context, state *pb.BeaconState) error {
-	if !featureconfig.Get().InitSyncCacheState {
-		return nil
-	}
-	finalizedRoot := bytesutil.ToBytes32(state.FinalizedCheckpoint.Root)
-	fs := s.initSyncState[finalizedRoot]
-
-	if err := s.db.SaveState(ctx, fs, finalizedRoot); err != nil {
-		return errors.Wrap(err, "could not save state")
-	}
-	for r, oldState := range s.initSyncState {
-		if oldState.Slot < state.FinalizedCheckpoint.Epoch*params.BeaconConfig().SlotsPerEpoch {
-			delete(s.initSyncState, r)
-		}
-	}
-	return nil
-}
-
-// This filters block roots that are not known as head root and finalized root in DB.
-// It serves as the last line of defence before we prune states.
-func (s *Store) filterBlockRoots(ctx context.Context, roots [][32]byte) ([][32]byte, error) {
-	f, err := s.db.FinalizedCheckpoint(ctx)
-	if err != nil {
-		return nil, err
-	}
-	fRoot := f.Root
-	h, err := s.db.HeadBlock(ctx)
-	if err != nil {
-		return nil, err
-	}
-	hRoot, err := ssz.SigningRoot(h)
-	if err != nil {
-		return nil, err
-	}
-
-	filtered := make([][32]byte, 0, len(roots))
-	for _, root := range roots {
-		if bytes.Equal(root[:], fRoot[:]) || bytes.Equal(root[:], hRoot[:]) {
-			continue
-		}
-		filtered = append(filtered, root)
-	}
-
-	return filtered, nil
-}

beacon-chain/blockchain/forkchoice/process_block_test.go

@@ -1,610 +0,0 @@
-package forkchoice
-
-import (
-	"bytes"
-	"context"
-	"reflect"
-	"strings"
-	"testing"
-	"time"
-
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-bitfield"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/prysmaticlabs/prysm/shared/stateutil"
-)
-
-func TestStore_OnBlock(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	genesisStateRoot, err := stateutil.HashTreeRootState(&pb.BeaconState{})
-	if err != nil {
-		t.Error(err)
-	}
-	genesis := blocks.NewGenesisBlock(genesisStateRoot[:])
-	if err := db.SaveBlock(ctx, genesis); err != nil {
-		t.Error(err)
-	}
-	validGenesisRoot, err := ssz.HashTreeRoot(genesis.Block)
-	if err != nil {
-		t.Error(err)
-	}
-	if err := store.db.SaveState(ctx, &pb.BeaconState{}, validGenesisRoot); err != nil {
-		t.Fatal(err)
-	}
-	roots, err := blockTree1(db, validGenesisRoot[:])
-	if err != nil {
-		t.Fatal(err)
-	}
-	random := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1, ParentRoot: validGenesisRoot[:]}}
-	if err := db.SaveBlock(ctx, random); err != nil {
-		t.Error(err)
-	}
-	randomParentRoot, err := ssz.HashTreeRoot(random.Block)
-	if err != nil {
-		t.Error(err)
-	}
-	if err := store.db.SaveState(ctx, &pb.BeaconState{}, randomParentRoot); err != nil {
-		t.Fatal(err)
-	}
-	randomParentRoot2 := roots[1]
-	if err := store.db.SaveState(ctx, &pb.BeaconState{}, bytesutil.ToBytes32(randomParentRoot2)); err != nil {
-		t.Fatal(err)
-	}
-
-	tests := []struct {
-		name          string
-		blk           *ethpb.BeaconBlock
-		s             *pb.BeaconState
-		time          uint64
-		wantErrString string
-	}{
-		{
-			name:          "parent block root does not have a state",
-			blk:           &ethpb.BeaconBlock{},
-			s:             &pb.BeaconState{},
-			wantErrString: "pre state of slot 0 does not exist",
-		},
-		{
-			name:          "block is from the feature",
-			blk:           &ethpb.BeaconBlock{ParentRoot: randomParentRoot[:], Slot: params.BeaconConfig().FarFutureEpoch},
-			s:             &pb.BeaconState{},
-			wantErrString: "could not process slot from the future",
-		},
-		{
-			name:          "could not get finalized block",
-			blk:           &ethpb.BeaconBlock{ParentRoot: randomParentRoot[:]},
-			s:             &pb.BeaconState{},
-			wantErrString: "block from slot 0 is not a descendent of the current finalized block",
-		},
-		{
-			name:          "same slot as finalized block",
-			blk:           &ethpb.BeaconBlock{Slot: 0, ParentRoot: randomParentRoot2},
-			s:             &pb.BeaconState{},
-			wantErrString: "block is equal or earlier than finalized block, slot 0 < slot 0",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if err := store.GenesisStore(ctx, &ethpb.Checkpoint{Root: validGenesisRoot[:]}, &ethpb.Checkpoint{Root: validGenesisRoot[:]}); err != nil {
-				t.Fatal(err)
-			}
-			store.finalizedCheckpt.Root = roots[0]
-
-			err := store.OnBlock(ctx, &ethpb.SignedBeaconBlock{Block: tt.blk})
-			if !strings.Contains(err.Error(), tt.wantErrString) {
-				t.Errorf("Store.OnBlock() error = %v, wantErr = %v", err, tt.wantErrString)
-			}
-		})
-	}
-}
-
-func TestStore_SaveNewValidators(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	preCount := 2 // validators 0 and validators 1
-	s := &pb.BeaconState{Validators: []*ethpb.Validator{
-		{PublicKey: []byte{0}}, {PublicKey: []byte{1}},
-		{PublicKey: []byte{2}}, {PublicKey: []byte{3}},
-	}}
-	if err := store.saveNewValidators(ctx, preCount, s); err != nil {
-		t.Fatal(err)
-	}
-
-	if !db.HasValidatorIndex(ctx, bytesutil.ToBytes48([]byte{2})) {
-		t.Error("Wanted validator saved in db")
-	}
-	if !db.HasValidatorIndex(ctx, bytesutil.ToBytes48([]byte{3})) {
-		t.Error("Wanted validator saved in db")
-	}
-	if db.HasValidatorIndex(ctx, bytesutil.ToBytes48([]byte{1})) {
-		t.Error("validator not suppose to be saved in db")
-	}
-}
-
-func TestStore_SavesNewBlockAttestations(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	a1 := &ethpb.Attestation{Data: &ethpb.AttestationData{}, AggregationBits: bitfield.Bitlist{0b101}}
-	a2 := &ethpb.Attestation{Data: &ethpb.AttestationData{BeaconBlockRoot: []byte{'A'}}, AggregationBits: bitfield.Bitlist{0b110}}
-	r1, _ := ssz.HashTreeRoot(a1.Data)
-	r2, _ := ssz.HashTreeRoot(a2.Data)
-
-	if err := store.saveNewBlockAttestations(ctx, []*ethpb.Attestation{a1, a2}); err != nil {
-		t.Fatal(err)
-	}
-
-	saved, err := store.db.AttestationsByDataRoot(ctx, r1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual([]*ethpb.Attestation{a1}, saved) {
-		t.Error("did not retrieve saved attestation")
-	}
-
-	saved, err = store.db.AttestationsByDataRoot(ctx, r2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual([]*ethpb.Attestation{a2}, saved) {
-		t.Error("did not retrieve saved attestation")
-	}
-
-	a1 = &ethpb.Attestation{Data: &ethpb.AttestationData{}, AggregationBits: bitfield.Bitlist{0b111}}
-	a2 = &ethpb.Attestation{Data: &ethpb.AttestationData{BeaconBlockRoot: []byte{'A'}}, AggregationBits: bitfield.Bitlist{0b111}}
-
-	if err := store.saveNewBlockAttestations(ctx, []*ethpb.Attestation{a1, a2}); err != nil {
-		t.Fatal(err)
-	}
-
-	saved, err = store.db.AttestationsByDataRoot(ctx, r1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual([]*ethpb.Attestation{a1}, saved) {
-		t.Error("did not retrieve saved attestation")
-	}
-
-	saved, err = store.db.AttestationsByDataRoot(ctx, r2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual([]*ethpb.Attestation{a2}, saved) {
-		t.Error("did not retrieve saved attestation")
-	}
-}
-
-func TestRemoveStateSinceLastFinalized(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	params.UseMinimalConfig()
-	defer params.UseMainnetConfig()
-
-	store := NewForkChoiceService(ctx, db)
-
-	// Save 100 blocks in DB, each has a state.
-	numBlocks := 100
-	totalBlocks := make([]*ethpb.SignedBeaconBlock, numBlocks)
-	blockRoots := make([][32]byte, 0)
-	for i := 0; i < len(totalBlocks); i++ {
-		totalBlocks[i] = &ethpb.SignedBeaconBlock{
-			Block: &ethpb.BeaconBlock{
-				Slot: uint64(i),
-			},
-		}
-		r, err := ssz.HashTreeRoot(totalBlocks[i].Block)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if err := store.db.SaveState(ctx, &pb.BeaconState{Slot: uint64(i)}, r); err != nil {
-			t.Fatal(err)
-		}
-		if err := store.db.SaveBlock(ctx, totalBlocks[i]); err != nil {
-			t.Fatal(err)
-		}
-		blockRoots = append(blockRoots, r)
-		if err := store.db.SaveHeadBlockRoot(ctx, r); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// New finalized epoch: 1
-	finalizedEpoch := uint64(1)
-	finalizedSlot := finalizedEpoch * params.BeaconConfig().SlotsPerEpoch
-	endSlot := helpers.StartSlot(finalizedEpoch+1) - 1 // Inclusive
-	if err := store.rmStatesOlderThanLastFinalized(ctx, 0, endSlot); err != nil {
-		t.Fatal(err)
-	}
-	for _, r := range blockRoots {
-		s, err := store.db.State(ctx, r)
-		if err != nil {
-			t.Fatal(err)
-		}
-		// Also verifies genesis state didnt get deleted
-		if s != nil && s.Slot != finalizedSlot && s.Slot != 0 && s.Slot < endSlot {
-			t.Errorf("State with slot %d should not be in DB", s.Slot)
-		}
-	}
-
-	// New finalized epoch: 5
-	newFinalizedEpoch := uint64(5)
-	newFinalizedSlot := newFinalizedEpoch * params.BeaconConfig().SlotsPerEpoch
-	endSlot = helpers.StartSlot(newFinalizedEpoch+1) - 1 // Inclusive
-	if err := store.rmStatesOlderThanLastFinalized(ctx, helpers.StartSlot(finalizedEpoch+1)-1, endSlot); err != nil {
-		t.Fatal(err)
-	}
-	for _, r := range blockRoots {
-		s, err := store.db.State(ctx, r)
-		if err != nil {
-			t.Fatal(err)
-		}
-		// Also verifies genesis state didnt get deleted
-		if s != nil && s.Slot != newFinalizedSlot && s.Slot != finalizedSlot && s.Slot != 0 && s.Slot < endSlot {
-			t.Errorf("State with slot %d should not be in DB", s.Slot)
-		}
-	}
-}
-
-func TestRemoveStateSinceLastFinalized_EmptyStartSlot(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	params.UseMinimalConfig()
-	defer params.UseMainnetConfig()
-
-	store := NewForkChoiceService(ctx, db)
-	store.genesisTime = uint64(time.Now().Unix())
-
-	update, err := store.shouldUpdateCurrentJustified(ctx, &ethpb.Checkpoint{})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !update {
-		t.Error("Should be able to update justified, received false")
-	}
-
-	lastJustifiedBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{ParentRoot: []byte{'G'}}}
-	lastJustifiedRoot, _ := ssz.HashTreeRoot(lastJustifiedBlk.Block)
-	newJustifiedBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1, ParentRoot: lastJustifiedRoot[:]}}
-	newJustifiedRoot, _ := ssz.HashTreeRoot(newJustifiedBlk.Block)
-	if err := store.db.SaveBlock(ctx, newJustifiedBlk); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveBlock(ctx, lastJustifiedBlk); err != nil {
-		t.Fatal(err)
-	}
-
-	diff := (params.BeaconConfig().SlotsPerEpoch - 1) * params.BeaconConfig().SecondsPerSlot
-	store.genesisTime = uint64(time.Now().Unix()) - diff
-	store.justifiedCheckpt = &ethpb.Checkpoint{Root: lastJustifiedRoot[:]}
-	update, err = store.shouldUpdateCurrentJustified(ctx, &ethpb.Checkpoint{Root: newJustifiedRoot[:]})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !update {
-		t.Error("Should be able to update justified, received false")
-	}
-}
-
-func TestShouldUpdateJustified_ReturnFalse(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	params.UseMinimalConfig()
-	defer params.UseMainnetConfig()
-
-	store := NewForkChoiceService(ctx, db)
-
-	lastJustifiedBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{ParentRoot: []byte{'G'}}}
-	lastJustifiedRoot, _ := ssz.HashTreeRoot(lastJustifiedBlk.Block)
-	newJustifiedBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{ParentRoot: lastJustifiedRoot[:]}}
-	newJustifiedRoot, _ := ssz.HashTreeRoot(newJustifiedBlk.Block)
-	if err := store.db.SaveBlock(ctx, newJustifiedBlk); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveBlock(ctx, lastJustifiedBlk); err != nil {
-		t.Fatal(err)
-	}
-
-	diff := (params.BeaconConfig().SlotsPerEpoch - 1) * params.BeaconConfig().SecondsPerSlot
-	store.genesisTime = uint64(time.Now().Unix()) - diff
-	store.justifiedCheckpt = &ethpb.Checkpoint{Root: lastJustifiedRoot[:]}
-
-	update, err := store.shouldUpdateCurrentJustified(ctx, &ethpb.Checkpoint{Root: newJustifiedRoot[:]})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if update {
-		t.Error("Should not be able to update justified, received true")
-	}
-}
-
-func TestUpdateJustifiedCheckpoint_Update(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	params.UseMinimalConfig()
-	defer params.UseMainnetConfig()
-
-	store := NewForkChoiceService(ctx, db)
-	store.genesisTime = uint64(time.Now().Unix())
-
-	store.justifiedCheckpt = &ethpb.Checkpoint{Root: []byte{'A'}}
-	store.bestJustifiedCheckpt = &ethpb.Checkpoint{Epoch: 1, Root: []byte{'B'}}
-	store.updateJustifiedCheckpoint()
-
-	if !bytes.Equal(store.justifiedCheckpt.Root, []byte{'B'}) {
-		t.Error("Justified check point root did not update")
-	}
-}
-
-func TestUpdateJustifiedCheckpoint_NoUpdate(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	params.UseMinimalConfig()
-	defer params.UseMainnetConfig()
-
-	store := NewForkChoiceService(ctx, db)
-	store.genesisTime = uint64(time.Now().Unix()) - params.BeaconConfig().SecondsPerSlot
-
-	store.justifiedCheckpt = &ethpb.Checkpoint{Root: []byte{'A'}}
-	store.bestJustifiedCheckpt = &ethpb.Checkpoint{Epoch: 1, Root: []byte{'B'}}
-	store.updateJustifiedCheckpoint()
-
-	if bytes.Equal(store.justifiedCheckpt.Root, []byte{'B'}) {
-		t.Error("Justified check point root was not suppose to update")
-
-		store := NewForkChoiceService(ctx, db)
-
-		// Save 5 blocks in DB, each has a state.
-		numBlocks := 5
-		totalBlocks := make([]*ethpb.SignedBeaconBlock, numBlocks)
-		blockRoots := make([][32]byte, 0)
-		for i := 0; i < len(totalBlocks); i++ {
-			totalBlocks[i] = &ethpb.SignedBeaconBlock{
-				Block: &ethpb.BeaconBlock{
-					Slot: uint64(i),
-				},
-			}
-			r, err := ssz.HashTreeRoot(totalBlocks[i].Block)
-			if err != nil {
-				t.Fatal(err)
-			}
-			if err := store.db.SaveState(ctx, &pb.BeaconState{Slot: uint64(i)}, r); err != nil {
-				t.Fatal(err)
-			}
-			if err := store.db.SaveBlock(ctx, totalBlocks[i]); err != nil {
-				t.Fatal(err)
-			}
-			blockRoots = append(blockRoots, r)
-		}
-		if err := store.db.SaveHeadBlockRoot(ctx, blockRoots[0]); err != nil {
-			t.Fatal(err)
-		}
-		if err := store.rmStatesOlderThanLastFinalized(ctx, 10, 11); err != nil {
-			t.Fatal(err)
-		}
-		// Since 5-10 are skip slots, block with slot 4 should be deleted
-		s, err := store.db.State(ctx, blockRoots[4])
-		if err != nil {
-			t.Fatal(err)
-		}
-		if s != nil {
-			t.Error("Did not delete state for start slot")
-		}
-	}
-}
-
-func TestCachedPreState_CanGetFromCache(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	s := &pb.BeaconState{Slot: 1}
-	r := [32]byte{'A'}
-	b := &ethpb.BeaconBlock{Slot: 1, ParentRoot: r[:]}
-	store.initSyncState[r] = s
-
-	wanted := "pre state of slot 1 does not exist"
-	if _, err := store.cachedPreState(ctx, b); !strings.Contains(err.Error(), wanted) {
-		t.Fatal("Not expected error")
-	}
-}
-
-func TestCachedPreState_CanGetFromCacheWithFeature(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	config := &featureconfig.Flags{
-		InitSyncCacheState: true,
-	}
-	featureconfig.Init(config)
-
-	store := NewForkChoiceService(ctx, db)
-	s := &pb.BeaconState{Slot: 1}
-	r := [32]byte{'A'}
-	b := &ethpb.BeaconBlock{Slot: 1, ParentRoot: r[:]}
-	store.initSyncState[r] = s
-
-	received, err := store.cachedPreState(ctx, b)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(s, received) {
-		t.Error("cached state not the same")
-	}
-}
-
-func TestCachedPreState_CanGetFromDB(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	r := [32]byte{'A'}
-	b := &ethpb.BeaconBlock{Slot: 1, ParentRoot: r[:]}
-
-	_, err := store.cachedPreState(ctx, b)
-	wanted := "pre state of slot 1 does not exist"
-	if err.Error() != wanted {
-		t.Error("Did not get wanted error")
-	}
-
-	s := &pb.BeaconState{Slot: 1}
-	store.db.SaveState(ctx, s, r)
-
-	received, err := store.cachedPreState(ctx, b)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(s, received) {
-		t.Error("cached state not the same")
-	}
-}
-
-func TestSaveInitState_CanSaveDelete(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	config := &featureconfig.Flags{
-		InitSyncCacheState: true,
-	}
-	featureconfig.Init(config)
-
-	for i := uint64(0); i < 64; i++ {
-		b := &ethpb.BeaconBlock{Slot: i}
-		s := &pb.BeaconState{Slot: i}
-		r, _ := ssz.HashTreeRoot(b)
-		store.initSyncState[r] = s
-	}
-
-	// Set finalized root as slot 32
-	finalizedRoot, _ := ssz.HashTreeRoot(&ethpb.BeaconBlock{Slot: 32})
-
-	if err := store.saveInitState(ctx, &pb.BeaconState{FinalizedCheckpoint: &ethpb.Checkpoint{
-		Epoch: 1, Root: finalizedRoot[:]}}); err != nil {
-		t.Fatal(err)
-	}
-
-	// Verify finalized state is saved in DB
-	finalizedState, err := store.db.State(ctx, finalizedRoot)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if finalizedState == nil {
-		t.Error("finalized state can't be nil")
-	}
-
-	// Verify cached state is properly pruned
-	if len(store.initSyncState) != int(params.BeaconConfig().SlotsPerEpoch) {
-		t.Errorf("wanted: %d, got: %d", len(store.initSyncState), params.BeaconConfig().SlotsPerEpoch)
-	}
-}
-
-func TestUpdateJustified_CouldUpdateBest(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	signedBlock := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{}}
-	if err := db.SaveBlock(ctx, signedBlock); err != nil {
-		t.Fatal(err)
-	}
-	r, err := ssz.HashTreeRoot(signedBlock.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	store.justifiedCheckpt = &ethpb.Checkpoint{Root: []byte{'A'}}
-	store.bestJustifiedCheckpt = &ethpb.Checkpoint{Root: []byte{'A'}}
-	store.initSyncState[r] = &pb.BeaconState{}
-	if err := db.SaveState(ctx, &pb.BeaconState{}, r); err != nil {
-		t.Fatal(err)
-	}
-
-	// Could update
-	s := &pb.BeaconState{CurrentJustifiedCheckpoint: &ethpb.Checkpoint{Epoch: 1, Root: r[:]}}
-	if err := store.updateJustified(context.Background(), s); err != nil {
-		t.Fatal(err)
-	}
-
-	if store.bestJustifiedCheckpt.Epoch != s.CurrentJustifiedCheckpoint.Epoch {
-		t.Error("Incorrect justified epoch in store")
-	}
-
-	// Could not update
-	store.bestJustifiedCheckpt.Epoch = 2
-	if err := store.updateJustified(context.Background(), s); err != nil {
-		t.Fatal(err)
-	}
-
-	if store.bestJustifiedCheckpt.Epoch != 2 {
-		t.Error("Incorrect justified epoch in store")
-	}
-}
-
-func TestFilterBlockRoots_CanFilter(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	fBlock := &ethpb.BeaconBlock{}
-	fRoot, _ := ssz.HashTreeRoot(fBlock)
-	hBlock := &ethpb.BeaconBlock{Slot: 1}
-	headRoot, _ := ssz.HashTreeRoot(hBlock)
-	if err := store.db.SaveBlock(ctx, &ethpb.SignedBeaconBlock{Block: fBlock}); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveState(ctx, &pb.BeaconState{}, fRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveFinalizedCheckpoint(ctx, &ethpb.Checkpoint{Root: fRoot[:]}); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveBlock(ctx, &ethpb.SignedBeaconBlock{Block: hBlock}); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveState(ctx, &pb.BeaconState{}, headRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveHeadBlockRoot(ctx, headRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	roots := [][32]byte{{'C'}, {'D'}, headRoot, {'E'}, fRoot, {'F'}}
-	wanted := [][32]byte{{'C'}, {'D'}, {'E'}, {'F'}}
-
-	received, err := store.filterBlockRoots(ctx, roots)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if !reflect.DeepEqual(wanted, received) {
-		t.Error("Did not filter correctly")
-	}
-}

beacon-chain/blockchain/forkchoice/service.go

@@ -1,433 +0,0 @@
-package forkchoice
-
-import (
-	"bytes"
-	"context"
-	"encoding/hex"
-	"fmt"
-	"sync"
-
-	"github.com/gogo/protobuf/proto"
-	"github.com/pkg/errors"
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"github.com/prysmaticlabs/prysm/shared/stateutil"
-	"go.opencensus.io/trace"
-)
-
-// ForkChoicer defines a common interface for methods useful for directly applying fork choice
-// to beacon blocks to compute head.
-type ForkChoicer interface {
-	Head(ctx context.Context) ([]byte, error)
-	OnBlock(ctx context.Context, b *ethpb.SignedBeaconBlock) error
-	OnBlockInitialSyncStateTransition(ctx context.Context, b *ethpb.SignedBeaconBlock) error
-	OnAttestation(ctx context.Context, a *ethpb.Attestation) error
-	GenesisStore(ctx context.Context, justifiedCheckpoint *ethpb.Checkpoint, finalizedCheckpoint *ethpb.Checkpoint) error
-	FinalizedCheckpt() *ethpb.Checkpoint
-}
-
-// Store represents a service struct that handles the forkchoice
-// logic of managing the full PoS beacon chain.
-type Store struct {
-	ctx                   context.Context
-	cancel                context.CancelFunc
-	db                    db.Database
-	justifiedCheckpt      *ethpb.Checkpoint
-	finalizedCheckpt      *ethpb.Checkpoint
-	prevFinalizedCheckpt  *ethpb.Checkpoint
-	checkpointState       *cache.CheckpointStateCache
-	checkpointStateLock   sync.Mutex
-	genesisTime           uint64
-	bestJustifiedCheckpt  *ethpb.Checkpoint
-	latestVoteMap         map[uint64]*pb.ValidatorLatestVote
-	voteLock              sync.RWMutex
-	initSyncState         map[[32]byte]*pb.BeaconState
-	initSyncStateLock     sync.RWMutex
-	nextEpochBoundarySlot uint64
-}
-
-// NewForkChoiceService instantiates a new service instance that will
-// be registered into a running beacon node.
-func NewForkChoiceService(ctx context.Context, db db.Database) *Store {
-	ctx, cancel := context.WithCancel(ctx)
-	return &Store{
-		ctx:             ctx,
-		cancel:          cancel,
-		db:              db,
-		checkpointState: cache.NewCheckpointStateCache(),
-		latestVoteMap:   make(map[uint64]*pb.ValidatorLatestVote),
-		initSyncState:   make(map[[32]byte]*pb.BeaconState),
-	}
-}
-
-// GenesisStore initializes the store struct before beacon chain
-// starts to advance.
-//
-// Spec pseudocode definition:
-//   def get_genesis_store(genesis_state: BeaconState) -> Store:
-//    genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))
-//    root = signing_root(genesis_block)
-//    justified_checkpoint = Checkpoint(epoch=GENESIS_EPOCH, root=root)
-//    finalized_checkpoint = Checkpoint(epoch=GENESIS_EPOCH, root=root)
-//    return Store(
-//        time=genesis_state.genesis_time,
-//        justified_checkpoint=justified_checkpoint,
-//        finalized_checkpoint=finalized_checkpoint,
-//        blocks={root: genesis_block},
-//        block_states={root: genesis_state.copy()},
-//        checkpoint_states={justified_checkpoint: genesis_state.copy()},
-//    )
-func (s *Store) GenesisStore(
-	ctx context.Context,
-	justifiedCheckpoint *ethpb.Checkpoint,
-	finalizedCheckpoint *ethpb.Checkpoint) error {
-
-	s.justifiedCheckpt = proto.Clone(justifiedCheckpoint).(*ethpb.Checkpoint)
-	s.bestJustifiedCheckpt = proto.Clone(justifiedCheckpoint).(*ethpb.Checkpoint)
-	s.finalizedCheckpt = proto.Clone(finalizedCheckpoint).(*ethpb.Checkpoint)
-	s.prevFinalizedCheckpt = proto.Clone(finalizedCheckpoint).(*ethpb.Checkpoint)
-
-	justifiedState, err := s.db.State(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root))
-	if err != nil {
-		return errors.Wrap(err, "could not retrieve last justified state")
-	}
-
-	if err := s.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: s.justifiedCheckpt,
-		State:      justifiedState,
-	}); err != nil {
-		return errors.Wrap(err, "could not save genesis state in check point cache")
-	}
-
-	s.genesisTime = justifiedState.GenesisTime
-	if err := s.cacheGenesisState(ctx); err != nil {
-		return errors.Wrap(err, "could not cache initial sync state")
-	}
-
-	return nil
-}
-
-// This sets up gensis for initial sync state cache.
-func (s *Store) cacheGenesisState(ctx context.Context) error {
-	if !featureconfig.Get().InitSyncCacheState {
-		return nil
-	}
-
-	genesisState, err := s.db.GenesisState(ctx)
-	if err != nil {
-		return err
-	}
-	stateRoot, err := stateutil.HashTreeRootState(genesisState)
-	if err != nil {
-		return errors.Wrap(err, "could not tree hash genesis state")
-	}
-	genesisBlk := blocks.NewGenesisBlock(stateRoot[:])
-	genesisBlkRoot, err := ssz.HashTreeRoot(genesisBlk.Block)
-	if err != nil {
-		return errors.Wrap(err, "could not get genesis block root")
-	}
-	s.initSyncState[genesisBlkRoot] = genesisState
-
-	return nil
-}
-
-// ancestor returns the block root of an ancestry block from the input block root.
-//
-// Spec pseudocode definition:
-//   def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash:
-//    block = store.blocks[root]
-//    if block.slot > slot:
-//      return get_ancestor(store, block.parent_root, slot)
-//    elif block.slot == slot:
-//      return root
-//    else:
-//      return Bytes32()  # root is older than queried slot: no results.
-func (s *Store) ancestor(ctx context.Context, root []byte, slot uint64) ([]byte, error) {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.ancestor")
-	defer span.End()
-
-	// Stop recursive ancestry lookup if context is cancelled.
-	if ctx.Err() != nil {
-		return nil, ctx.Err()
-	}
-
-	signed, err := s.db.Block(ctx, bytesutil.ToBytes32(root))
-	if err != nil {
-		return nil, errors.Wrap(err, "could not get ancestor block")
-	}
-	if signed == nil || signed.Block == nil {
-		return nil, errors.New("nil block")
-	}
-	b := signed.Block
-
-	// If we dont have the ancestor in the DB, simply return nil so rest of fork choice
-	// operation can proceed. This is not an error condition.
-	if b == nil || b.Slot < slot {
-		return nil, nil
-	}
-
-	if b.Slot == slot {
-		return root, nil
-	}
-
-	return s.ancestor(ctx, b.ParentRoot, slot)
-}
-
-// latestAttestingBalance returns the staked balance of a block from the input block root.
-//
-// Spec pseudocode definition:
-//   def get_latest_attesting_balance(store: Store, root: Hash) -> Gwei:
-//    state = store.checkpoint_states[store.justified_checkpoint]
-//    active_indices = get_active_validator_indices(state, get_current_epoch(state))
-//    return Gwei(sum(
-//        state.validators[i].effective_balance for i in active_indices
-//        if (i in store.latest_messages
-//            and get_ancestor(store, store.latest_messages[i].root, store.blocks[root].slot) == root)
-//    ))
-func (s *Store) latestAttestingBalance(ctx context.Context, root []byte) (uint64, error) {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.latestAttestingBalance")
-	defer span.End()
-
-	lastJustifiedState, err := s.checkpointState.StateByCheckpoint(s.JustifiedCheckpt())
-	if err != nil {
-		return 0, errors.Wrap(err, "could not retrieve cached state via last justified check point")
-	}
-	if lastJustifiedState == nil {
-		return 0, errors.Wrapf(err, "could not get justified state at epoch %d", s.JustifiedCheckpt().Epoch)
-	}
-
-	lastJustifiedEpoch := helpers.CurrentEpoch(lastJustifiedState)
-	activeIndices, err := helpers.ActiveValidatorIndices(lastJustifiedState, lastJustifiedEpoch)
-	if err != nil {
-		return 0, errors.Wrap(err, "could not get active indices for last justified checkpoint")
-	}
-
-	wantedBlkSigned, err := s.db.Block(ctx, bytesutil.ToBytes32(root))
-	if err != nil {
-		return 0, errors.Wrap(err, "could not get target block")
-	}
-	if wantedBlkSigned == nil || wantedBlkSigned.Block == nil {
-		return 0, errors.New("nil wanted block")
-	}
-	wantedBlk := wantedBlkSigned.Block
-
-	balances := uint64(0)
-	s.voteLock.RLock()
-	defer s.voteLock.RUnlock()
-	for _, i := range activeIndices {
-		vote, ok := s.latestVoteMap[i]
-		if !ok {
-			continue
-		}
-
-		wantedRoot, err := s.ancestor(ctx, vote.Root, wantedBlk.Slot)
-		if err != nil {
-			return 0, errors.Wrapf(err, "could not get ancestor root for slot %d", wantedBlk.Slot)
-		}
-		if bytes.Equal(wantedRoot, root) {
-			balances += lastJustifiedState.Validators[i].EffectiveBalance
-		}
-	}
-	return balances, nil
-}
-
-// Head returns the head of the beacon chain.
-//
-// Spec pseudocode definition:
-//   def get_head(store: Store) -> Root:
-//    # Get filtered block tree that only includes viable branches
-//    blocks = get_filtered_block_tree(store)
-//    # Execute the LMD-GHOST fork choice
-//    head = store.justified_checkpoint.root
-//    justified_slot = compute_start_slot_at_epoch(store.justified_checkpoint.epoch)
-//    while True:
-//        children = [
-//            root for root in blocks.keys()
-//            if blocks[root].parent_root == head and blocks[root].slot > justified_slot
-//        ]
-//        if len(children) == 0:
-//            return head
-//        # Sort by latest attesting balance with ties broken lexicographically
-//        head = max(children, key=lambda root: (get_latest_attesting_balance(store, root), root))
-func (s *Store) Head(ctx context.Context) ([]byte, error) {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.head")
-	defer span.End()
-
-	head := s.JustifiedCheckpt().Root
-	filteredBlocks, err := s.getFilterBlockTree(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	justifiedSlot := helpers.StartSlot(s.justifiedCheckpt.Epoch)
-	for {
-		children := make([][32]byte, 0, len(filteredBlocks))
-		for root, block := range filteredBlocks {
-			if bytes.Equal(block.ParentRoot, head) && block.Slot > justifiedSlot {
-				children = append(children, root)
-			}
-		}
-
-		if len(children) == 0 {
-			return head, nil
-		}
-
-		// if a block has one child, then we don't have to lookup anything to
-		// know that this child will be the best child.
-		head = children[0][:]
-		if len(children) > 1 {
-			highest, err := s.latestAttestingBalance(ctx, head)
-			if err != nil {
-				return nil, errors.Wrap(err, "could not get latest balance")
-			}
-			for _, child := range children[1:] {
-				balance, err := s.latestAttestingBalance(ctx, child[:])
-				if err != nil {
-					return nil, errors.Wrap(err, "could not get latest balance")
-				}
-				// When there's a tie, it's broken lexicographically to favor the higher one.
-				if balance > highest ||
-					balance == highest && bytes.Compare(child[:], head) > 0 {
-					highest = balance
-					head = child[:]
-				}
-			}
-		}
-	}
-}
-
-// getFilterBlockTree retrieves a filtered block tree from store, it only returns branches
-// whose leaf state's justified and finalized info agrees with what's in the store.
-// Rationale: https://notes.ethereum.org/Fj-gVkOSTpOyUx-zkWjuwg?view
-//
-// Spec pseudocode definition:
-//   def get_filtered_block_tree(store: Store) -> Dict[Root, BeaconBlock]:
-//    """
-//    Retrieve a filtered block true from ``store``, only returning branches
-//    whose leaf state's justified/finalized info agrees with that in ``store``.
-//    """
-//    base = store.justified_checkpoint.root
-//    blocks: Dict[Root, BeaconBlock] = {}
-//    filter_block_tree(store, base, blocks)
-//    return blocks
-func (s *Store) getFilterBlockTree(ctx context.Context) (map[[32]byte]*ethpb.BeaconBlock, error) {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.getFilterBlockTree")
-	defer span.End()
-
-	baseRoot := bytesutil.ToBytes32(s.justifiedCheckpt.Root)
-	filteredBlocks := make(map[[32]byte]*ethpb.BeaconBlock)
-	if _, err := s.filterBlockTree(ctx, baseRoot, filteredBlocks); err != nil {
-		return nil, err
-	}
-
-	return filteredBlocks, nil
-}
-
-// filterBlockTree filters for branches that see latest finalized and justified info as correct on-chain
-// before running Head.
-//
-// Spec pseudocode definition:
-//   def filter_block_tree(store: Store, block_root: Root, blocks: Dict[Root, BeaconBlock]) -> bool:
-//    block = store.blocks[block_root]
-//    children = [
-//        root for root in store.blocks.keys()
-//        if store.blocks[root].parent_root == block_root
-//    ]
-//    # If any children branches contain expected finalized/justified checkpoints,
-//    # add to filtered block-tree and signal viability to parent.
-//    if any(children):
-//        filter_block_tree_result = [filter_block_tree(store, child, blocks) for child in children]
-//        if any(filter_block_tree_result):
-//            blocks[block_root] = block
-//            return True
-//        return False
-//    # If leaf block, check finalized/justified checkpoints as matching latest.
-//    head_state = store.block_states[block_root]
-//    correct_justified = (
-//        store.justified_checkpoint.epoch == GENESIS_EPOCH
-//        or head_state.current_justified_checkpoint == store.justified_checkpoint
-//    )
-//    correct_finalized = (
-//        store.finalized_checkpoint.epoch == GENESIS_EPOCH
-//        or head_state.finalized_checkpoint == store.finalized_checkpoint
-//    )
-//    # If expected finalized/justified, add to viable block-tree and signal viability to parent.
-//    if correct_justified and correct_finalized:
-//        blocks[block_root] = block
-//        return True
-//    # Otherwise, branch not viable
-//    return False
-func (s *Store) filterBlockTree(ctx context.Context, blockRoot [32]byte, filteredBlocks map[[32]byte]*ethpb.BeaconBlock) (bool, error) {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.filterBlockTree")
-	defer span.End()
-	signed, err := s.db.Block(ctx, blockRoot)
-	if err != nil {
-		return false, err
-	}
-	if signed == nil || signed.Block == nil {
-		return false, errors.New("nil block")
-	}
-	block := signed.Block
-
-	filter := filters.NewFilter().SetParentRoot(blockRoot[:])
-	childrenRoots, err := s.db.BlockRoots(ctx, filter)
-	if err != nil {
-		return false, err
-	}
-
-	if len(childrenRoots) != 0 {
-		var filtered bool
-		for _, childRoot := range childrenRoots {
-			didFilter, err := s.filterBlockTree(ctx, childRoot, filteredBlocks)
-			if err != nil {
-				return false, err
-			}
-			if didFilter {
-				filtered = true
-			}
-		}
-		if filtered {
-			filteredBlocks[blockRoot] = block
-			return true, nil
-		}
-		return false, nil
-	}
-
-	headState, err := s.db.State(ctx, blockRoot)
-	if err != nil {
-		return false, err
-	}
-
-	if headState == nil {
-		return false, fmt.Errorf("no state matching block root %v", hex.EncodeToString(blockRoot[:]))
-	}
-
-	correctJustified := s.justifiedCheckpt.Epoch == 0 ||
-		proto.Equal(s.justifiedCheckpt, headState.CurrentJustifiedCheckpoint)
-	correctFinalized := s.finalizedCheckpt.Epoch == 0 ||
-		proto.Equal(s.finalizedCheckpt, headState.FinalizedCheckpoint)
-	if correctJustified && correctFinalized {
-		filteredBlocks[blockRoot] = block
-		return true, nil
-	}
-
-	return false, nil
-}
-
-// JustifiedCheckpt returns the latest justified check point from fork choice store.
-func (s *Store) JustifiedCheckpt() *ethpb.Checkpoint {
-	return proto.Clone(s.justifiedCheckpt).(*ethpb.Checkpoint)
-}
-
-// FinalizedCheckpt returns the latest finalized check point from fork choice store.
-func (s *Store) FinalizedCheckpt() *ethpb.Checkpoint {
-	return proto.Clone(s.finalizedCheckpt).(*ethpb.Checkpoint)
-}

beacon-chain/blockchain/forkchoice/service_test.go

@@ -1,517 +0,0 @@
-package forkchoice
-
-import (
-	"bytes"
-	"context"
-	"reflect"
-	"testing"
-	"time"
-
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
-	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"github.com/prysmaticlabs/prysm/shared/stateutil"
-)
-
-func TestStore_GenesisStoreOk(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	genesisTime := time.Unix(9999, 0)
-	genesisState := &pb.BeaconState{GenesisTime: uint64(genesisTime.Unix())}
-	genesisStateRoot, err := stateutil.HashTreeRootState(genesisState)
-	if err != nil {
-		t.Fatal(err)
-	}
-	genesisBlk := blocks.NewGenesisBlock(genesisStateRoot[:])
-	genesisBlkRoot, err := ssz.HashTreeRoot(genesisBlk.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveState(ctx, genesisState, genesisBlkRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveGenesisBlockRoot(ctx, genesisBlkRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	checkPoint := &ethpb.Checkpoint{Root: genesisBlkRoot[:]}
-	if err := store.GenesisStore(ctx, checkPoint, checkPoint); err != nil {
-		t.Fatal(err)
-	}
-
-	if !reflect.DeepEqual(store.justifiedCheckpt, checkPoint) {
-		t.Error("Justified check point from genesis store did not match")
-	}
-	if !reflect.DeepEqual(store.finalizedCheckpt, checkPoint) {
-		t.Error("Finalized check point from genesis store did not match")
-	}
-
-	cachedState, err := store.checkpointState.StateByCheckpoint(checkPoint)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(cachedState, genesisState) {
-		t.Error("Incorrect genesis state cached")
-	}
-}
-
-func TestStore_AncestorOk(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db, []byte{'g'})
-	if err != nil {
-		t.Fatal(err)
-	}
-	type args struct {
-		root []byte
-		slot uint64
-	}
-
-	//    /- B1
-	// B0           /- B5 - B7
-	//    \- B3 - B4 - B6 - B8
-	tests := []struct {
-		args *args
-		want []byte
-	}{
-		{args: &args{roots[1], 0}, want: roots[0]},
-		{args: &args{roots[8], 0}, want: roots[0]},
-		{args: &args{roots[8], 4}, want: roots[4]},
-		{args: &args{roots[7], 4}, want: roots[4]},
-		{args: &args{roots[7], 0}, want: roots[0]},
-	}
-	for _, tt := range tests {
-		got, err := store.ancestor(ctx, tt.args.root, tt.args.slot)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if !reflect.DeepEqual(got, tt.want) {
-			t.Errorf("Store.ancestor(ctx, ) = %v, want %v", got, tt.want)
-		}
-	}
-}
-
-func TestStore_AncestorNotPartOfTheChain(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db, []byte{'g'})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	//    /- B1
-	// B0           /- B5 - B7
-	//    \- B3 - B4 - B6 - B8
-	root, err := store.ancestor(ctx, roots[8], 1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if root != nil {
-		t.Error("block at slot 1 is not part of the chain")
-	}
-	root, err = store.ancestor(ctx, roots[8], 2)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if root != nil {
-		t.Error("block at slot 2 is not part of the chain")
-	}
-}
-
-func TestStore_LatestAttestingBalance(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db, []byte{'g'})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	validators := make([]*ethpb.Validator, 100)
-	for i := 0; i < len(validators); i++ {
-		validators[i] = &ethpb.Validator{ExitEpoch: 2, EffectiveBalance: 1e9}
-	}
-
-	s := &pb.BeaconState{Validators: validators}
-	stateRoot, err := stateutil.HashTreeRootState(s)
-	if err != nil {
-		t.Fatal(err)
-	}
-	b := blocks.NewGenesisBlock(stateRoot[:])
-	blkRoot, err := ssz.HashTreeRoot(b.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveState(ctx, s, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveGenesisBlockRoot(ctx, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	checkPoint := &ethpb.Checkpoint{Root: blkRoot[:]}
-	if err := store.GenesisStore(ctx, checkPoint, checkPoint); err != nil {
-		t.Fatal(err)
-	}
-
-	//    /- B1 (33 votes)
-	// B0           /- B5 - B7 (33 votes)
-	//    \- B3 - B4 - B6 - B8 (34 votes)
-	for i := 0; i < len(validators); i++ {
-		switch {
-		case i < 33:
-			store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[1]}
-		case i > 66:
-			store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[7]}
-		default:
-			store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[8]}
-		}
-	}
-
-	tests := []struct {
-		root []byte
-		want uint64
-	}{
-		{root: roots[0], want: 100 * 1e9},
-		{root: roots[1], want: 33 * 1e9},
-		{root: roots[3], want: 67 * 1e9},
-		{root: roots[4], want: 67 * 1e9},
-		{root: roots[7], want: 33 * 1e9},
-		{root: roots[8], want: 34 * 1e9},
-	}
-	for _, tt := range tests {
-		got, err := store.latestAttestingBalance(ctx, tt.root)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if got != tt.want {
-			t.Errorf("Store.latestAttestingBalance(ctx, ) = %v, want %v", got, tt.want)
-		}
-	}
-}
-
-func TestStore_ChildrenBlocksFromParentRoot(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db, []byte{'g'})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	filter := filters.NewFilter().SetParentRoot(roots[0]).SetStartSlot(0)
-	children, err := store.db.BlockRoots(ctx, filter)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(children, [][32]byte{bytesutil.ToBytes32(roots[1]), bytesutil.ToBytes32(roots[3])}) {
-		t.Error("Did not receive correct children roots")
-	}
-
-	filter = filters.NewFilter().SetParentRoot(roots[0]).SetStartSlot(2)
-	children, err = store.db.BlockRoots(ctx, filter)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !reflect.DeepEqual(children, [][32]byte{bytesutil.ToBytes32(roots[3])}) {
-		t.Error("Did not receive correct children roots")
-	}
-}
-
-func TestStore_GetHead(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db, []byte{'g'})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	validators := make([]*ethpb.Validator, 100)
-	for i := 0; i < len(validators); i++ {
-		validators[i] = &ethpb.Validator{ExitEpoch: 2, EffectiveBalance: 1e9}
-	}
-
-	s := &pb.BeaconState{Validators: validators}
-	stateRoot, err := stateutil.HashTreeRootState(s)
-	if err != nil {
-		t.Fatal(err)
-	}
-	b := blocks.NewGenesisBlock(stateRoot[:])
-	blkRoot, err := ssz.HashTreeRoot(b.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveState(ctx, s, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveGenesisBlockRoot(ctx, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	checkPoint := &ethpb.Checkpoint{Root: blkRoot[:]}
-
-	if err := store.GenesisStore(ctx, checkPoint, checkPoint); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveState(ctx, s, bytesutil.ToBytes32(roots[0])); err != nil {
-		t.Fatal(err)
-	}
-	store.justifiedCheckpt.Root = roots[0]
-	if err := store.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: store.justifiedCheckpt,
-		State:      s,
-	}); err != nil {
-		t.Fatal(err)
-	}
-
-	//    /- B1 (33 votes)
-	// B0           /- B5 - B7 (33 votes)
-	//    \- B3 - B4 - B6 - B8 (34 votes)
-	for i := 0; i < len(validators); i++ {
-		switch {
-		case i < 33:
-			store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[1]}
-		case i > 66:
-			store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[7]}
-		default:
-			store.latestVoteMap[uint64(i)] = &pb.ValidatorLatestVote{Root: roots[8]}
-		}
-	}
-
-	// Default head is B8
-	head, err := store.Head(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !bytes.Equal(head, roots[8]) {
-		t.Error("Incorrect head")
-	}
-
-	// 1 validator switches vote to B7 to gain 34%, enough to switch head
-	store.latestVoteMap[uint64(50)] = &pb.ValidatorLatestVote{Root: roots[7]}
-
-	head, err = store.Head(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !bytes.Equal(head, roots[7]) {
-		t.Error("Incorrect head")
-	}
-
-	// 18 validators switches vote to B1 to gain 51%, enough to switch head
-	for i := 0; i < 18; i++ {
-		idx := 50 + uint64(i)
-		store.latestVoteMap[uint64(idx)] = &pb.ValidatorLatestVote{Root: roots[1]}
-	}
-	head, err = store.Head(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !bytes.Equal(head, roots[1]) {
-		t.Log(head)
-		t.Error("Incorrect head")
-	}
-}
-
-func TestCacheGenesisState_Correct(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	config := &featureconfig.Flags{
-		InitSyncCacheState: true,
-	}
-	featureconfig.Init(config)
-
-	b := &ethpb.BeaconBlock{Slot: 1}
-	r, _ := ssz.HashTreeRoot(b)
-	s := &pb.BeaconState{GenesisTime: 99}
-
-	store.db.SaveState(ctx, s, r)
-	store.db.SaveGenesisBlockRoot(ctx, r)
-
-	if err := store.cacheGenesisState(ctx); err != nil {
-		t.Fatal(err)
-	}
-
-	for _, state := range store.initSyncState {
-		if !reflect.DeepEqual(s, state) {
-			t.Error("Did not get wanted state")
-		}
-	}
-}
-
-func TestStore_GetFilterBlockTree_CorrectLeaf(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db, []byte{'g'})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	s := &pb.BeaconState{}
-	stateRoot, err := stateutil.HashTreeRootState(s)
-	if err != nil {
-		t.Fatal(err)
-	}
-	b := blocks.NewGenesisBlock(stateRoot[:])
-	blkRoot, err := ssz.HashTreeRoot(b.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveState(ctx, s, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveGenesisBlockRoot(ctx, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	checkPoint := &ethpb.Checkpoint{Root: blkRoot[:]}
-
-	if err := store.GenesisStore(ctx, checkPoint, checkPoint); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveState(ctx, s, bytesutil.ToBytes32(roots[0])); err != nil {
-		t.Fatal(err)
-	}
-	store.justifiedCheckpt.Root = roots[0]
-	if err := store.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: store.justifiedCheckpt,
-		State:      s,
-	}); err != nil {
-		t.Fatal(err)
-	}
-
-	tree, err := store.getFilterBlockTree(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	wanted := make(map[[32]byte]*ethpb.BeaconBlock)
-	for _, root := range roots {
-		root32 := bytesutil.ToBytes32(root)
-		b, _ := store.db.Block(ctx, root32)
-		if b != nil {
-			wanted[root32] = b.Block
-		}
-	}
-	if !reflect.DeepEqual(tree, wanted) {
-		t.Error("Did not filter tree correctly")
-	}
-}
-
-func TestStore_GetFilterBlockTree_IncorrectLeaf(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-
-	roots, err := blockTree1(db, []byte{'g'})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	s := &pb.BeaconState{}
-	stateRoot, err := stateutil.HashTreeRootState(s)
-	if err != nil {
-		t.Fatal(err)
-	}
-	b := blocks.NewGenesisBlock(stateRoot[:])
-	blkRoot, err := ssz.HashTreeRoot(b.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveState(ctx, s, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveGenesisBlockRoot(ctx, blkRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	checkPoint := &ethpb.Checkpoint{Root: blkRoot[:]}
-
-	if err := store.GenesisStore(ctx, checkPoint, checkPoint); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.db.SaveState(ctx, s, bytesutil.ToBytes32(roots[0])); err != nil {
-		t.Fatal(err)
-	}
-	store.justifiedCheckpt.Root = roots[0]
-	if err := store.checkpointState.AddCheckpointState(&cache.CheckpointState{
-		Checkpoint: store.justifiedCheckpt,
-		State:      s,
-	}); err != nil {
-		t.Fatal(err)
-	}
-	// Filter for incorrect leaves for 1, 7 and 8
-	store.db.SaveState(ctx, &pb.BeaconState{CurrentJustifiedCheckpoint: &ethpb.Checkpoint{}}, bytesutil.ToBytes32(roots[1]))
-	store.db.SaveState(ctx, &pb.BeaconState{CurrentJustifiedCheckpoint: &ethpb.Checkpoint{}}, bytesutil.ToBytes32(roots[7]))
-	store.db.SaveState(ctx, &pb.BeaconState{CurrentJustifiedCheckpoint: &ethpb.Checkpoint{}}, bytesutil.ToBytes32(roots[8]))
-	store.justifiedCheckpt.Epoch = 1
-	tree, err := store.getFilterBlockTree(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(tree) != 0 {
-		t.Error("filtered tree should be 0 length")
-	}
-
-	// Set leave 1 as correct
-	store.db.SaveState(ctx, &pb.BeaconState{CurrentJustifiedCheckpoint: &ethpb.Checkpoint{Epoch: 1, Root: store.justifiedCheckpt.Root}}, bytesutil.ToBytes32(roots[1]))
-	tree, err = store.getFilterBlockTree(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	wanted := make(map[[32]byte]*ethpb.BeaconBlock)
-	root32 := bytesutil.ToBytes32(roots[0])
-	b, err = store.db.Block(ctx, root32)
-	if err != nil {
-		t.Fatal(err)
-	}
-	wanted[root32] = b.Block
-	root32 = bytesutil.ToBytes32(roots[1])
-	b, err = store.db.Block(ctx, root32)
-	if err != nil {
-		t.Fatal(err)
-	}
-	wanted[root32] = b.Block
-
-	if !reflect.DeepEqual(tree, wanted) {
-		t.Error("Did not filter tree correctly")
-	}
-}

beacon-chain/blockchain/forkchoice/tree_test.go

@@ -1,153 +0,0 @@
-package forkchoice
-
-import (
-	"context"
-
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-)
-
-// blockTree1 constructs the following tree:
-//    /- B1
-// B0           /- B5 - B7
-//    \- B3 - B4 - B6 - B8
-// (B1, and B3 are all from the same slots)
-func blockTree1(db db.Database, genesisRoot []byte) ([][]byte, error) {
-	b0 := &ethpb.BeaconBlock{Slot: 0, ParentRoot: genesisRoot}
-	r0, _ := ssz.HashTreeRoot(b0)
-	b1 := &ethpb.BeaconBlock{Slot: 1, ParentRoot: r0[:]}
-	r1, _ := ssz.HashTreeRoot(b1)
-	b3 := &ethpb.BeaconBlock{Slot: 3, ParentRoot: r0[:]}
-	r3, _ := ssz.HashTreeRoot(b3)
-	b4 := &ethpb.BeaconBlock{Slot: 4, ParentRoot: r3[:]}
-	r4, _ := ssz.HashTreeRoot(b4)
-	b5 := &ethpb.BeaconBlock{Slot: 5, ParentRoot: r4[:]}
-	r5, _ := ssz.HashTreeRoot(b5)
-	b6 := &ethpb.BeaconBlock{Slot: 6, ParentRoot: r4[:]}
-	r6, _ := ssz.HashTreeRoot(b6)
-	b7 := &ethpb.BeaconBlock{Slot: 7, ParentRoot: r5[:]}
-	r7, _ := ssz.HashTreeRoot(b7)
-	b8 := &ethpb.BeaconBlock{Slot: 8, ParentRoot: r6[:]}
-	r8, _ := ssz.HashTreeRoot(b8)
-	for _, b := range []*ethpb.BeaconBlock{b0, b1, b3, b4, b5, b6, b7, b8} {
-		if err := db.SaveBlock(context.Background(), &ethpb.SignedBeaconBlock{Block: b}); err != nil {
-			return nil, err
-		}
-		if err := db.SaveState(context.Background(), &pb.BeaconState{}, bytesutil.ToBytes32(b.ParentRoot)); err != nil {
-			return nil, err
-		}
-	}
-	if err := db.SaveState(context.Background(), &pb.BeaconState{}, r1); err != nil {
-		return nil, err
-	}
-	if err := db.SaveState(context.Background(), &pb.BeaconState{}, r7); err != nil {
-		return nil, err
-	}
-	if err := db.SaveState(context.Background(), &pb.BeaconState{}, r8); err != nil {
-		return nil, err
-	}
-	return [][]byte{r0[:], r1[:], nil, r3[:], r4[:], r5[:], r6[:], r7[:], r8[:]}, nil
-}
-
-// blockTree2 constructs the following tree:
-// Scenario graph: shorturl.at/loyP6
-//
-//digraph G {
-//    rankdir=LR;
-//    node [shape="none"];
-//
-//    subgraph blocks {
-//        rankdir=LR;
-//        node [shape="box"];
-//        a->b;
-//        a->c;
-//        b->d;
-//        b->e;
-//        c->f;
-//        c->g;
-//        d->h
-//        d->i
-//        d->j
-//        d->k
-//        h->l
-//        h->m
-//        g->n
-//        g->o
-//        e->p
-//    }
-//}
-func blockTree2(db db.Database) ([][]byte, error) {
-	b0 := &ethpb.BeaconBlock{Slot: 0, ParentRoot: []byte{'g'}}
-	r0, _ := ssz.HashTreeRoot(b0)
-	b1 := &ethpb.BeaconBlock{Slot: 1, ParentRoot: r0[:]}
-	r1, _ := ssz.HashTreeRoot(b1)
-	b2 := &ethpb.BeaconBlock{Slot: 2, ParentRoot: r0[:]}
-	r2, _ := ssz.HashTreeRoot(b2)
-	b3 := &ethpb.BeaconBlock{Slot: 3, ParentRoot: r1[:]}
-	r3, _ := ssz.HashTreeRoot(b3)
-	b4 := &ethpb.BeaconBlock{Slot: 4, ParentRoot: r1[:]}
-	r4, _ := ssz.HashTreeRoot(b4)
-	b5 := &ethpb.BeaconBlock{Slot: 5, ParentRoot: r2[:]}
-	r5, _ := ssz.HashTreeRoot(b5)
-	b6 := &ethpb.BeaconBlock{Slot: 6, ParentRoot: r2[:]}
-	r6, _ := ssz.HashTreeRoot(b6)
-	b7 := &ethpb.BeaconBlock{Slot: 7, ParentRoot: r3[:]}
-	r7, _ := ssz.HashTreeRoot(b7)
-	b8 := &ethpb.BeaconBlock{Slot: 8, ParentRoot: r3[:]}
-	r8, _ := ssz.HashTreeRoot(b8)
-	b9 := &ethpb.BeaconBlock{Slot: 9, ParentRoot: r3[:]}
-	r9, _ := ssz.HashTreeRoot(b9)
-	b10 := &ethpb.BeaconBlock{Slot: 10, ParentRoot: r3[:]}
-	r10, _ := ssz.HashTreeRoot(b10)
-	b11 := &ethpb.BeaconBlock{Slot: 11, ParentRoot: r4[:]}
-	r11, _ := ssz.HashTreeRoot(b11)
-	b12 := &ethpb.BeaconBlock{Slot: 12, ParentRoot: r6[:]}
-	r12, _ := ssz.HashTreeRoot(b12)
-	b13 := &ethpb.BeaconBlock{Slot: 13, ParentRoot: r6[:]}
-	r13, _ := ssz.HashTreeRoot(b13)
-	b14 := &ethpb.BeaconBlock{Slot: 14, ParentRoot: r7[:]}
-	r14, _ := ssz.HashTreeRoot(b14)
-	b15 := &ethpb.BeaconBlock{Slot: 15, ParentRoot: r7[:]}
-	r15, _ := ssz.HashTreeRoot(b15)
-	for _, b := range []*ethpb.BeaconBlock{b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12, b13, b14, b15} {
-		if err := db.SaveBlock(context.Background(), &ethpb.SignedBeaconBlock{Block: b}); err != nil {
-			return nil, err
-		}
-		if err := db.SaveState(context.Background(), &pb.BeaconState{}, bytesutil.ToBytes32(b.ParentRoot)); err != nil {
-			return nil, err
-		}
-	}
-	return [][]byte{r0[:], r1[:], r2[:], r3[:], r4[:], r5[:], r6[:], r7[:], r8[:], r9[:], r10[:], r11[:], r12[:], r13[:], r14[:], r15[:]}, nil
-}
-
-// blockTree3 constructs a tree that is 512 blocks in a row.
-// B0 - B1 - B2 - B3 - .... - B512
-func blockTree3(db db.Database) ([][]byte, error) {
-	blkCount := 512
-	roots := make([][]byte, 0, blkCount)
-	blks := make([]*ethpb.BeaconBlock, 0, blkCount)
-	b0 := &ethpb.BeaconBlock{Slot: 0, ParentRoot: []byte{'g'}}
-	r0, _ := ssz.HashTreeRoot(b0)
-	roots = append(roots, r0[:])
-	blks = append(blks, b0)
-
-	for i := 1; i < blkCount; i++ {
-		b := &ethpb.BeaconBlock{Slot: uint64(i), ParentRoot: roots[len(roots)-1]}
-		r, _ := ssz.HashTreeRoot(b)
-		roots = append(roots, r[:])
-		blks = append(blks, b)
-	}
-
-	for _, b := range blks {
-		if err := db.SaveBlock(context.Background(), &ethpb.SignedBeaconBlock{Block: b}); err != nil {
-			return nil, err
-		}
-		if err := db.SaveState(context.Background(), &pb.BeaconState{}, bytesutil.ToBytes32(b.ParentRoot)); err != nil {
-			return nil, err
-		}
-	}
-	return roots, nil
-}

beacon-chain/blockchain/head.go

@@ -0,0 +1,192 @@
+package blockchain
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"go.opencensus.io/trace"
+)
+
+// This defines the current chain service's view of head.
+type head struct {
+	slot  uint64                   // current head slot.
+	root  [32]byte                 // current head root.
+	block *ethpb.SignedBeaconBlock // current head block.
+	state *state.BeaconState       // current head state.
+}
+
+// This gets head from the fork choice service and saves head related items
+// (ie root, block, state) to the local service cache.
+func (s *Service) updateHead(ctx context.Context, balances []uint64) error {
+	ctx, span := trace.StartSpan(ctx, "blockchain.updateHead")
+	defer span.End()
+
+	// To get the proper head update, a node first checks its best justified
+	// can become justified. This is designed to prevent bounce attack and
+	// ensure head gets its best justified info.
+	if s.bestJustifiedCheckpt.Epoch > s.justifiedCheckpt.Epoch {
+		s.justifiedCheckpt = s.bestJustifiedCheckpt
+	}
+
+	// Get head from the fork choice service.
+	f := s.finalizedCheckpt
+	j := s.justifiedCheckpt
+	headRoot, err := s.forkChoiceStore.Head(ctx, j.Epoch, bytesutil.ToBytes32(j.Root), balances, f.Epoch)
+	if err != nil {
+		return err
+	}
+
+	// Save head to the local service cache.
+	return s.saveHead(ctx, headRoot)
+}
+
+// This saves head info to the local service cache, it also saves the
+// new head root to the DB.
+func (s *Service) saveHead(ctx context.Context, headRoot [32]byte) error {
+	ctx, span := trace.StartSpan(ctx, "blockchain.saveHead")
+	defer span.End()
+
+	// Do nothing if head hasn't changed.
+	if headRoot == s.headRoot() {
+		return nil
+	}
+
+	// If the head state is not available, just return nil.
+	// There's nothing to cache
+	_, cached := s.initSyncState[headRoot]
+	if !cached && !s.beaconDB.HasState(ctx, headRoot) {
+		return nil
+	}
+
+	// Get the new head block from DB.
+	newHeadBlock, err := s.beaconDB.Block(ctx, headRoot)
+	if err != nil {
+		return err
+	}
+	if newHeadBlock == nil || newHeadBlock.Block == nil {
+		return errors.New("cannot save nil head block")
+	}
+
+	// Get the new head state from cached state or DB.
+	var newHeadState *state.BeaconState
+	var exists bool
+	newHeadState, exists = s.initSyncState[headRoot]
+	if !exists {
+		newHeadState, err = s.beaconDB.State(ctx, headRoot)
+		if err != nil {
+			return errors.Wrap(err, "could not retrieve head state in DB")
+		}
+		if newHeadState == nil {
+			return errors.New("cannot save nil head state")
+		}
+	}
+	if newHeadState == nil {
+		return errors.New("cannot save nil head state")
+	}
+
+	// Cache the new head info.
+	s.setHead(headRoot, newHeadBlock, newHeadState)
+
+	// Save the new head root to DB.
+	if err := s.beaconDB.SaveHeadBlockRoot(ctx, headRoot); err != nil {
+		return errors.Wrap(err, "could not save head root in DB")
+	}
+
+	return nil
+}
+
+// This gets called to update canonical root mapping. It does not save head block
+// root in DB. With the inception of inital-sync-cache-state flag, it uses finalized
+// check point as anchors to resume sync therefore head is no longer needed to be saved on per slot basis.
+func (s *Service) saveHeadNoDB(ctx context.Context, b *ethpb.SignedBeaconBlock, r [32]byte) error {
+	if b == nil || b.Block == nil {
+		return errors.New("cannot save nil head block")
+	}
+
+	headState, err := s.beaconDB.State(ctx, r)
+	if err != nil {
+		return errors.Wrap(err, "could not retrieve head state in DB")
+	}
+	if headState == nil {
+		s.initSyncStateLock.RLock()
+		cachedHeadState, ok := s.initSyncState[r]
+		if ok {
+			headState = cachedHeadState
+		}
+		s.initSyncStateLock.RUnlock()
+	}
+
+	if headState == nil {
+		return errors.New("nil head state")
+	}
+
+	s.setHead(r, stateTrie.CopySignedBeaconBlock(b), headState)
+
+	return nil
+}
+
+// This sets head view object which is used to track the head slot, root, block and state.
+func (s *Service) setHead(root [32]byte, block *ethpb.SignedBeaconBlock, state *state.BeaconState) {
+	s.headLock.Lock()
+	defer s.headLock.Unlock()
+
+	// This does a full copy of the block and state.
+	s.head = &head{
+		slot:  block.Block.Slot,
+		root:  root,
+		block: stateTrie.CopySignedBeaconBlock(block),
+		state: state.Copy(),
+	}
+}
+
+// This returns the head slot.
+func (s *Service) headSlot() uint64 {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	return s.head.slot
+}
+
+// This returns the head root.
+// It does a full copy on head root for immutability.
+func (s *Service) headRoot() [32]byte {
+	if s.head == nil {
+		return params.BeaconConfig().ZeroHash
+	}
+
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	return s.head.root
+}
+
+// This returns the head block.
+// It does a full copy on head block for immutability.
+func (s *Service) headBlock() *ethpb.SignedBeaconBlock {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	return stateTrie.CopySignedBeaconBlock(s.head.block)
+}
+
+// This returns the head state.
+// It does a full copy on head state for immutability.
+func (s *Service) headState() *state.BeaconState {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	return s.head.state.Copy()
+}
+
+// Returns true if head state exists.
+func (s *Service) hasHeadState() bool {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	return s.head != nil && s.head.state != nil
+}

beacon-chain/blockchain/head_test.go

@@ -0,0 +1,72 @@
+package blockchain
+
+import (
+	"bytes"
+	"context"
+	"reflect"
+	"testing"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/go-ssz"
+	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+)
+
+func TestSaveHead_Same(t *testing.T) {
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+	service := setupBeaconChain(t, db)
+
+	r := [32]byte{'A'}
+	service.head = &head{slot: 0, root: r}
+
+	if err := service.saveHead(context.Background(), r); err != nil {
+		t.Fatal(err)
+	}
+
+	if service.headSlot() != 0 {
+		t.Error("Head did not stay the same")
+	}
+
+	if service.headRoot() != r {
+		t.Error("Head did not stay the same")
+	}
+}
+
+func TestSaveHead_Different(t *testing.T) {
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+	service := setupBeaconChain(t, db)
+
+	oldRoot := [32]byte{'A'}
+	service.head = &head{slot: 0, root: oldRoot}
+
+	newHeadBlock := &ethpb.BeaconBlock{Slot: 1}
+	newHeadSignedBlock := &ethpb.SignedBeaconBlock{Block: newHeadBlock}
+	service.beaconDB.SaveBlock(context.Background(), newHeadSignedBlock)
+	newRoot, _ := ssz.HashTreeRoot(newHeadBlock)
+	headState, _ := state.InitializeFromProto(&pb.BeaconState{Slot: 1})
+	service.beaconDB.SaveState(context.Background(), headState, newRoot)
+	if err := service.saveHead(context.Background(), newRoot); err != nil {
+		t.Fatal(err)
+	}
+
+	if service.HeadSlot() != 1 {
+		t.Error("Head did not change")
+	}
+
+	cachedRoot, err := service.HeadRoot(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(cachedRoot, newRoot[:]) {
+		t.Error("Head did not change")
+	}
+	if !reflect.DeepEqual(service.headBlock(), newHeadSignedBlock) {
+		t.Error("Head did not change")
+	}
+	if !reflect.DeepEqual(service.headState().CloneInnerState(), headState.CloneInnerState()) {
+		t.Error("Head did not change")
+	}
+}

beacon-chain/blockchain/info.go

@@ -1,57 +1,83 @@
 package blockchain
 
 import (
-	"bytes"
-	"encoding/hex"
 	"fmt"
 	"net/http"
-	"sort"
+	"strconv"
 
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/sirupsen/logrus"
+	"github.com/emicklei/dot"
 )
 
-const latestSlotCount = 10
+const template = `<html>
+<head>
+    <script src="//cdnjs.cloudflare.com/ajax/libs/viz.js/2.1.2/viz.js"></script>
+    <script src="//cdnjs.cloudflare.com/ajax/libs/viz.js/2.1.2/full.render.js"></script>
+<body>
+    <script type="application/javascript">
+        var graph = ` + "`%s`;" + `
+        var viz = new Viz();
+        viz.renderSVGElement(graph) // reading the graph.
+            .then(function(element) {
+                document.body.appendChild(element); // appends to document.
+            })
+            .catch(error => {
+                // Create a new Viz instance (@see Caveats page for more info)
+                viz = new Viz();
+                // Possibly display the error
+                console.error(error);
+            });
+    </script>
+</head>
+</body>
+</html>`
 
-// HeadsHandler is a handler to serve /heads page in metrics.
-func (s *Service) HeadsHandler(w http.ResponseWriter, _ *http.Request) {
-	buf := new(bytes.Buffer)
-
-	if _, err := fmt.Fprintf(w, "\n %s\t%s\t", "Head slot", "Head root"); err != nil {
-		logrus.WithError(err).Error("Failed to render chain heads page")
-		return
+// TreeHandler is a handler to serve /tree page in metrics.
+func (s *Service) TreeHandler(w http.ResponseWriter, _ *http.Request) {
+	if s.headState() == nil {
+		if _, err := w.Write([]byte("Unavailable during initial syncing")); err != nil {
+			log.WithError(err).Error("Failed to render p2p info page")
+		}
 	}
 
-	if _, err := fmt.Fprintf(w, "\n %s\t%s\t", "---------", "---------"); err != nil {
-		logrus.WithError(err).Error("Failed to render chain heads page")
-		return
+	nodes := s.forkChoiceStore.Nodes()
+
+	graph := dot.NewGraph(dot.Directed)
+	graph.Attr("rankdir", "RL")
+	graph.Attr("labeljust", "l")
+
+	dotNodes := make([]*dot.Node, len(nodes))
+	avgBalance := uint64(averageBalance(s.headState().Balances()))
+
+	for i := len(nodes) - 1; i >= 0; i-- {
+		// Construct label for each node.
+		slot := strconv.Itoa(int(nodes[i].Slot))
+		weight := strconv.Itoa(int(nodes[i].Weight / 1e9)) // Convert unit Gwei to unit ETH.
+		votes := strconv.Itoa(int(nodes[i].Weight / 1e9 / avgBalance))
+		bestDescendent := strconv.Itoa(int(nodes[i].BestDescendent))
+		index := strconv.Itoa(int(i))
+		label := "slot: " + slot + "\n index: " + index + "\n bestDescendent: " + bestDescendent + "\n votes: " + votes + "\n weight: " + weight
+		var dotN dot.Node
+		if nodes[i].Parent != ^uint64(0) {
+			dotN = graph.Node(index).Box().Attr("label", label)
+		}
+
+		if nodes[i].Slot == s.headSlot() &&
+			nodes[i].BestDescendent == ^uint64(0) {
+			dotN = dotN.Attr("color", "green")
+		}
+
+		dotNodes[i] = &dotN
 	}
 
-	slots := s.latestHeadSlots()
-	for _, slot := range slots {
-		r := hex.EncodeToString(bytesutil.Trunc(s.canonicalRoots[uint64(slot)]))
-		if _, err := fmt.Fprintf(w, "\n %d\t\t%s\t", slot, r); err != nil {
-			logrus.WithError(err).Error("Failed to render chain heads page")
-			return
+	for i := len(nodes) - 1; i >= 0; i-- {
+		if nodes[i].Parent != ^uint64(0) && nodes[i].Parent < uint64(len(dotNodes)) {
+			graph.Edge(*dotNodes[i], *dotNodes[nodes[i].Parent])
 		}
 	}
 
 	w.WriteHeader(http.StatusOK)
-	if _, err := w.Write(buf.Bytes()); err != nil {
-		log.WithError(err).Error("Failed to render chain heads page")
+	w.Header().Set("Content-Type", "text/html")
+	if _, err := fmt.Fprintf(w, template, graph.String()); err != nil {
+		log.WithError(err).Error("Failed to render p2p info page")
 	}
-
-}
-
-// This returns the latest head slots in a slice and up to latestSlotCount
-func (s *Service) latestHeadSlots() []int {
-	slots := make([]int, 0, len(s.canonicalRoots))
-	for k := range s.canonicalRoots {
-		slots = append(slots, int(k))
-	}
-	sort.Ints(slots)
-	if (len(slots)) > latestSlotCount {
-		return slots[len(slots)-latestSlotCount:]
-	}
-	return slots
 }

beacon-chain/blockchain/init_sync_process_block.go

@@ -0,0 +1,191 @@
+package blockchain
+
+import (
+	"context"
+	"sort"
+
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+)
+
+const maxCacheSize = 70
+const initialSyncCacheSize = 45
+const minimumCacheSize = initialSyncCacheSize / 3
+
+func (s *Service) persistCachedStates(ctx context.Context, numOfStates int) error {
+	oldStates := make([]*stateTrie.BeaconState, 0, numOfStates)
+
+	// Add slots to the map and add epoch boundary states to the slice.
+	for _, rt := range s.boundaryRoots[:numOfStates-minimumCacheSize] {
+		oldStates = append(oldStates, s.initSyncState[rt])
+	}
+
+	err := s.beaconDB.SaveStates(ctx, oldStates, s.boundaryRoots[:numOfStates-minimumCacheSize])
+	if err != nil {
+		return err
+	}
+	for _, rt := range s.boundaryRoots[:numOfStates-minimumCacheSize] {
+		delete(s.initSyncState, rt)
+	}
+	s.boundaryRoots = s.boundaryRoots[numOfStates-minimumCacheSize:]
+	return nil
+}
+
+// filter out boundary candidates from our currently processed batch of states.
+func (s *Service) filterBoundaryCandidates(ctx context.Context, root [32]byte, postState *stateTrie.BeaconState) {
+	// Only trigger on epoch start.
+	if !helpers.IsEpochStart(postState.Slot()) {
+		return
+	}
+
+	stateSlice := make([][32]byte, 0, len(s.initSyncState))
+	// Add epoch boundary roots to slice.
+	for rt := range s.initSyncState {
+		stateSlice = append(stateSlice, rt)
+	}
+
+	sort.Slice(stateSlice, func(i int, j int) bool {
+		return s.initSyncState[stateSlice[i]].Slot() < s.initSyncState[stateSlice[j]].Slot()
+	})
+	epochLength := params.BeaconConfig().SlotsPerEpoch
+
+	if len(s.boundaryRoots) > 0 {
+		// Retrieve previous boundary root.
+		previousBoundaryRoot := s.boundaryRoots[len(s.boundaryRoots)-1]
+		previousState, ok := s.initSyncState[previousBoundaryRoot]
+		if !ok {
+			// Remove the non-existent root and exit filtering.
+			s.boundaryRoots = s.boundaryRoots[:len(s.boundaryRoots)-1]
+			return
+		}
+		previousSlot := previousState.Slot()
+
+		// Round up slot number to account for skipped slots.
+		previousSlot = helpers.RoundUpToNearestEpoch(previousSlot)
+		if postState.Slot()-previousSlot >= epochLength {
+			targetSlot := postState.Slot()
+			tempRoots := s.loopThroughCandidates(stateSlice, previousBoundaryRoot, previousSlot, targetSlot)
+			s.boundaryRoots = append(s.boundaryRoots, tempRoots...)
+		}
+	}
+	s.boundaryRoots = append(s.boundaryRoots, root)
+	s.pruneOldStates()
+	s.pruneNonBoundaryStates()
+}
+
+// loop-through the provided candidate roots to filter out which would be appropriate boundary roots.
+func (s *Service) loopThroughCandidates(stateSlice [][32]byte, previousBoundaryRoot [32]byte,
+	previousSlot uint64, targetSlot uint64) [][32]byte {
+	tempRoots := [][32]byte{}
+	epochLength := params.BeaconConfig().SlotsPerEpoch
+
+	// Loop through current states to filter for valid boundary states.
+	for i := len(stateSlice) - 1; stateSlice[i] != previousBoundaryRoot && i >= 0; i-- {
+		currentSlot := s.initSyncState[stateSlice[i]].Slot()
+		// Skip if the current slot is larger than the previous epoch
+		// boundary.
+		if currentSlot > targetSlot-epochLength {
+			continue
+		}
+		tempRoots = append(tempRoots, stateSlice[i])
+
+		// Switch target slot if the current slot is greater than
+		// 1 epoch boundary from the previously saved boundary slot.
+		if currentSlot > previousSlot+epochLength {
+			currentSlot = helpers.RoundUpToNearestEpoch(currentSlot)
+			targetSlot = currentSlot
+			continue
+		}
+		break
+	}
+	// Reverse to append the roots in ascending order corresponding
+	// to the respective slots.
+	tempRoots = bytesutil.ReverseBytes32Slice(tempRoots)
+	return tempRoots
+}
+
+// prune for states past the current finalized checkpoint.
+func (s *Service) pruneOldStates() {
+	prunedBoundaryRoots := [][32]byte{}
+	for _, rt := range s.boundaryRoots {
+		st, ok := s.initSyncState[rt]
+		// Skip non-existent roots.
+		if !ok {
+			continue
+		}
+		if st.Slot() < helpers.StartSlot(s.FinalizedCheckpt().Epoch) {
+			delete(s.initSyncState, rt)
+			continue
+		}
+		prunedBoundaryRoots = append(prunedBoundaryRoots, rt)
+	}
+	s.boundaryRoots = prunedBoundaryRoots
+}
+
+// prune cache for non-boundary states.
+func (s *Service) pruneNonBoundaryStates() {
+	boundaryMap := make(map[[32]byte]bool)
+	for i := range s.boundaryRoots {
+		boundaryMap[s.boundaryRoots[i]] = true
+	}
+	for rt := range s.initSyncState {
+		if !boundaryMap[rt] {
+			delete(s.initSyncState, rt)
+		}
+	}
+}
+
+func (s *Service) pruneOldNonFinalizedStates() {
+	stateSlice := make([][32]byte, 0, len(s.initSyncState))
+	// Add epoch boundary roots to slice.
+	for rt := range s.initSyncState {
+		stateSlice = append(stateSlice, rt)
+	}
+
+	// Sort by slots.
+	sort.Slice(stateSlice, func(i int, j int) bool {
+		return s.initSyncState[stateSlice[i]].Slot() < s.initSyncState[stateSlice[j]].Slot()
+	})
+
+	boundaryMap := make(map[[32]byte]bool)
+	for i := range s.boundaryRoots {
+		boundaryMap[s.boundaryRoots[i]] = true
+	}
+	for _, rt := range stateSlice[:initialSyncCacheSize] {
+		if boundaryMap[rt] {
+			continue
+		}
+		delete(s.initSyncState, rt)
+	}
+}
+
+func (s *Service) generateState(ctx context.Context, startRoot [32]byte, endRoot [32]byte) (*stateTrie.BeaconState, error) {
+	preState, err := s.beaconDB.State(ctx, startRoot)
+	if err != nil {
+		return nil, err
+	}
+	if preState == nil {
+		return nil, errors.New("finalized state does not exist in db")
+	}
+	endBlock, err := s.beaconDB.Block(ctx, endRoot)
+	if err != nil {
+		return nil, err
+	}
+	if endBlock == nil {
+		return nil, errors.New("provided block root does not have block saved in the db")
+	}
+	log.Warnf("Generating missing state of slot %d and root %#x", endBlock.Block.Slot, endRoot)
+
+	blocks, err := s.stateGen.LoadBlocks(ctx, preState.Slot()+1, endBlock.Block.Slot, endRoot)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not load the required blocks")
+	}
+	postState, err := s.stateGen.ReplayBlocks(ctx, preState, blocks, endBlock.Block.Slot)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not replay the blocks to generate the resultant state")
+	}
+	return postState, nil
+}

beacon-chain/blockchain/init_sync_process_block_test.go

@@ -0,0 +1,279 @@
+package blockchain
+
+import (
+	"context"
+	"testing"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/go-ssz"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
+	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
+	"gopkg.in/d4l3k/messagediff.v1"
+)
+
+func TestFilterBoundaryCandidates_FilterCorrect(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	st, _ := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
+
+	for i := uint64(0); i < 500; i++ {
+		st.SetSlot(i)
+		root := [32]byte{}
+		copy(root[:], bytesutil.Bytes32(i))
+		service.initSyncState[root] = st.Copy()
+		if helpers.IsEpochStart(i) {
+			service.boundaryRoots = append(service.boundaryRoots, root)
+		}
+	}
+	lastIndex := len(service.boundaryRoots) - 1
+	for i := uint64(500); i < 2000; i++ {
+		st.SetSlot(i)
+		root := [32]byte{}
+		copy(root[:], bytesutil.Bytes32(i))
+		service.initSyncState[root] = st.Copy()
+	}
+	// Set current state.
+	latestSlot := helpers.RoundUpToNearestEpoch(2000)
+	st.SetSlot(latestSlot)
+	lastRoot := [32]byte{}
+	copy(lastRoot[:], bytesutil.Bytes32(latestSlot))
+
+	service.initSyncState[lastRoot] = st.Copy()
+	service.finalizedCheckpt = &ethpb.Checkpoint{
+		Epoch: 0,
+		Root:  []byte{},
+	}
+	service.filterBoundaryCandidates(context.Background(), lastRoot, st)
+	if len(service.boundaryRoots[lastIndex+1:]) == 0 {
+		t.Fatal("Wanted non zero added boundary roots")
+	}
+	for _, rt := range service.boundaryRoots[lastIndex+1:] {
+		st, ok := service.initSyncState[rt]
+		if !ok {
+			t.Error("Root doen't exist in cache map")
+			continue
+		}
+		if !(helpers.IsEpochStart(st.Slot()) || helpers.IsEpochStart(st.Slot()-1) || helpers.IsEpochStart(st.Slot()+1)) {
+			t.Errorf("boundary roots not validly stored. They have slot %d", st.Slot())
+		}
+	}
+}
+
+func TestFilterBoundaryCandidates_HandleSkippedSlots(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	st, _ := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
+
+	for i := uint64(0); i < 500; i++ {
+		st.SetSlot(i)
+		root := [32]byte{}
+		copy(root[:], bytesutil.Bytes32(i))
+		service.initSyncState[root] = st.Copy()
+		if helpers.IsEpochStart(i) {
+			service.boundaryRoots = append(service.boundaryRoots, root)
+		}
+	}
+	lastIndex := len(service.boundaryRoots) - 1
+	for i := uint64(500); i < 2000; i++ {
+		st.SetSlot(i)
+		root := [32]byte{}
+		copy(root[:], bytesutil.Bytes32(i))
+		// save only for offsetted slots
+		if helpers.IsEpochStart(i + 10) {
+			service.initSyncState[root] = st.Copy()
+		}
+	}
+	// Set current state.
+	latestSlot := helpers.RoundUpToNearestEpoch(2000)
+	st.SetSlot(latestSlot)
+	lastRoot := [32]byte{}
+	copy(lastRoot[:], bytesutil.Bytes32(latestSlot))
+
+	service.initSyncState[lastRoot] = st.Copy()
+	service.finalizedCheckpt = &ethpb.Checkpoint{
+		Epoch: 0,
+		Root:  []byte{},
+	}
+	service.filterBoundaryCandidates(context.Background(), lastRoot, st)
+	if len(service.boundaryRoots[lastIndex+1:]) == 0 {
+		t.Fatal("Wanted non zero added boundary roots")
+	}
+	for _, rt := range service.boundaryRoots[lastIndex+1:] {
+		st, ok := service.initSyncState[rt]
+		if !ok {
+			t.Error("Root doen't exist in cache map")
+			continue
+		}
+		if st.Slot() >= 500 {
+			// Ignore head boundary root.
+			if st.Slot() == 2016 {
+				continue
+			}
+			if !helpers.IsEpochStart(st.Slot() + 10) {
+				t.Errorf("boundary roots not validly stored. They have slot %d "+
+					"instead of the offset from epoch start", st.Slot())
+			}
+		}
+	}
+}
+
+func TestPruneOldStates_AlreadyFinalized(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	st, _ := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
+
+	for i := uint64(100); i < 200; i++ {
+		st.SetSlot(i)
+		root := [32]byte{}
+		copy(root[:], bytesutil.Bytes32(i))
+		service.initSyncState[root] = st.Copy()
+		service.boundaryRoots = append(service.boundaryRoots, root)
+	}
+	finalizedEpoch := uint64(5)
+	service.finalizedCheckpt = &ethpb.Checkpoint{Epoch: finalizedEpoch}
+	service.pruneOldStates()
+	for _, rt := range service.boundaryRoots {
+		st, ok := service.initSyncState[rt]
+		if !ok {
+			t.Error("Root doen't exist in cache map")
+			continue
+		}
+		if st.Slot() < helpers.StartSlot(finalizedEpoch) {
+			t.Errorf("State with slot %d still exists and not pruned", st.Slot())
+		}
+	}
+}
+
+func TestPruneNonBoundary_CanPrune(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	st, _ := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
+
+	for i := uint64(0); i < 2000; i++ {
+		st.SetSlot(i)
+		root := [32]byte{}
+		copy(root[:], bytesutil.Bytes32(i))
+		service.initSyncState[root] = st.Copy()
+		if helpers.IsEpochStart(i) {
+			service.boundaryRoots = append(service.boundaryRoots, root)
+		}
+	}
+	service.pruneNonBoundaryStates()
+	for _, rt := range service.boundaryRoots {
+		st, ok := service.initSyncState[rt]
+		if !ok {
+			t.Error("Root doesn't exist in cache map")
+			continue
+		}
+		if !helpers.IsEpochStart(st.Slot()) {
+			t.Errorf("Non boundary state with slot %d still exists and not pruned", st.Slot())
+		}
+	}
+}
+
+func TestGenerateState_CorrectlyGenerated(t *testing.T) {
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(context.Background(), cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	beaconState, privs := testutil.DeterministicGenesisState(t, 32)
+	genesisBlock := blocks.NewGenesisBlock([]byte{})
+	bodyRoot, err := ssz.HashTreeRoot(genesisBlock.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+	beaconState.SetLatestBlockHeader(&ethpb.BeaconBlockHeader{
+		Slot:       genesisBlock.Block.Slot,
+		ParentRoot: genesisBlock.Block.ParentRoot,
+		StateRoot:  params.BeaconConfig().ZeroHash[:],
+		BodyRoot:   bodyRoot[:],
+	})
+	beaconState.SetSlashings(make([]uint64, params.BeaconConfig().EpochsPerSlashingsVector))
+	cp := beaconState.CurrentJustifiedCheckpoint()
+	mockRoot := [32]byte{}
+	copy(mockRoot[:], "hello-world")
+	cp.Root = mockRoot[:]
+	beaconState.SetCurrentJustifiedCheckpoint(cp)
+	beaconState.SetCurrentEpochAttestations([]*pb.PendingAttestation{})
+	err = db.SaveBlock(context.Background(), genesisBlock)
+	if err != nil {
+		t.Fatal(err)
+	}
+	genRoot, err := ssz.HashTreeRoot(genesisBlock)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = db.SaveState(context.Background(), beaconState, genRoot)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	lastBlock := &ethpb.SignedBeaconBlock{}
+	for i := uint64(1); i < 10; i++ {
+		block, err := testutil.GenerateFullBlock(beaconState, privs, testutil.DefaultBlockGenConfig(), i)
+		if err != nil {
+			t.Fatal(err)
+		}
+		beaconState, err = state.ExecuteStateTransition(context.Background(), beaconState, block)
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = db.SaveBlock(context.Background(), block)
+		if err != nil {
+			t.Fatal(err)
+		}
+		lastBlock = block
+	}
+	root, err := ssz.HashTreeRoot(lastBlock.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	newState, err := service.generateState(context.Background(), genRoot, root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !ssz.DeepEqual(newState.InnerStateUnsafe(), beaconState.InnerStateUnsafe()) {
+		diff, _ := messagediff.PrettyDiff(newState.InnerStateUnsafe(), beaconState.InnerStateUnsafe())
+		t.Errorf("Generated state is different from what is expected: %s", diff)
+	}
+}

beacon-chain/blockchain/log.go

@@ -1,17 +1,50 @@
 package blockchain
 
 import (
+	"fmt"
+
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/sirupsen/logrus"
 )
 
 var log = logrus.WithField("prefix", "blockchain")
 
 // logs state transition related data every slot.
-func logStateTransitionData(b *ethpb.BeaconBlock, r []byte) {
+func logStateTransitionData(b *ethpb.BeaconBlock) {
 	log.WithFields(logrus.Fields{
-		"slot":         b.Slot,
-		"attestations": len(b.Body.Attestations),
-		"deposits":     len(b.Body.Deposits),
+		"slot":              b.Slot,
+		"attestations":      len(b.Body.Attestations),
+		"deposits":          len(b.Body.Deposits),
+		"attesterSlashings": len(b.Body.AttesterSlashings),
 	}).Info("Finished applying state transition")
 }
+
+func logEpochData(beaconState *stateTrie.BeaconState) {
+	log.WithFields(logrus.Fields{
+		"epoch":                  helpers.CurrentEpoch(beaconState),
+		"finalizedEpoch":         beaconState.FinalizedCheckpointEpoch(),
+		"justifiedEpoch":         beaconState.CurrentJustifiedCheckpoint().Epoch,
+		"previousJustifiedEpoch": beaconState.PreviousJustifiedCheckpoint().Epoch,
+	}).Info("Starting next epoch")
+	activeVals, err := helpers.ActiveValidatorIndices(beaconState, helpers.CurrentEpoch(beaconState))
+	if err != nil {
+		log.WithError(err).Error("Could not get active validator indices")
+		return
+	}
+	log.WithFields(logrus.Fields{
+		"totalValidators":  len(beaconState.Validators()),
+		"activeValidators": len(activeVals),
+		"averageBalance":   fmt.Sprintf("%.5f ETH", averageBalance(beaconState.Balances())),
+	}).Info("Validator registry information")
+}
+
+func averageBalance(balances []uint64) float64 {
+	total := uint64(0)
+	for i := 0; i < len(balances); i++ {
+		total += balances[i]
+	}
+	return float64(total) / float64(len(balances)) / float64(params.BeaconConfig().GweiPerEth)
+}

beacon-chain/blockchain/metrics.go

@@ -3,7 +3,11 @@ package blockchain
 import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
 var (
@@ -15,38 +19,10 @@ var (
 		Name: "beacon_head_slot",
 		Help: "Slot of the head block of the beacon chain",
 	})
-	beaconHeadRoot = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "beacon_head_root",
-		Help: "Root of the head block of the beacon chain, it returns the lowest 8 bytes interpreted as little endian",
-	})
-	competingAtts = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "competing_attestations",
-		Help: "The # of attestations received and processed from a competing chain",
-	})
 	competingBlks = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "competing_blocks",
 		Help: "The # of blocks received and processed from a competing chain",
 	})
-	processedBlkNoPubsub = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "processed_no_pubsub_block_counter",
-		Help: "The # of processed block without pubsub, this usually means the blocks from sync",
-	})
-	processedBlkNoPubsubForkchoice = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "processed_no_pubsub_forkchoice_block_counter",
-		Help: "The # of processed block without pubsub and forkchoice, this means indicate blocks from initial sync",
-	})
-	processedBlk = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "processed_block_counter",
-		Help: "The # of total processed in block chain service, with fork choice and pubsub",
-	})
-	processedAttNoPubsub = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "processed_no_pubsub_attestation_counter",
-		Help: "The # of processed attestation without pubsub, this usually means the attestations from sync",
-	})
-	processedAtt = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "processed_attestation_counter",
-		Help: "The # of processed attestation with pubsub and fork choice, this ususally means attestations from rpc",
-	})
 	headFinalizedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
 		Name: "head_finalized_epoch",
 		Help: "Last finalized epoch of the head state",
@@ -55,14 +31,150 @@ var (
 		Name: "head_finalized_root",
 		Help: "Last finalized root of the head state",
 	})
+	beaconFinalizedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_finalized_epoch",
+		Help: "Last finalized epoch of the processed state",
+	})
+	beaconFinalizedRoot = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_finalized_root",
+		Help: "Last finalized root of the processed state",
+	})
+	beaconCurrentJustifiedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_current_justified_epoch",
+		Help: "Current justified epoch of the processed state",
+	})
+	beaconCurrentJustifiedRoot = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_current_justified_root",
+		Help: "Current justified root of the processed state",
+	})
+	beaconPrevJustifiedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_previous_justified_epoch",
+		Help: "Previous justified epoch of the processed state",
+	})
+	beaconPrevJustifiedRoot = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_previous_justified_root",
+		Help: "Previous justified root of the processed state",
+	})
+	validatorsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "validator_count",
+		Help: "The total number of validators",
+	}, []string{"state"})
+	validatorsBalance = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "validators_total_balance",
+		Help: "The total balance of validators, in GWei",
+	}, []string{"state"})
+	validatorsEffectiveBalance = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "validators_total_effective_balance",
+		Help: "The total effective balance of validators, in GWei",
+	}, []string{"state"})
+	currentEth1DataDepositCount = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "current_eth1_data_deposit_count",
+		Help: "The current eth1 deposit count in the last processed state eth1data field.",
+	})
+	totalEligibleBalances = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "total_eligible_balances",
+		Help: "The total amount of ether, in gwei, that has been used in voting attestation target of previous epoch",
+	})
+	totalVotedTargetBalances = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "total_voted_target_balances",
+		Help: "The total amount of ether, in gwei, that is eligible for voting of previous epoch",
+	})
 )
 
-func (s *Service) reportSlotMetrics(currentSlot uint64) {
+// reportSlotMetrics reports slot related metrics.
+func reportSlotMetrics(currentSlot uint64, headSlot uint64, finalizedCheckpoint *ethpb.Checkpoint) {
 	beaconSlot.Set(float64(currentSlot))
-	beaconHeadSlot.Set(float64(s.HeadSlot()))
-	beaconHeadRoot.Set(float64(bytesutil.ToLowInt64(s.HeadRoot())))
-	if s.headState != nil {
-		headFinalizedEpoch.Set(float64(s.headState.FinalizedCheckpoint.Epoch))
-		headFinalizedRoot.Set(float64(bytesutil.ToLowInt64(s.headState.FinalizedCheckpoint.Root)))
+	beaconHeadSlot.Set(float64(headSlot))
+	if finalizedCheckpoint != nil {
+		headFinalizedEpoch.Set(float64(finalizedCheckpoint.Epoch))
+		headFinalizedRoot.Set(float64(bytesutil.ToLowInt64(finalizedCheckpoint.Root)))
+	}
+}
+
+// reportEpochMetrics reports epoch related metrics.
+func reportEpochMetrics(state *stateTrie.BeaconState) {
+	currentEpoch := state.Slot() / params.BeaconConfig().SlotsPerEpoch
+
+	// Validator instances
+	pendingInstances := 0
+	activeInstances := 0
+	slashingInstances := 0
+	slashedInstances := 0
+	exitingInstances := 0
+	exitedInstances := 0
+	// Validator balances
+	pendingBalance := uint64(0)
+	activeBalance := uint64(0)
+	activeEffectiveBalance := uint64(0)
+	exitingBalance := uint64(0)
+	exitingEffectiveBalance := uint64(0)
+	slashingBalance := uint64(0)
+	slashingEffectiveBalance := uint64(0)
+
+	for i, validator := range state.Validators() {
+		bal, err := state.BalanceAtIndex(uint64(i))
+		if err != nil {
+			continue
+		}
+		if validator.Slashed {
+			if currentEpoch < validator.ExitEpoch {
+				slashingInstances++
+				slashingBalance += bal
+				slashingEffectiveBalance += validator.EffectiveBalance
+			} else {
+				slashedInstances++
+			}
+			continue
+		}
+		if validator.ExitEpoch != params.BeaconConfig().FarFutureEpoch {
+			if currentEpoch < validator.ExitEpoch {
+				exitingInstances++
+				exitingBalance += bal
+				exitingEffectiveBalance += validator.EffectiveBalance
+			} else {
+				exitedInstances++
+			}
+			continue
+		}
+		if currentEpoch < validator.ActivationEpoch {
+			pendingInstances++
+			pendingBalance += bal
+			continue
+		}
+		activeInstances++
+		activeBalance += bal
+		activeEffectiveBalance += validator.EffectiveBalance
+	}
+	validatorsCount.WithLabelValues("Pending").Set(float64(pendingInstances))
+	validatorsCount.WithLabelValues("Active").Set(float64(activeInstances))
+	validatorsCount.WithLabelValues("Exiting").Set(float64(exitingInstances))
+	validatorsCount.WithLabelValues("Exited").Set(float64(exitedInstances))
+	validatorsCount.WithLabelValues("Slashing").Set(float64(slashingInstances))
+	validatorsCount.WithLabelValues("Slashed").Set(float64(slashedInstances))
+	validatorsBalance.WithLabelValues("Pending").Set(float64(pendingBalance))
+	validatorsBalance.WithLabelValues("Active").Set(float64(activeBalance))
+	validatorsBalance.WithLabelValues("Exiting").Set(float64(exitingBalance))
+	validatorsBalance.WithLabelValues("Slashing").Set(float64(slashingBalance))
+	validatorsEffectiveBalance.WithLabelValues("Active").Set(float64(activeEffectiveBalance))
+	validatorsEffectiveBalance.WithLabelValues("Exiting").Set(float64(exitingEffectiveBalance))
+	validatorsEffectiveBalance.WithLabelValues("Slashing").Set(float64(slashingEffectiveBalance))
+
+	// Last justified slot
+	beaconCurrentJustifiedEpoch.Set(float64(state.CurrentJustifiedCheckpoint().Epoch))
+	beaconCurrentJustifiedRoot.Set(float64(bytesutil.ToLowInt64(state.CurrentJustifiedCheckpoint().Root)))
+
+	// Last previous justified slot
+	beaconPrevJustifiedEpoch.Set(float64(state.PreviousJustifiedCheckpoint().Epoch))
+	beaconPrevJustifiedRoot.Set(float64(bytesutil.ToLowInt64(state.PreviousJustifiedCheckpoint().Root)))
+
+	// Last finalized slot
+	beaconFinalizedEpoch.Set(float64(state.FinalizedCheckpointEpoch()))
+	beaconFinalizedRoot.Set(float64(bytesutil.ToLowInt64(state.FinalizedCheckpoint().Root)))
+
+	currentEth1DataDepositCount.Set(float64(state.Eth1Data().DepositCount))
+
+	if precompute.Balances != nil {
+		totalEligibleBalances.Set(float64(precompute.Balances.PrevEpoch))
+		totalVotedTargetBalances.Set(float64(precompute.Balances.PrevEpochTargetAttesters))
 	}
 }

beacon-chain/blockchain/process_attestation.go

@@ -0,0 +1,132 @@
+package blockchain
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/flags"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"go.opencensus.io/trace"
+)
+
+// ErrTargetRootNotInDB returns when the target block root of an attestation cannot be found in the
+// beacon database.
+var ErrTargetRootNotInDB = errors.New("target root does not exist in db")
+
+// onAttestation is called whenever an attestation is received, verifies the attestation is valid and saves
+/// it to the DB.
+//
+// Spec pseudocode definition:
+//   def on_attestation(store: Service, attestation: Attestation) -> None:
+//    """
+//    Run ``on_attestation`` upon receiving a new ``attestation`` from either within a block or directly on the wire.
+//
+//    An ``attestation`` that is asserted as invalid may be valid at a later time,
+//    consider scheduling it for later processing in such case.
+//    """
+//    target = attestation.data.target
+//
+//    # Attestations must be from the current or previous epoch
+//    current_epoch = compute_epoch_at_slot(get_current_slot(store))
+//    # Use GENESIS_EPOCH for previous when genesis to avoid underflow
+//    previous_epoch = current_epoch - 1 if current_epoch > GENESIS_EPOCH else GENESIS_EPOCH
+//    assert target.epoch in [current_epoch, previous_epoch]
+//    assert target.epoch == compute_epoch_at_slot(attestation.data.slot)
+//
+//    # Attestations target be for a known block. If target block is unknown, delay consideration until the block is found
+//    assert target.root in store.blocks
+//    # Attestations cannot be from future epochs. If they are, delay consideration until the epoch arrives
+//    base_state = store.block_states[target.root].copy()
+//    assert store.time >= base_state.genesis_time + compute_start_slot_at_epoch(target.epoch) * SECONDS_PER_SLOT
+//
+//    # Attestations must be for a known block. If block is unknown, delay consideration until the block is found
+//    assert attestation.data.beacon_block_root in store.blocks
+//    # Attestations must not be for blocks in the future. If not, the attestation should not be considered
+//    assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot
+//
+//    # Service target checkpoint state if not yet seen
+//    if target not in store.checkpoint_states:
+//        process_slots(base_state, compute_start_slot_at_epoch(target.epoch))
+//        store.checkpoint_states[target] = base_state
+//    target_state = store.checkpoint_states[target]
+//
+//    # Attestations can only affect the fork choice of subsequent slots.
+//    # Delay consideration in the fork choice until their slot is in the past.
+//    assert store.time >= (attestation.data.slot + 1) * SECONDS_PER_SLOT
+//
+//    # Get state at the `target` to validate attestation and calculate the committees
+//    indexed_attestation = get_indexed_attestation(target_state, attestation)
+//    assert is_valid_indexed_attestation(target_state, indexed_attestation)
+//
+//    # Update latest messages
+//    for i in indexed_attestation.attesting_indices:
+//        if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch:
+//            store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=attestation.data.beacon_block_root)
+func (s *Service) onAttestation(ctx context.Context, a *ethpb.Attestation) ([]uint64, error) {
+	ctx, span := trace.StartSpan(ctx, "blockchain.onAttestation")
+	defer span.End()
+
+	tgt := stateTrie.CopyCheckpoint(a.Data.Target)
+	tgtSlot := helpers.StartSlot(tgt.Epoch)
+
+	if helpers.SlotToEpoch(a.Data.Slot) != a.Data.Target.Epoch {
+		return nil, fmt.Errorf("data slot is not in the same epoch as target %d != %d", helpers.SlotToEpoch(a.Data.Slot), a.Data.Target.Epoch)
+	}
+
+	// Verify beacon node has seen the target block before.
+	if !s.hasBlock(ctx, bytesutil.ToBytes32(tgt.Root)) {
+		return nil, ErrTargetRootNotInDB
+	}
+
+	// Retrieve attestation's data beacon block pre state. Advance pre state to latest epoch if necessary and
+	// save it to the cache.
+	baseState, err := s.getAttPreState(ctx, tgt)
+	if err != nil {
+		return nil, err
+	}
+
+	genesisTime := baseState.GenesisTime()
+
+	// Verify attestation target is from current epoch or previous epoch.
+	if err := s.verifyAttTargetEpoch(ctx, genesisTime, uint64(time.Now().Unix()), tgt); err != nil {
+		return nil, err
+	}
+
+	// Verify Attestations cannot be from future epochs.
+	if err := helpers.VerifySlotTime(genesisTime, tgtSlot, helpers.TimeShiftTolerance); err != nil {
+		return nil, errors.Wrap(err, "could not verify attestation target slot")
+	}
+
+	// Verify attestation beacon block is known and not from the future.
+	if err := s.verifyBeaconBlock(ctx, a.Data); err != nil {
+		return nil, errors.Wrap(err, "could not verify attestation beacon block")
+	}
+
+	// Verify attestations can only affect the fork choice of subsequent slots.
+	if err := helpers.VerifySlotTime(genesisTime, a.Data.Slot+1, helpers.TimeShiftTolerance); err != nil {
+		return nil, err
+	}
+
+	// Use the target state to to validate attestation and calculate the committees.
+	indexedAtt, err := s.verifyAttestation(ctx, baseState, a)
+	if err != nil {
+		return nil, err
+	}
+
+	// Only save attestation in DB for archival node.
+	if flags.Get().EnableArchive {
+		if err := s.beaconDB.SaveAttestation(ctx, a); err != nil {
+			return nil, err
+		}
+	}
+
+	// Update forkchoice store with the new attestation for updating weight.
+	s.forkChoiceStore.ProcessAttestation(ctx, indexedAtt.AttestingIndices, bytesutil.ToBytes32(a.Data.BeaconBlockRoot), a.Data.Target.Epoch)
+
+	return indexedAtt.AttestingIndices, nil
+}

beacon-chain/blockchain/process_attestation_helpers.go

@@ -0,0 +1,146 @@
+package blockchain
+
+import (
+	"bytes"
+	"context"
+	"encoding/hex"
+	"fmt"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/attestationutil"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+	"github.com/prysmaticlabs/prysm/shared/params"
+)
+
+// getAttPreState retrieves the att pre state by either from the cache or the DB.
+func (s *Service) getAttPreState(ctx context.Context, c *ethpb.Checkpoint) (*stateTrie.BeaconState, error) {
+	s.checkpointStateLock.Lock()
+	defer s.checkpointStateLock.Unlock()
+	cachedState, err := s.checkpointState.StateByCheckpoint(c)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get cached checkpoint state")
+	}
+	if cachedState != nil {
+		return cachedState, nil
+	}
+	if featureconfig.Get().CheckHeadState {
+		headRoot, err := s.HeadRoot(ctx)
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not get head root")
+		}
+		if bytes.Equal(headRoot, c.Root) {
+			st, err := s.HeadState(ctx)
+			if err != nil {
+				return nil, errors.Wrapf(err, "could not get head state")
+			}
+			if err := s.checkpointState.AddCheckpointState(&cache.CheckpointState{
+				Checkpoint: c,
+				State:      st.Copy(),
+			}); err != nil {
+				return nil, errors.Wrap(err, "could not saved checkpoint state to cache")
+			}
+			return st, nil
+		}
+	}
+
+	baseState, err := s.beaconDB.State(ctx, bytesutil.ToBytes32(c.Root))
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not get pre state for slot %d", helpers.StartSlot(c.Epoch))
+	}
+	if baseState == nil {
+		return nil, fmt.Errorf("pre state of target block %d does not exist", helpers.StartSlot(c.Epoch))
+	}
+
+	if helpers.StartSlot(c.Epoch) > baseState.Slot() {
+		baseState, err = state.ProcessSlots(ctx, baseState, helpers.StartSlot(c.Epoch))
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not process slots up to %d", helpers.StartSlot(c.Epoch))
+		}
+	}
+
+	if err := s.checkpointState.AddCheckpointState(&cache.CheckpointState{
+		Checkpoint: c,
+		State:      baseState.Copy(),
+	}); err != nil {
+		return nil, errors.Wrap(err, "could not saved checkpoint state to cache")
+	}
+
+	return baseState, nil
+}
+
+// verifyAttTargetEpoch validates attestation is from the current or previous epoch.
+func (s *Service) verifyAttTargetEpoch(ctx context.Context, genesisTime uint64, nowTime uint64, c *ethpb.Checkpoint) error {
+	currentSlot := (nowTime - genesisTime) / params.BeaconConfig().SecondsPerSlot
+	currentEpoch := helpers.SlotToEpoch(currentSlot)
+	var prevEpoch uint64
+	// Prevents previous epoch under flow
+	if currentEpoch > 1 {
+		prevEpoch = currentEpoch - 1
+	}
+	if c.Epoch != prevEpoch && c.Epoch != currentEpoch {
+		return fmt.Errorf("target epoch %d does not match current epoch %d or prev epoch %d", c.Epoch, currentEpoch, prevEpoch)
+	}
+	return nil
+}
+
+// verifyBeaconBlock verifies beacon head block is known and not from the future.
+func (s *Service) verifyBeaconBlock(ctx context.Context, data *ethpb.AttestationData) error {
+	b, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(data.BeaconBlockRoot))
+	if err != nil {
+		return err
+	}
+	if b == nil || b.Block == nil {
+		return fmt.Errorf("beacon block %#x does not exist", bytesutil.Trunc(data.BeaconBlockRoot))
+	}
+	if b.Block.Slot > data.Slot {
+		return fmt.Errorf("could not process attestation for future block, %d > %d", b.Block.Slot, data.Slot)
+	}
+	return nil
+}
+
+// verifyAttestation validates input attestation is valid.
+func (s *Service) verifyAttestation(ctx context.Context, baseState *stateTrie.BeaconState, a *ethpb.Attestation) (*ethpb.IndexedAttestation, error) {
+	committee, err := helpers.BeaconCommitteeFromState(baseState, a.Data.Slot, a.Data.CommitteeIndex)
+	if err != nil {
+		return nil, err
+	}
+	indexedAtt, err := attestationutil.ConvertToIndexed(ctx, a, committee)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not convert attestation to indexed attestation")
+	}
+
+	if err := blocks.VerifyIndexedAttestation(ctx, baseState, indexedAtt); err != nil {
+		if err == blocks.ErrSigFailedToVerify {
+			// When sig fails to verify, check if there's a differences in committees due to
+			// different seeds.
+			aState, err := s.beaconDB.State(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
+			if err != nil {
+				return nil, err
+			}
+			epoch := helpers.SlotToEpoch(a.Data.Slot)
+			origSeed, err := helpers.Seed(baseState, epoch, params.BeaconConfig().DomainBeaconAttester)
+			if err != nil {
+				return nil, errors.Wrap(err, "could not get original seed")
+			}
+
+			aSeed, err := helpers.Seed(aState, epoch, params.BeaconConfig().DomainBeaconAttester)
+			if err != nil {
+				return nil, errors.Wrap(err, "could not get attester's seed")
+			}
+			if origSeed != aSeed {
+				return nil, fmt.Errorf("could not verify indexed attestation due to differences in seeds: %v != %v",
+					hex.EncodeToString(bytesutil.Trunc(origSeed[:])), hex.EncodeToString(bytesutil.Trunc(aSeed[:])))
+			}
+		}
+		return nil, errors.Wrap(err, "could not verify indexed attestation")
+	}
+
+	return indexedAtt, nil
+}

beacon-chain/blockchain/process_attestation_test.go

@@ -1,4 +1,4 @@
-package forkchoice
+package blockchain
 
 import (
 	"context"
@@ -7,12 +7,15 @@ import (
 	"testing"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-bitfield"
 	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
@@ -22,9 +25,13 @@ func TestStore_OnAttestation(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 
-	store := NewForkChoiceService(ctx, db)
+	cfg := &Config{BeaconDB: db, ForkChoiceStore: protoarray.New(0, 0, [32]byte{})}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
 
-	_, err := blockTree1(db, []byte{'g'})
+	_, err = blockTree1(db, []byte{'g'})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -40,7 +47,12 @@ func TestStore_OnAttestation(t *testing.T) {
 		t.Fatal(err)
 	}
 	BlkWithStateBadAttRoot, _ := ssz.HashTreeRoot(BlkWithStateBadAtt.Block)
-	if err := store.db.SaveState(ctx, &pb.BeaconState{}, BlkWithStateBadAttRoot); err != nil {
+
+	s, err := beaconstate.InitializeFromProto(&pb.BeaconState{})
+	if err := s.SetSlot(100 * params.BeaconConfig().SlotsPerEpoch); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveState(ctx, s, BlkWithStateBadAttRoot); err != nil {
 		t.Fatal(err)
 	}
 
@@ -49,14 +61,15 @@ func TestStore_OnAttestation(t *testing.T) {
 		t.Fatal(err)
 	}
 	BlkWithValidStateRoot, _ := ssz.HashTreeRoot(BlkWithValidState.Block)
-	if err := store.db.SaveState(ctx, &pb.BeaconState{
+	s, _ = stateTrie.InitializeFromProto(&pb.BeaconState{
 		Fork: &pb.Fork{
 			Epoch:           0,
 			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
 			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
 		},
 		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
-	}, BlkWithValidStateRoot); err != nil {
+	})
+	if err := service.beaconDB.SaveState(ctx, s, BlkWithValidStateRoot); err != nil {
 		t.Fatal(err)
 	}
 
@@ -92,7 +105,7 @@ func TestStore_OnAttestation(t *testing.T) {
 			name: "process attestation doesn't match current epoch",
 			a: &ethpb.Attestation{Data: &ethpb.AttestationData{Slot: 100 * params.BeaconConfig().SlotsPerEpoch, Target: &ethpb.Checkpoint{Epoch: 100,
 				Root: BlkWithStateBadAttRoot[:]}}},
-			s:             &pb.BeaconState{},
+			s:             &pb.BeaconState{Slot: 100 * params.BeaconConfig().SlotsPerEpoch},
 			wantErr:       true,
 			wantErrString: "does not match current epoch",
 		},
@@ -100,17 +113,10 @@ func TestStore_OnAttestation(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if err := store.GenesisStore(
-				ctx,
-				&ethpb.Checkpoint{Root: BlkWithValidStateRoot[:]},
-				&ethpb.Checkpoint{Root: BlkWithValidStateRoot[:]}); err != nil {
-				t.Fatal(err)
-			}
-
-			err := store.OnAttestation(ctx, tt.a)
+			_, err := service.onAttestation(ctx, tt.a)
 			if tt.wantErr {
 				if !strings.Contains(err.Error(), tt.wantErrString) {
-					t.Errorf("Store.OnAttestation() error = %v, wantErr = %v", err, tt.wantErrString)
+					t.Errorf("Store.onAttestation() error = %v, wantErr = %v", err, tt.wantErrString)
 				}
 			} else {
 				t.Error(err)
@@ -125,106 +131,92 @@ func TestStore_SaveCheckpointState(t *testing.T) {
 	defer testDB.TeardownDB(t, db)
 	params.UseDemoBeaconConfig()
 
-	store := NewForkChoiceService(ctx, db)
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
 
-	s := &pb.BeaconState{
+	s, _ := stateTrie.InitializeFromProto(&pb.BeaconState{
 		Fork: &pb.Fork{
 			Epoch:           0,
 			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
 			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
 		},
 		RandaoMixes:         make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
-		StateRoots:          make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+		StateRoots:          make([][]byte, params.BeaconConfig().SlotsPerHistoricalRoot),
 		BlockRoots:          make([][]byte, params.BeaconConfig().SlotsPerHistoricalRoot),
 		LatestBlockHeader:   &ethpb.BeaconBlockHeader{},
 		JustificationBits:   []byte{0},
 		Slashings:           make([]uint64, params.BeaconConfig().EpochsPerSlashingsVector),
 		FinalizedCheckpoint: &ethpb.Checkpoint{},
-	}
+	})
 	r := [32]byte{'g'}
-	if err := store.db.SaveState(ctx, s, r); err != nil {
-		t.Fatal(err)
-	}
-	if err := store.GenesisStore(ctx, &ethpb.Checkpoint{Root: r[:]}, &ethpb.Checkpoint{Root: r[:]}); err != nil {
+	if err := service.beaconDB.SaveState(ctx, s, r); err != nil {
 		t.Fatal(err)
 	}
+	service.justifiedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	service.bestJustifiedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	service.finalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	service.prevFinalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
 
 	cp1 := &ethpb.Checkpoint{Epoch: 1, Root: []byte{'A'}}
-	s1, err := store.saveCheckpointState(ctx, s, cp1)
+	service.beaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'A'}))
+	s1, err := service.getAttPreState(ctx, cp1)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if s1.Slot != 1*params.BeaconConfig().SlotsPerEpoch {
-		t.Errorf("Wanted state slot: %d, got: %d", 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot)
+	if s1.Slot() != 1*params.BeaconConfig().SlotsPerEpoch {
+		t.Errorf("Wanted state slot: %d, got: %d", 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot())
 	}
 
 	cp2 := &ethpb.Checkpoint{Epoch: 2, Root: []byte{'B'}}
-	s2, err := store.saveCheckpointState(ctx, s, cp2)
+	service.beaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'B'}))
+	s2, err := service.getAttPreState(ctx, cp2)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if s2.Slot != 2*params.BeaconConfig().SlotsPerEpoch {
-		t.Errorf("Wanted state slot: %d, got: %d", 2*params.BeaconConfig().SlotsPerEpoch, s2.Slot)
+	if s2.Slot() != 2*params.BeaconConfig().SlotsPerEpoch {
+		t.Errorf("Wanted state slot: %d, got: %d", 2*params.BeaconConfig().SlotsPerEpoch, s2.Slot())
 	}
 
-	s1, err = store.saveCheckpointState(ctx, nil, cp1)
+	s1, err = service.getAttPreState(ctx, cp1)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if s1.Slot != 1*params.BeaconConfig().SlotsPerEpoch {
-		t.Errorf("Wanted state slot: %d, got: %d", 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot)
+	if s1.Slot() != 1*params.BeaconConfig().SlotsPerEpoch {
+		t.Errorf("Wanted state slot: %d, got: %d", 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot())
 	}
 
-	s1, err = store.checkpointState.StateByCheckpoint(cp1)
+	s1, err = service.checkpointState.StateByCheckpoint(cp1)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if s1.Slot != 1*params.BeaconConfig().SlotsPerEpoch {
-		t.Errorf("Wanted state slot: %d, got: %d", 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot)
+	if s1.Slot() != 1*params.BeaconConfig().SlotsPerEpoch {
+		t.Errorf("Wanted state slot: %d, got: %d", 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot())
 	}
 
-	s2, err = store.checkpointState.StateByCheckpoint(cp2)
+	s2, err = service.checkpointState.StateByCheckpoint(cp2)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if s2.Slot != 2*params.BeaconConfig().SlotsPerEpoch {
-		t.Errorf("Wanted state slot: %d, got: %d", 2*params.BeaconConfig().SlotsPerEpoch, s2.Slot)
+	if s2.Slot() != 2*params.BeaconConfig().SlotsPerEpoch {
+		t.Errorf("Wanted state slot: %d, got: %d", 2*params.BeaconConfig().SlotsPerEpoch, s2.Slot())
 	}
 
-	s.Slot = params.BeaconConfig().SlotsPerEpoch + 1
-	if err := store.GenesisStore(ctx, &ethpb.Checkpoint{Root: r[:]}, &ethpb.Checkpoint{Root: r[:]}); err != nil {
-		t.Fatal(err)
-	}
+	s.SetSlot(params.BeaconConfig().SlotsPerEpoch + 1)
+	service.justifiedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	service.bestJustifiedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	service.finalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	service.prevFinalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
 	cp3 := &ethpb.Checkpoint{Epoch: 1, Root: []byte{'C'}}
-	s3, err := store.saveCheckpointState(ctx, s, cp3)
+	service.beaconDB.SaveState(ctx, s, bytesutil.ToBytes32([]byte{'C'}))
+	s3, err := service.getAttPreState(ctx, cp3)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if s3.Slot != s.Slot {
-		t.Errorf("Wanted state slot: %d, got: %d", s.Slot, s3.Slot)
-	}
-}
-
-func TestStore_ReturnAggregatedAttestation(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	store := NewForkChoiceService(ctx, db)
-	a1 := &ethpb.Attestation{Data: &ethpb.AttestationData{}, AggregationBits: bitfield.Bitlist{0x02}}
-	err := store.db.SaveAttestation(ctx, a1)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	a2 := &ethpb.Attestation{Data: &ethpb.AttestationData{}, AggregationBits: bitfield.Bitlist{0x03}}
-	saved, err := store.aggregatedAttestations(ctx, a2)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if !reflect.DeepEqual([]*ethpb.Attestation{a2}, saved) {
-		t.Error("did not retrieve saved attestation")
+	if s3.Slot() != s.Slot() {
+		t.Errorf("Wanted state slot: %d, got: %d", s.Slot(), s3.Slot())
 	}
 }
 
@@ -233,31 +225,37 @@ func TestStore_UpdateCheckpointState(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 
-	store := NewForkChoiceService(ctx, db)
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	epoch := uint64(1)
 	baseState, _ := testutil.DeterministicGenesisState(t, 1)
-	baseState.Slot = epoch * params.BeaconConfig().SlotsPerEpoch
+	baseState.SetSlot(epoch * params.BeaconConfig().SlotsPerEpoch)
 	checkpoint := &ethpb.Checkpoint{Epoch: epoch}
-	returned, err := store.saveCheckpointState(ctx, baseState, checkpoint)
+	service.beaconDB.SaveState(ctx, baseState, bytesutil.ToBytes32(checkpoint.Root))
+	returned, err := service.getAttPreState(ctx, checkpoint)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(baseState, returned) {
+	if baseState.Slot() != returned.Slot() {
 		t.Error("Incorrectly returned base state")
 	}
 
-	cached, err := store.checkpointState.StateByCheckpoint(checkpoint)
+	cached, err := service.checkpointState.StateByCheckpoint(checkpoint)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if cached != nil {
-		t.Error("State shouldn't have been cached")
+	if cached == nil {
+		t.Error("State should have been cached")
 	}
 
 	epoch = uint64(2)
 	newCheckpoint := &ethpb.Checkpoint{Epoch: epoch}
-	returned, err = store.saveCheckpointState(ctx, baseState, newCheckpoint)
+	service.beaconDB.SaveState(ctx, baseState, bytesutil.ToBytes32(newCheckpoint.Root))
+	returned, err = service.getAttPreState(ctx, newCheckpoint)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -265,11 +263,11 @@ func TestStore_UpdateCheckpointState(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(baseState, returned) {
+	if baseState.Slot() != returned.Slot() {
 		t.Error("Incorrectly returned base state")
 	}
 
-	cached, err = store.checkpointState.StateByCheckpoint(newCheckpoint)
+	cached, err = service.checkpointState.StateByCheckpoint(newCheckpoint)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -283,8 +281,13 @@ func TestAttEpoch_MatchPrevEpoch(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 
-	store := NewForkChoiceService(ctx, db)
-	if err := store.verifyAttTargetEpoch(
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := service.verifyAttTargetEpoch(
 		ctx,
 		0,
 		params.BeaconConfig().SlotsPerEpoch*params.BeaconConfig().SecondsPerSlot,
@@ -298,8 +301,13 @@ func TestAttEpoch_MatchCurrentEpoch(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 
-	store := NewForkChoiceService(ctx, db)
-	if err := store.verifyAttTargetEpoch(
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := service.verifyAttTargetEpoch(
 		ctx,
 		0,
 		params.BeaconConfig().SlotsPerEpoch*params.BeaconConfig().SecondsPerSlot,
@@ -313,8 +321,13 @@ func TestAttEpoch_NotMatch(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 
-	store := NewForkChoiceService(ctx, db)
-	if err := store.verifyAttTargetEpoch(
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := service.verifyAttTargetEpoch(
 		ctx,
 		0,
 		2*params.BeaconConfig().SlotsPerEpoch*params.BeaconConfig().SecondsPerSlot,
@@ -329,9 +342,14 @@ func TestVerifyBeaconBlock_NoBlock(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 
-	s := NewForkChoiceService(ctx, db)
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	d := &ethpb.AttestationData{}
-	if err := s.verifyBeaconBlock(ctx, d); !strings.Contains(err.Error(), "beacon block  does not exist") {
+	if err := service.verifyBeaconBlock(ctx, d); !strings.Contains(err.Error(), "beacon block  does not exist") {
 		t.Error("Did not receive the wanted error")
 	}
 }
@@ -341,13 +359,18 @@ func TestVerifyBeaconBlock_futureBlock(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 
-	s := NewForkChoiceService(ctx, db)
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 2}}
-	s.db.SaveBlock(ctx, b)
+	service.beaconDB.SaveBlock(ctx, b)
 	r, _ := ssz.HashTreeRoot(b.Block)
 	d := &ethpb.AttestationData{Slot: 1, BeaconBlockRoot: r[:]}
 
-	if err := s.verifyBeaconBlock(ctx, d); !strings.Contains(err.Error(), "could not process attestation for future block") {
+	if err := service.verifyBeaconBlock(ctx, d); !strings.Contains(err.Error(), "could not process attestation for future block") {
 		t.Error("Did not receive the wanted error")
 	}
 }
@@ -357,13 +380,18 @@ func TestVerifyBeaconBlock_OK(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 
-	s := NewForkChoiceService(ctx, db)
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 2}}
-	s.db.SaveBlock(ctx, b)
+	service.beaconDB.SaveBlock(ctx, b)
 	r, _ := ssz.HashTreeRoot(b.Block)
 	d := &ethpb.AttestationData{Slot: 2, BeaconBlockRoot: r[:]}
 
-	if err := s.verifyBeaconBlock(ctx, d); err != nil {
+	if err := service.verifyBeaconBlock(ctx, d); err != nil {
 		t.Error("Did not receive the wanted error")
 	}
 }

beacon-chain/blockchain/process_block.go

@@ -0,0 +1,314 @@
+package blockchain
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/go-ssz"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/flags"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/attestationutil"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/sirupsen/logrus"
+	"go.opencensus.io/trace"
+)
+
+// onBlock is called when a gossip block is received. It runs regular state transition on the block.
+//
+// Spec pseudocode definition:
+//   def on_block(store: Store, block: BeaconBlock) -> None:
+//    # Make a copy of the state to avoid mutability issues
+//    assert block.parent_root in store.block_states
+//    pre_state = store.block_states[block.parent_root].copy()
+//    # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past.
+//    assert store.time >= pre_state.genesis_time + block.slot * SECONDS_PER_SLOT
+//    # Add new block to the store
+//    store.blocks[signing_root(block)] = block
+//    # Check block is a descendant of the finalized block
+//    assert (
+//        get_ancestor(store, signing_root(block), store.blocks[store.finalized_checkpoint.root].slot) ==
+//        store.finalized_checkpoint.root
+//    )
+//    # Check that block is later than the finalized epoch slot
+//    assert block.slot > compute_start_slot_of_epoch(store.finalized_checkpoint.epoch)
+//    # Check the block is valid and compute the post-state
+//    state = state_transition(pre_state, block)
+//    # Add new state for this block to the store
+//    store.block_states[signing_root(block)] = state
+//
+//    # Update justified checkpoint
+//    if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch:
+//        if state.current_justified_checkpoint.epoch > store.best_justified_checkpoint.epoch:
+//            store.best_justified_checkpoint = state.current_justified_checkpoint
+//
+//    # Update finalized checkpoint
+//    if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch:
+//        store.finalized_checkpoint = state.finalized_checkpoint
+func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock) (*stateTrie.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "blockchain.onBlock")
+	defer span.End()
+
+	if signed == nil || signed.Block == nil {
+		return nil, errors.New("nil block")
+	}
+
+	b := signed.Block
+
+	// Retrieve incoming block's pre state.
+	preState, err := s.getBlockPreState(ctx, b)
+	if err != nil {
+		return nil, err
+	}
+	preStateValidatorCount := preState.NumValidators()
+
+	root, err := ssz.HashTreeRoot(b)
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not get signing root of block %d", b.Slot)
+	}
+	log.WithFields(logrus.Fields{
+		"slot": b.Slot,
+		"root": fmt.Sprintf("0x%s...", hex.EncodeToString(root[:])[:8]),
+	}).Info("Executing state transition on block")
+
+	postState, err := state.ExecuteStateTransition(ctx, preState, signed)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not execute state transition")
+	}
+
+	if err := s.beaconDB.SaveBlock(ctx, signed); err != nil {
+		return nil, errors.Wrapf(err, "could not save block from slot %d", b.Slot)
+	}
+
+	if err := s.insertBlockToForkChoiceStore(ctx, b, root, postState); err != nil {
+		return nil, errors.Wrapf(err, "could not insert block %d to fork choice store", b.Slot)
+	}
+
+	if err := s.beaconDB.SaveState(ctx, postState, root); err != nil {
+		return nil, errors.Wrap(err, "could not save state")
+	}
+
+	// Update justified check point.
+	if postState.CurrentJustifiedCheckpoint().Epoch > s.justifiedCheckpt.Epoch {
+		if err := s.updateJustified(ctx, postState); err != nil {
+			return nil, err
+		}
+	}
+
+	// Update finalized check point. Prune the block cache and helper caches on every new finalized epoch.
+	if postState.FinalizedCheckpointEpoch() > s.finalizedCheckpt.Epoch {
+		if err := s.beaconDB.SaveFinalizedCheckpoint(ctx, postState.FinalizedCheckpoint()); err != nil {
+			return nil, errors.Wrap(err, "could not save finalized checkpoint")
+		}
+
+		startSlot := helpers.StartSlot(s.prevFinalizedCheckpt.Epoch)
+		endSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
+		if endSlot > startSlot {
+			if err := s.rmStatesOlderThanLastFinalized(ctx, startSlot, endSlot); err != nil {
+				return nil, errors.Wrapf(err, "could not delete states prior to finalized check point, range: %d, %d",
+					startSlot, endSlot)
+			}
+		}
+
+		// Prune proto array fork choice nodes, all nodes before finalized check point will
+		// be pruned.
+		s.forkChoiceStore.Prune(ctx, bytesutil.ToBytes32(postState.FinalizedCheckpoint().Root))
+
+		s.prevFinalizedCheckpt = s.finalizedCheckpt
+		s.finalizedCheckpt = postState.FinalizedCheckpoint()
+
+		if err := s.finalizedImpliesNewJustified(ctx, postState); err != nil {
+			return nil, errors.Wrap(err, "could not save new justified")
+		}
+	}
+
+	// Update validator indices in database as needed.
+	if err := s.saveNewValidators(ctx, preStateValidatorCount, postState); err != nil {
+		return nil, errors.Wrap(err, "could not save finalized checkpoint")
+	}
+
+	// Epoch boundary bookkeeping such as logging epoch summaries.
+	if postState.Slot() >= s.nextEpochBoundarySlot {
+		logEpochData(postState)
+		reportEpochMetrics(postState)
+
+		// Update committees cache at epoch boundary slot.
+		if err := helpers.UpdateCommitteeCache(postState, helpers.CurrentEpoch(postState)); err != nil {
+			return nil, err
+		}
+		if err := helpers.UpdateProposerIndicesInCache(postState, helpers.CurrentEpoch(postState)); err != nil {
+			return nil, err
+		}
+
+		s.nextEpochBoundarySlot = helpers.StartSlot(helpers.NextEpoch(postState))
+	}
+
+	// Delete the processed block attestations from attestation pool.
+	if err := s.deletePoolAtts(b.Body.Attestations); err != nil {
+		return nil, err
+	}
+
+	// Delete the processed block attester slashings from slashings pool.
+	for i := 0; i < len(b.Body.AttesterSlashings); i++ {
+		s.slashingPool.MarkIncludedAttesterSlashing(b.Body.AttesterSlashings[i])
+	}
+
+	return postState, nil
+}
+
+// onBlockInitialSyncStateTransition is called when an initial sync block is received.
+// It runs state transition on the block and without any BLS verification. The excluded BLS verification
+// includes attestation's aggregated signature. It also does not save attestations.
+func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed *ethpb.SignedBeaconBlock) error {
+	ctx, span := trace.StartSpan(ctx, "blockchain.onBlock")
+	defer span.End()
+
+	if signed == nil || signed.Block == nil {
+		return errors.New("nil block")
+	}
+
+	b := signed.Block
+
+	s.initSyncStateLock.Lock()
+	defer s.initSyncStateLock.Unlock()
+
+	// Retrieve incoming block's pre state.
+	preState, err := s.verifyBlkPreState(ctx, b)
+	if err != nil {
+		return err
+	}
+	preStateValidatorCount := preState.NumValidators()
+
+	postState, err := state.ExecuteStateTransitionNoVerifyAttSigs(ctx, preState, signed)
+	if err != nil {
+		return errors.Wrap(err, "could not execute state transition")
+	}
+
+	if err := s.beaconDB.SaveBlock(ctx, signed); err != nil {
+		return errors.Wrapf(err, "could not save block from slot %d", b.Slot)
+	}
+	root, err := ssz.HashTreeRoot(b)
+	if err != nil {
+		return errors.Wrapf(err, "could not get signing root of block %d", b.Slot)
+	}
+
+	if err := s.insertBlockToForkChoiceStore(ctx, b, root, postState); err != nil {
+		return errors.Wrapf(err, "could not insert block %d to fork choice store", b.Slot)
+	}
+
+	s.initSyncState[root] = postState.Copy()
+	s.filterBoundaryCandidates(ctx, root, postState)
+
+	if flags.Get().EnableArchive {
+		atts := signed.Block.Body.Attestations
+		if err := s.beaconDB.SaveAttestations(ctx, atts); err != nil {
+			return errors.Wrapf(err, "could not save block attestations from slot %d", b.Slot)
+		}
+	}
+
+	// Update justified check point.
+	if postState.CurrentJustifiedCheckpoint().Epoch > s.justifiedCheckpt.Epoch {
+		if err := s.updateJustified(ctx, postState); err != nil {
+			return err
+		}
+	}
+
+	// Update finalized check point. Prune the block cache and helper caches on every new finalized epoch.
+	if postState.FinalizedCheckpointEpoch() > s.finalizedCheckpt.Epoch {
+		startSlot := helpers.StartSlot(s.prevFinalizedCheckpt.Epoch)
+		endSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
+		if endSlot > startSlot {
+			if err := s.rmStatesOlderThanLastFinalized(ctx, startSlot, endSlot); err != nil {
+				return errors.Wrapf(err, "could not delete states prior to finalized check point, range: %d, %d",
+					startSlot, endSlot)
+			}
+		}
+
+		if err := s.saveInitState(ctx, postState); err != nil {
+			return errors.Wrap(err, "could not save init sync finalized state")
+		}
+
+		if err := s.beaconDB.SaveFinalizedCheckpoint(ctx, postState.FinalizedCheckpoint()); err != nil {
+			return errors.Wrap(err, "could not save finalized checkpoint")
+		}
+
+		s.prevFinalizedCheckpt = s.finalizedCheckpt
+		s.finalizedCheckpt = postState.FinalizedCheckpoint()
+
+		if err := s.finalizedImpliesNewJustified(ctx, postState); err != nil {
+			return errors.Wrap(err, "could not save new justified")
+		}
+	}
+
+	// Update validator indices in database as needed.
+	if err := s.saveNewValidators(ctx, preStateValidatorCount, postState); err != nil {
+		return errors.Wrap(err, "could not save finalized checkpoint")
+	}
+
+	numOfStates := len(s.boundaryRoots)
+	if numOfStates > initialSyncCacheSize {
+		if err = s.persistCachedStates(ctx, numOfStates); err != nil {
+			return err
+		}
+	}
+	if len(s.initSyncState) > maxCacheSize {
+		s.pruneOldNonFinalizedStates()
+	}
+
+	// Epoch boundary bookkeeping such as logging epoch summaries.
+	if postState.Slot() >= s.nextEpochBoundarySlot {
+		reportEpochMetrics(postState)
+		s.nextEpochBoundarySlot = helpers.StartSlot(helpers.NextEpoch(postState))
+
+		// Update committees cache at epoch boundary slot.
+		if err := helpers.UpdateCommitteeCache(postState, helpers.CurrentEpoch(postState)); err != nil {
+			return err
+		}
+		if err := helpers.UpdateProposerIndicesInCache(postState, helpers.CurrentEpoch(postState)); err != nil {
+			return err
+		}
+
+		if helpers.IsEpochStart(postState.Slot()) {
+			if err := s.beaconDB.SaveState(ctx, postState, root); err != nil {
+				return errors.Wrap(err, "could not save state")
+			}
+		}
+	}
+
+	return nil
+}
+
+// This feeds in the block and block's attestations to fork choice store. It's allows fork choice store
+// to gain information on the most current chain.
+func (s *Service) insertBlockToForkChoiceStore(ctx context.Context, blk *ethpb.BeaconBlock, root [32]byte, state *stateTrie.BeaconState) error {
+	if err := s.fillInForkChoiceMissingBlocks(ctx, blk, state); err != nil {
+		return err
+	}
+
+	// Feed in block to fork choice store.
+	if err := s.forkChoiceStore.ProcessBlock(ctx,
+		blk.Slot, root, bytesutil.ToBytes32(blk.ParentRoot),
+		state.CurrentJustifiedCheckpoint().Epoch,
+		state.FinalizedCheckpointEpoch()); err != nil {
+		return errors.Wrap(err, "could not process block for proto array fork choice")
+	}
+
+	// Feed in block's attestations to fork choice store.
+	for _, a := range blk.Body.Attestations {
+		committee, err := helpers.BeaconCommitteeFromState(state, a.Data.Slot, a.Data.CommitteeIndex)
+		if err != nil {
+			return err
+		}
+		indices, err := attestationutil.AttestingIndices(a.AggregationBits, committee)
+		if err != nil {
+			return err
+		}
+		s.forkChoiceStore.ProcessAttestation(ctx, indices, bytesutil.ToBytes32(a.Data.BeaconBlockRoot), a.Data.Target.Epoch)
+	}
+
+	return nil
+}

beacon-chain/blockchain/process_block_helpers.go

@@ -0,0 +1,467 @@
+package blockchain
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/pkg/errors"
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/go-ssz"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/traceutil"
+	"github.com/sirupsen/logrus"
+	"go.opencensus.io/trace"
+)
+
+// CurrentSlot returns the current slot based on time.
+func (s *Service) CurrentSlot() uint64 {
+	return uint64(time.Now().Unix()-s.genesisTime.Unix()) / params.BeaconConfig().SecondsPerSlot
+}
+
+// getBlockPreState returns the pre state of an incoming block. It uses the parent root of the block
+// to retrieve the state in DB. It verifies the pre state's validity and the incoming block
+// is in the correct time window.
+func (s *Service) getBlockPreState(ctx context.Context, b *ethpb.BeaconBlock) (*stateTrie.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "forkchoice.getBlockPreState")
+	defer span.End()
+
+	// Verify incoming block has a valid pre state.
+	preState, err := s.verifyBlkPreState(ctx, b)
+	if err != nil {
+		return nil, err
+	}
+
+	// Verify block slot time is not from the feature.
+	if err := helpers.VerifySlotTime(preState.GenesisTime(), b.Slot, helpers.TimeShiftTolerance); err != nil {
+		return nil, err
+	}
+
+	// Verify block is a descendent of a finalized block.
+	if err := s.verifyBlkDescendant(ctx, bytesutil.ToBytes32(b.ParentRoot), b.Slot); err != nil {
+		return nil, err
+	}
+
+	// Verify block is later than the finalized epoch slot.
+	if err := s.verifyBlkFinalizedSlot(b); err != nil {
+		return nil, err
+	}
+
+	return preState, nil
+}
+
+// verifyBlkPreState validates input block has a valid pre-state.
+func (s *Service) verifyBlkPreState(ctx context.Context, b *ethpb.BeaconBlock) (*stateTrie.BeaconState, error) {
+	preState := s.initSyncState[bytesutil.ToBytes32(b.ParentRoot)]
+	var err error
+	if preState == nil {
+		if featureconfig.Get().CheckHeadState {
+			headRoot, err := s.HeadRoot(ctx)
+			if err != nil {
+				return nil, errors.Wrapf(err, "could not get head root")
+			}
+			if bytes.Equal(headRoot, b.ParentRoot) {
+				return s.HeadState(ctx)
+			}
+		}
+		preState, err = s.beaconDB.State(ctx, bytesutil.ToBytes32(b.ParentRoot))
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not get pre state for slot %d", b.Slot)
+		}
+		if preState == nil {
+			if bytes.Equal(s.finalizedCheckpt.Root, b.ParentRoot) {
+				return nil, fmt.Errorf("pre state of slot %d does not exist", b.Slot)
+			}
+			preState, err = s.generateState(ctx, bytesutil.ToBytes32(s.finalizedCheckpt.Root), bytesutil.ToBytes32(b.ParentRoot))
+			if err != nil {
+				return nil, err
+			}
+		}
+		return preState, nil // No copy needed from newly hydrated DB object.
+	}
+	return preState.Copy(), nil
+}
+
+// verifyBlkDescendant validates input block root is a descendant of the
+// current finalized block root.
+func (s *Service) verifyBlkDescendant(ctx context.Context, root [32]byte, slot uint64) error {
+	ctx, span := trace.StartSpan(ctx, "forkchoice.verifyBlkDescendant")
+	defer span.End()
+
+	finalizedBlkSigned, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(s.finalizedCheckpt.Root))
+	if err != nil || finalizedBlkSigned == nil || finalizedBlkSigned.Block == nil {
+		return errors.Wrap(err, "could not get finalized block")
+	}
+	finalizedBlk := finalizedBlkSigned.Block
+
+	bFinalizedRoot, err := s.ancestor(ctx, root[:], finalizedBlk.Slot)
+	if err != nil {
+		return errors.Wrap(err, "could not get finalized block root")
+	}
+	if bFinalizedRoot == nil {
+		return fmt.Errorf("no finalized block known for block from slot %d", slot)
+	}
+
+	if !bytes.Equal(bFinalizedRoot, s.finalizedCheckpt.Root) {
+		err := fmt.Errorf("block from slot %d is not a descendent of the current finalized block slot %d, %#x != %#x",
+			slot, finalizedBlk.Slot, bytesutil.Trunc(bFinalizedRoot), bytesutil.Trunc(s.finalizedCheckpt.Root))
+		traceutil.AnnotateError(span, err)
+		return err
+	}
+	return nil
+}
+
+// verifyBlkFinalizedSlot validates input block is not less than or equal
+// to current finalized slot.
+func (s *Service) verifyBlkFinalizedSlot(b *ethpb.BeaconBlock) error {
+	finalizedSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
+	if finalizedSlot >= b.Slot {
+		return fmt.Errorf("block is equal or earlier than finalized block, slot %d < slot %d", b.Slot, finalizedSlot)
+	}
+	return nil
+}
+
+// saveNewValidators saves newly added validator indices from the state to db.
+// Does nothing if validator count has not changed.
+func (s *Service) saveNewValidators(ctx context.Context, preStateValidatorCount int, postState *stateTrie.BeaconState) error {
+	postStateValidatorCount := postState.NumValidators()
+	if preStateValidatorCount != postStateValidatorCount {
+		indices := make([]uint64, 0)
+		pubKeys := make([][48]byte, 0)
+		for i := preStateValidatorCount; i < postStateValidatorCount; i++ {
+			indices = append(indices, uint64(i))
+			pubKeys = append(pubKeys, postState.PubkeyAtIndex(uint64(i)))
+		}
+		if err := s.beaconDB.SaveValidatorIndices(ctx, pubKeys, indices); err != nil {
+			return errors.Wrapf(err, "could not save activated validators: %v", indices)
+		}
+		log.WithFields(logrus.Fields{
+			"indices":             indices,
+			"totalValidatorCount": postStateValidatorCount - preStateValidatorCount,
+		}).Trace("Validator indices saved in DB")
+	}
+	return nil
+}
+
+// rmStatesOlderThanLastFinalized deletes the states in db since last finalized check point.
+func (s *Service) rmStatesOlderThanLastFinalized(ctx context.Context, startSlot uint64, endSlot uint64) error {
+	ctx, span := trace.StartSpan(ctx, "forkchoice.rmStatesBySlots")
+	defer span.End()
+
+	// Make sure start slot is not a skipped slot
+	for i := startSlot; i > 0; i-- {
+		filter := filters.NewFilter().SetStartSlot(i).SetEndSlot(i)
+		b, err := s.beaconDB.Blocks(ctx, filter)
+		if err != nil {
+			return err
+		}
+		if len(b) > 0 {
+			startSlot = i
+			break
+		}
+	}
+
+	// Make sure finalized slot is not a skipped slot.
+	for i := endSlot; i > 0; i-- {
+		filter := filters.NewFilter().SetStartSlot(i).SetEndSlot(i)
+		b, err := s.beaconDB.Blocks(ctx, filter)
+		if err != nil {
+			return err
+		}
+		if len(b) > 0 {
+			endSlot = i - 1
+			break
+		}
+	}
+
+	// Do not remove genesis state
+	if startSlot == 0 {
+		startSlot++
+	}
+	// If end slot comes less than start slot
+	if endSlot < startSlot {
+		endSlot = startSlot
+	}
+
+	filter := filters.NewFilter().SetStartSlot(startSlot).SetEndSlot(endSlot)
+	roots, err := s.beaconDB.BlockRoots(ctx, filter)
+	if err != nil {
+		return err
+	}
+
+	roots, err = s.filterBlockRoots(ctx, roots)
+	if err != nil {
+		return err
+	}
+
+	if err := s.beaconDB.DeleteStates(ctx, roots); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// shouldUpdateCurrentJustified prevents bouncing attack, by only update conflicting justified
+// checkpoints in the fork choice if in the early slots of the epoch.
+// Otherwise, delay incorporation of new justified checkpoint until next epoch boundary.
+// See https://ethresear.ch/t/prevention-of-bouncing-attack-on-ffg/6114 for more detailed analysis and discussion.
+func (s *Service) shouldUpdateCurrentJustified(ctx context.Context, newJustifiedCheckpt *ethpb.Checkpoint) (bool, error) {
+	if helpers.SlotsSinceEpochStarts(s.CurrentSlot()) < params.BeaconConfig().SafeSlotsToUpdateJustified {
+		return true, nil
+	}
+	newJustifiedBlockSigned, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(newJustifiedCheckpt.Root))
+	if err != nil {
+		return false, err
+	}
+	if newJustifiedBlockSigned == nil || newJustifiedBlockSigned.Block == nil {
+		return false, errors.New("nil new justified block")
+	}
+	newJustifiedBlock := newJustifiedBlockSigned.Block
+	if newJustifiedBlock.Slot <= helpers.StartSlot(s.justifiedCheckpt.Epoch) {
+		return false, nil
+	}
+	justifiedBlockSigned, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root))
+	if err != nil {
+		return false, err
+	}
+	if justifiedBlockSigned == nil || justifiedBlockSigned.Block == nil {
+		return false, errors.New("nil justified block")
+	}
+	justifiedBlock := justifiedBlockSigned.Block
+	b, err := s.ancestor(ctx, newJustifiedCheckpt.Root, justifiedBlock.Slot)
+	if err != nil {
+		return false, err
+	}
+	if !bytes.Equal(b, s.justifiedCheckpt.Root) {
+		return false, nil
+	}
+	return true, nil
+}
+
+func (s *Service) updateJustified(ctx context.Context, state *stateTrie.BeaconState) error {
+	cpt := state.CurrentJustifiedCheckpoint()
+	if cpt.Epoch > s.bestJustifiedCheckpt.Epoch {
+		s.bestJustifiedCheckpt = cpt
+	}
+	canUpdate, err := s.shouldUpdateCurrentJustified(ctx, cpt)
+	if err != nil {
+		return err
+	}
+	if canUpdate {
+		s.prevJustifiedCheckpt = s.justifiedCheckpt
+		s.justifiedCheckpt = cpt
+	}
+
+	justifiedRoot := bytesutil.ToBytes32(cpt.Root)
+
+	justifiedState := s.initSyncState[justifiedRoot]
+	// If justified state is nil, resume back to normal syncing process and save
+	// justified check point.
+	if justifiedState == nil {
+		if s.beaconDB.HasState(ctx, justifiedRoot) {
+			return s.beaconDB.SaveJustifiedCheckpoint(ctx, cpt)
+		}
+		justifiedState, err = s.generateState(ctx, bytesutil.ToBytes32(s.finalizedCheckpt.Root), justifiedRoot)
+		if err != nil {
+			log.Error(err)
+			return s.beaconDB.SaveJustifiedCheckpoint(ctx, cpt)
+		}
+	}
+	if err := s.beaconDB.SaveState(ctx, justifiedState, justifiedRoot); err != nil {
+		return errors.Wrap(err, "could not save justified state")
+	}
+
+	return s.beaconDB.SaveJustifiedCheckpoint(ctx, cpt)
+}
+
+// This saves every finalized state in DB during initial sync, needed as part of optimization to
+// use cache state during initial sync in case of restart.
+func (s *Service) saveInitState(ctx context.Context, state *stateTrie.BeaconState) error {
+	cpt := state.FinalizedCheckpoint()
+	finalizedRoot := bytesutil.ToBytes32(cpt.Root)
+	fs := s.initSyncState[finalizedRoot]
+	if fs == nil {
+		var err error
+		fs, err = s.beaconDB.State(ctx, finalizedRoot)
+		if err != nil {
+			return err
+		}
+		if fs == nil {
+			fs, err = s.generateState(ctx, bytesutil.ToBytes32(s.prevFinalizedCheckpt.Root), finalizedRoot)
+			if err != nil {
+				// This might happen if the client was in sync and is now re-syncing for whatever reason.
+				log.Warn("Initial sync cache did not have finalized state root cached")
+				return err
+			}
+		}
+	}
+
+	if err := s.beaconDB.SaveState(ctx, fs, finalizedRoot); err != nil {
+		return errors.Wrap(err, "could not save state")
+	}
+	return nil
+}
+
+// This filters block roots that are not known as head root and finalized root in DB.
+// It serves as the last line of defence before we prune states.
+func (s *Service) filterBlockRoots(ctx context.Context, roots [][32]byte) ([][32]byte, error) {
+	f, err := s.beaconDB.FinalizedCheckpoint(ctx)
+	if err != nil {
+		return nil, err
+	}
+	fRoot := f.Root
+	h, err := s.beaconDB.HeadBlock(ctx)
+	if err != nil {
+		return nil, err
+	}
+	hRoot, err := ssz.HashTreeRoot(h.Block)
+	if err != nil {
+		return nil, err
+	}
+
+	filtered := make([][32]byte, 0, len(roots))
+	for _, root := range roots {
+		if bytes.Equal(root[:], fRoot[:]) || bytes.Equal(root[:], hRoot[:]) {
+			continue
+		}
+		filtered = append(filtered, root)
+	}
+
+	return filtered, nil
+}
+
+// ancestor returns the block root of an ancestry block from the input block root.
+//
+// Spec pseudocode definition:
+//   def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash:
+//    block = store.blocks[root]
+//    if block.slot > slot:
+//      return get_ancestor(store, block.parent_root, slot)
+//    elif block.slot == slot:
+//      return root
+//    else:
+//      return Bytes32()  # root is older than queried slot: no results.
+func (s *Service) ancestor(ctx context.Context, root []byte, slot uint64) ([]byte, error) {
+	ctx, span := trace.StartSpan(ctx, "forkchoice.ancestor")
+	defer span.End()
+
+	// Stop recursive ancestry lookup if context is cancelled.
+	if ctx.Err() != nil {
+		return nil, ctx.Err()
+	}
+
+	signed, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(root))
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get ancestor block")
+	}
+	if signed == nil || signed.Block == nil {
+		return nil, errors.New("nil block")
+	}
+	b := signed.Block
+
+	// If we dont have the ancestor in the DB, simply return nil so rest of fork choice
+	// operation can proceed. This is not an error condition.
+	if b == nil || b.Slot < slot {
+		return nil, nil
+	}
+
+	if b.Slot == slot {
+		return root, nil
+	}
+
+	return s.ancestor(ctx, b.ParentRoot, slot)
+}
+
+// This updates justified check point in store, if the new justified is later than stored justified or
+// the store's justified is not in chain with finalized check point.
+//
+// Spec definition:
+//   if (
+//            state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch
+//            or get_ancestor(store, store.justified_checkpoint.root, finalized_slot) != store.finalized_checkpoint.root
+//        ):
+//            store.justified_checkpoint = state.current_justified_checkpoint
+func (s *Service) finalizedImpliesNewJustified(ctx context.Context, state *stateTrie.BeaconState) error {
+	finalizedBlkSigned, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(s.finalizedCheckpt.Root))
+	if err != nil || finalizedBlkSigned == nil || finalizedBlkSigned.Block == nil {
+		return errors.Wrap(err, "could not get finalized block")
+	}
+	finalizedBlk := finalizedBlkSigned.Block
+
+	anc, err := s.ancestor(ctx, s.justifiedCheckpt.Root, finalizedBlk.Slot)
+	if err != nil {
+		return err
+	}
+
+	// Either the new justified is later than stored justified or not in chain with finalized check pint.
+	if cpt := state.CurrentJustifiedCheckpoint(); cpt != nil && cpt.Epoch > s.justifiedCheckpt.Epoch || !bytes.Equal(anc, s.finalizedCheckpt.Root) {
+		s.justifiedCheckpt = state.CurrentJustifiedCheckpoint()
+	}
+
+	return nil
+}
+
+// This retrieves missing blocks from DB (ie. the blocks that couldn't received over sync) and inserts them to fork choice store.
+// This is useful for block tree visualizer and additional vote accounting.
+func (s *Service) fillInForkChoiceMissingBlocks(ctx context.Context, blk *ethpb.BeaconBlock, state *stateTrie.BeaconState) error {
+	pendingNodes := make([]*ethpb.BeaconBlock, 0)
+
+	parentRoot := bytesutil.ToBytes32(blk.ParentRoot)
+	slot := blk.Slot
+	// Fork choice only matters from last finalized slot.
+	higherThanFinalized := slot > helpers.StartSlot(s.finalizedCheckpt.Epoch)
+	// As long as parent node is not in fork choice store, and parent node is in DB.
+	for !s.forkChoiceStore.HasNode(parentRoot) && s.beaconDB.HasBlock(ctx, parentRoot) && higherThanFinalized {
+		b, err := s.beaconDB.Block(ctx, parentRoot)
+		if err != nil {
+			return err
+		}
+
+		pendingNodes = append(pendingNodes, b.Block)
+		parentRoot = bytesutil.ToBytes32(b.Block.ParentRoot)
+		slot = b.Block.Slot
+		higherThanFinalized = slot > helpers.StartSlot(s.finalizedCheckpt.Epoch)
+	}
+
+	// Insert parent nodes to fork choice store in reverse order.
+	// Lower slots should be at the end of the list.
+	for i := len(pendingNodes) - 1; i >= 0; i-- {
+		b := pendingNodes[i]
+		r, err := ssz.HashTreeRoot(b)
+		if err != nil {
+			return err
+		}
+
+		if err := s.forkChoiceStore.ProcessBlock(ctx,
+			b.Slot, r, bytesutil.ToBytes32(b.ParentRoot),
+			state.CurrentJustifiedCheckpoint().Epoch,
+			state.FinalizedCheckpointEpoch()); err != nil {
+			return errors.Wrap(err, "could not process block for proto array fork choice")
+		}
+	}
+
+	return nil
+}
+
+// The deletes input attestations from the attestation pool, so proposers don't include them in a block for the future.
+func (s *Service) deletePoolAtts(atts []*ethpb.Attestation) error {
+	for _, att := range atts {
+		if helpers.IsAggregated(att) {
+			if err := s.attPool.DeleteAggregatedAttestation(att); err != nil {
+				return err
+			}
+		} else {
+			if err := s.attPool.DeleteUnaggregatedAttestation(att); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

beacon-chain/blockchain/process_block_test.go

@@ -0,0 +1,701 @@
+package blockchain
+
+import (
+	"context"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/go-ssz"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
+)
+
+func TestStore_OnBlock(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	genesisStateRoot := [32]byte{}
+	genesis := blocks.NewGenesisBlock(genesisStateRoot[:])
+	if err := db.SaveBlock(ctx, genesis); err != nil {
+		t.Error(err)
+	}
+	validGenesisRoot, err := ssz.HashTreeRoot(genesis.Block)
+	if err != nil {
+		t.Error(err)
+	}
+	st, err := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveState(ctx, st.Copy(), validGenesisRoot); err != nil {
+		t.Fatal(err)
+	}
+	roots, err := blockTree1(db, validGenesisRoot[:])
+	if err != nil {
+		t.Fatal(err)
+	}
+	random := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1, ParentRoot: validGenesisRoot[:]}}
+	if err := db.SaveBlock(ctx, random); err != nil {
+		t.Error(err)
+	}
+	randomParentRoot, err := ssz.HashTreeRoot(random.Block)
+	if err != nil {
+		t.Error(err)
+	}
+	if err := service.beaconDB.SaveState(ctx, st.Copy(), randomParentRoot); err != nil {
+		t.Fatal(err)
+	}
+	randomParentRoot2 := roots[1]
+	if err := service.beaconDB.SaveState(ctx, st.Copy(), bytesutil.ToBytes32(randomParentRoot2)); err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct {
+		name          string
+		blk           *ethpb.BeaconBlock
+		s             *stateTrie.BeaconState
+		time          uint64
+		wantErrString string
+	}{
+		{
+			name:          "parent block root does not have a state",
+			blk:           &ethpb.BeaconBlock{},
+			s:             st.Copy(),
+			wantErrString: "provided block root does not have block saved in the db",
+		},
+		{
+			name:          "block is from the feature",
+			blk:           &ethpb.BeaconBlock{ParentRoot: randomParentRoot[:], Slot: params.BeaconConfig().FarFutureEpoch},
+			s:             st.Copy(),
+			wantErrString: "could not process slot from the future",
+		},
+		{
+			name:          "could not get finalized block",
+			blk:           &ethpb.BeaconBlock{ParentRoot: randomParentRoot[:]},
+			s:             st.Copy(),
+			wantErrString: "block from slot 0 is not a descendent of the current finalized block",
+		},
+		{
+			name:          "same slot as finalized block",
+			blk:           &ethpb.BeaconBlock{Slot: 0, ParentRoot: randomParentRoot2},
+			s:             st.Copy(),
+			wantErrString: "block is equal or earlier than finalized block, slot 0 < slot 0",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			service.justifiedCheckpt = &ethpb.Checkpoint{Root: validGenesisRoot[:]}
+			service.bestJustifiedCheckpt = &ethpb.Checkpoint{Root: validGenesisRoot[:]}
+			service.finalizedCheckpt = &ethpb.Checkpoint{Root: validGenesisRoot[:]}
+			service.prevFinalizedCheckpt = &ethpb.Checkpoint{Root: validGenesisRoot[:]}
+			service.finalizedCheckpt.Root = roots[0]
+
+			_, err := service.onBlock(ctx, &ethpb.SignedBeaconBlock{Block: tt.blk})
+			if !strings.Contains(err.Error(), tt.wantErrString) {
+				t.Errorf("Store.OnBlock() error = %v, wantErr = %v", err, tt.wantErrString)
+			}
+		})
+	}
+}
+
+func TestStore_SaveNewValidators(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	preCount := 2 // validators 0 and validators 1
+	s, _ := stateTrie.InitializeFromProto(&pb.BeaconState{Validators: []*ethpb.Validator{
+		{PublicKey: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}},
+		{PublicKey: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}},
+		{PublicKey: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}},
+		{PublicKey: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3}},
+	}})
+	if err := service.saveNewValidators(ctx, preCount, s); err != nil {
+		t.Fatal(err)
+	}
+
+	if !db.HasValidatorIndex(ctx, []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}) {
+		t.Error("Wanted validator saved in db")
+	}
+	if !db.HasValidatorIndex(ctx, []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3}) {
+		t.Error("Wanted validator saved in db")
+	}
+	if db.HasValidatorIndex(ctx, []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}) {
+		t.Error("validator not suppose to be saved in db")
+	}
+}
+
+func TestRemoveStateSinceLastFinalized(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+	params.UseMinimalConfig()
+	defer params.UseMainnetConfig()
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Save 100 blocks in DB, each has a state.
+	numBlocks := 100
+	totalBlocks := make([]*ethpb.SignedBeaconBlock, numBlocks)
+	blockRoots := make([][32]byte, 0)
+	for i := 0; i < len(totalBlocks); i++ {
+		totalBlocks[i] = &ethpb.SignedBeaconBlock{
+			Block: &ethpb.BeaconBlock{
+				Slot: uint64(i),
+			},
+		}
+		r, err := ssz.HashTreeRoot(totalBlocks[i].Block)
+		if err != nil {
+			t.Fatal(err)
+		}
+		s, _ := stateTrie.InitializeFromProto(&pb.BeaconState{Slot: uint64(i)})
+		if err := service.beaconDB.SaveState(ctx, s, r); err != nil {
+			t.Fatal(err)
+		}
+		if err := service.beaconDB.SaveBlock(ctx, totalBlocks[i]); err != nil {
+			t.Fatal(err)
+		}
+		blockRoots = append(blockRoots, r)
+		if err := service.beaconDB.SaveHeadBlockRoot(ctx, r); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// New finalized epoch: 1
+	finalizedEpoch := uint64(1)
+	finalizedSlot := finalizedEpoch * params.BeaconConfig().SlotsPerEpoch
+	endSlot := helpers.StartSlot(finalizedEpoch+1) - 1 // Inclusive
+	if err := service.rmStatesOlderThanLastFinalized(ctx, 0, endSlot); err != nil {
+		t.Fatal(err)
+	}
+	for _, r := range blockRoots {
+		s, err := service.beaconDB.State(ctx, r)
+		if err != nil {
+			t.Fatal(err)
+		}
+		// Also verifies genesis state didnt get deleted
+		if s != nil && s.Slot() != finalizedSlot && s.Slot() != 0 && s.Slot() < endSlot {
+			t.Errorf("State with slot %d should not be in DB", s.Slot())
+		}
+	}
+
+	// New finalized epoch: 5
+	newFinalizedEpoch := uint64(5)
+	newFinalizedSlot := newFinalizedEpoch * params.BeaconConfig().SlotsPerEpoch
+	endSlot = helpers.StartSlot(newFinalizedEpoch+1) - 1 // Inclusive
+	if err := service.rmStatesOlderThanLastFinalized(ctx, helpers.StartSlot(finalizedEpoch+1)-1, endSlot); err != nil {
+		t.Fatal(err)
+	}
+	for _, r := range blockRoots {
+		s, err := service.beaconDB.State(ctx, r)
+		if err != nil {
+			t.Fatal(err)
+		}
+		// Also verifies genesis state didnt get deleted
+		if s != nil && s.Slot() != newFinalizedSlot && s.Slot() != finalizedSlot && s.Slot() != 0 && s.Slot() < endSlot {
+			t.Errorf("State with slot %d should not be in DB", s.Slot())
+		}
+	}
+}
+
+func TestRemoveStateSinceLastFinalized_EmptyStartSlot(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+	params.UseMinimalConfig()
+	defer params.UseMainnetConfig()
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	service.genesisTime = time.Now()
+
+	update, err := service.shouldUpdateCurrentJustified(ctx, &ethpb.Checkpoint{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !update {
+		t.Error("Should be able to update justified, received false")
+	}
+
+	lastJustifiedBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{ParentRoot: []byte{'G'}}}
+	lastJustifiedRoot, _ := ssz.HashTreeRoot(lastJustifiedBlk.Block)
+	newJustifiedBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1, ParentRoot: lastJustifiedRoot[:]}}
+	newJustifiedRoot, _ := ssz.HashTreeRoot(newJustifiedBlk.Block)
+	if err := service.beaconDB.SaveBlock(ctx, newJustifiedBlk); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveBlock(ctx, lastJustifiedBlk); err != nil {
+		t.Fatal(err)
+	}
+
+	diff := (params.BeaconConfig().SlotsPerEpoch - 1) * params.BeaconConfig().SecondsPerSlot
+	service.genesisTime = time.Unix(time.Now().Unix()-int64(diff), 0)
+	service.justifiedCheckpt = &ethpb.Checkpoint{Root: lastJustifiedRoot[:]}
+	update, err = service.shouldUpdateCurrentJustified(ctx, &ethpb.Checkpoint{Root: newJustifiedRoot[:]})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !update {
+		t.Error("Should be able to update justified, received false")
+	}
+}
+
+func TestShouldUpdateJustified_ReturnFalse(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+	params.UseMinimalConfig()
+	defer params.UseMainnetConfig()
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	lastJustifiedBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{ParentRoot: []byte{'G'}}}
+	lastJustifiedRoot, _ := ssz.HashTreeRoot(lastJustifiedBlk.Block)
+	newJustifiedBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{ParentRoot: lastJustifiedRoot[:]}}
+	newJustifiedRoot, _ := ssz.HashTreeRoot(newJustifiedBlk.Block)
+	if err := service.beaconDB.SaveBlock(ctx, newJustifiedBlk); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveBlock(ctx, lastJustifiedBlk); err != nil {
+		t.Fatal(err)
+	}
+
+	diff := (params.BeaconConfig().SlotsPerEpoch - 1) * params.BeaconConfig().SecondsPerSlot
+	service.genesisTime = time.Unix(time.Now().Unix()-int64(diff), 0)
+	service.justifiedCheckpt = &ethpb.Checkpoint{Root: lastJustifiedRoot[:]}
+
+	update, err := service.shouldUpdateCurrentJustified(ctx, &ethpb.Checkpoint{Root: newJustifiedRoot[:]})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if update {
+		t.Error("Should not be able to update justified, received true")
+	}
+}
+
+func TestCachedPreState_CanGetFromCache(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s, _ := stateTrie.InitializeFromProto(&pb.BeaconState{Slot: 1})
+	r := [32]byte{'A'}
+	b := &ethpb.BeaconBlock{Slot: 1, ParentRoot: r[:]}
+	service.initSyncState[r] = s
+
+	received, err := service.verifyBlkPreState(ctx, b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(s.InnerStateUnsafe(), received.InnerStateUnsafe()) {
+		t.Error("cached state not the same")
+	}
+}
+
+func TestCachedPreState_CanGetFromDB(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	r := [32]byte{'A'}
+	b := &ethpb.BeaconBlock{Slot: 1, ParentRoot: r[:]}
+
+	service.finalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	_, err = service.verifyBlkPreState(ctx, b)
+	wanted := "pre state of slot 1 does not exist"
+	if err.Error() != wanted {
+		t.Error("Did not get wanted error")
+	}
+
+	s, _ := stateTrie.InitializeFromProto(&pb.BeaconState{Slot: 1})
+	service.beaconDB.SaveState(ctx, s, r)
+
+	received, err := service.verifyBlkPreState(ctx, b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(s, received) {
+		t.Error("cached state not the same")
+	}
+}
+
+func TestSaveInitState_CanSaveDelete(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := uint64(0); i < 64; i++ {
+		b := &ethpb.BeaconBlock{Slot: i}
+		s, _ := stateTrie.InitializeFromProto(&pb.BeaconState{Slot: i})
+		r, _ := ssz.HashTreeRoot(b)
+		service.initSyncState[r] = s
+	}
+
+	// Set finalized root as slot 32
+	finalizedRoot, _ := ssz.HashTreeRoot(&ethpb.BeaconBlock{Slot: 32})
+
+	s, _ := stateTrie.InitializeFromProto(&pb.BeaconState{FinalizedCheckpoint: &ethpb.Checkpoint{
+		Epoch: 1, Root: finalizedRoot[:]}})
+	if err := service.saveInitState(ctx, s); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify finalized state is saved in DB
+	finalizedState, err := service.beaconDB.State(ctx, finalizedRoot)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if finalizedState == nil {
+		t.Error("finalized state can't be nil")
+	}
+}
+
+func TestUpdateJustified_CouldUpdateBest(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	signedBlock := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{}}
+	if err := db.SaveBlock(ctx, signedBlock); err != nil {
+		t.Fatal(err)
+	}
+	r, err := ssz.HashTreeRoot(signedBlock.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+	service.justifiedCheckpt = &ethpb.Checkpoint{Root: []byte{'A'}}
+	service.bestJustifiedCheckpt = &ethpb.Checkpoint{Root: []byte{'A'}}
+	st, err := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	service.initSyncState[r] = st.Copy()
+	if err := db.SaveState(ctx, st.Copy(), r); err != nil {
+		t.Fatal(err)
+	}
+
+	// Could update
+	s, _ := stateTrie.InitializeFromProto(&pb.BeaconState{CurrentJustifiedCheckpoint: &ethpb.Checkpoint{Epoch: 1, Root: r[:]}})
+	if err := service.updateJustified(context.Background(), s); err != nil {
+		t.Fatal(err)
+	}
+
+	if service.bestJustifiedCheckpt.Epoch != s.CurrentJustifiedCheckpoint().Epoch {
+		t.Error("Incorrect justified epoch in service")
+	}
+
+	// Could not update
+	service.bestJustifiedCheckpt.Epoch = 2
+	if err := service.updateJustified(context.Background(), s); err != nil {
+		t.Fatal(err)
+	}
+
+	if service.bestJustifiedCheckpt.Epoch != 2 {
+		t.Error("Incorrect justified epoch in service")
+	}
+}
+
+func TestFilterBlockRoots_CanFilter(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fBlock := &ethpb.BeaconBlock{}
+	fRoot, _ := ssz.HashTreeRoot(fBlock)
+	hBlock := &ethpb.BeaconBlock{Slot: 1}
+	headRoot, _ := ssz.HashTreeRoot(hBlock)
+	st, _ := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
+	if err := service.beaconDB.SaveBlock(ctx, &ethpb.SignedBeaconBlock{Block: fBlock}); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveState(ctx, st.Copy(), fRoot); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveFinalizedCheckpoint(ctx, &ethpb.Checkpoint{Root: fRoot[:]}); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveBlock(ctx, &ethpb.SignedBeaconBlock{Block: hBlock}); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveState(ctx, st.Copy(), headRoot); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveHeadBlockRoot(ctx, headRoot); err != nil {
+		t.Fatal(err)
+	}
+
+	roots := [][32]byte{{'C'}, {'D'}, headRoot, {'E'}, fRoot, {'F'}}
+	wanted := [][32]byte{{'C'}, {'D'}, {'E'}, {'F'}}
+
+	received, err := service.filterBlockRoots(ctx, roots)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(wanted, received) {
+		t.Error("Did not filter correctly")
+	}
+}
+
+func TestPersistCache_CanSave(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	st, _ := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
+
+	for i := uint64(0); i < initialSyncCacheSize; i++ {
+		st.SetSlot(i)
+		root := [32]byte{}
+		copy(root[:], bytesutil.Bytes32(i))
+		service.initSyncState[root] = st.Copy()
+		service.boundaryRoots = append(service.boundaryRoots, root)
+	}
+
+	if err = service.persistCachedStates(ctx, initialSyncCacheSize); err != nil {
+		t.Fatal(err)
+	}
+
+	for i := uint64(0); i < initialSyncCacheSize-minimumCacheSize; i++ {
+		root := [32]byte{}
+		copy(root[:], bytesutil.Bytes32(i))
+		state, err := db.State(context.Background(), root)
+		if err != nil {
+			t.Errorf("State with root of %#x , could not be retrieved: %v", root, err)
+		}
+		if state == nil {
+			t.Errorf("State with root of %#x , does not exist", root)
+		}
+		if state.Slot() != i {
+			t.Errorf("Incorrect slot retrieved. Wanted %d but got %d", i, state.Slot())
+		}
+	}
+}
+
+func TestFillForkChoiceMissingBlocks_CanSave(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	service.forkChoiceStore = protoarray.New(0, 0, [32]byte{'A'})
+	service.finalizedCheckpt = &ethpb.Checkpoint{}
+
+	genesisStateRoot := [32]byte{}
+	genesis := blocks.NewGenesisBlock(genesisStateRoot[:])
+	if err := db.SaveBlock(ctx, genesis); err != nil {
+		t.Error(err)
+	}
+	validGenesisRoot, err := ssz.HashTreeRoot(genesis.Block)
+	if err != nil {
+		t.Error(err)
+	}
+	st, _ := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
+	if err := service.beaconDB.SaveState(ctx, st.Copy(), validGenesisRoot); err != nil {
+		t.Fatal(err)
+	}
+	roots, err := blockTree1(db, validGenesisRoot[:])
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	beaconState, _ := testutil.DeterministicGenesisState(t, 32)
+	block := &ethpb.BeaconBlock{Slot: 9, ParentRoot: roots[8]}
+	if err := service.fillInForkChoiceMissingBlocks(context.Background(), block, beaconState); err != nil {
+		t.Fatal(err)
+	}
+
+	// 5 nodes from the block tree 1. B0 - B3 - B4 - B6 - B8
+	if len(service.forkChoiceStore.Nodes()) != 5 {
+		t.Error("Miss match nodes")
+	}
+
+	if !service.forkChoiceStore.HasNode(bytesutil.ToBytes32(roots[4])) {
+		t.Error("Didn't save node")
+	}
+	if !service.forkChoiceStore.HasNode(bytesutil.ToBytes32(roots[6])) {
+		t.Error("Didn't save node")
+	}
+	if !service.forkChoiceStore.HasNode(bytesutil.ToBytes32(roots[8])) {
+		t.Error("Didn't save node")
+	}
+}
+
+func TestFillForkChoiceMissingBlocks_FilterFinalized(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	service.forkChoiceStore = protoarray.New(0, 0, [32]byte{'A'})
+	// Set finalized epoch to 1.
+	service.finalizedCheckpt = &ethpb.Checkpoint{Epoch: 1}
+
+	genesisStateRoot := [32]byte{}
+	genesis := blocks.NewGenesisBlock(genesisStateRoot[:])
+	if err := db.SaveBlock(ctx, genesis); err != nil {
+		t.Error(err)
+	}
+	validGenesisRoot, err := ssz.HashTreeRoot(genesis.Block)
+	if err != nil {
+		t.Error(err)
+	}
+	st, _ := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
+	if err := service.beaconDB.SaveState(ctx, st.Copy(), validGenesisRoot); err != nil {
+		t.Fatal(err)
+	}
+
+	// Define a tree branch, slot 63 <- 64 <- 65
+	b63 := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 63}}
+	if err := service.beaconDB.SaveBlock(ctx, b63); err != nil {
+		t.Fatal(err)
+	}
+	r63, _ := ssz.HashTreeRoot(b63.Block)
+	b64 := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 64, ParentRoot: r63[:]}}
+	if err := service.beaconDB.SaveBlock(ctx, b64); err != nil {
+		t.Fatal(err)
+	}
+	r64, _ := ssz.HashTreeRoot(b64.Block)
+	b65 := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 65, ParentRoot: r64[:]}}
+	if err := service.beaconDB.SaveBlock(ctx, b65); err != nil {
+		t.Fatal(err)
+	}
+
+	beaconState, _ := testutil.DeterministicGenesisState(t, 32)
+	if err := service.fillInForkChoiceMissingBlocks(context.Background(), b65.Block, beaconState); err != nil {
+		t.Fatal(err)
+	}
+
+	// There should be 2 nodes, block 65 and block 64.
+	if len(service.forkChoiceStore.Nodes()) != 2 {
+		t.Error("Miss match nodes")
+	}
+
+	// Block with slot 63 should be in fork choice because it's less than finalized epoch 1.
+	if !service.forkChoiceStore.HasNode(r63) {
+		t.Error("Didn't save node")
+	}
+}
+
+// blockTree1 constructs the following tree:
+//    /- B1
+// B0           /- B5 - B7
+//    \- B3 - B4 - B6 - B8
+// (B1, and B3 are all from the same slots)
+func blockTree1(db db.Database, genesisRoot []byte) ([][]byte, error) {
+	b0 := &ethpb.BeaconBlock{Slot: 0, ParentRoot: genesisRoot}
+	r0, _ := ssz.HashTreeRoot(b0)
+	b1 := &ethpb.BeaconBlock{Slot: 1, ParentRoot: r0[:]}
+	r1, _ := ssz.HashTreeRoot(b1)
+	b3 := &ethpb.BeaconBlock{Slot: 3, ParentRoot: r0[:]}
+	r3, _ := ssz.HashTreeRoot(b3)
+	b4 := &ethpb.BeaconBlock{Slot: 4, ParentRoot: r3[:]}
+	r4, _ := ssz.HashTreeRoot(b4)
+	b5 := &ethpb.BeaconBlock{Slot: 5, ParentRoot: r4[:]}
+	r5, _ := ssz.HashTreeRoot(b5)
+	b6 := &ethpb.BeaconBlock{Slot: 6, ParentRoot: r4[:]}
+	r6, _ := ssz.HashTreeRoot(b6)
+	b7 := &ethpb.BeaconBlock{Slot: 7, ParentRoot: r5[:]}
+	r7, _ := ssz.HashTreeRoot(b7)
+	b8 := &ethpb.BeaconBlock{Slot: 8, ParentRoot: r6[:]}
+	r8, _ := ssz.HashTreeRoot(b8)
+	st, err := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
+	if err != nil {
+		return nil, err
+	}
+	for _, b := range []*ethpb.BeaconBlock{b0, b1, b3, b4, b5, b6, b7, b8} {
+		if err := db.SaveBlock(context.Background(), &ethpb.SignedBeaconBlock{Block: b}); err != nil {
+			return nil, err
+		}
+		if err := db.SaveState(context.Background(), st.Copy(), bytesutil.ToBytes32(b.ParentRoot)); err != nil {
+			return nil, err
+		}
+	}
+	if err := db.SaveState(context.Background(), st.Copy(), r1); err != nil {
+		return nil, err
+	}
+	if err := db.SaveState(context.Background(), st.Copy(), r7); err != nil {
+		return nil, err
+	}
+	if err := db.SaveState(context.Background(), st.Copy(), r8); err != nil {
+		return nil, err
+	}
+	return [][]byte{r0[:], r1[:], nil, r3[:], r4[:], r5[:], r6[:], r7[:], r8[:]}, nil
+}

beacon-chain/blockchain/receive_attestation.go

@@ -1,14 +1,17 @@
 package blockchain
 
 import (
-	"bytes"
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/slotutil"
 	"github.com/sirupsen/logrus"
@@ -18,6 +21,7 @@ import (
 // AttestationReceiver interface defines the methods of chain service receive and processing new attestations.
 type AttestationReceiver interface {
 	ReceiveAttestationNoPubsub(ctx context.Context, att *ethpb.Attestation) error
+	IsValidAttestation(ctx context.Context, att *ethpb.Attestation) bool
 }
 
 // ReceiveAttestationNoPubsub is a function that defines the operations that are preformed on
@@ -29,39 +33,51 @@ func (s *Service) ReceiveAttestationNoPubsub(ctx context.Context, att *ethpb.Att
 	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveAttestationNoPubsub")
 	defer span.End()
 
-	// Update forkchoice store for the new attestation
-	if err := s.forkChoiceStore.OnAttestation(ctx, att); err != nil {
-		return errors.Wrap(err, "could not process attestation from fork choice service")
-	}
-
-	// Run fork choice for head block after updating fork choice store.
-	headRoot, err := s.forkChoiceStore.Head(ctx)
+	_, err := s.onAttestation(ctx, att)
 	if err != nil {
-		return errors.Wrap(err, "could not get head from fork choice service")
+		return errors.Wrap(err, "could not process attestation")
 	}
-	// Only save head if it's different than the current head.
-	if !bytes.Equal(headRoot, s.HeadRoot()) {
-		signed, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(headRoot))
+
+	if !featureconfig.Get().DisableUpdateHeadPerAttestation {
+		baseState, err := s.getAttPreState(ctx, att.Data.Target)
 		if err != nil {
-			return errors.Wrap(err, "could not compute state from block head")
+			return err
 		}
-		if signed == nil || signed.Block == nil {
-			return errors.New("nil head block")
-		}
-		if err := s.saveHead(ctx, signed, bytesutil.ToBytes32(headRoot)); err != nil {
-			return errors.Wrap(err, "could not save head")
+
+		// This updates fork choice head, if a new head could not be updated due to
+		// long range or intermediate forking. It simply logs a warning and returns nil
+		// as that's more appropriate than returning errors.
+		if err := s.updateHead(ctx, baseState.Balances()); err != nil {
+			log.Warnf("Resolving fork due to new attestation: %v", err)
+			return nil
 		}
 	}
 
-	processedAttNoPubsub.Inc()
 	return nil
 }
 
+// IsValidAttestation returns true if the attestation can be verified against its pre-state.
+func (s *Service) IsValidAttestation(ctx context.Context, att *ethpb.Attestation) bool {
+	baseState, err := s.getAttPreState(ctx, att.Data.Target)
+	if err != nil {
+		log.WithError(err).Error("Failed to validate attestation")
+		return false
+	}
+
+	if err := blocks.VerifyAttestation(ctx, baseState, att); err != nil {
+		log.WithError(err).Error("Failed to validate attestation")
+		return false
+	}
+
+	return true
+}
+
 // This processes attestations from the attestation pool to account for validator votes and fork choice.
-func (s *Service) processAttestation() {
+func (s *Service) processAttestation(subscribedToStateEvents chan struct{}) {
 	// Wait for state to be initialized.
 	stateChannel := make(chan *feed.Event, 1)
 	stateSub := s.stateNotifier.StateFeed().Subscribe(stateChannel)
+	subscribedToStateEvents <- struct{}{}
 	<-stateChannel
 	stateSub.Unsubscribe()
 
@@ -74,16 +90,50 @@ func (s *Service) processAttestation() {
 			ctx := context.Background()
 			atts := s.attPool.ForkchoiceAttestations()
 			for _, a := range atts {
+				hasState := s.beaconDB.HasState(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot)) && s.beaconDB.HasState(ctx, bytesutil.ToBytes32(a.Data.Target.Root))
+				hasBlock := s.hasBlock(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
+				if !(hasState && hasBlock) {
+					continue
+				}
+
 				if err := s.attPool.DeleteForkchoiceAttestation(a); err != nil {
 					log.WithError(err).Error("Could not delete fork choice attestation in pool")
 				}
 
+				if !s.verifyCheckpointEpoch(a.Data.Target) {
+					continue
+				}
+
 				if err := s.ReceiveAttestationNoPubsub(ctx, a); err != nil {
 					log.WithFields(logrus.Fields{
-						"targetRoot": fmt.Sprintf("%#x", a.Data.Target.Root),
-					}).WithError(err).Error("Could not receive attestation in chain service")
+						"slot":             a.Data.Slot,
+						"committeeIndex":   a.Data.CommitteeIndex,
+						"beaconBlockRoot":  fmt.Sprintf("%#x", bytesutil.Trunc(a.Data.BeaconBlockRoot)),
+						"targetRoot":       fmt.Sprintf("%#x", bytesutil.Trunc(a.Data.Target.Root)),
+						"aggregationCount": a.AggregationBits.Count(),
+					}).WithError(err).Warn("Could not receive attestation in chain service")
 				}
 			}
 		}
 	}
 }
+
+// This verifies the epoch of input checkpoint is within current epoch and previous epoch
+// with respect to current time. Returns true if it's within, false if it's not.
+func (s *Service) verifyCheckpointEpoch(c *ethpb.Checkpoint) bool {
+	now := uint64(time.Now().Unix())
+	genesisTime := uint64(s.genesisTime.Unix())
+	currentSlot := (now - genesisTime) / params.BeaconConfig().SecondsPerSlot
+	currentEpoch := helpers.SlotToEpoch(currentSlot)
+
+	var prevEpoch uint64
+	if currentEpoch > 1 {
+		prevEpoch = currentEpoch - 1
+	}
+
+	if c.Epoch != prevEpoch && c.Epoch != currentEpoch {
+		return false
+	}
+
+	return true
+}

beacon-chain/blockchain/receive_attestation_test.go

@@ -2,45 +2,24 @@ package blockchain
 
 import (
 	"testing"
+	"time"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/testutil"
-	logTest "github.com/sirupsen/logrus/hooks/test"
-	"golang.org/x/net/context"
 )
 
-func TestReceiveAttestationNoPubsub_ProcessCorrectly(t *testing.T) {
-	hook := logTest.NewGlobal()
+func TestVerifyCheckpointEpoch_Ok(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
-	ctx := context.Background()
 
 	chainService := setupBeaconChain(t, db)
-	r, _ := ssz.HashTreeRoot(&ethpb.BeaconBlock{})
-	chainService.forkChoiceStore = &store{headRoot: r[:]}
+	chainService.genesisTime = time.Now()
 
-	b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{}}
-	if err := chainService.beaconDB.SaveBlock(ctx, b); err != nil {
-		t.Fatal(err)
-	}
-	root, err := ssz.HashTreeRoot(b.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := chainService.beaconDB.SaveState(ctx, &pb.BeaconState{}, root); err != nil {
-		t.Fatal(err)
+	if !chainService.verifyCheckpointEpoch(&ethpb.Checkpoint{}) {
+		t.Error("Wanted true, got false")
 	}
 
-	a := &ethpb.Attestation{Data: &ethpb.AttestationData{
-		Target: &ethpb.Checkpoint{Root: root[:]},
-	}}
-	if err := chainService.ReceiveAttestationNoPubsub(ctx, a); err != nil {
-		t.Fatal(err)
+	if chainService.verifyCheckpointEpoch(&ethpb.Checkpoint{Epoch: 1}) {
+		t.Error("Wanted false, got true")
 	}
-
-	testutil.AssertLogsContain(t, hook, "Saved new head info")
-	testutil.AssertLogsDoNotContain(t, hook, "Broadcasting attestation")
 }

beacon-chain/blockchain/receive_block.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"encoding/hex"
 
-	"github.com/gogo/protobuf/proto"
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/go-ssz"
@@ -13,7 +12,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/traceutil"
 	"github.com/sirupsen/logrus"
@@ -55,7 +54,6 @@ func (s *Service) ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlo
 		return err
 	}
 
-	processedBlk.Inc()
 	return nil
 }
 
@@ -67,35 +65,40 @@ func (s *Service) ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlo
 func (s *Service) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
 	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlockNoPubsub")
 	defer span.End()
-	blockCopy := proto.Clone(block).(*ethpb.SignedBeaconBlock)
+	blockCopy := stateTrie.CopySignedBeaconBlock(block)
 
 	// Apply state transition on the new block.
-	if err := s.forkChoiceStore.OnBlock(ctx, blockCopy); err != nil {
-		err := errors.Wrap(err, "could not process block from fork choice service")
+	postState, err := s.onBlock(ctx, blockCopy)
+	if err != nil {
+		err := errors.Wrap(err, "could not process block")
 		traceutil.AnnotateError(span, err)
 		return err
 	}
+
+	// Add attestations from the block to the pool for fork choice.
+	if err := s.attPool.SaveBlockAttestations(blockCopy.Block.Body.Attestations); err != nil {
+		log.Errorf("Could not save attestation for fork choice: %v", err)
+		return nil
+	}
+	for _, exit := range block.Block.Body.VoluntaryExits {
+		s.exitPool.MarkIncluded(exit)
+	}
+
+	s.epochParticipationLock.Lock()
+	defer s.epochParticipationLock.Unlock()
+	s.epochParticipation[helpers.SlotToEpoch(blockCopy.Block.Slot)] = precompute.Balances
+
 	root, err := ssz.HashTreeRoot(blockCopy.Block)
 	if err != nil {
 		return errors.Wrap(err, "could not get signing root on received block")
 	}
 
-	// Run fork choice after applying state transition on the new block.
-	headRoot, err := s.forkChoiceStore.Head(ctx)
-	if err != nil {
-		return errors.Wrap(err, "could not get head from fork choice service")
-	}
-	signedHeadBlock, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(headRoot))
-	if err != nil {
-		return errors.Wrap(err, "could not compute state from block head")
-	}
-	if signedHeadBlock == nil || signedHeadBlock.Block == nil {
-		return errors.New("nil head block")
-	}
-
-	// Only save head if it's different than the current head.
-	if !bytes.Equal(headRoot, s.HeadRoot()) {
-		if err := s.saveHead(ctx, signedHeadBlock, bytesutil.ToBytes32(headRoot)); err != nil {
+	if featureconfig.Get().DisableForkChoice && block.Block.Slot > s.headSlot() {
+		if err := s.saveHead(ctx, root); err != nil {
+			return errors.Wrap(err, "could not save head")
+		}
+	} else {
+		if err := s.updateHead(ctx, postState.Balances()); err != nil {
 			return errors.Wrap(err, "could not save head")
 		}
 	}
@@ -104,31 +107,17 @@ func (s *Service) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedB
 	s.stateNotifier.StateFeed().Send(&feed.Event{
 		Type: statefeed.BlockProcessed,
 		Data: &statefeed.BlockProcessedData{
+			Slot:      blockCopy.Block.Slot,
 			BlockRoot: root,
 			Verified:  true,
 		},
 	})
 
-	// Add attestations from the block to the pool for fork choice.
-	if err := s.attPool.SaveBlockAttestations(blockCopy.Block.Body.Attestations); err != nil {
-		log.Errorf("Could not save attestation for fork choice: %v", err)
-		return nil
-	}
-
 	// Reports on block and fork choice metrics.
-	s.reportSlotMetrics(blockCopy.Block.Slot)
-
-	// Log if block is a competing block.
-	isCompetingBlock(root[:], blockCopy.Block.Slot, headRoot, signedHeadBlock.Block.Slot)
+	reportSlotMetrics(blockCopy.Block.Slot, s.headSlot(), s.finalizedCheckpt)
 
 	// Log state transition data.
-	logStateTransitionData(blockCopy.Block, root[:])
-
-	s.epochParticipationLock.Lock()
-	defer s.epochParticipationLock.Unlock()
-	s.epochParticipation[helpers.SlotToEpoch(blockCopy.Block.Slot)] = precompute.Balances
-
-	processedBlkNoPubsub.Inc()
+	logStateTransitionData(blockCopy.Block)
 
 	return nil
 }
@@ -140,21 +129,26 @@ func (s *Service) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedB
 func (s *Service) ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
 	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlockNoForkchoice")
 	defer span.End()
-	blockCopy := proto.Clone(block).(*ethpb.SignedBeaconBlock)
+	blockCopy := stateTrie.CopySignedBeaconBlock(block)
 
-	// Apply state transition on the incoming newly received block.
-	if err := s.forkChoiceStore.OnBlock(ctx, blockCopy); err != nil {
-		err := errors.Wrap(err, "could not process block from fork choice service")
+	// Apply state transition on the new block.
+	_, err := s.onBlock(ctx, blockCopy)
+	if err != nil {
+		err := errors.Wrap(err, "could not process block")
 		traceutil.AnnotateError(span, err)
 		return err
 	}
+
 	root, err := ssz.HashTreeRoot(blockCopy.Block)
 	if err != nil {
 		return errors.Wrap(err, "could not get signing root on received block")
 	}
-
-	if !bytes.Equal(root[:], s.HeadRoot()) {
-		if err := s.saveHead(ctx, blockCopy, root); err != nil {
+	cachedHeadRoot, err := s.HeadRoot(ctx)
+	if err != nil {
+		return errors.Wrap(err, "could not get head root from cache")
+	}
+	if !bytes.Equal(root[:], cachedHeadRoot) {
+		if err := s.saveHead(ctx, root); err != nil {
 			return errors.Wrap(err, "could not save head")
 		}
 	}
@@ -163,22 +157,22 @@ func (s *Service) ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *eth
 	s.stateNotifier.StateFeed().Send(&feed.Event{
 		Type: statefeed.BlockProcessed,
 		Data: &statefeed.BlockProcessedData{
+			Slot:      blockCopy.Block.Slot,
 			BlockRoot: root,
 			Verified:  true,
 		},
 	})
 
 	// Reports on block and fork choice metrics.
-	s.reportSlotMetrics(blockCopy.Block.Slot)
+	reportSlotMetrics(blockCopy.Block.Slot, s.headSlot(), s.finalizedCheckpt)
 
 	// Log state transition data.
-	logStateTransitionData(blockCopy.Block, root[:])
+	logStateTransitionData(blockCopy.Block)
 
 	s.epochParticipationLock.Lock()
 	defer s.epochParticipationLock.Unlock()
 	s.epochParticipation[helpers.SlotToEpoch(blockCopy.Block.Slot)] = precompute.Balances
 
-	processedBlkNoPubsubForkchoice.Inc()
 	return nil
 }
 
@@ -188,32 +182,30 @@ func (s *Service) ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *eth
 func (s *Service) ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
 	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlockNoVerify")
 	defer span.End()
-	blockCopy := proto.Clone(block).(*ethpb.SignedBeaconBlock)
+	blockCopy := stateTrie.CopySignedBeaconBlock(block)
 
 	// Apply state transition on the incoming newly received blockCopy without verifying its BLS contents.
-	if err := s.forkChoiceStore.OnBlockInitialSyncStateTransition(ctx, blockCopy); err != nil {
-		return errors.Wrap(err, "could not process blockCopy from fork choice service")
+	if err := s.onBlockInitialSyncStateTransition(ctx, blockCopy); err != nil {
+		err := errors.Wrap(err, "could not process block")
+		traceutil.AnnotateError(span, err)
+		return err
 	}
+
 	root, err := ssz.HashTreeRoot(blockCopy.Block)
 	if err != nil {
 		return errors.Wrap(err, "could not get signing root on received blockCopy")
 	}
 
-	if featureconfig.Get().InitSyncCacheState {
-		if !bytes.Equal(root[:], s.HeadRoot()) {
-			if err := s.saveHeadNoDB(ctx, blockCopy, root); err != nil {
-				err := errors.Wrap(err, "could not save head")
-				traceutil.AnnotateError(span, err)
-				return err
-			}
-		}
-	} else {
-		if !bytes.Equal(root[:], s.HeadRoot()) {
-			if err := s.saveHead(ctx, blockCopy, root); err != nil {
-				err := errors.Wrap(err, "could not save head")
-				traceutil.AnnotateError(span, err)
-				return err
-			}
+	cachedHeadRoot, err := s.HeadRoot(ctx)
+	if err != nil {
+		return errors.Wrap(err, "could not get head root from cache")
+	}
+
+	if !bytes.Equal(root[:], cachedHeadRoot) {
+		if err := s.saveHeadNoDB(ctx, blockCopy, root); err != nil {
+			err := errors.Wrap(err, "could not save head")
+			traceutil.AnnotateError(span, err)
+			return err
 		}
 	}
 
@@ -221,13 +213,14 @@ func (s *Service) ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedB
 	s.stateNotifier.StateFeed().Send(&feed.Event{
 		Type: statefeed.BlockProcessed,
 		Data: &statefeed.BlockProcessedData{
+			Slot:      blockCopy.Block.Slot,
 			BlockRoot: root,
 			Verified:  false,
 		},
 	})
 
 	// Reports on blockCopy and fork choice metrics.
-	s.reportSlotMetrics(blockCopy.Block.Slot)
+	reportSlotMetrics(blockCopy.Block.Slot, s.headSlot(), s.finalizedCheckpt)
 
 	// Log state transition data.
 	log.WithFields(logrus.Fields{
@@ -242,16 +235,3 @@ func (s *Service) ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedB
 
 	return nil
 }
-
-// This checks if the block is from a competing chain, emits warning and updates metrics.
-func isCompetingBlock(root []byte, slot uint64, headRoot []byte, headSlot uint64) {
-	if !bytes.Equal(root[:], headRoot) {
-		log.WithFields(logrus.Fields{
-			"blkSlot":  slot,
-			"blkRoot":  hex.EncodeToString(root[:]),
-			"headSlot": headSlot,
-			"headRoot": hex.EncodeToString(headRoot),
-		}).Warn("Calculated head diffs from new block")
-		competingBlks.Inc()
-	}
-}

beacon-chain/blockchain/receive_block_test.go

@@ -1,195 +0,0 @@
-package blockchain
-
-import (
-	"bytes"
-	"context"
-	"reflect"
-	"testing"
-
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	b "github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
-	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/stateutil"
-	"github.com/prysmaticlabs/prysm/shared/testutil"
-	logTest "github.com/sirupsen/logrus/hooks/test"
-)
-
-func TestReceiveBlock_ProcessCorrectly(t *testing.T) {
-	hook := logTest.NewGlobal()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	ctx := context.Background()
-
-	chainService := setupBeaconChain(t, db)
-
-	beaconState, privKeys := testutil.DeterministicGenesisState(t, 100)
-	genesis, _ := testutil.GenerateFullBlock(beaconState, privKeys, nil, beaconState.Slot+1)
-	beaconState, err := state.ExecuteStateTransition(ctx, beaconState, genesis)
-	if err != nil {
-		t.Fatal(err)
-	}
-	genesisBlkRoot, err := ssz.HashTreeRoot(genesis.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveState(ctx, beaconState, genesisBlkRoot); err != nil {
-		t.Fatal(err)
-	}
-	cp := &ethpb.Checkpoint{Root: genesisBlkRoot[:]}
-	if err := chainService.forkChoiceStore.GenesisStore(ctx, cp, cp); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := chainService.beaconDB.SaveBlock(ctx, genesis); err != nil {
-		t.Fatalf("Could not save block to db: %v", err)
-	}
-
-	if err := db.SaveState(ctx, beaconState, genesisBlkRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	slot := beaconState.Slot + 1
-	block, err := testutil.GenerateFullBlock(beaconState, privKeys, nil, slot)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := chainService.beaconDB.SaveBlock(ctx, block); err != nil {
-		t.Fatal(err)
-	}
-	if err := chainService.ReceiveBlock(context.Background(), block); err != nil {
-		t.Errorf("Block failed processing: %v", err)
-	}
-	testutil.AssertLogsContain(t, hook, "Finished applying state transition")
-}
-
-func TestReceiveReceiveBlockNoPubsub_CanSaveHeadInfo(t *testing.T) {
-	hook := logTest.NewGlobal()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	ctx := context.Background()
-
-	chainService := setupBeaconChain(t, db)
-
-	headBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 100}}
-	if err := db.SaveBlock(ctx, headBlk); err != nil {
-		t.Fatal(err)
-	}
-	r, err := ssz.HashTreeRoot(headBlk.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	head := &pb.BeaconState{Slot: 100, FinalizedCheckpoint: &ethpb.Checkpoint{Root: r[:]}}
-	if err := db.SaveState(ctx, head, r); err != nil {
-		t.Fatal(err)
-	}
-	chainService.forkChoiceStore = &store{headRoot: r[:]}
-
-	if err := chainService.ReceiveBlockNoPubsub(ctx, &ethpb.SignedBeaconBlock{
-		Block: &ethpb.BeaconBlock{
-			Slot: 1,
-			Body: &ethpb.BeaconBlockBody{},
-		},
-	}); err != nil {
-		t.Fatal(err)
-	}
-
-	if !bytes.Equal(r[:], chainService.HeadRoot()) {
-		t.Error("Incorrect head root saved")
-	}
-
-	if !reflect.DeepEqual(headBlk, chainService.HeadBlock()) {
-		t.Error("Incorrect head block saved")
-	}
-
-	testutil.AssertLogsContain(t, hook, "Saved new head info")
-}
-
-func TestReceiveReceiveBlockNoPubsub_SameHead(t *testing.T) {
-	hook := logTest.NewGlobal()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	ctx := context.Background()
-
-	chainService := setupBeaconChain(t, db)
-
-	headBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{}}
-	if err := db.SaveBlock(ctx, headBlk); err != nil {
-		t.Fatal(err)
-	}
-	newBlk := &ethpb.SignedBeaconBlock{
-		Block: &ethpb.BeaconBlock{
-			Slot: 1,
-			Body: &ethpb.BeaconBlockBody{},
-		},
-	}
-	newRoot, _ := ssz.HashTreeRoot(newBlk.Block)
-	if err := db.SaveBlock(ctx, newBlk); err != nil {
-		t.Fatal(err)
-	}
-
-	chainService.forkChoiceStore = &store{headRoot: newRoot[:]}
-	chainService.canonicalRoots[0] = newRoot[:]
-
-	if err := chainService.ReceiveBlockNoPubsub(ctx, newBlk); err != nil {
-		t.Fatal(err)
-	}
-
-	testutil.AssertLogsDoNotContain(t, hook, "Saved new head info")
-}
-
-func TestReceiveBlockNoPubsubForkchoice_ProcessCorrectly(t *testing.T) {
-	hook := logTest.NewGlobal()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	ctx := context.Background()
-
-	chainService := setupBeaconChain(t, db)
-	beaconState, privKeys := testutil.DeterministicGenesisState(t, 100)
-
-	block, err := testutil.GenerateFullBlock(beaconState, privKeys, nil, beaconState.Slot)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	stateRoot, err := stateutil.HashTreeRootState(beaconState)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	genesis := b.NewGenesisBlock(stateRoot[:])
-	parentRoot, err := ssz.HashTreeRoot(genesis.Block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveState(ctx, beaconState, parentRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := chainService.forkChoiceStore.GenesisStore(ctx, &ethpb.Checkpoint{Root: parentRoot[:]}, &ethpb.Checkpoint{Root: parentRoot[:]}); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := chainService.beaconDB.SaveBlock(ctx, block); err != nil {
-		t.Fatalf("Could not save block to db: %v", err)
-	}
-
-	block, err = testutil.GenerateFullBlock(beaconState, privKeys, nil, beaconState.Slot)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveState(ctx, beaconState, bytesutil.ToBytes32(block.Block.ParentRoot)); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := chainService.beaconDB.SaveBlock(ctx, block); err != nil {
-		t.Fatal(err)
-	}
-	if err := chainService.ReceiveBlockNoPubsubForkchoice(context.Background(), block); err != nil {
-		t.Errorf("Block failed processing: %v", err)
-	}
-	testutil.AssertLogsContain(t, hook, "Finished applying state transition")
-	testutil.AssertLogsDoNotContain(t, hook, "Finished fork choice")
-}

beacon-chain/blockchain/service.go

@@ -13,7 +13,7 @@ import (
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/blockchain/forkchoice"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
@@ -22,13 +22,20 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
+	"github.com/prysmaticlabs/prysm/beacon-chain/flags"
+	f "github.com/prysmaticlabs/prysm/beacon-chain/forkchoice"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/attestations"
+	"github.com/prysmaticlabs/prysm/beacon-chain/operations/slashings"
+	"github.com/prysmaticlabs/prysm/beacon-chain/operations/voluntaryexits"
 	"github.com/prysmaticlabs/prysm/beacon-chain/p2p"
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"github.com/sirupsen/logrus"
+	"github.com/prysmaticlabs/prysm/shared/params"
 	"go.opencensus.io/trace"
 )
 
@@ -37,42 +44,56 @@ import (
 type Service struct {
 	ctx                    context.Context
 	cancel                 context.CancelFunc
-	beaconDB               db.Database
+	beaconDB               db.HeadAccessDatabase
 	depositCache           *depositcache.DepositCache
 	chainStartFetcher      powchain.ChainStartFetcher
 	attPool                attestations.Pool
-	forkChoiceStore        forkchoice.ForkChoicer
+	slashingPool           *slashings.Pool
+	exitPool               *voluntaryexits.Pool
 	genesisTime            time.Time
 	p2p                    p2p.Broadcaster
 	maxRoutines            int64
-	headSlot               uint64
-	headBlock              *ethpb.SignedBeaconBlock
-	headState              *pb.BeaconState
-	canonicalRoots         map[uint64][]byte
+	head                   *head
 	headLock               sync.RWMutex
 	stateNotifier          statefeed.Notifier
 	genesisRoot            [32]byte
 	epochParticipation     map[uint64]*precompute.Balance
 	epochParticipationLock sync.RWMutex
+	forkChoiceStore        f.ForkChoicer
+	justifiedCheckpt       *ethpb.Checkpoint
+	prevJustifiedCheckpt   *ethpb.Checkpoint
+	bestJustifiedCheckpt   *ethpb.Checkpoint
+	finalizedCheckpt       *ethpb.Checkpoint
+	prevFinalizedCheckpt   *ethpb.Checkpoint
+	nextEpochBoundarySlot  uint64
+	voteLock               sync.RWMutex
+	initSyncState          map[[32]byte]*stateTrie.BeaconState
+	boundaryRoots          [][32]byte
+	initSyncStateLock      sync.RWMutex
+	checkpointState        *cache.CheckpointStateCache
+	checkpointStateLock    sync.Mutex
+	stateGen               *stategen.State
 }
 
 // Config options for the service.
 type Config struct {
 	BeaconBlockBuf    int
 	ChainStartFetcher powchain.ChainStartFetcher
-	BeaconDB          db.Database
+	BeaconDB          db.HeadAccessDatabase
 	DepositCache      *depositcache.DepositCache
 	AttPool           attestations.Pool
+	ExitPool          *voluntaryexits.Pool
+	SlashingPool      *slashings.Pool
 	P2p               p2p.Broadcaster
 	MaxRoutines       int64
 	StateNotifier     statefeed.Notifier
+	ForkChoiceStore   f.ForkChoicer
 }
 
 // NewService instantiates a new block service instance that will
 // be registered into a running beacon node.
 func NewService(ctx context.Context, cfg *Config) (*Service, error) {
 	ctx, cancel := context.WithCancel(ctx)
-	store := forkchoice.NewForkChoiceService(ctx, cfg.BeaconDB)
 	return &Service{
 		ctx:                ctx,
 		cancel:             cancel,
@@ -80,12 +101,17 @@ func NewService(ctx context.Context, cfg *Config) (*Service, error) {
 		depositCache:       cfg.DepositCache,
 		chainStartFetcher:  cfg.ChainStartFetcher,
 		attPool:            cfg.AttPool,
-		forkChoiceStore:    store,
+		exitPool:           cfg.ExitPool,
+		slashingPool:       cfg.SlashingPool,
 		p2p:                cfg.P2p,
-		canonicalRoots:     make(map[uint64][]byte),
 		maxRoutines:        cfg.MaxRoutines,
 		stateNotifier:      cfg.StateNotifier,
 		epochParticipation: make(map[uint64]*precompute.Balance),
+		forkChoiceStore:    cfg.ForkChoiceStore,
+		initSyncState:      make(map[[32]byte]*stateTrie.BeaconState),
+		boundaryRoots:      [][32]byte{},
+		checkpointState:    cache.NewCheckpointStateCache(),
+		stateGen:           stategen.New(cfg.BeaconDB),
 	}, nil
 }
 
@@ -100,23 +126,24 @@ func (s *Service) Start() {
 	// For running initial sync with state cache, in an event of restart, we use
 	// last finalized check point as start point to sync instead of head
 	// state. This is because we no longer save state every slot during sync.
-	if featureconfig.Get().InitSyncCacheState {
-		cp, err := s.beaconDB.FinalizedCheckpoint(ctx)
+	cp, err := s.beaconDB.FinalizedCheckpoint(ctx)
+	if err != nil {
+		log.Fatalf("Could not fetch finalized cp: %v", err)
+	}
+	if beaconState == nil {
+		beaconState, err = s.beaconDB.State(ctx, bytesutil.ToBytes32(cp.Root))
 		if err != nil {
-			log.Fatalf("Could not fetch finalized cp: %v", err)
-		}
-		if beaconState == nil {
-			beaconState, err = s.beaconDB.State(ctx, bytesutil.ToBytes32(cp.Root))
-			if err != nil {
-				log.Fatalf("Could not fetch beacon state: %v", err)
-			}
+			log.Fatalf("Could not fetch beacon state: %v", err)
 		}
 	}
 
+	// Make sure that attestation processor is subscribed and ready for state initializing event.
+	attestationProcessorSubscribed := make(chan struct{}, 1)
+
 	// If the chain has already been initialized, simply start the block processing routine.
 	if beaconState != nil {
 		log.Info("Blockchain data already exists in DB, initializing...")
-		s.genesisTime = time.Unix(int64(beaconState.GenesisTime), 0)
+		s.genesisTime = time.Unix(int64(beaconState.GenesisTime()), 0)
 		if err := s.initializeChainInfo(ctx); err != nil {
 			log.Fatalf("Could not set up chain info: %v", err)
 		}
@@ -128,9 +155,21 @@ func (s *Service) Start() {
 		if err != nil {
 			log.Fatalf("Could not get finalized checkpoint: %v", err)
 		}
-		if err := s.forkChoiceStore.GenesisStore(ctx, justifiedCheckpoint, finalizedCheckpoint); err != nil {
-			log.Fatalf("Could not start fork choice service: %v", err)
+
+		// Resume fork choice.
+		s.justifiedCheckpt = stateTrie.CopyCheckpoint(justifiedCheckpoint)
+		s.prevJustifiedCheckpt = stateTrie.CopyCheckpoint(justifiedCheckpoint)
+		s.bestJustifiedCheckpt = stateTrie.CopyCheckpoint(justifiedCheckpoint)
+		s.finalizedCheckpt = stateTrie.CopyCheckpoint(finalizedCheckpoint)
+		s.prevFinalizedCheckpt = stateTrie.CopyCheckpoint(finalizedCheckpoint)
+		s.resumeForkChoice(justifiedCheckpoint, finalizedCheckpoint)
+
+		if finalizedCheckpoint.Epoch > 1 {
+			if err := s.pruneGarbageState(ctx, helpers.StartSlot(finalizedCheckpoint.Epoch)-params.BeaconConfig().SlotsPerEpoch); err != nil {
+				log.WithError(err).Warn("Could not prune old states")
+			}
 		}
+
 		s.stateNotifier.StateFeed().Send(&feed.Event{
 			Type: statefeed.Initialized,
 			Data: &statefeed.InitializedData{
@@ -147,6 +186,7 @@ func (s *Service) Start() {
 			stateChannel := make(chan *feed.Event, 1)
 			stateSub := s.stateNotifier.StateFeed().Subscribe(stateChannel)
 			defer stateSub.Unsubscribe()
+			<-attestationProcessorSubscribed
 			for {
 				select {
 				case event := <-stateChannel:
@@ -167,7 +207,7 @@ func (s *Service) Start() {
 		}()
 	}
 
-	go s.processAttestation()
+	go s.processAttestation(attestationProcessorSubscribed)
 }
 
 // processChainStartTime initializes a series of deposits from the ChainStart deposits in the eth1
@@ -191,11 +231,10 @@ func (s *Service) processChainStartTime(ctx context.Context, genesisTime time.Ti
 func (s *Service) initializeBeaconChain(
 	ctx context.Context,
 	genesisTime time.Time,
-	preGenesisState *pb.BeaconState,
+	preGenesisState *stateTrie.BeaconState,
 	eth1data *ethpb.Eth1Data) error {
 	_, span := trace.StartSpan(context.Background(), "beacon-chain.Service.initializeBeaconChain")
 	defer span.End()
-	log.Info("Genesis time reached, starting the beacon chain")
 	s.genesisTime = genesisTime
 	unixTime := uint64(genesisTime.Unix())
 
@@ -208,11 +247,17 @@ func (s *Service) initializeBeaconChain(
 		return errors.Wrap(err, "could not save genesis data")
 	}
 
+	log.Info("Initialized beacon chain genesis state")
+
+	// Clear out all pre-genesis data now that the state is initialized.
+	s.chainStartFetcher.ClearPreGenesisData()
+
 	// Update committee shuffled indices for genesis epoch.
-	if featureconfig.Get().EnableNewCache {
-		if err := helpers.UpdateCommitteeCache(genesisState, 0 /* genesis epoch */); err != nil {
-			return err
-		}
+	if err := helpers.UpdateCommitteeCache(genesisState, 0 /* genesis epoch */); err != nil {
+		return err
+	}
+	if err := helpers.UpdateProposerIndicesInCache(genesisState, 0 /* genesis epoch */); err != nil {
+		return err
 	}
 
 	return nil
@@ -233,81 +278,29 @@ func (s *Service) Status() error {
 	return nil
 }
 
-// This gets called to update canonical root mapping.
-func (s *Service) saveHead(ctx context.Context, signed *ethpb.SignedBeaconBlock, r [32]byte) error {
-	s.headLock.Lock()
-	defer s.headLock.Unlock()
-
-	if signed == nil || signed.Block == nil {
-		return errors.New("cannot save nil head block")
-	}
-
-	s.headSlot = signed.Block.Slot
-
-	s.canonicalRoots[signed.Block.Slot] = r[:]
-
-	if err := s.beaconDB.SaveHeadBlockRoot(ctx, r); err != nil {
-		return errors.Wrap(err, "could not save head root in DB")
-	}
-	s.headBlock = signed
-
-	headState, err := s.beaconDB.State(ctx, r)
-	if err != nil {
-		return errors.Wrap(err, "could not retrieve head state in DB")
-	}
-	s.headState = headState
-
-	log.WithFields(logrus.Fields{
-		"slot":     signed.Block.Slot,
-		"headRoot": fmt.Sprintf("%#x", r),
-	}).Debug("Saved new head info")
-	return nil
+// ClearCachedStates removes all stored caches states. This is done after the node
+// is synced.
+func (s *Service) ClearCachedStates() {
+	s.initSyncState = map[[32]byte]*stateTrie.BeaconState{}
 }
 
-// This gets called to update canonical root mapping. It does not save head block
-// root in DB. With the inception of inital-sync-cache-state flag, it uses finalized
-// check point as anchors to resume sync therefore head is no longer needed to be saved on per slot basis.
-func (s *Service) saveHeadNoDB(ctx context.Context, b *ethpb.SignedBeaconBlock, r [32]byte) error {
-	s.headLock.Lock()
-	defer s.headLock.Unlock()
+// This gets called when beacon chain is first initialized to save validator indices and public keys in db.
+func (s *Service) saveGenesisValidators(ctx context.Context, state *stateTrie.BeaconState) error {
+	pubkeys := make([][48]byte, state.NumValidators())
+	indices := make([]uint64, state.NumValidators())
 
-	s.headSlot = b.Block.Slot
-
-	s.canonicalRoots[b.Block.Slot] = r[:]
-
-	s.headBlock = b
-
-	headState, err := s.beaconDB.State(ctx, r)
-	if err != nil {
-		return errors.Wrap(err, "could not retrieve head state in DB")
+	for i := 0; i < state.NumValidators(); i++ {
+		pubkeys[i] = state.PubkeyAtIndex(uint64(i))
+		indices[i] = uint64(i)
 	}
-	s.headState = headState
-
-	log.WithFields(logrus.Fields{
-		"slot":     b.Block.Slot,
-		"headRoot": fmt.Sprintf("%#x", r),
-	}).Debug("Saved new head info")
-	return nil
+	return s.beaconDB.SaveValidatorIndices(ctx, pubkeys, indices)
 }
 
-// This gets called when beacon chain is first initialized to save validator indices and pubkeys in db
-func (s *Service) saveGenesisValidators(ctx context.Context, state *pb.BeaconState) error {
-	for i, v := range state.Validators {
-		if err := s.beaconDB.SaveValidatorIndex(ctx, bytesutil.ToBytes48(v.PublicKey), uint64(i)); err != nil {
-			return errors.Wrapf(err, "could not save validator index: %d", i)
-		}
-	}
-	return nil
-}
-
-// This gets called when beacon chain is first initialized to save genesis data (state, block, and more) in db
-func (s *Service) saveGenesisData(ctx context.Context, genesisState *pb.BeaconState) error {
-	s.headLock.Lock()
-	defer s.headLock.Unlock()
-
-	stateRoot, err := ssz.HashTreeRoot(genesisState)
+// This gets called when beacon chain is first initialized to save genesis data (state, block, and more) in db.
+func (s *Service) saveGenesisData(ctx context.Context, genesisState *stateTrie.BeaconState) error {
+	stateRoot, err := genesisState.HashTreeRoot()
 	if err != nil {
-		return errors.Wrap(err, "could not tree hash genesis state")
+		return err
 	}
 	genesisBlk := blocks.NewGenesisBlock(stateRoot[:])
 	genesisBlkRoot, err := ssz.HashTreeRoot(genesisBlk.Block)
@@ -332,23 +325,30 @@ func (s *Service) saveGenesisData(ctx context.Context, genesisState *pb.BeaconSt
 	}
 
 	genesisCheckpoint := &ethpb.Checkpoint{Root: genesisBlkRoot[:]}
-	if err := s.forkChoiceStore.GenesisStore(ctx, genesisCheckpoint, genesisCheckpoint); err != nil {
-		return errors.Wrap(err, "Could not start fork choice service: %v")
+
+	// Add the genesis block to the fork choice store.
+	s.justifiedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
+	s.prevJustifiedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
+	s.bestJustifiedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
+	s.finalizedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
+	s.prevFinalizedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
+
+	if err := s.forkChoiceStore.ProcessBlock(ctx,
+		genesisBlk.Block.Slot,
+		genesisBlkRoot,
+		params.BeaconConfig().ZeroHash,
+		genesisCheckpoint.Epoch,
+		genesisCheckpoint.Epoch); err != nil {
+		log.Fatalf("Could not process genesis block for fork choice: %v", err)
 	}
 
-	s.genesisRoot = genesisBlkRoot
-	s.headBlock = genesisBlk
-	s.headState = genesisState
-	s.canonicalRoots[genesisState.Slot] = genesisBlkRoot[:]
+	s.setHead(genesisBlkRoot, genesisBlk, genesisState)
 
 	return nil
 }
 
 // This gets called to initialize chain info variables using the finalized checkpoint stored in DB
 func (s *Service) initializeChainInfo(ctx context.Context) error {
-	s.headLock.Lock()
-	defer s.headLock.Unlock()
-
 	genesisBlock, err := s.beaconDB.GenesisBlock(ctx)
 	if err != nil {
 		return errors.Wrap(err, "could not get genesis block from db")
@@ -362,6 +362,23 @@ func (s *Service) initializeChainInfo(ctx context.Context) error {
 	}
 	s.genesisRoot = genesisBlkRoot
 
+	if flags.Get().UnsafeSync {
+		headBlock, err := s.beaconDB.HeadBlock(ctx)
+		if err != nil {
+			return errors.Wrap(err, "could not retrieve head block")
+		}
+		headRoot, err := ssz.HashTreeRoot(headBlock.Block)
+		if err != nil {
+			return errors.Wrap(err, "could not hash head block")
+		}
+		headState, err := s.beaconDB.HeadState(ctx)
+		if err != nil {
+			return errors.Wrap(err, "could not retrieve head state")
+		}
+		s.setHead(headRoot, headBlock, headState)
+		return nil
+	}
+
 	finalized, err := s.beaconDB.FinalizedCheckpoint(ctx)
 	if err != nil {
 		return errors.Wrap(err, "could not get finalized checkpoint from db")
@@ -371,19 +388,58 @@ func (s *Service) initializeChainInfo(ctx context.Context) error {
 		// would be the genesis state and block.
 		return errors.New("no finalized epoch in the database")
 	}
-	s.headState, err = s.beaconDB.State(ctx, bytesutil.ToBytes32(finalized.Root))
+	finalizedState, err := s.beaconDB.State(ctx, bytesutil.ToBytes32(finalized.Root))
 	if err != nil {
 		return errors.Wrap(err, "could not get finalized state from db")
 	}
-	s.headBlock, err = s.beaconDB.Block(ctx, bytesutil.ToBytes32(finalized.Root))
+	finalizedBlock, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(finalized.Root))
 	if err != nil {
 		return errors.Wrap(err, "could not get finalized block from db")
 	}
 
-	if s.headBlock != nil && s.headBlock.Block != nil {
-		s.headSlot = s.headBlock.Block.Slot
+	if finalizedState == nil || finalizedBlock == nil {
+		return errors.New("finalized state and block can't be nil")
 	}
-	s.canonicalRoots[s.headSlot] = finalized.Root
+
+	s.setHead(bytesutil.ToBytes32(finalized.Root), finalizedBlock, finalizedState)
 
 	return nil
 }
+
+// This is called when a client starts from a non-genesis slot. It deletes the states in DB
+// from slot 1 (avoid genesis state) to `slot`.
+func (s *Service) pruneGarbageState(ctx context.Context, slot uint64) error {
+	if featureconfig.Get().DontPruneStateStartUp {
+		return nil
+	}
+
+	filter := filters.NewFilter().SetStartSlot(1).SetEndSlot(slot)
+	roots, err := s.beaconDB.BlockRoots(ctx, filter)
+	if err != nil {
+		return err
+	}
+	if err := s.beaconDB.DeleteStates(ctx, roots); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// This is called when a client starts from non-genesis slot. This passes last justified and finalized
+// information to fork choice service to initializes fork choice store.
+func (s *Service) resumeForkChoice(justifiedCheckpoint *ethpb.Checkpoint, finalizedCheckpoint *ethpb.Checkpoint) {
+	store := protoarray.New(justifiedCheckpoint.Epoch, finalizedCheckpoint.Epoch, bytesutil.ToBytes32(finalizedCheckpoint.Root))
+	s.forkChoiceStore = store
+}
+
+// This returns true if block has been processed before. Two ways to verify the block has been processed:
+// 1.) Check fork choice store.
+// 2.) Check DB.
+// Checking 1.) is ten times faster than checking 2.)
+func (s *Service) hasBlock(ctx context.Context, root [32]byte) bool {
+	if s.forkChoiceStore.HasNode(root) {
+		return true
+	}
+
+	return s.beaconDB.HasBlock(ctx, root)
+}

beacon-chain/blockchain/service_norace_test.go

@@ -5,7 +5,6 @@ import (
 	"io/ioutil"
 	"testing"
 
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/sirupsen/logrus"
 )
@@ -19,19 +18,16 @@ func TestChainService_SaveHead_DataRace(t *testing.T) {
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)
 	s := &Service{
-		beaconDB:       db,
-		canonicalRoots: make(map[uint64][]byte),
+		beaconDB: db,
 	}
 	go func() {
 		s.saveHead(
 			context.Background(),
-			&ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 777}},
 			[32]byte{},
 		)
 	}()
 	s.saveHead(
 		context.Background(),
-		&ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 888}},
 		[32]byte{},
 	)
 }

beacon-chain/blockchain/service_test.go

@@ -20,12 +20,15 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/attestations"
 	"github.com/prysmaticlabs/prysm/beacon-chain/p2p"
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
+	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	protodb "github.com/prysmaticlabs/prysm/proto/beacon/db"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/event"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
@@ -42,16 +45,20 @@ type store struct {
 	headRoot []byte
 }
 
-func (s *store) OnBlock(ctx context.Context, b *ethpb.SignedBeaconBlock) error {
-	return nil
+func (s *store) OnBlock(ctx context.Context, b *ethpb.SignedBeaconBlock) (*beaconstate.BeaconState, error) {
+	return nil, nil
 }
 
-func (s *store) OnBlockInitialSyncStateTransition(ctx context.Context, b *ethpb.SignedBeaconBlock) error {
-	return nil
+func (s *store) OnBlockCacheFilteredTree(ctx context.Context, b *ethpb.SignedBeaconBlock) (*beaconstate.BeaconState, error) {
+	return nil, nil
 }
 
-func (s *store) OnAttestation(ctx context.Context, a *ethpb.Attestation) error {
-	return nil
+func (s *store) OnBlockInitialSyncStateTransition(ctx context.Context, b *ethpb.SignedBeaconBlock) (*beaconstate.BeaconState, error) {
+	return nil, nil
+}
+
+func (s *store) OnAttestation(ctx context.Context, a *ethpb.Attestation) ([]uint64, error) {
+	return nil, nil
 }
 
 func (s *store) GenesisStore(ctx context.Context, justifiedCheckpoint *ethpb.Checkpoint, finalizedCheckpoint *ethpb.Checkpoint) error {
@@ -62,6 +69,10 @@ func (s *store) FinalizedCheckpt() *ethpb.Checkpoint {
 	return nil
 }
 
+func (s *store) JustifiedCheckpt() *ethpb.Checkpoint {
+	return nil
+}
+
 func (s *store) Head(ctx context.Context) ([]byte, error) {
 	return s.headRoot, nil
 }
@@ -94,6 +105,25 @@ func setupBeaconChain(t *testing.T, beaconDB db.Database) *Service {
 	ctx := context.Background()
 	var web3Service *powchain.Service
 	var err error
+	bState, _ := testutil.DeterministicGenesisState(t, 10)
+	err = beaconDB.SavePowchainData(ctx, &protodb.ETH1ChainData{
+		BeaconState: bState.InnerStateUnsafe(),
+		Trie:        &protodb.SparseMerkleTrie{},
+		CurrentEth1Data: &protodb.LatestETH1Data{
+			BlockHash: make([]byte, 32),
+		},
+		ChainstartData: &protodb.ChainStartData{
+			Eth1Data: &ethpb.Eth1Data{
+				DepositRoot:  make([]byte, 32),
+				DepositCount: 0,
+				BlockHash:    make([]byte, 32),
+			},
+		},
+		DepositContainers: []*protodb.DepositContainer{},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
 	web3Service, err = powchain.NewService(ctx, &powchain.Web3ServiceConfig{
 		BeaconDB:        beaconDB,
 		ETH1Endpoint:    endpoint,
@@ -111,6 +141,7 @@ func setupBeaconChain(t *testing.T, beaconDB db.Database) *Service {
 		P2p:               &mockBroadcaster{},
 		StateNotifier:     &mockBeaconNode{},
 		AttPool:           attestations.NewPool(),
+		ForkChoiceStore:   protoarray.New(0, 0, params.BeaconConfig().ZeroHash),
 	}
 	if err != nil {
 		t.Fatalf("could not register blockchain service: %v", err)
@@ -165,7 +196,7 @@ func TestChainStartStop_Uninitialized(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if beaconState == nil || beaconState.Slot != 0 {
+	if beaconState == nil || beaconState.Slot() != 0 {
 		t.Error("Expected canonical state feed to send a state with genesis block")
 	}
 	if err := chainService.Stop(); err != nil {
@@ -176,7 +207,7 @@ func TestChainStartStop_Uninitialized(t *testing.T) {
 		t.Error("Context was not canceled")
 	}
 	testutil.AssertLogsContain(t, hook, "Waiting")
-	testutil.AssertLogsContain(t, hook, "Genesis time reached")
+	testutil.AssertLogsContain(t, hook, "Initialized beacon chain genesis state")
 }
 
 func TestChainStartStop_Initialized(t *testing.T) {
@@ -195,7 +226,11 @@ func TestChainStartStop_Initialized(t *testing.T) {
 	if err := db.SaveBlock(ctx, genesisBlk); err != nil {
 		t.Fatal(err)
 	}
-	if err := db.SaveState(ctx, &pb.BeaconState{Slot: 1}, blkRoot); err != nil {
+	s, err := beaconstate.InitializeFromProto(&pb.BeaconState{Slot: 1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := db.SaveState(ctx, s, blkRoot); err != nil {
 		t.Fatal(err)
 	}
 	if err := db.SaveHeadBlockRoot(ctx, blkRoot); err != nil {
@@ -238,11 +273,14 @@ func TestChainService_InitializeBeaconChain(t *testing.T) {
 		t.Fatal(err)
 	}
 	hashTreeRoot := trie.HashTreeRoot()
-	genState := state.EmptyGenesisState()
-	genState.Eth1Data = &ethpb.Eth1Data{
+	genState, err := state.EmptyGenesisState()
+	if err != nil {
+		t.Fatal(err)
+	}
+	genState.SetEth1Data(&ethpb.Eth1Data{
 		DepositRoot:  hashTreeRoot[:],
 		DepositCount: uint64(len(deposits)),
-	}
+	})
 	genState, err = b.ProcessDeposits(ctx, genState, &ethpb.BeaconBlockBody{Deposits: deposits})
 	if err != nil {
 		t.Fatal(err)
@@ -253,13 +291,13 @@ func TestChainService_InitializeBeaconChain(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	s, err := bc.beaconDB.State(ctx, bytesutil.ToBytes32(bc.canonicalRoots[0]))
+	s, err := bc.beaconDB.State(ctx, bc.headRoot())
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	for _, v := range s.Validators {
-		if !db.HasValidatorIndex(ctx, bytesutil.ToBytes48(v.PublicKey)) {
+	for _, v := range s.Validators() {
+		if !db.HasValidatorIndex(ctx, v.PublicKey) {
 			t.Errorf("Validator %s missing from db", hex.EncodeToString(v.PublicKey))
 		}
 	}
@@ -267,11 +305,15 @@ func TestChainService_InitializeBeaconChain(t *testing.T) {
 	if _, err := bc.HeadState(ctx); err != nil {
 		t.Error(err)
 	}
-	if bc.HeadBlock() == nil {
+	headBlk, err := bc.HeadBlock(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if headBlk == nil {
 		t.Error("Head state can't be nil after initialize beacon chain")
 	}
-	if bc.CanonicalRoot(0) == nil {
-		t.Error("Canonical root for slot 0 can't be nil after initialize beacon chain")
+	if bc.headRoot() == params.BeaconConfig().ZeroHash {
+		t.Error("Canonical root for slot 0 can't be zeros after initialize beacon chain")
 	}
 }
 
@@ -294,7 +336,10 @@ func TestChainService_InitializeChainInfo(t *testing.T) {
 
 	finalizedSlot := params.BeaconConfig().SlotsPerEpoch*2 + 1
 	headBlock := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: finalizedSlot, ParentRoot: genesisRoot[:]}}
-	headState := &pb.BeaconState{Slot: finalizedSlot}
+	headState, err := beaconstate.InitializeFromProto(&pb.BeaconState{Slot: finalizedSlot})
+	if err != nil {
+		t.Fatal(err)
+	}
 	headRoot, _ := ssz.HashTreeRoot(headBlock.Block)
 	if err := db.SaveState(ctx, headState, headRoot); err != nil {
 		t.Fatal(err)
@@ -311,24 +356,32 @@ func TestChainService_InitializeChainInfo(t *testing.T) {
 	if err := db.SaveBlock(ctx, headBlock); err != nil {
 		t.Fatal(err)
 	}
-	c := &Service{beaconDB: db, canonicalRoots: make(map[uint64][]byte)}
+	c := &Service{beaconDB: db}
 	if err := c.initializeChainInfo(ctx); err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(c.HeadBlock(), headBlock) {
+	headBlk, err := c.HeadBlock(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(headBlk, headBlock) {
 		t.Error("head block incorrect")
 	}
 	s, err := c.HeadState(ctx)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(s, headState) {
+	if !reflect.DeepEqual(s.InnerStateUnsafe(), headState.InnerStateUnsafe()) {
 		t.Error("head state incorrect")
 	}
 	if headBlock.Block.Slot != c.HeadSlot() {
 		t.Error("head slot incorrect")
 	}
-	if !bytes.Equal(headRoot[:], c.HeadRoot()) {
+	r, err := c.HeadRoot(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(headRoot[:], r) {
 		t.Error("head slot incorrect")
 	}
 	if c.genesisRoot != genesisRoot {
@@ -341,11 +394,13 @@ func TestChainService_SaveHeadNoDB(t *testing.T) {
 	defer testDB.TeardownDB(t, db)
 	ctx := context.Background()
 	s := &Service{
-		beaconDB:       db,
-		canonicalRoots: make(map[uint64][]byte),
+		beaconDB: db,
 	}
 	b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1}}
 	r, _ := ssz.HashTreeRoot(b)
+	state := &pb.BeaconState{}
+	newState, err := beaconstate.InitializeFromProto(state)
+	s.beaconDB.SaveState(ctx, newState, r)
 	if err := s.saveHeadNoDB(ctx, b, r); err != nil {
 		t.Fatal(err)
 	}
@@ -358,3 +413,124 @@ func TestChainService_SaveHeadNoDB(t *testing.T) {
 		t.Error("head block should not be equal")
 	}
 }
+
+func TestChainService_PruneOldStates(t *testing.T) {
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+	ctx := context.Background()
+	s := &Service{
+		beaconDB: db,
+	}
+
+	for i := 0; i < 100; i++ {
+		block := &ethpb.BeaconBlock{Slot: uint64(i)}
+		if err := s.beaconDB.SaveBlock(ctx, &ethpb.SignedBeaconBlock{Block: block}); err != nil {
+			t.Fatal(err)
+		}
+		r, err := ssz.HashTreeRoot(block)
+		if err != nil {
+			t.Fatal(err)
+		}
+		state := &pb.BeaconState{Slot: uint64(i)}
+		newState, err := beaconstate.InitializeFromProto(state)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err := s.beaconDB.SaveState(ctx, newState, r); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// Delete half of the states.
+	if err := s.pruneGarbageState(ctx, 50); err != nil {
+		t.Fatal(err)
+	}
+
+	filter := filters.NewFilter().SetStartSlot(1).SetEndSlot(100)
+	roots, err := s.beaconDB.BlockRoots(ctx, filter)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := 1; i < 50; i++ {
+		s, err := s.beaconDB.State(ctx, roots[i])
+		if err != nil {
+			t.Fatal(err)
+		}
+		if s != nil {
+			t.Errorf("wanted nil for slot %d", i)
+		}
+	}
+}
+
+func TestHasBlock_ForkChoiceAndDB(t *testing.T) {
+	ctx := context.Background()
+	db := testDB.SetupDB(t)
+	defer testDB.TeardownDB(t, db)
+	s := &Service{
+		forkChoiceStore:  protoarray.New(0, 0, [32]byte{}),
+		finalizedCheckpt: &ethpb.Checkpoint{},
+		beaconDB:         db,
+	}
+	block := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Body: &ethpb.BeaconBlockBody{}}}
+	r, _ := ssz.HashTreeRoot(block.Block)
+	bs := &pb.BeaconState{FinalizedCheckpoint: &ethpb.Checkpoint{}, CurrentJustifiedCheckpoint: &ethpb.Checkpoint{}}
+	state, _ := beaconstate.InitializeFromProto(bs)
+	if err := s.insertBlockToForkChoiceStore(ctx, block.Block, r, state); err != nil {
+		t.Fatal(err)
+	}
+
+	if s.hasBlock(ctx, [32]byte{}) {
+		t.Error("Should not have block")
+	}
+
+	if !s.hasBlock(ctx, r) {
+		t.Error("Should have block")
+	}
+}
+
+func BenchmarkHasBlockDB(b *testing.B) {
+	db := testDB.SetupDB(b)
+	defer testDB.TeardownDB(b, db)
+	ctx := context.Background()
+	s := &Service{
+		beaconDB: db,
+	}
+	block := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{}}
+	if err := s.beaconDB.SaveBlock(ctx, block); err != nil {
+		b.Fatal(err)
+	}
+	r, _ := ssz.HashTreeRoot(block.Block)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if !s.beaconDB.HasBlock(ctx, r) {
+			b.Fatal("Block is not in DB")
+		}
+	}
+}
+
+func BenchmarkHasBlockForkChoiceStore(b *testing.B) {
+	ctx := context.Background()
+	db := testDB.SetupDB(b)
+	defer testDB.TeardownDB(b, db)
+	s := &Service{
+		forkChoiceStore:  protoarray.New(0, 0, [32]byte{}),
+		finalizedCheckpt: &ethpb.Checkpoint{},
+		beaconDB:         db,
+	}
+	block := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Body: &ethpb.BeaconBlockBody{}}}
+	r, _ := ssz.HashTreeRoot(block.Block)
+	bs := &pb.BeaconState{FinalizedCheckpoint: &ethpb.Checkpoint{}, CurrentJustifiedCheckpoint: &ethpb.Checkpoint{}}
+	state, _ := beaconstate.InitializeFromProto(bs)
+	if err := s.insertBlockToForkChoiceStore(ctx, block.Block, r, state); err != nil {
+		b.Fatal(err)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if !s.forkChoiceStore.HasNode(r) {
+			b.Fatal("Block is not in fork choice store")
+		}
+	}
+}

beacon-chain/blockchain/testing/BUILD.bazel

@@ -8,10 +8,12 @@ go_library(
     visibility = ["//beacon-chain:__subpackages__"],
     deps = [
         "//beacon-chain/core/epoch/precompute:go_default_library",
+        "//beacon-chain/core/feed/block:go_default_library",
         "//beacon-chain/core/feed/operation:go_default_library",
         "//beacon-chain/core/feed/state:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/db:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/event:go_default_library",
         "//shared/params:go_default_library",

beacon-chain/blockchain/testing/mock.go

@@ -9,10 +9,12 @@ import (
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
+	blockfeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/block"
 	opfeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/operation"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/event"
 	"github.com/prysmaticlabs/prysm/shared/params"
@@ -21,7 +23,7 @@ import (
 
 // ChainService defines the mock interface for testing
 type ChainService struct {
-	State                       *pb.BeaconState
+	State                       *stateTrie.BeaconState
 	Root                        []byte
 	Block                       *ethpb.SignedBeaconBlock
 	FinalizedCheckPoint         *ethpb.Checkpoint
@@ -33,7 +35,9 @@ type ChainService struct {
 	Fork                        *pb.Fork
 	DB                          db.Database
 	stateNotifier               statefeed.Notifier
+	blockNotifier               blockfeed.Notifier
 	opNotifier                  opfeed.Notifier
+	ValidAttestation            bool
 }
 
 // StateNotifier mocks the same method in the chain service.
@@ -44,6 +48,27 @@ func (ms *ChainService) StateNotifier() statefeed.Notifier {
 	return ms.stateNotifier
 }
 
+// BlockNotifier mocks the same method in the chain service.
+func (ms *ChainService) BlockNotifier() blockfeed.Notifier {
+	if ms.blockNotifier == nil {
+		ms.blockNotifier = &MockBlockNotifier{}
+	}
+	return ms.blockNotifier
+}
+
+// MockBlockNotifier mocks the block notifier.
+type MockBlockNotifier struct {
+	feed *event.Feed
+}
+
+// BlockFeed returns a block feed.
+func (msn *MockBlockNotifier) BlockFeed() *event.Feed {
+	if msn.feed == nil {
+		msn.feed = new(event.Feed)
+	}
+	return msn.feed
+}
+
 // MockStateNotifier mocks the state notifier.
 type MockStateNotifier struct {
 	feed *event.Feed
@@ -96,12 +121,14 @@ func (ms *ChainService) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.S
 // ReceiveBlockNoPubsubForkchoice mocks ReceiveBlockNoPubsubForkchoice method in chain service.
 func (ms *ChainService) ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
 	if ms.State == nil {
-		ms.State = &pb.BeaconState{}
+		ms.State = &stateTrie.BeaconState{}
 	}
 	if !bytes.Equal(ms.Root, block.Block.ParentRoot) {
 		return errors.Errorf("wanted %#x but got %#x", ms.Root, block.Block.ParentRoot)
 	}
-	ms.State.Slot = block.Block.Slot
+	if err := ms.State.SetSlot(block.Block.Slot); err != nil {
+		return err
+	}
 	ms.BlocksReceived = append(ms.BlocksReceived, block)
 	signingRoot, err := ssz.HashTreeRoot(block.Block)
 	if err != nil {
@@ -123,23 +150,22 @@ func (ms *ChainService) HeadSlot() uint64 {
 	if ms.State == nil {
 		return 0
 	}
-	return ms.State.Slot
-
+	return ms.State.Slot()
 }
 
 // HeadRoot mocks HeadRoot method in chain service.
-func (ms *ChainService) HeadRoot() []byte {
-	return ms.Root
+func (ms *ChainService) HeadRoot(ctx context.Context) ([]byte, error) {
+	return ms.Root, nil
 
 }
 
 // HeadBlock mocks HeadBlock method in chain service.
-func (ms *ChainService) HeadBlock() *ethpb.SignedBeaconBlock {
-	return ms.Block
+func (ms *ChainService) HeadBlock(context.Context) (*ethpb.SignedBeaconBlock, error) {
+	return ms.Block, nil
 }
 
 // HeadState mocks HeadState method in chain service.
-func (ms *ChainService) HeadState(context.Context) (*pb.BeaconState, error) {
+func (ms *ChainService) HeadState(context.Context) (*stateTrie.BeaconState, error) {
 	return ms.State, nil
 }
 
@@ -191,7 +217,20 @@ func (ms *ChainService) GenesisTime() time.Time {
 	return ms.Genesis
 }
 
+// CurrentSlot mocks the same method in the chain service.
+func (ms *ChainService) CurrentSlot() uint64 {
+	return 0
+}
+
 // Participation mocks the same method in the chain service.
 func (ms *ChainService) Participation(epoch uint64) *precompute.Balance {
 	return ms.Balance
 }
+
+// IsValidAttestation always returns true.
+func (ms *ChainService) IsValidAttestation(ctx context.Context, att *ethpb.Attestation) bool {
+	return ms.ValidAttestation
+}
+
+// ClearCachedStates does nothing.
+func (ms *ChainService) ClearCachedStates() {}

beacon-chain/cache/BUILD.bazel

@@ -8,16 +8,18 @@ go_library(
         "committee.go",
         "common.go",
         "eth1_data.go",
+        "hot_state_cache.go",
+        "skip_slot_cache.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/cache",
     visibility = ["//beacon-chain:__subpackages__"],
     deps = [
-        "//proto/beacon/p2p/v1:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//shared/featureconfig:go_default_library",
         "//shared/hashutil:go_default_library",
         "//shared/params:go_default_library",
         "//shared/sliceutil:go_default_library",
-        "@com_github_gogo_protobuf//proto:go_default_library",
+        "@com_github_hashicorp_golang_lru//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
@@ -31,19 +33,23 @@ go_test(
     srcs = [
         "attestation_data_test.go",
         "checkpoint_state_test.go",
+        "committee_fuzz_test.go",
         "committee_test.go",
         "eth1_data_test.go",
         "feature_flag_test.go",
+        "hot_state_cache_test.go",
+        "skip_slot_cache_test.go",
     ],
     embed = [":go_default_library"],
-    race = "on",
     deps = [
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/bytesutil:go_default_library",
         "//shared/featureconfig:go_default_library",
         "//shared/hashutil:go_default_library",
         "//shared/params:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
+        "@com_github_google_gofuzz//:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
     ],
 )

beacon-chain/cache/attestation_data.go

@@ -11,7 +11,6 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"k8s.io/client-go/tools/cache"
 )
 
@@ -59,12 +58,6 @@ func NewAttestationCache() *AttestationCache {
 // Get waits for any in progress calculation to complete before returning a
 // cached response, if any.
 func (c *AttestationCache) Get(ctx context.Context, req *ethpb.AttestationDataRequest) (*ethpb.AttestationData, error) {
-	if !featureconfig.Get().EnableAttestationCache {
-		// Return a miss result if cache is not enabled.
-		attestationCacheMiss.Inc()
-		return nil, nil
-	}
-
 	if req == nil {
 		return nil, errors.New("nil attestation data request")
 	}
@@ -113,10 +106,6 @@ func (c *AttestationCache) Get(ctx context.Context, req *ethpb.AttestationDataRe
 // MarkInProgress a request so that any other similar requests will block on
 // Get until MarkNotInProgress is called.
 func (c *AttestationCache) MarkInProgress(req *ethpb.AttestationDataRequest) error {
-	if !featureconfig.Get().EnableAttestationCache {
-		return nil
-	}
-
 	c.lock.Lock()
 	defer c.lock.Unlock()
 	s, e := reqToKey(req)
@@ -126,19 +115,13 @@ func (c *AttestationCache) MarkInProgress(req *ethpb.AttestationDataRequest) err
 	if c.inProgress[s] {
 		return ErrAlreadyInProgress
 	}
-	if featureconfig.Get().EnableAttestationCache {
-		c.inProgress[s] = true
-	}
+	c.inProgress[s] = true
 	return nil
 }
 
 // MarkNotInProgress will release the lock on a given request. This should be
 // called after put.
 func (c *AttestationCache) MarkNotInProgress(req *ethpb.AttestationDataRequest) error {
-	if !featureconfig.Get().EnableAttestationCache {
-		return nil
-	}
-
 	c.lock.Lock()
 	defer c.lock.Unlock()
 	s, e := reqToKey(req)
@@ -151,10 +134,6 @@ func (c *AttestationCache) MarkNotInProgress(req *ethpb.AttestationDataRequest)
 
 // Put the response in the cache.
 func (c *AttestationCache) Put(ctx context.Context, req *ethpb.AttestationDataRequest, res *ethpb.AttestationData) error {
-	if !featureconfig.Get().EnableAttestationCache {
-		return nil
-	}
-
 	data := &attestationReqResWrapper{
 		req,
 		res,

beacon-chain/cache/checkpoint_state.go

@@ -4,11 +4,10 @@ import (
 	"errors"
 	"sync"
 
-	"github.com/gogo/protobuf/proto"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/hashutil"
 	"k8s.io/client-go/tools/cache"
 )
@@ -19,7 +18,9 @@ var (
 	ErrNotCheckpointState = errors.New("object is not a state by check point struct")
 
 	// maxCheckpointStateSize defines the max number of entries check point to state cache can contain.
-	maxCheckpointStateSize = 4
+	// Choosing 10 to account for multiple forks, this allows 5 forks per epoch boundary with 2 epochs
+	// window to accept attestation based on latest spec.
+	maxCheckpointStateSize = 10
 
 	// Metrics.
 	checkpointStateMiss = promauto.NewCounter(prometheus.CounterOpts{
@@ -35,7 +36,7 @@ var (
 // CheckpointState defines the active validator indices per epoch.
 type CheckpointState struct {
 	Checkpoint *ethpb.Checkpoint
-	State      *pb.BeaconState
+	State      *stateTrie.BeaconState
 }
 
 // CheckpointStateCache is a struct with 1 queue for looking up state by checkpoint.
@@ -67,7 +68,7 @@ func NewCheckpointStateCache() *CheckpointStateCache {
 
 // StateByCheckpoint fetches state by checkpoint. Returns true with a
 // reference to the CheckpointState info, if exists. Otherwise returns false, nil.
-func (c *CheckpointStateCache) StateByCheckpoint(cp *ethpb.Checkpoint) (*pb.BeaconState, error) {
+func (c *CheckpointStateCache) StateByCheckpoint(cp *ethpb.Checkpoint) (*stateTrie.BeaconState, error) {
 	c.lock.RLock()
 	defer c.lock.RUnlock()
 	h, err := hashutil.HashProto(cp)
@@ -92,7 +93,7 @@ func (c *CheckpointStateCache) StateByCheckpoint(cp *ethpb.Checkpoint) (*pb.Beac
 		return nil, ErrNotCheckpointState
 	}
 
-	return proto.Clone(info.State).(*pb.BeaconState), nil
+	return info.State.Copy(), nil
 }
 
 // AddCheckpointState adds CheckpointState object to the cache. This method also trims the least
@@ -100,7 +101,10 @@ func (c *CheckpointStateCache) StateByCheckpoint(cp *ethpb.Checkpoint) (*pb.Beac
 func (c *CheckpointStateCache) AddCheckpointState(cp *CheckpointState) error {
 	c.lock.Lock()
 	defer c.lock.Unlock()
-	if err := c.cache.AddIfNotPresent(cp); err != nil {
+	if err := c.cache.AddIfNotPresent(&CheckpointState{
+		Checkpoint: stateTrie.CopyCheckpoint(cp.Checkpoint),
+		State:      cp.State.Copy(),
+	}); err != nil {
 		return err
 	}
 

beacon-chain/cache/checkpoint_state_test.go

@@ -5,15 +5,22 @@ import (
 	"testing"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/hashutil"
 )
 
 func TestCheckpointStateCacheKeyFn_OK(t *testing.T) {
 	cp := &ethpb.Checkpoint{Epoch: 1, Root: []byte{'A'}}
+	st, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot: 64,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
 	info := &CheckpointState{
 		Checkpoint: cp,
-		State:      &pb.BeaconState{Slot: 64},
+		State:      st,
 	}
 	key, err := checkpointState(info)
 	if err != nil {
@@ -39,9 +46,15 @@ func TestCheckpointStateCache_StateByCheckpoint(t *testing.T) {
 	cache := NewCheckpointStateCache()
 
 	cp1 := &ethpb.Checkpoint{Epoch: 1, Root: []byte{'A'}}
+	st, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot: 64,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
 	info1 := &CheckpointState{
 		Checkpoint: cp1,
-		State:      &pb.BeaconState{Slot: 64},
+		State:      st,
 	}
 	state, err := cache.StateByCheckpoint(cp1)
 	if err != nil {
@@ -58,14 +71,20 @@ func TestCheckpointStateCache_StateByCheckpoint(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(state, info1.State) {
+	if !reflect.DeepEqual(state.InnerStateUnsafe(), info1.State.InnerStateUnsafe()) {
 		t.Error("incorrectly cached state")
 	}
 
 	cp2 := &ethpb.Checkpoint{Epoch: 2, Root: []byte{'B'}}
+	st2, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot: 128,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
 	info2 := &CheckpointState{
 		Checkpoint: cp2,
-		State:      &pb.BeaconState{Slot: 128},
+		State:      st2,
 	}
 	if err := cache.AddCheckpointState(info2); err != nil {
 		t.Fatal(err)
@@ -74,7 +93,7 @@ func TestCheckpointStateCache_StateByCheckpoint(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(state, info2.State) {
+	if !reflect.DeepEqual(state.CloneInnerState(), info2.State.CloneInnerState()) {
 		t.Error("incorrectly cached state")
 	}
 
@@ -82,18 +101,26 @@ func TestCheckpointStateCache_StateByCheckpoint(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(state, info1.State) {
+	if !reflect.DeepEqual(state.CloneInnerState(), info1.State.CloneInnerState()) {
 		t.Error("incorrectly cached state")
 	}
 }
 
-func TestCheckpointStateCache__MaxSize(t *testing.T) {
+func TestCheckpointStateCache_MaxSize(t *testing.T) {
 	c := NewCheckpointStateCache()
-
+	st, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot: 0,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
 	for i := 0; i < maxCheckpointStateSize+100; i++ {
+		if err := st.SetSlot(uint64(i)); err != nil {
+			t.Fatal(err)
+		}
 		info := &CheckpointState{
 			Checkpoint: &ethpb.Checkpoint{Epoch: uint64(i)},
-			State:      &pb.BeaconState{Slot: uint64(i)},
+			State:      st,
 		}
 		if err := c.AddCheckpointState(info); err != nil {
 			t.Fatal(err)

beacon-chain/cache/committee.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/sliceutil"
 	"k8s.io/client-go/tools/cache"
@@ -40,6 +39,7 @@ type Committees struct {
 	Seed            [32]byte
 	ShuffledIndices []uint64
 	SortedIndices   []uint64
+	ProposerIndices []uint64
 }
 
 // CommitteeCache is a struct with 1 queue for looking up shuffled indices list by seed.
@@ -68,9 +68,6 @@ func NewCommitteesCache() *CommitteeCache {
 // Committee fetches the shuffled indices by slot and committee index. Every list of indices
 // represent one committee. Returns true if the list exists with slot and committee index. Otherwise returns false, nil.
 func (c *CommitteeCache) Committee(slot uint64, seed [32]byte, index uint64) ([]uint64, error) {
-	if !featureconfig.Get().EnableShuffledIndexCache && !featureconfig.Get().EnableNewCache {
-		return nil, nil
-	}
 	c.lock.RLock()
 	defer c.lock.RUnlock()
 
@@ -99,21 +96,50 @@ func (c *CommitteeCache) Committee(slot uint64, seed [32]byte, index uint64) ([]
 	indexOffSet := index + (slot%params.BeaconConfig().SlotsPerEpoch)*committeeCountPerSlot
 	start, end := startEndIndices(item, indexOffSet)
 
+	if int(end) > len(item.ShuffledIndices) {
+		return nil, errors.New("requested index out of bound")
+	}
+
 	return item.ShuffledIndices[start:end], nil
 }
 
 // AddCommitteeShuffledList adds Committee shuffled list object to the cache. T
 // his method also trims the least recently list if the cache size has ready the max cache size limit.
 func (c *CommitteeCache) AddCommitteeShuffledList(committees *Committees) error {
-	if !featureconfig.Get().EnableShuffledIndexCache && !featureconfig.Get().EnableNewCache {
-		return nil
-	}
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
 	if err := c.CommitteeCache.AddIfNotPresent(committees); err != nil {
 		return err
 	}
+	trim(c.CommitteeCache, maxCommitteesCacheSize)
+	return nil
+}
+
+// AddProposerIndicesList updates the committee shuffled list with proposer indices.
+func (c *CommitteeCache) AddProposerIndicesList(seed [32]byte, indices []uint64) error {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	obj, exists, err := c.CommitteeCache.GetByKey(key(seed))
+	if err != nil {
+		return err
+	}
+	if !exists {
+		committees := &Committees{ProposerIndices: indices}
+		if err := c.CommitteeCache.Add(committees); err != nil {
+			return err
+		}
+	} else {
+		committees, ok := obj.(*Committees)
+		if !ok {
+			return ErrNotCommittee
+		}
+		committees.ProposerIndices = indices
+		if err := c.CommitteeCache.Add(committees); err != nil {
+			return err
+		}
+	}
 
 	trim(c.CommitteeCache, maxCommitteesCacheSize)
 	return nil
@@ -121,10 +147,6 @@ func (c *CommitteeCache) AddCommitteeShuffledList(committees *Committees) error
 
 // ActiveIndices returns the active indices of a given seed stored in cache.
 func (c *CommitteeCache) ActiveIndices(seed [32]byte) ([]uint64, error) {
-	if !featureconfig.Get().EnableShuffledIndexCache && !featureconfig.Get().EnableNewCache {
-		return nil, nil
-	}
-
 	c.lock.RLock()
 	defer c.lock.RUnlock()
 	obj, exists, err := c.CommitteeCache.GetByKey(key(seed))
@@ -147,6 +169,30 @@ func (c *CommitteeCache) ActiveIndices(seed [32]byte) ([]uint64, error) {
 	return item.SortedIndices, nil
 }
 
+// ProposerIndices returns the proposer indices of a given seed.
+func (c *CommitteeCache) ProposerIndices(seed [32]byte) ([]uint64, error) {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	obj, exists, err := c.CommitteeCache.GetByKey(key(seed))
+	if err != nil {
+		return nil, err
+	}
+
+	if exists {
+		CommitteeCacheHit.Inc()
+	} else {
+		CommitteeCacheMiss.Inc()
+		return nil, nil
+	}
+
+	item, ok := obj.(*Committees)
+	if !ok {
+		return nil, ErrNotCommittee
+	}
+
+	return item.ProposerIndices, nil
+}
+
 func startEndIndices(c *Committees, index uint64) (uint64, uint64) {
 	validatorCount := uint64(len(c.ShuffledIndices))
 	start := sliceutil.SplitOffset(validatorCount, c.CommitteeCount, index)
@@ -157,7 +203,7 @@ func startEndIndices(c *Committees, index uint64) (uint64, uint64) {
 // Using seed as source for key to handle reorgs in the same epoch.
 // The seed is derived from state's array of randao mixes and epoch value
 // hashed together. This avoids collisions on different validator set. Spec definition:
-// https://github.com/ethereum/eth2.0-specs/blob/v0.9.2/specs/core/0_beacon-chain.md#get_seed
+// https://github.com/ethereum/eth2.0-specs/blob/v0.9.3/specs/core/0_beacon-chain.md#get_seed
 func key(seed [32]byte) string {
 	return string(seed[:])
 }

beacon-chain/cache/committee_fuzz_test.go

@@ -0,0 +1,68 @@
+package cache
+
+import (
+	"reflect"
+	"testing"
+
+	fuzz "github.com/google/gofuzz"
+)
+
+func TestCommitteeKeyFuzz_OK(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	c := &Committees{}
+
+	for i := 0; i < 100000; i++ {
+		fuzzer.Fuzz(c)
+		k, err := committeeKeyFn(c)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if k != key(c.Seed) {
+			t.Errorf("Incorrect hash k: %s, expected %s", k, key(c.Seed))
+		}
+	}
+}
+
+func TestCommitteeCache_FuzzCommitteesByEpoch(t *testing.T) {
+	cache := NewCommitteesCache()
+	fuzzer := fuzz.NewWithSeed(0)
+	c := &Committees{}
+
+	for i := 0; i < 100000; i++ {
+		fuzzer.Fuzz(c)
+		if err := cache.AddCommitteeShuffledList(c); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := cache.Committee(0, c.Seed, 0); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if len(cache.CommitteeCache.ListKeys()) != maxCommitteesCacheSize {
+		t.Error("Incorrect key size")
+	}
+}
+
+func TestCommitteeCache_FuzzActiveIndices(t *testing.T) {
+	cache := NewCommitteesCache()
+	fuzzer := fuzz.NewWithSeed(0)
+	c := &Committees{}
+
+	for i := 0; i < 100000; i++ {
+		fuzzer.Fuzz(c)
+		if err := cache.AddCommitteeShuffledList(c); err != nil {
+			t.Fatal(err)
+		}
+		indices, err := cache.ActiveIndices(c.Seed)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !reflect.DeepEqual(indices, c.SortedIndices) {
+			t.Error("Saved indices not the same")
+		}
+	}
+
+	if len(cache.CommitteeCache.ListKeys()) != maxCommitteesCacheSize {
+		t.Error("Incorrect key size")
+	}
+}

beacon-chain/cache/committee_test.go

@@ -96,6 +96,53 @@ func TestCommitteeCache_ActiveIndices(t *testing.T) {
 	}
 }
 
+func TestCommitteeCache_AddProposerIndicesList(t *testing.T) {
+	cache := NewCommitteesCache()
+
+	seed := [32]byte{'A'}
+	indices := []uint64{1, 2, 3, 4, 5}
+	indices, err := cache.ProposerIndices(seed)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if indices != nil {
+		t.Error("Expected committee count not to exist in empty cache")
+	}
+	if err := cache.AddProposerIndicesList(seed, indices); err != nil {
+		t.Fatal(err)
+	}
+	received, err := cache.ProposerIndices(seed)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(indices, received) {
+		t.Error("Did not receive correct proposer indices from cache")
+	}
+
+	item := &Committees{Seed: [32]byte{'B'}, SortedIndices: []uint64{1, 2, 3, 4, 5, 6}}
+	if err := cache.AddCommitteeShuffledList(item); err != nil {
+		t.Fatal(err)
+	}
+	indices, err = cache.ProposerIndices(item.Seed)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if indices != nil {
+		t.Error("Expected committee count not to exist in empty cache")
+	}
+	if err := cache.AddProposerIndicesList(item.Seed, indices); err != nil {
+		t.Fatal(err)
+	}
+	received, err = cache.ProposerIndices(item.Seed)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(indices, received) {
+		t.Error("Did not receive correct proposer indices from cache")
+	}
+
+}
+
 func TestCommitteeCache_CanRotate(t *testing.T) {
 	cache := NewCommitteesCache()
 

beacon-chain/cache/eth1_data_test.go

@@ -91,7 +91,7 @@ func TestEth1Data_MaxSize(t *testing.T) {
 
 	for i := 0; i < maxEth1DataVoteSize+1; i++ {
 		var hash [32]byte
-		copy(hash[:], []byte(strconv.Itoa(i)))
+		copy(hash[:], strconv.Itoa(i))
 		eInfo := &Eth1DataVote{
 			Eth1DataHash: hash,
 		}

beacon-chain/cache/feature_flag_test.go

@@ -4,8 +4,6 @@ import "github.com/prysmaticlabs/prysm/shared/featureconfig"
 
 func init() {
 	featureconfig.Init(&featureconfig.Flags{
-		EnableAttestationCache:   true,
-		EnableEth1DataVoteCache:  true,
-		EnableShuffledIndexCache: true,
+		EnableEth1DataVoteCache: true,
 	})
 }

beacon-chain/cache/hot_state_cache.go

@@ -0,0 +1,61 @@
+package cache
+
+import (
+	lru "github.com/hashicorp/golang-lru"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+)
+
+var (
+	// hotStateCacheSize defines the max number of hot state this can cache.
+	hotStateCacheSize = 16
+	// Metrics
+	hotStateCacheHit = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "hot_state_cache_hit",
+		Help: "The total number of cache hits on the hot state cache.",
+	})
+	hotStateCacheMiss = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "hot_state_cache_miss",
+		Help: "The total number of cache misses on the hot state cache.",
+	})
+)
+
+// HotStateCache is used to store the processed beacon state after finalized check point..
+type HotStateCache struct {
+	cache *lru.Cache
+}
+
+// NewHotStateCache initializes the map and underlying cache.
+func NewHotStateCache() *HotStateCache {
+	cache, err := lru.New(hotStateCacheSize)
+	if err != nil {
+		panic(err)
+	}
+	return &HotStateCache{
+		cache: cache,
+	}
+}
+
+// Get returns a cached response via input block root, if any.
+// The response is copied by default.
+func (c *HotStateCache) Get(root [32]byte) *stateTrie.BeaconState {
+	item, exists := c.cache.Get(root)
+
+	if exists && item != nil {
+		hotStateCacheHit.Inc()
+		return item.(*stateTrie.BeaconState).Copy()
+	}
+	hotStateCacheMiss.Inc()
+	return nil
+}
+
+// Put the response in the cache.
+func (c *HotStateCache) Put(root [32]byte, state *stateTrie.BeaconState) {
+	c.cache.Add(root, state)
+}
+
+// Has returns true if the key exists in the cache.
+func (c *HotStateCache) Has(root [32]byte) bool {
+	return c.cache.Contains(root)
+}

beacon-chain/cache/hot_state_cache_test.go

@@ -0,0 +1,41 @@
+package cache_test
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+)
+
+func TestHotStateCache_RoundTrip(t *testing.T) {
+	c := cache.NewHotStateCache()
+	root := [32]byte{'A'}
+	state := c.Get(root)
+	if state != nil {
+		t.Errorf("Empty cache returned an object: %v", state)
+	}
+	if c.Has(root) {
+		t.Error("Empty cache has an object")
+	}
+
+	state, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot: 10,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	c.Put(root, state)
+
+	if !c.Has(root) {
+		t.Error("Empty cache does not have an object")
+	}
+	res := c.Get(root)
+	if state == nil {
+		t.Errorf("Empty cache returned an object: %v", state)
+	}
+	if !reflect.DeepEqual(state.CloneInnerState(), res.CloneInnerState()) {
+		t.Error("Expected equal protos to return from cache")
+	}
+}

beacon-chain/cache/skip_slot_cache.go

@@ -0,0 +1,130 @@
+package cache
+
+import (
+	"context"
+	"math"
+	"sync"
+	"time"
+
+	lru "github.com/hashicorp/golang-lru"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+)
+
+var (
+	// Metrics
+	skipSlotCacheHit = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "skip_slot_cache_hit",
+		Help: "The total number of cache hits on the skip slot cache.",
+	})
+	skipSlotCacheMiss = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "skip_slot_cache_miss",
+		Help: "The total number of cache misses on the skip slot cache.",
+	})
+)
+
+// SkipSlotCache is used to store the cached results of processing skip slots in state.ProcessSlots.
+type SkipSlotCache struct {
+	cache      *lru.Cache
+	lock       sync.RWMutex
+	inProgress map[uint64]bool
+}
+
+// NewSkipSlotCache initializes the map and underlying cache.
+func NewSkipSlotCache() *SkipSlotCache {
+	cache, err := lru.New(8)
+	if err != nil {
+		panic(err)
+	}
+	return &SkipSlotCache{
+		cache:      cache,
+		inProgress: make(map[uint64]bool),
+	}
+}
+
+// Get waits for any in progress calculation to complete before returning a
+// cached response, if any.
+func (c *SkipSlotCache) Get(ctx context.Context, slot uint64) (*stateTrie.BeaconState, error) {
+	if !featureconfig.Get().EnableSkipSlotsCache {
+		// Return a miss result if cache is not enabled.
+		skipSlotCacheMiss.Inc()
+		return nil, nil
+	}
+
+	delay := minDelay
+
+	// Another identical request may be in progress already. Let's wait until
+	// any in progress request resolves or our timeout is exceeded.
+	for {
+		if ctx.Err() != nil {
+			return nil, ctx.Err()
+		}
+
+		c.lock.RLock()
+		if !c.inProgress[slot] {
+			c.lock.RUnlock()
+			break
+		}
+		c.lock.RUnlock()
+
+		// This increasing backoff is to decrease the CPU cycles while waiting
+		// for the in progress boolean to flip to false.
+		time.Sleep(time.Duration(delay) * time.Nanosecond)
+		delay *= delayFactor
+		delay = math.Min(delay, maxDelay)
+	}
+
+	item, exists := c.cache.Get(slot)
+
+	if exists && item != nil {
+		skipSlotCacheHit.Inc()
+		return item.(*stateTrie.BeaconState).Copy(), nil
+	}
+	skipSlotCacheMiss.Inc()
+	return nil, nil
+}
+
+// MarkInProgress a request so that any other similar requests will block on
+// Get until MarkNotInProgress is called.
+func (c *SkipSlotCache) MarkInProgress(slot uint64) error {
+	if !featureconfig.Get().EnableSkipSlotsCache {
+		return nil
+	}
+
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	if c.inProgress[slot] {
+		return ErrAlreadyInProgress
+	}
+	c.inProgress[slot] = true
+	return nil
+}
+
+// MarkNotInProgress will release the lock on a given request. This should be
+// called after put.
+func (c *SkipSlotCache) MarkNotInProgress(slot uint64) error {
+	if !featureconfig.Get().EnableSkipSlotsCache {
+		return nil
+	}
+
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	delete(c.inProgress, slot)
+	return nil
+}
+
+// Put the response in the cache.
+func (c *SkipSlotCache) Put(ctx context.Context, slot uint64, state *stateTrie.BeaconState) error {
+	if !featureconfig.Get().EnableSkipSlotsCache {
+		return nil
+	}
+
+	// Copy state so cached value is not mutated.
+	c.cache.Add(slot, state.Copy())
+
+	return nil
+}

beacon-chain/cache/skip_slot_cache_test.go

@@ -0,0 +1,57 @@
+package cache_test
+
+import (
+	"context"
+	"reflect"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+)
+
+func TestSkipSlotCache_RoundTrip(t *testing.T) {
+	ctx := context.Background()
+	c := cache.NewSkipSlotCache()
+	fc := featureconfig.Get()
+	fc.EnableSkipSlotsCache = true
+	featureconfig.Init(fc)
+
+	state, err := c.Get(ctx, 5)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if state != nil {
+		t.Errorf("Empty cache returned an object: %v", state)
+	}
+
+	if err := c.MarkInProgress(5); err != nil {
+		t.Error(err)
+	}
+
+	state, err = stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot: 10,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err = c.Put(ctx, 5, state); err != nil {
+		t.Error(err)
+	}
+
+	if err := c.MarkNotInProgress(5); err != nil {
+		t.Error(err)
+	}
+
+	res, err := c.Get(ctx, 5)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !reflect.DeepEqual(state.CloneInnerState(), res.CloneInnerState()) {
+		t.Error("Expected equal protos to return from cache")
+	}
+}

beacon-chain/core/blocks/BUILD.bazel

@@ -14,9 +14,11 @@ go_library(
     deps = [
         "//beacon-chain/cache:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
-        "//beacon-chain/core/state/stateutils:go_default_library",
         "//beacon-chain/core/validators:go_default_library",
+        "//beacon-chain/state:go_default_library",
+        "//beacon-chain/state/stateutil:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
+        "//shared/attestationutil:go_default_library",
         "//shared/bls:go_default_library",
         "//shared/bytesutil:go_default_library",
         "//shared/featureconfig:go_default_library",
@@ -46,9 +48,11 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//beacon-chain/core/helpers:go_default_library",
-        "//beacon-chain/core/state/stateutils:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
+        "//shared/attestationutil:go_default_library",
         "//shared/bls:go_default_library",
+        "//shared/bytesutil:go_default_library",
         "//shared/params:go_default_library",
         "//shared/testutil:go_default_library",
         "//shared/trieutil:go_default_library",

beacon-chain/core/blocks/block_operations.go

@@ -14,9 +14,11 @@ import (
 	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/state/stateutils"
 	v "github.com/prysmaticlabs/prysm/beacon-chain/core/validators"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/attestationutil"
 	"github.com/prysmaticlabs/prysm/shared/bls"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/featureconfig"
@@ -37,7 +39,7 @@ var eth1DataCache = cache.NewEth1DataVoteCache()
 // failed to verify.
 var ErrSigFailedToVerify = errors.New("signature did not verify")
 
-func verifySigningRoot(obj interface{}, pub []byte, signature []byte, domain uint64) error {
+func verifySigningRoot(obj interface{}, pub []byte, signature []byte, domain []byte) error {
 	publicKey, err := bls.PublicKeyFromBytes(pub)
 	if err != nil {
 		return errors.Wrap(err, "could not convert bytes to public key")
@@ -46,18 +48,18 @@ func verifySigningRoot(obj interface{}, pub []byte, signature []byte, domain uin
 	if err != nil {
 		return errors.Wrap(err, "could not convert bytes to signature")
 	}
-	root, err := ssz.HashTreeRoot(obj)
+	root, err := helpers.ComputeSigningRoot(obj, domain)
 	if err != nil {
-		return errors.Wrap(err, "could not get signing root")
+		return errors.Wrap(err, "could not compute signing root")
 	}
-	if !sig.Verify(root[:], publicKey, domain) {
+	if !sig.Verify(root[:], publicKey) {
 		return ErrSigFailedToVerify
 	}
 	return nil
 }
 
 // Deprecated: This method uses deprecated ssz.SigningRoot.
-func verifyDepositDataSigningRoot(obj *ethpb.Deposit_Data, pub []byte, signature []byte, domain uint64) error {
+func verifyDepositDataSigningRoot(obj *ethpb.Deposit_Data, pub []byte, signature []byte, domain []byte) error {
 	publicKey, err := bls.PublicKeyFromBytes(pub)
 	if err != nil {
 		return errors.Wrap(err, "could not convert bytes to public key")
@@ -70,13 +72,21 @@ func verifyDepositDataSigningRoot(obj *ethpb.Deposit_Data, pub []byte, signature
 	if err != nil {
 		return errors.Wrap(err, "could not get signing root")
 	}
-	if !sig.Verify(root[:], publicKey, domain) {
+	sigRoot := &pb.SigningRoot{
+		ObjectRoot: root[:],
+		Domain:     domain,
+	}
+	ctrRoot, err := ssz.HashTreeRoot(sigRoot)
+	if err != nil {
+		return errors.Wrap(err, "could not get container root")
+	}
+	if !sig.Verify(ctrRoot[:], publicKey) {
 		return ErrSigFailedToVerify
 	}
 	return nil
 }
 
-func verifySignature(signedData []byte, pub []byte, signature []byte, domain uint64) error {
+func verifySignature(signedData []byte, pub []byte, signature []byte, domain []byte) error {
 	publicKey, err := bls.PublicKeyFromBytes(pub)
 	if err != nil {
 		return errors.Wrap(err, "could not convert bytes to public key")
@@ -85,7 +95,15 @@ func verifySignature(signedData []byte, pub []byte, signature []byte, domain uin
 	if err != nil {
 		return errors.Wrap(err, "could not convert bytes to signature")
 	}
-	if !sig.Verify(signedData, publicKey, domain) {
+	ctr := &pb.SigningRoot{
+		ObjectRoot: signedData,
+		Domain:     domain,
+	}
+	root, err := ssz.HashTreeRoot(ctr)
+	if err != nil {
+		return errors.Wrap(err, "could not hash container")
+	}
+	if !sig.Verify(root[:], publicKey) {
 		return ErrSigFailedToVerify
 	}
 	return nil
@@ -100,29 +118,46 @@ func verifySignature(signedData []byte, pub []byte, signature []byte, domain uin
 //    state.eth1_data_votes.append(body.eth1_data)
 //    if state.eth1_data_votes.count(body.eth1_data) * 2 > SLOTS_PER_ETH1_VOTING_PERIOD:
 //        state.latest_eth1_data = body.eth1_data
-func ProcessEth1DataInBlock(beaconState *pb.BeaconState, block *ethpb.BeaconBlock) (*pb.BeaconState, error) {
-	beaconState.Eth1DataVotes = append(beaconState.Eth1DataVotes, block.Body.Eth1Data)
-
+func ProcessEth1DataInBlock(beaconState *stateTrie.BeaconState, block *ethpb.BeaconBlock) (*stateTrie.BeaconState, error) {
+	if beaconState == nil {
+		return nil, errors.New("nil state")
+	}
+	if block == nil || block.Body == nil {
+		return nil, errors.New("nil block or block withought body")
+	}
+	if err := beaconState.AppendEth1DataVotes(block.Body.Eth1Data); err != nil {
+		return nil, err
+	}
 	hasSupport, err := Eth1DataHasEnoughSupport(beaconState, block.Body.Eth1Data)
 	if err != nil {
 		return nil, err
 	}
-
 	if hasSupport {
-		beaconState.Eth1Data = block.Body.Eth1Data
+		if err := beaconState.SetEth1Data(block.Body.Eth1Data); err != nil {
+			return nil, err
+		}
 	}
-
 	return beaconState, nil
 }
 
+func areEth1DataEqual(a, b *ethpb.Eth1Data) bool {
+	if a == nil || b == nil {
+		return false
+	}
+	return a.DepositCount == b.DepositCount &&
+		bytes.Equal(a.BlockHash, b.BlockHash) &&
+		bytes.Equal(a.DepositRoot, b.DepositRoot)
+}
+
 // Eth1DataHasEnoughSupport returns true when the given eth1data has more than 50% votes in the
 // eth1 voting period. A vote is cast by including eth1data in a block and part of state processing
 // appends eth1data to the state in the Eth1DataVotes list. Iterating through this list checks the
 // votes to see if they match the eth1data.
-func Eth1DataHasEnoughSupport(beaconState *pb.BeaconState, data *ethpb.Eth1Data) (bool, error) {
+func Eth1DataHasEnoughSupport(beaconState *stateTrie.BeaconState, data *ethpb.Eth1Data) (bool, error) {
 	voteCount := uint64(0)
 	var eth1DataHash [32]byte
 	var err error
+	data = stateTrie.CopyETH1Data(data)
 	if featureconfig.Get().EnableEth1DataVoteCache {
 		eth1DataHash, err = hashutil.HashProto(data)
 		if err != nil {
@@ -135,8 +170,8 @@ func Eth1DataHasEnoughSupport(beaconState *pb.BeaconState, data *ethpb.Eth1Data)
 
 	}
 	if voteCount == 0 {
-		for _, vote := range beaconState.Eth1DataVotes {
-			if proto.Equal(vote, data) {
+		for _, vote := range beaconState.Eth1DataVotes() {
+			if areEth1DataEqual(vote, data) {
 				voteCount++
 			}
 		}
@@ -181,9 +216,9 @@ func Eth1DataHasEnoughSupport(beaconState *pb.BeaconState, data *ethpb.Eth1Data)
 //    # Verify proposer signature
 //    assert bls_verify(proposer.pubkey, signing_root(block), block.signature, get_domain(state, DOMAIN_BEACON_PROPOSER))
 func ProcessBlockHeader(
-	beaconState *pb.BeaconState,
+	beaconState *stateTrie.BeaconState,
 	block *ethpb.SignedBeaconBlock,
-) (*pb.BeaconState, error) {
+) (*stateTrie.BeaconState, error) {
 	beaconState, err := ProcessBlockHeaderNoVerify(beaconState, block.Block)
 	if err != nil {
 		return nil, err
@@ -193,11 +228,17 @@ func ProcessBlockHeader(
 	if err != nil {
 		return nil, err
 	}
-	proposer := beaconState.Validators[idx]
+	proposer, err := beaconState.ValidatorAtIndex(idx)
+	if err != nil {
+		return nil, err
+	}
 
 	// Verify proposer signature.
-	currentEpoch := helpers.CurrentEpoch(beaconState)
-	domain := helpers.Domain(beaconState.Fork, currentEpoch, params.BeaconConfig().DomainBeaconProposer)
+	currentEpoch := helpers.SlotToEpoch(beaconState.Slot())
+	domain, err := helpers.Domain(beaconState.Fork(), currentEpoch, params.BeaconConfig().DomainBeaconProposer)
+	if err != nil {
+		return nil, err
+	}
 	if err := verifySigningRoot(block.Block, proposer.PublicKey, block.Signature, domain); err != nil {
 		return nil, ErrSigFailedToVerify
 	}
@@ -229,20 +270,20 @@ func ProcessBlockHeader(
 //    proposer = state.validators[get_beacon_proposer_index(state)]
 //    assert not proposer.slashed
 func ProcessBlockHeaderNoVerify(
-	beaconState *pb.BeaconState,
+	beaconState *stateTrie.BeaconState,
 	block *ethpb.BeaconBlock,
-) (*pb.BeaconState, error) {
+) (*stateTrie.BeaconState, error) {
 	if block == nil {
 		return nil, errors.New("nil block")
 	}
-	if beaconState.Slot != block.Slot {
-		return nil, fmt.Errorf("state slot: %d is different then block slot: %d", beaconState.Slot, block.Slot)
+	if beaconState.Slot() != block.Slot {
+		return nil, fmt.Errorf("state slot: %d is different then block slot: %d", beaconState.Slot(), block.Slot)
 	}
-
-	parentRoot, err := ssz.HashTreeRoot(beaconState.LatestBlockHeader)
+	parentRoot, err := stateutil.BlockHeaderRoot(beaconState.LatestBlockHeader())
 	if err != nil {
 		return nil, err
 	}
+
 	if !bytes.Equal(block.ParentRoot, parentRoot[:]) {
 		return nil, fmt.Errorf(
 			"parent root %#x does not match the latest block header signing root in state %#x",
@@ -253,7 +294,10 @@ func ProcessBlockHeaderNoVerify(
 	if err != nil {
 		return nil, err
 	}
-	proposer := beaconState.Validators[idx]
+	proposer, err := beaconState.ValidatorAtIndex(idx)
+	if err != nil {
+		return nil, err
+	}
 	if proposer.Slashed {
 		return nil, fmt.Errorf("proposer at index %d was previously slashed", idx)
 	}
@@ -262,11 +306,13 @@ func ProcessBlockHeaderNoVerify(
 	if err != nil {
 		return nil, err
 	}
-	beaconState.LatestBlockHeader = &ethpb.BeaconBlockHeader{
+	if err := beaconState.SetLatestBlockHeader(&ethpb.BeaconBlockHeader{
 		Slot:       block.Slot,
 		ParentRoot: block.ParentRoot,
 		StateRoot:  params.BeaconConfig().ZeroHash[:],
 		BodyRoot:   bodyRoot[:],
+	}); err != nil {
+		return nil, err
 	}
 	return beaconState, nil
 }
@@ -291,21 +337,24 @@ func ProcessBlockHeaderNoVerify(
 //             hash(body.randao_reveal))
 //     )
 func ProcessRandao(
-	beaconState *pb.BeaconState,
+	beaconState *stateTrie.BeaconState,
 	body *ethpb.BeaconBlockBody,
-) (*pb.BeaconState, error) {
+) (*stateTrie.BeaconState, error) {
 	proposerIdx, err := helpers.BeaconProposerIndex(beaconState)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not get beacon proposer index")
 	}
-	proposerPub := beaconState.Validators[proposerIdx].PublicKey
+	proposerPub := beaconState.PubkeyAtIndex(proposerIdx)
 
-	currentEpoch := helpers.CurrentEpoch(beaconState)
+	currentEpoch := helpers.SlotToEpoch(beaconState.Slot())
 	buf := make([]byte, 32)
 	binary.LittleEndian.PutUint64(buf, currentEpoch)
 
-	domain := helpers.Domain(beaconState.Fork, currentEpoch, params.BeaconConfig().DomainRandao)
-	if err := verifySignature(buf, proposerPub, body.RandaoReveal, domain); err != nil {
+	domain, err := helpers.Domain(beaconState.Fork(), currentEpoch, params.BeaconConfig().DomainRandao)
+	if err != nil {
+		return nil, err
+	}
+	if err := verifySignature(buf, proposerPub[:], body.RandaoReveal, domain); err != nil {
 		return nil, errors.Wrap(err, "could not verify block randao")
 	}
 
@@ -326,19 +375,27 @@ func ProcessRandao(
 //             hash(body.randao_reveal))
 //     )
 func ProcessRandaoNoVerify(
-	beaconState *pb.BeaconState,
+	beaconState *stateTrie.BeaconState,
 	body *ethpb.BeaconBlockBody,
-) (*pb.BeaconState, error) {
-	currentEpoch := helpers.CurrentEpoch(beaconState)
+) (*stateTrie.BeaconState, error) {
+	currentEpoch := helpers.SlotToEpoch(beaconState.Slot())
 	// If block randao passed verification, we XOR the state's latest randao mix with the block's
 	// randao and update the state's corresponding latest randao mix value.
 	latestMixesLength := params.BeaconConfig().EpochsPerHistoricalVector
-	latestMixSlice := beaconState.RandaoMixes[currentEpoch%latestMixesLength]
+	latestMixSlice, err := beaconState.RandaoMixAtIndex(currentEpoch % latestMixesLength)
+	if err != nil {
+		return nil, err
+	}
 	blockRandaoReveal := hashutil.Hash(body.RandaoReveal)
+	if len(blockRandaoReveal) != len(latestMixSlice) {
+		return nil, errors.New("blockRandaoReveal length doesnt match latestMixSlice length")
+	}
 	for i, x := range blockRandaoReveal {
 		latestMixSlice[i] ^= x
 	}
-	beaconState.RandaoMixes[currentEpoch%latestMixesLength] = latestMixSlice
+	if err := beaconState.UpdateRandaoMixesAtIndex(latestMixSlice, currentEpoch%latestMixesLength); err != nil {
+		return nil, err
+	}
 	return beaconState, nil
 }
 
@@ -364,10 +421,17 @@ func ProcessRandaoNoVerify(
 //        assert bls_verify(proposer.pubkey, signing_root(header), header.signature, domain)
 //
 //    slash_validator(state, proposer_slashing.proposer_index)
-func ProcessProposerSlashings(ctx context.Context, beaconState *pb.BeaconState, body *ethpb.BeaconBlockBody) (*pb.BeaconState, error) {
+func ProcessProposerSlashings(
+	ctx context.Context,
+	beaconState *stateTrie.BeaconState,
+	body *ethpb.BeaconBlockBody,
+) (*stateTrie.BeaconState, error) {
 	var err error
 	for idx, slashing := range body.ProposerSlashings {
-		if int(slashing.ProposerIndex) >= len(beaconState.Validators) {
+		if slashing == nil {
+			return nil, errors.New("nil proposer slashings in block body")
+		}
+		if int(slashing.ProposerIndex) >= beaconState.NumValidators() {
 			return nil, fmt.Errorf("invalid proposer index given in slashing %d", slashing.ProposerIndex)
 		}
 		if err = VerifyProposerSlashing(beaconState, slashing); err != nil {
@@ -385,22 +449,30 @@ func ProcessProposerSlashings(ctx context.Context, beaconState *pb.BeaconState,
 
 // VerifyProposerSlashing verifies that the data provided fro slashing is valid.
 func VerifyProposerSlashing(
-	beaconState *pb.BeaconState,
+	beaconState *stateTrie.BeaconState,
 	slashing *ethpb.ProposerSlashing,
 ) error {
-	proposer := beaconState.Validators[slashing.ProposerIndex]
-
+	proposer, err := beaconState.ValidatorAtIndex(slashing.ProposerIndex)
+	if err != nil {
+		return err
+	}
+	if slashing.Header_1 == nil || slashing.Header_1.Header == nil || slashing.Header_2 == nil || slashing.Header_2.Header == nil {
+		return errors.New("nil header cannot be verified")
+	}
 	if slashing.Header_1.Header.Slot != slashing.Header_2.Header.Slot {
 		return fmt.Errorf("mismatched header slots, received %d == %d", slashing.Header_1.Header.Slot, slashing.Header_2.Header.Slot)
 	}
 	if proto.Equal(slashing.Header_1, slashing.Header_2) {
 		return errors.New("expected slashing headers to differ")
 	}
-	if !helpers.IsSlashableValidator(proposer, helpers.CurrentEpoch(beaconState)) {
+	if !helpers.IsSlashableValidator(proposer, helpers.SlotToEpoch(beaconState.Slot())) {
 		return fmt.Errorf("validator with key %#x is not slashable", proposer.PublicKey)
 	}
 	// Using headerEpoch1 here because both of the headers should have the same epoch.
-	domain := helpers.Domain(beaconState.Fork, helpers.StartSlot(slashing.Header_1.Header.Slot), params.BeaconConfig().DomainBeaconProposer)
+	domain, err := helpers.Domain(beaconState.Fork(), helpers.StartSlot(slashing.Header_1.Header.Slot), params.BeaconConfig().DomainBeaconProposer)
+	if err != nil {
+		return err
+	}
 	headers := []*ethpb.SignedBeaconBlockHeader{slashing.Header_1, slashing.Header_2}
 	for _, header := range headers {
 		if err := verifySigningRoot(header.Header, proposer.PublicKey, header.Signature, domain); err != nil {
@@ -429,7 +501,11 @@ func VerifyProposerSlashing(
 //            slash_validator(state, index)
 //            slashed_any = True
 //    assert slashed_any
-func ProcessAttesterSlashings(ctx context.Context, beaconState *pb.BeaconState, body *ethpb.BeaconBlockBody) (*pb.BeaconState, error) {
+func ProcessAttesterSlashings(
+	ctx context.Context,
+	beaconState *stateTrie.BeaconState,
+	body *ethpb.BeaconBlockBody,
+) (*stateTrie.BeaconState, error) {
 	for idx, slashing := range body.AttesterSlashings {
 		if err := VerifyAttesterSlashing(ctx, beaconState, slashing); err != nil {
 			return nil, errors.Wrapf(err, "could not verify attester slashing %d", idx)
@@ -438,11 +514,16 @@ func ProcessAttesterSlashings(ctx context.Context, beaconState *pb.BeaconState,
 		sort.SliceStable(slashableIndices, func(i, j int) bool {
 			return slashableIndices[i] < slashableIndices[j]
 		})
-		currentEpoch := helpers.CurrentEpoch(beaconState)
+		currentEpoch := helpers.SlotToEpoch(beaconState.Slot())
 		var err error
 		var slashedAny bool
+		var val *ethpb.Validator
 		for _, validatorIndex := range slashableIndices {
-			if helpers.IsSlashableValidator(beaconState.Validators[validatorIndex], currentEpoch) {
+			val, err = beaconState.ValidatorAtIndex(validatorIndex)
+			if err != nil {
+				return nil, err
+			}
+			if helpers.IsSlashableValidator(val, currentEpoch) {
 				beaconState, err = v.SlashValidator(beaconState, validatorIndex, 0)
 				if err != nil {
 					return nil, errors.Wrapf(err, "could not slash validator index %d",
@@ -459,7 +540,16 @@ func ProcessAttesterSlashings(ctx context.Context, beaconState *pb.BeaconState,
 }
 
 // VerifyAttesterSlashing validates the attestation data in both attestations in the slashing object.
-func VerifyAttesterSlashing(ctx context.Context, beaconState *pb.BeaconState, slashing *ethpb.AttesterSlashing) error {
+func VerifyAttesterSlashing(ctx context.Context, beaconState *stateTrie.BeaconState, slashing *ethpb.AttesterSlashing) error {
+	if slashing == nil {
+		return errors.New("nil slashing")
+	}
+	if slashing.Attestation_1 == nil || slashing.Attestation_2 == nil {
+		return errors.New("nil attestation")
+	}
+	if slashing.Attestation_1.Data == nil || slashing.Attestation_2.Data == nil {
+		return errors.New("nil attestation data")
+	}
 	att1 := slashing.Attestation_1
 	att2 := slashing.Attestation_2
 	data1 := att1.Data
@@ -490,12 +580,18 @@ func VerifyAttesterSlashing(ctx context.Context, beaconState *pb.BeaconState, sl
 //        (data_1.source.epoch < data_2.source.epoch and data_2.target.epoch < data_1.target.epoch)
 //    )
 func IsSlashableAttestationData(data1 *ethpb.AttestationData, data2 *ethpb.AttestationData) bool {
+	if data1 == nil || data2 == nil || data1.Target == nil || data2.Target == nil || data1.Source == nil || data2.Source == nil {
+		return false
+	}
 	isDoubleVote := !proto.Equal(data1, data2) && data1.Target.Epoch == data2.Target.Epoch
 	isSurroundVote := data1.Source.Epoch < data2.Source.Epoch && data2.Target.Epoch < data1.Target.Epoch
 	return isDoubleVote || isSurroundVote
 }
 
 func slashableAttesterIndices(slashing *ethpb.AttesterSlashing) []uint64 {
+	if slashing == nil || slashing.Attestation_1 == nil || slashing.Attestation_2 == nil {
+		return nil
+	}
 	indices1 := slashing.Attestation_1.AttestingIndices
 	indices2 := slashing.Attestation_1.AttestingIndices
 	return sliceutil.IntersectionUint64(indices1, indices2)
@@ -504,7 +600,11 @@ func slashableAttesterIndices(slashing *ethpb.AttesterSlashing) []uint64 {
 // ProcessAttestations applies processing operations to a block's inner attestation
 // records. This function returns a list of pending attestations which can then be
 // appended to the BeaconState's latest attestations.
-func ProcessAttestations(ctx context.Context, beaconState *pb.BeaconState, body *ethpb.BeaconBlockBody) (*pb.BeaconState, error) {
+func ProcessAttestations(
+	ctx context.Context,
+	beaconState *stateTrie.BeaconState,
+	body *ethpb.BeaconBlockBody,
+) (*stateTrie.BeaconState, error) {
 	var err error
 	for idx, attestation := range body.Attestations {
 		beaconState, err = ProcessAttestation(ctx, beaconState, attestation)
@@ -517,7 +617,11 @@ func ProcessAttestations(ctx context.Context, beaconState *pb.BeaconState, body
 
 // ProcessAttestationsNoVerify applies processing operations to a block's inner attestation
 // records. The only difference would be that the attestation signature would not be verified.
-func ProcessAttestationsNoVerify(ctx context.Context, beaconState *pb.BeaconState, body *ethpb.BeaconBlockBody) (*pb.BeaconState, error) {
+func ProcessAttestationsNoVerify(
+	ctx context.Context,
+	beaconState *stateTrie.BeaconState,
+	body *ethpb.BeaconBlockBody,
+) (*stateTrie.BeaconState, error) {
 	var err error
 	for idx, attestation := range body.Attestations {
 		beaconState, err = ProcessAttestationNoVerify(ctx, beaconState, attestation)
@@ -557,7 +661,11 @@ func ProcessAttestationsNoVerify(ctx context.Context, beaconState *pb.BeaconStat
 //
 //    # Check signature
 //    assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation))
-func ProcessAttestation(ctx context.Context, beaconState *pb.BeaconState, att *ethpb.Attestation) (*pb.BeaconState, error) {
+func ProcessAttestation(
+	ctx context.Context,
+	beaconState *stateTrie.BeaconState,
+	att *ethpb.Attestation,
+) (*stateTrie.BeaconState, error) {
 	beaconState, err := ProcessAttestationNoVerify(ctx, beaconState, att)
 	if err != nil {
 		return nil, err
@@ -567,7 +675,11 @@ func ProcessAttestation(ctx context.Context, beaconState *pb.BeaconState, att *e
 
 // ProcessAttestationNoVerify processes the attestation without verifying the attestation signature. This
 // method is used to validate attestations whose signatures have already been verified.
-func ProcessAttestationNoVerify(ctx context.Context, beaconState *pb.BeaconState, att *ethpb.Attestation) (*pb.BeaconState, error) {
+func ProcessAttestationNoVerify(
+	ctx context.Context,
+	beaconState *stateTrie.BeaconState,
+	att *ethpb.Attestation,
+) (*stateTrie.BeaconState, error) {
 	ctx, span := trace.StartSpan(ctx, "core.ProcessAttestationNoVerify")
 	defer span.End()
 
@@ -575,13 +687,20 @@ func ProcessAttestationNoVerify(ctx context.Context, beaconState *pb.BeaconState
 		return nil, errors.New("nil attestation data target")
 	}
 
+	currEpoch := helpers.SlotToEpoch(beaconState.Slot())
+	var prevEpoch uint64
+	if currEpoch == 0 {
+		prevEpoch = 0
+	} else {
+		prevEpoch = currEpoch - 1
+	}
 	data := att.Data
-	if data.Target.Epoch != helpers.PrevEpoch(beaconState) && data.Target.Epoch != helpers.CurrentEpoch(beaconState) {
+	if data.Target.Epoch != prevEpoch && data.Target.Epoch != currEpoch {
 		return nil, fmt.Errorf(
 			"expected target epoch (%d) to be the previous epoch (%d) or the current epoch (%d)",
 			data.Target.Epoch,
-			helpers.PrevEpoch(beaconState),
-			helpers.CurrentEpoch(beaconState),
+			prevEpoch,
+			currEpoch,
 		)
 	}
 	if helpers.SlotToEpoch(data.Slot) != data.Target.Epoch {
@@ -589,20 +708,20 @@ func ProcessAttestationNoVerify(ctx context.Context, beaconState *pb.BeaconState
 	}
 
 	s := att.Data.Slot
-	minInclusionCheck := s+params.BeaconConfig().MinAttestationInclusionDelay <= beaconState.Slot
-	epochInclusionCheck := beaconState.Slot <= s+params.BeaconConfig().SlotsPerEpoch
+	minInclusionCheck := s+params.BeaconConfig().MinAttestationInclusionDelay <= beaconState.Slot()
+	epochInclusionCheck := beaconState.Slot() <= s+params.BeaconConfig().SlotsPerEpoch
 	if !minInclusionCheck {
 		return nil, fmt.Errorf(
 			"attestation slot %d + inclusion delay %d > state slot %d",
 			s,
 			params.BeaconConfig().MinAttestationInclusionDelay,
-			beaconState.Slot,
+			beaconState.Slot(),
 		)
 	}
 	if !epochInclusionCheck {
 		return nil, fmt.Errorf(
 			"state slot %d > attestation slot %d + SLOTS_PER_EPOCH %d",
-			beaconState.Slot,
+			beaconState.Slot(),
 			s,
 			params.BeaconConfig().SlotsPerEpoch,
 		)
@@ -619,23 +738,27 @@ func ProcessAttestationNoVerify(ctx context.Context, beaconState *pb.BeaconState
 	pendingAtt := &pb.PendingAttestation{
 		Data:            data,
 		AggregationBits: att.AggregationBits,
-		InclusionDelay:  beaconState.Slot - s,
+		InclusionDelay:  beaconState.Slot() - s,
 		ProposerIndex:   proposerIndex,
 	}
 
 	var ffgSourceEpoch uint64
 	var ffgSourceRoot []byte
 	var ffgTargetEpoch uint64
-	if data.Target.Epoch == helpers.CurrentEpoch(beaconState) {
-		ffgSourceEpoch = beaconState.CurrentJustifiedCheckpoint.Epoch
-		ffgSourceRoot = beaconState.CurrentJustifiedCheckpoint.Root
-		ffgTargetEpoch = helpers.CurrentEpoch(beaconState)
-		beaconState.CurrentEpochAttestations = append(beaconState.CurrentEpochAttestations, pendingAtt)
+	if data.Target.Epoch == currEpoch {
+		ffgSourceEpoch = beaconState.CurrentJustifiedCheckpoint().Epoch
+		ffgSourceRoot = beaconState.CurrentJustifiedCheckpoint().Root
+		ffgTargetEpoch = currEpoch
+		if err := beaconState.AppendCurrentEpochAttestations(pendingAtt); err != nil {
+			return nil, err
+		}
 	} else {
-		ffgSourceEpoch = beaconState.PreviousJustifiedCheckpoint.Epoch
-		ffgSourceRoot = beaconState.PreviousJustifiedCheckpoint.Root
-		ffgTargetEpoch = helpers.PrevEpoch(beaconState)
-		beaconState.PreviousEpochAttestations = append(beaconState.PreviousEpochAttestations, pendingAtt)
+		ffgSourceEpoch = beaconState.PreviousJustifiedCheckpoint().Epoch
+		ffgSourceRoot = beaconState.PreviousJustifiedCheckpoint().Root
+		ffgTargetEpoch = prevEpoch
+		if err := beaconState.AppendPreviousEpochAttestations(pendingAtt); err != nil {
+			return nil, err
+		}
 	}
 	if data.Source.Epoch != ffgSourceEpoch {
 		return nil, fmt.Errorf("expected source epoch %d, received %d", ffgSourceEpoch, data.Source.Epoch)
@@ -650,44 +773,6 @@ func ProcessAttestationNoVerify(ctx context.Context, beaconState *pb.BeaconState
 	return beaconState, nil
 }
 
-// ConvertToIndexed converts attestation to (almost) indexed-verifiable form.
-//
-// Note about spec pseudocode definition. The state was used by get_attesting_indices to determine
-// the attestation committee. Now that we provide this as an argument, we no longer need to provide
-// a state.
-//
-// Spec pseudocode definition:
-//   def get_indexed_attestation(state: BeaconState, attestation: Attestation) -> IndexedAttestation:
-//    """
-//    Return the indexed attestation corresponding to ``attestation``.
-//    """
-//    attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bits)
-//
-//    return IndexedAttestation(
-//        attesting_indices=sorted(attesting_indices),
-//        data=attestation.data,
-//        signature=attestation.signature,
-//    )
-func ConvertToIndexed(ctx context.Context, attestation *ethpb.Attestation, committee []uint64) (*ethpb.IndexedAttestation, error) {
-	ctx, span := trace.StartSpan(ctx, "core.ConvertToIndexed")
-	defer span.End()
-
-	attIndices, err := helpers.AttestingIndices(attestation.AggregationBits, committee)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not get attesting indices")
-	}
-
-	sort.Slice(attIndices, func(i, j int) bool {
-		return attIndices[i] < attIndices[j]
-	})
-	inAtt := &ethpb.IndexedAttestation{
-		Data:             attestation.Data,
-		Signature:        attestation.Signature,
-		AttestingIndices: attIndices,
-	}
-	return inAtt, nil
-}
-
 // VerifyIndexedAttestation determines the validity of an indexed attestation.
 //
 // Spec pseudocode definition:
@@ -711,10 +796,12 @@ func ConvertToIndexed(ctx context.Context, attestation *ethpb.Attestation, commi
 //    ):
 //        return False
 //    return True
-func VerifyIndexedAttestation(ctx context.Context, beaconState *pb.BeaconState, indexedAtt *ethpb.IndexedAttestation) error {
+func VerifyIndexedAttestation(ctx context.Context, beaconState *stateTrie.BeaconState, indexedAtt *ethpb.IndexedAttestation) error {
 	ctx, span := trace.StartSpan(ctx, "core.VerifyIndexedAttestation")
 	defer span.End()
-
+	if indexedAtt == nil || indexedAtt.Data == nil || indexedAtt.Data.Target == nil {
+		return errors.New("nil or missing indexed attestation data")
+	}
 	indices := indexedAtt.AttestingIndices
 
 	if uint64(len(indices)) > params.BeaconConfig().MaxValidatorsPerCommittee {
@@ -737,26 +824,25 @@ func VerifyIndexedAttestation(ctx context.Context, beaconState *pb.BeaconState,
 		return errors.New("attesting indices is not uniquely sorted")
 	}
 
-	domain := helpers.Domain(beaconState.Fork, indexedAtt.Data.Target.Epoch, params.BeaconConfig().DomainBeaconAttester)
-	var pubkey *bls.PublicKey
-	var err error
+	domain, err := helpers.Domain(beaconState.Fork(), indexedAtt.Data.Target.Epoch, params.BeaconConfig().DomainBeaconAttester)
+	if err != nil {
+		return err
+	}
+	pubkeys := []*bls.PublicKey{}
 	if len(indices) > 0 {
-		pubkey, err = bls.PublicKeyFromBytes(beaconState.Validators[indices[0]].PublicKey)
-		if err != nil {
-			return errors.Wrap(err, "could not deserialize validator public key")
-		}
-		for _, i := range indices[1:] {
-			pk, err := bls.PublicKeyFromBytes(beaconState.Validators[i].PublicKey)
+		for i := 0; i < len(indices); i++ {
+			pubkeyAtIdx := beaconState.PubkeyAtIndex(indices[i])
+			pk, err := bls.PublicKeyFromBytes(pubkeyAtIdx[:])
 			if err != nil {
 				return errors.Wrap(err, "could not deserialize validator public key")
 			}
-			pubkey.Aggregate(pk)
+			pubkeys = append(pubkeys, pk)
 		}
 	}
 
-	messageHash, err := ssz.HashTreeRoot(indexedAtt.Data)
+	messageHash, err := helpers.ComputeSigningRoot(indexedAtt.Data, domain)
 	if err != nil {
-		return errors.Wrap(err, "could not tree hash att data")
+		return errors.Wrap(err, "could not get signing root of object")
 	}
 
 	sig, err := bls.SignatureFromBytes(indexedAtt.Signature)
@@ -765,7 +851,7 @@ func VerifyIndexedAttestation(ctx context.Context, beaconState *pb.BeaconState,
 	}
 
 	voted := len(indices) > 0
-	if voted && !sig.Verify(messageHash[:], pubkey, domain) {
+	if voted && !sig.FastAggregateVerify(pubkeys, messageHash) {
 		return ErrSigFailedToVerify
 	}
 	return nil
@@ -773,12 +859,15 @@ func VerifyIndexedAttestation(ctx context.Context, beaconState *pb.BeaconState,
 
 // VerifyAttestation converts and attestation into an indexed attestation and verifies
 // the signature in that attestation.
-func VerifyAttestation(ctx context.Context, beaconState *pb.BeaconState, att *ethpb.Attestation) error {
+func VerifyAttestation(ctx context.Context, beaconState *stateTrie.BeaconState, att *ethpb.Attestation) error {
+	if att == nil || att.Data == nil {
+		return fmt.Errorf("nil or missing attestation data: %v", att)
+	}
 	committee, err := helpers.BeaconCommitteeFromState(beaconState, att.Data.Slot, att.Data.CommitteeIndex)
 	if err != nil {
 		return err
 	}
-	indexedAtt, err := ConvertToIndexed(ctx, att, committee)
+	indexedAtt, err := attestationutil.ConvertToIndexed(ctx, att, committee)
 	if err != nil {
 		return errors.Wrap(err, "could not convert to indexed attestation")
 	}
@@ -792,13 +881,18 @@ func VerifyAttestation(ctx context.Context, beaconState *pb.BeaconState, att *et
 // Spec pseudocode definition:
 //   For each deposit in block.body.deposits:
 //     process_deposit(state, deposit)
-func ProcessDeposits(ctx context.Context, beaconState *pb.BeaconState, body *ethpb.BeaconBlockBody) (*pb.BeaconState, error) {
+func ProcessDeposits(
+	ctx context.Context,
+	beaconState *stateTrie.BeaconState,
+	body *ethpb.BeaconBlockBody,
+) (*stateTrie.BeaconState, error) {
 	var err error
 	deposits := body.Deposits
-
-	valIndexMap := stateutils.ValidatorIndexMap(beaconState)
 	for _, deposit := range deposits {
-		beaconState, err = ProcessDeposit(beaconState, deposit, valIndexMap)
+		if deposit == nil || deposit.Data == nil {
+			return nil, errors.New("got a nil deposit in block")
+		}
+		beaconState, err = ProcessDeposit(beaconState, deposit)
 		if err != nil {
 			return nil, errors.Wrapf(err, "could not process deposit from %#x", bytesutil.Trunc(deposit.Data.PublicKey))
 		}
@@ -807,24 +901,37 @@ func ProcessDeposits(ctx context.Context, beaconState *pb.BeaconState, body *eth
 }
 
 // ProcessPreGenesisDeposit processes a deposit for the beacon state before chainstart.
-func ProcessPreGenesisDeposit(ctx context.Context, beaconState *pb.BeaconState,
-	deposit *ethpb.Deposit, validatorIndices map[[48]byte]int) (*pb.BeaconState, error) {
+func ProcessPreGenesisDeposit(
+	ctx context.Context,
+	beaconState *stateTrie.BeaconState,
+	deposit *ethpb.Deposit,
+) (*stateTrie.BeaconState, error) {
 	var err error
-	beaconState, err = ProcessDeposit(beaconState, deposit, validatorIndices)
+	beaconState, err = ProcessDeposit(beaconState, deposit)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not process deposit")
 	}
 	pubkey := deposit.Data.PublicKey
-	index, ok := validatorIndices[bytesutil.ToBytes48(pubkey)]
+	index, ok := beaconState.ValidatorIndexByPubkey(bytesutil.ToBytes48(pubkey))
 	if !ok {
 		return beaconState, nil
 	}
-	balance := beaconState.Balances[index]
-	beaconState.Validators[index].EffectiveBalance = mathutil.Min(balance-balance%params.BeaconConfig().EffectiveBalanceIncrement, params.BeaconConfig().MaxEffectiveBalance)
-	if beaconState.Validators[index].EffectiveBalance ==
+	balance, err := beaconState.BalanceAtIndex(index)
+	if err != nil {
+		return nil, err
+	}
+	validator, err := beaconState.ValidatorAtIndex(index)
+	if err != nil {
+		return nil, err
+	}
+	validator.EffectiveBalance = mathutil.Min(balance-balance%params.BeaconConfig().EffectiveBalanceIncrement, params.BeaconConfig().MaxEffectiveBalance)
+	if validator.EffectiveBalance ==
 		params.BeaconConfig().MaxEffectiveBalance {
-		beaconState.Validators[index].ActivationEligibilityEpoch = 0
-		beaconState.Validators[index].ActivationEpoch = 0
+		validator.ActivationEligibilityEpoch = 0
+		validator.ActivationEpoch = 0
+	}
+	if err := beaconState.UpdateValidatorAtIndex(uint64(index), validator); err != nil {
+		return nil, err
 	}
 	return beaconState, nil
 }
@@ -872,16 +979,25 @@ func ProcessPreGenesisDeposit(ctx context.Context, beaconState *pb.BeaconState,
 //        # Increase balance by deposit amount
 //        index = ValidatorIndex(validator_pubkeys.index(pubkey))
 //        increase_balance(state, index, amount)
-func ProcessDeposit(beaconState *pb.BeaconState, deposit *ethpb.Deposit, valIndexMap map[[48]byte]int) (*pb.BeaconState, error) {
+func ProcessDeposit(
+	beaconState *stateTrie.BeaconState,
+	deposit *ethpb.Deposit,
+) (*stateTrie.BeaconState, error) {
 	if err := verifyDeposit(beaconState, deposit); err != nil {
+		if deposit == nil || deposit.Data == nil {
+			return nil, err
+		}
 		return nil, errors.Wrapf(err, "could not verify deposit from %#x", bytesutil.Trunc(deposit.Data.PublicKey))
 	}
-	beaconState.Eth1DepositIndex++
+	if err := beaconState.SetEth1DepositIndex(beaconState.Eth1DepositIndex() + 1); err != nil {
+		return nil, err
+	}
 	pubKey := deposit.Data.PublicKey
 	amount := deposit.Data.Amount
-	index, ok := valIndexMap[bytesutil.ToBytes48(pubKey)]
+	index, ok := beaconState.ValidatorIndexByPubkey(bytesutil.ToBytes48(pubKey))
+	numVals := beaconState.NumValidators()
 	if !ok {
-		domain := bls.ComputeDomain(params.BeaconConfig().DomainDeposit)
+		domain := bls.ComputeDomain(params.BeaconConfig().DomainDeposit, nil)
 		depositSig := deposit.Data.Signature
 		if err := verifyDepositDataSigningRoot(deposit.Data, pubKey, depositSig, domain); err != nil {
 			// Ignore this error as in the spec pseudo code.
@@ -893,7 +1009,7 @@ func ProcessDeposit(beaconState *pb.BeaconState, deposit *ethpb.Deposit, valInde
 		if params.BeaconConfig().MaxEffectiveBalance < effectiveBalance {
 			effectiveBalance = params.BeaconConfig().MaxEffectiveBalance
 		}
-		beaconState.Validators = append(beaconState.Validators, &ethpb.Validator{
+		if err := beaconState.AppendValidator(&ethpb.Validator{
 			PublicKey:                  pubKey,
 			WithdrawalCredentials:      deposit.Data.WithdrawalCredentials,
 			ActivationEligibilityEpoch: params.BeaconConfig().FarFutureEpoch,
@@ -901,27 +1017,42 @@ func ProcessDeposit(beaconState *pb.BeaconState, deposit *ethpb.Deposit, valInde
 			ExitEpoch:                  params.BeaconConfig().FarFutureEpoch,
 			WithdrawableEpoch:          params.BeaconConfig().FarFutureEpoch,
 			EffectiveBalance:           effectiveBalance,
-		})
-		beaconState.Balances = append(beaconState.Balances, amount)
-		valIndexMap[bytesutil.ToBytes48(pubKey)] = len(beaconState.Validators) - 1
+		}); err != nil {
+			return nil, err
+		}
+		if err := beaconState.AppendBalance(amount); err != nil {
+			return nil, err
+		}
+		numVals++
+		beaconState.SetValidatorIndexByPubkey(bytesutil.ToBytes48(pubKey), uint64(numVals-1))
 	} else {
-		beaconState = helpers.IncreaseBalance(beaconState, uint64(index), amount)
+		if err := helpers.IncreaseBalance(beaconState, uint64(index), amount); err != nil {
+			return nil, err
+		}
 	}
 
 	return beaconState, nil
 }
 
-func verifyDeposit(beaconState *pb.BeaconState, deposit *ethpb.Deposit) error {
+func verifyDeposit(beaconState *stateTrie.BeaconState, deposit *ethpb.Deposit) error {
 	// Verify Merkle proof of deposit and deposit trie root.
-	receiptRoot := beaconState.Eth1Data.DepositRoot
+	if deposit == nil || deposit.Data == nil {
+		return errors.New("received nil deposit or nil deposit data")
+	}
+	eth1Data := beaconState.Eth1Data()
+	if eth1Data == nil {
+		return errors.New("received nil eth1data in the beacon state")
+	}
+
+	receiptRoot := eth1Data.DepositRoot
 	leaf, err := ssz.HashTreeRoot(deposit.Data)
 	if err != nil {
 		return errors.Wrap(err, "could not tree hash deposit data")
 	}
-	if ok := trieutil.VerifyMerkleProof(
+	if ok := trieutil.VerifyMerkleBranch(
 		receiptRoot,
 		leaf[:],
-		int(beaconState.Eth1DepositIndex),
+		int(beaconState.Eth1DepositIndex()),
 		deposit.Proof,
 	); !ok {
 		return fmt.Errorf(
@@ -955,12 +1086,28 @@ func verifyDeposit(beaconState *pb.BeaconState, deposit *ethpb.Deposit) error {
 //    assert bls_verify(validator.pubkey, signing_root(exit), exit.signature, domain)
 //    # Initiate exit
 //    initiate_validator_exit(state, exit.validator_index)
-func ProcessVoluntaryExits(ctx context.Context, beaconState *pb.BeaconState, body *ethpb.BeaconBlockBody) (*pb.BeaconState, error) {
-	var err error
+func ProcessVoluntaryExits(
+	ctx context.Context,
+	beaconState *stateTrie.BeaconState,
+	body *ethpb.BeaconBlockBody,
+) (*stateTrie.BeaconState, error) {
 	exits := body.VoluntaryExits
-
 	for idx, exit := range exits {
-		if err := VerifyExit(beaconState, exit); err != nil {
+		if exit == nil || exit.Exit == nil {
+			return nil, errors.New("nil voluntary exit in block body")
+		}
+		if int(exit.Exit.ValidatorIndex) >= beaconState.NumValidators() {
+			return nil, fmt.Errorf(
+				"validator index out of bound %d > %d",
+				exit.Exit.ValidatorIndex,
+				beaconState.NumValidators(),
+			)
+		}
+		val, err := beaconState.ValidatorAtIndex(exit.Exit.ValidatorIndex)
+		if err != nil {
+			return nil, err
+		}
+		if err := VerifyExit(val, beaconState.Slot(), beaconState.Fork(), exit); err != nil {
 			return nil, errors.Wrapf(err, "could not verify exit %d", idx)
 		}
 		beaconState, err = v.InitiateValidatorExit(beaconState, exit.Exit.ValidatorIndex)
@@ -974,13 +1121,16 @@ func ProcessVoluntaryExits(ctx context.Context, beaconState *pb.BeaconState, bod
 // ProcessVoluntaryExitsNoVerify processes all the voluntary exits in
 // a block body, without verifying their BLS signatures.
 func ProcessVoluntaryExitsNoVerify(
-	beaconState *pb.BeaconState,
+	beaconState *stateTrie.BeaconState,
 	body *ethpb.BeaconBlockBody,
-) (*pb.BeaconState, error) {
+) (*stateTrie.BeaconState, error) {
 	var err error
 	exits := body.VoluntaryExits
 
 	for idx, exit := range exits {
+		if exit == nil || exit.Exit == nil {
+			return nil, errors.New("nil exit")
+		}
 		beaconState, err = v.InitiateValidatorExit(beaconState, exit.Exit.ValidatorIndex)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to process voluntary exit at index %d", idx)
@@ -1008,18 +1158,13 @@ func ProcessVoluntaryExitsNoVerify(
 //    # Verify signature
 //    domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, exit.epoch)
 //    assert bls_verify(validator.pubkey, signing_root(exit), exit.signature, domain)
-func VerifyExit(beaconState *pb.BeaconState, signed *ethpb.SignedVoluntaryExit) error {
+func VerifyExit(validator *ethpb.Validator, currentSlot uint64, fork *pb.Fork, signed *ethpb.SignedVoluntaryExit) error {
 	if signed == nil || signed.Exit == nil {
 		return errors.New("nil exit")
 	}
 
 	exit := signed.Exit
-	if int(exit.ValidatorIndex) >= len(beaconState.Validators) {
-		return fmt.Errorf("validator index out of bound %d > %d", exit.ValidatorIndex, len(beaconState.Validators))
-	}
-
-	validator := beaconState.Validators[exit.ValidatorIndex]
-	currentEpoch := helpers.CurrentEpoch(beaconState)
+	currentEpoch := helpers.SlotToEpoch(currentSlot)
 	// Verify the validator is active.
 	if !helpers.IsActiveValidator(validator, currentEpoch) {
 		return errors.New("non-active validator cannot exit")
@@ -1040,7 +1185,10 @@ func VerifyExit(beaconState *pb.BeaconState, signed *ethpb.SignedVoluntaryExit)
 			validator.ActivationEpoch+params.BeaconConfig().PersistentCommitteePeriod,
 		)
 	}
-	domain := helpers.Domain(beaconState.Fork, exit.Epoch, params.BeaconConfig().DomainVoluntaryExit)
+	domain, err := helpers.Domain(fork, exit.Epoch, params.BeaconConfig().DomainVoluntaryExit)
+	if err != nil {
+		return err
+	}
 	if err := verifySigningRoot(exit, validator.PublicKey, signed.Signature, domain); err != nil {
 		return ErrSigFailedToVerify
 	}

beacon-chain/core/blocks/block_operations_fuzz_test.go

@@ -1,16 +1,20 @@
-package blocks_test
+package blocks
 
 import (
 	"context"
 	"testing"
 
-	fuzz "github.com/google/gofuzz"
 	eth "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+
+	fuzz "github.com/google/gofuzz"
+	//"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	ethereum_beacon_p2p_v1 "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 )
 
-func TestFuzzProcessAttestation_10000(t *testing.T) {
+func TestFuzzProcessAttestationNoVerify_10000(t *testing.T) {
 	fuzzer := fuzz.NewWithSeed(0)
 	ctx := context.Background()
 	state := &ethereum_beacon_p2p_v1.BeaconState{}
@@ -19,7 +23,8 @@ func TestFuzzProcessAttestation_10000(t *testing.T) {
 	for i := 0; i < 10000; i++ {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(att)
-		_, _ = blocks.ProcessAttestationNoVerify(ctx, state, att)
+		s, _ := beaconstate.InitializeFromProtoUnsafe(state)
+		_, _ = ProcessAttestationNoVerify(ctx, s, att)
 	}
 }
 
@@ -31,6 +36,402 @@ func TestFuzzProcessBlockHeader_10000(t *testing.T) {
 	for i := 0; i < 10000; i++ {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(block)
-		_, _ = blocks.ProcessBlockHeader(state, block)
+
+		s, _ := beaconstate.InitializeFromProtoUnsafe(state)
+		_, _ = ProcessBlockHeader(s, block)
+	}
+}
+
+func TestFuzzverifySigningRoot_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	pubkey := [48]byte{}
+	sig := [96]byte{}
+	domain := [4]byte{}
+	p := []byte{}
+	s := []byte{}
+	d := []byte{}
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(&pubkey)
+		fuzzer.Fuzz(&sig)
+		fuzzer.Fuzz(&domain)
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(&p)
+		fuzzer.Fuzz(&s)
+		fuzzer.Fuzz(&d)
+		verifySigningRoot(state, pubkey[:], sig[:], domain[:])
+		verifySigningRoot(state, p, s, d)
+
+	}
+}
+
+func TestFuzzverifyDepositDataSigningRoot_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	ba := []byte{}
+	pubkey := [48]byte{}
+	sig := [96]byte{}
+	domain := [4]byte{}
+	p := []byte{}
+	s := []byte{}
+	d := []byte{}
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(&ba)
+		fuzzer.Fuzz(&pubkey)
+		fuzzer.Fuzz(&sig)
+		fuzzer.Fuzz(&domain)
+		fuzzer.Fuzz(&p)
+		fuzzer.Fuzz(&s)
+		fuzzer.Fuzz(&d)
+		verifySignature(ba, pubkey[:], sig[:], domain[:])
+		verifySignature(ba, p, s, d)
+	}
+}
+
+func TestFuzzProcessEth1DataInBlock_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	block := &eth.BeaconBlock{}
+	state := &stateTrie.BeaconState{}
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(block)
+		s, err := ProcessEth1DataInBlock(state, block)
+		if err != nil && s != nil {
+			t.Fatalf("state should be nil on err. found: %v on error: %v for state: %v and block: %v", s, err, state, block)
+		}
+	}
+}
+
+func TestFuzzareEth1DataEqual_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	eth1data := &eth.Eth1Data{}
+	eth1data2 := &eth.Eth1Data{}
+
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(eth1data)
+		fuzzer.Fuzz(eth1data2)
+		areEth1DataEqual(eth1data, eth1data2)
+		areEth1DataEqual(eth1data, eth1data)
+	}
+}
+
+func TestFuzzEth1DataHasEnoughSupport_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	eth1data := &eth.Eth1Data{}
+	stateVotes := []*eth.Eth1Data{}
+	for i := 0; i < 100000; i++ {
+		fuzzer.Fuzz(eth1data)
+		fuzzer.Fuzz(&stateVotes)
+		s, _ := beaconstate.InitializeFromProto(&ethereum_beacon_p2p_v1.BeaconState{
+			Eth1DataVotes: stateVotes,
+		})
+		Eth1DataHasEnoughSupport(s, eth1data)
+	}
+
+}
+
+func TestFuzzProcessBlockHeaderNoVerify_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	block := &eth.BeaconBlock{}
+
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(block)
+		s, _ := beaconstate.InitializeFromProtoUnsafe(state)
+		_, _ = ProcessBlockHeaderNoVerify(s, block)
+	}
+}
+
+func TestFuzzProcessRandao_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	blockBody := &eth.BeaconBlockBody{}
+
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(blockBody)
+		s, _ := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessRandao(s, blockBody)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
+		}
+	}
+}
+
+func TestFuzzProcessRandaoNoVerify_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	blockBody := &eth.BeaconBlockBody{}
+
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(blockBody)
+		s, _ := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessRandaoNoVerify(s, blockBody)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
+		}
+	}
+}
+
+func TestFuzzProcessProposerSlashings_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	blockBody := &eth.BeaconBlockBody{}
+	ctx := context.Background()
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(blockBody)
+		s, _ := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessProposerSlashings(ctx, s, blockBody)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
+		}
+	}
+}
+
+func TestFuzzVerifyProposerSlashing_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	proposerSlashing := &eth.ProposerSlashing{}
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(proposerSlashing)
+		s, _ := beaconstate.InitializeFromProtoUnsafe(state)
+		VerifyProposerSlashing(s, proposerSlashing)
+	}
+}
+
+func TestFuzzProcessAttesterSlashings_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	blockBody := &eth.BeaconBlockBody{}
+	ctx := context.Background()
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(blockBody)
+		s, _ := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessAttesterSlashings(ctx, s, blockBody)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
+		}
+	}
+}
+
+func TestFuzzVerifyAttesterSlashing_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	attesterSlashing := &eth.AttesterSlashing{}
+	ctx := context.Background()
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(attesterSlashing)
+		s, _ := beaconstate.InitializeFromProtoUnsafe(state)
+		VerifyAttesterSlashing(ctx, s, attesterSlashing)
+	}
+}
+
+func TestFuzzIsSlashableAttestationData_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	attestationData := &eth.AttestationData{}
+	attestationData2 := &eth.AttestationData{}
+
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(attestationData)
+		fuzzer.Fuzz(attestationData2)
+		IsSlashableAttestationData(attestationData, attestationData2)
+	}
+}
+
+func TestFuzzslashableAttesterIndices_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	attesterSlashing := &eth.AttesterSlashing{}
+
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(attesterSlashing)
+		slashableAttesterIndices(attesterSlashing)
+	}
+}
+
+func TestFuzzProcessAttestations_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	blockBody := &eth.BeaconBlockBody{}
+	ctx := context.Background()
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(blockBody)
+		s, _ := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessAttestations(ctx, s, blockBody)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
+		}
+	}
+}
+
+func TestFuzzProcessAttestationsNoVerify_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	blockBody := &eth.BeaconBlockBody{}
+	ctx := context.Background()
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(blockBody)
+		s, _ := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessAttestationsNoVerify(ctx, s, blockBody)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
+		}
+	}
+}
+
+func TestFuzzProcessAttestation_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	attestation := &eth.Attestation{}
+	ctx := context.Background()
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(attestation)
+		s, _ := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessAttestation(ctx, s, attestation)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, attestation)
+		}
+	}
+}
+
+func TestFuzzVerifyIndexedAttestationn_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	idxAttestation := &eth.IndexedAttestation{}
+	ctx := context.Background()
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(idxAttestation)
+		s, _ := beaconstate.InitializeFromProtoUnsafe(state)
+		VerifyIndexedAttestation(ctx, s, idxAttestation)
+	}
+}
+
+func TestFuzzVerifyAttestation_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	attestation := &eth.Attestation{}
+	ctx := context.Background()
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(attestation)
+		s, _ := beaconstate.InitializeFromProtoUnsafe(state)
+		VerifyAttestation(ctx, s, attestation)
+	}
+}
+
+func TestFuzzProcessDeposits_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	blockBody := &eth.BeaconBlockBody{}
+	ctx := context.Background()
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(blockBody)
+		s, _ := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessDeposits(ctx, s, blockBody)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
+		}
+	}
+}
+
+func TestFuzzProcessPreGenesisDeposit_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	deposit := &eth.Deposit{}
+	ctx := context.Background()
+
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(deposit)
+		s, _ := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessPreGenesisDeposit(ctx, s, deposit)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, deposit)
+		}
+	}
+}
+
+func TestFuzzProcessDeposit_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	deposit := &eth.Deposit{}
+
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(deposit)
+		s, _ := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessDeposit(s, deposit)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, deposit)
+		}
+	}
+}
+
+func TestFuzzverifyDeposit_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	deposit := &eth.Deposit{}
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(deposit)
+		s, _ := beaconstate.InitializeFromProtoUnsafe(state)
+		verifyDeposit(s, deposit)
+	}
+}
+
+func TestFuzzProcessVoluntaryExits_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	blockBody := &eth.BeaconBlockBody{}
+	ctx := context.Background()
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(blockBody)
+		s, _ := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessVoluntaryExits(ctx, s, blockBody)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
+		}
+	}
+}
+
+func TestFuzzProcessVoluntaryExitsNoVerify_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	blockBody := &eth.BeaconBlockBody{}
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(state)
+		fuzzer.Fuzz(blockBody)
+		s, _ := beaconstate.InitializeFromProtoUnsafe(state)
+		r, err := ProcessVoluntaryExitsNoVerify(s, blockBody)
+		if err != nil && r != nil {
+			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
+		}
+	}
+}
+
+func TestFuzzVerifyExit_10000(t *testing.T) {
+	fuzzer := fuzz.NewWithSeed(0)
+	ve := &eth.SignedVoluntaryExit{}
+	val := &eth.Validator{}
+	fork := &pb.Fork{}
+	var slot uint64
+
+	for i := 0; i < 10000; i++ {
+		fuzzer.Fuzz(ve)
+		fuzzer.Fuzz(val)
+		fuzzer.Fuzz(fork)
+		fuzzer.Fuzz(&slot)
+		VerifyExit(val, slot, fork, ve)
 	}
 }

beacon-chain/core/blocks/eth1_data_test.go

@@ -6,6 +6,7 @@ import (
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
@@ -92,9 +93,9 @@ func TestEth1DataHasEnoughSupport(t *testing.T) {
 			c.SlotsPerEth1VotingPeriod = tt.votingPeriodLength
 			params.OverrideBeaconConfig(c)
 
-			s := &pb.BeaconState{
+			s, _ := beaconstate.InitializeFromProto(&pb.BeaconState{
 				Eth1DataVotes: tt.stateVotes,
-			}
+			})
 			result, err := blocks.Eth1DataHasEnoughSupport(s, tt.data)
 			if err != nil {
 				t.Fatal(err)

beacon-chain/core/blocks/spectest/BUILD.bazel

@@ -35,6 +35,7 @@ go_test(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/state:go_default_library",
         "//beacon-chain/core/state/stateutils:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/params/spectest:go_default_library",
         "//shared/testutil:go_default_library",
@@ -68,6 +69,7 @@ go_test(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/state:go_default_library",
         "//beacon-chain/core/state/stateutils:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/params/spectest:go_default_library",
         "//shared/testutil:go_default_library",

beacon-chain/core/blocks/spectest/block_header_test.go

@@ -11,6 +11,7 @@ import (
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params/spectest"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
@@ -38,10 +39,14 @@ func runBlockHeaderTest(t *testing.T, config string) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			preBeaconState := &pb.BeaconState{}
-			if err := ssz.Unmarshal(preBeaconStateFile, preBeaconState); err != nil {
+			preBeaconStateBase := &pb.BeaconState{}
+			if err := ssz.Unmarshal(preBeaconStateFile, preBeaconStateBase); err != nil {
 				t.Fatalf("Failed to unmarshal: %v", err)
 			}
+			preBeaconState, err := beaconstate.InitializeFromProto(preBeaconStateBase)
+			if err != nil {
+				t.Fatal(err)
+			}
 
 			// If the post.ssz is not present, it means the test should fail on our end.
 			postSSZFilepath, err := bazel.Runfile(path.Join(testsFolderPath, folder.Name(), "post.ssz"))
@@ -68,9 +73,8 @@ func runBlockHeaderTest(t *testing.T, config string) {
 				if err := ssz.Unmarshal(postBeaconStateFile, postBeaconState); err != nil {
 					t.Fatalf("Failed to unmarshal: %v", err)
 				}
-
-				if !proto.Equal(beaconState, postBeaconState) {
-					diff, _ := messagediff.PrettyDiff(beaconState, postBeaconState)
+				if !proto.Equal(beaconState.CloneInnerState(), postBeaconState) {
+					diff, _ := messagediff.PrettyDiff(beaconState.CloneInnerState(), postBeaconState)
 					t.Log(diff)
 					t.Fatal("Post state does not match expected")
 				}

beacon-chain/core/blocks/spectest/block_processing_test.go

@@ -12,7 +12,9 @@ import (
 	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/go-ssz"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
+	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params/spectest"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
@@ -27,14 +29,19 @@ func runBlockProcessingTest(t *testing.T, config string) {
 	testFolders, testsFolderPath := testutil.TestFolders(t, config, "sanity/blocks/pyspec_tests")
 	for _, folder := range testFolders {
 		t.Run(folder.Name(), func(t *testing.T) {
+			helpers.ClearCache()
 			preBeaconStateFile, err := testutil.BazelFileBytes(testsFolderPath, folder.Name(), "pre.ssz")
 			if err != nil {
 				t.Fatal(err)
 			}
-			beaconState := &pb.BeaconState{}
-			if err := ssz.Unmarshal(preBeaconStateFile, beaconState); err != nil {
+			beaconStateBase := &pb.BeaconState{}
+			if err := ssz.Unmarshal(preBeaconStateFile, beaconStateBase); err != nil {
 				t.Fatalf("Failed to unmarshal: %v", err)
 			}
+			beaconState, err := beaconstate.InitializeFromProto(beaconStateBase)
+			if err != nil {
+				t.Fatal(err)
+			}
 
 			file, err := testutil.BazelFileBytes(testsFolderPath, folder.Name(), "meta.yaml")
 			if err != nil {
@@ -87,8 +94,8 @@ func runBlockProcessingTest(t *testing.T, config string) {
 					t.Fatalf("Failed to unmarshal: %v", err)
 				}
 
-				if !proto.Equal(beaconState, postBeaconState) {
-					diff, _ := messagediff.PrettyDiff(beaconState, postBeaconState)
+				if !proto.Equal(beaconState.InnerStateUnsafe(), postBeaconState) {
+					diff, _ := messagediff.PrettyDiff(beaconState.InnerStateUnsafe(), postBeaconState)
 					t.Log(diff)
 					t.Fatal("Post state does not match expected")
 				}

beacon-chain/core/epoch/BUILD.bazel

@@ -8,7 +8,9 @@ go_library(
     deps = [
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/validators:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
+        "//shared/attestationutil:go_default_library",
         "//shared/mathutil:go_default_library",
         "//shared/params:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
@@ -27,6 +29,7 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//beacon-chain/core/helpers:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/params:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",

beacon-chain/core/epoch/epoch_processing.go

@@ -13,24 +13,27 @@ import (
 	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/validators"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/attestationutil"
 	"github.com/prysmaticlabs/prysm/shared/mathutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
-var epochState *pb.BeaconState
-
 // sortableIndices implements the Sort interface to sort newly activated validator indices
 // by activation epoch and by index number.
-type sortableIndices []uint64
+type sortableIndices struct {
+	indices    []uint64
+	validators []*ethpb.Validator
+}
 
-func (s sortableIndices) Len() int      { return len(s) }
-func (s sortableIndices) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s sortableIndices) Len() int      { return len(s.indices) }
+func (s sortableIndices) Swap(i, j int) { s.indices[i], s.indices[j] = s.indices[j], s.indices[i] }
 func (s sortableIndices) Less(i, j int) bool {
-	if epochState.Validators[s[i]].ActivationEligibilityEpoch == epochState.Validators[s[j]].ActivationEligibilityEpoch {
-		return s[i] < s[j]
+	if s.validators[s.indices[i]].ActivationEligibilityEpoch == s.validators[s.indices[j]].ActivationEligibilityEpoch {
+		return s.indices[i] < s.indices[j]
 	}
-	return epochState.Validators[s[i]].ActivationEligibilityEpoch < epochState.Validators[s[j]].ActivationEligibilityEpoch
+	return s.validators[s.indices[i]].ActivationEligibilityEpoch < s.validators[s.indices[j]].ActivationEligibilityEpoch
 }
 
 // AttestingBalance returns the total balance from all the attesting indices.
@@ -42,7 +45,7 @@ func (s sortableIndices) Less(i, j int) bool {
 // Spec pseudocode definition:
 //  def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestation]) -> Gwei:
 //    return get_total_balance(state, get_unslashed_attesting_indices(state, attestations))
-func AttestingBalance(state *pb.BeaconState, atts []*pb.PendingAttestation) (uint64, error) {
+func AttestingBalance(state *stateTrie.BeaconState, atts []*pb.PendingAttestation) (uint64, error) {
 	indices, err := unslashedAttestingIndices(state, atts)
 	if err != nil {
 		return 0, errors.Wrap(err, "could not get attesting indices")
@@ -73,14 +76,17 @@ func AttestingBalance(state *pb.BeaconState, atts []*pb.PendingAttestation) (uin
 //    for index in activation_queue[:get_validator_churn_limit(state)]:
 //        validator = state.validators[index]
 //        validator.activation_epoch = compute_activation_exit_epoch(get_current_epoch(state))
-func ProcessRegistryUpdates(state *pb.BeaconState) (*pb.BeaconState, error) {
+func ProcessRegistryUpdates(state *stateTrie.BeaconState) (*stateTrie.BeaconState, error) {
 	currentEpoch := helpers.CurrentEpoch(state)
-
+	vals := state.Validators()
 	var err error
-	for idx, validator := range state.Validators {
+	for idx, validator := range vals {
 		// Process the validators for activation eligibility.
 		if helpers.IsEligibleForActivationQueue(validator) {
 			validator.ActivationEligibilityEpoch = helpers.CurrentEpoch(state) + 1
+			if err := state.UpdateValidatorAtIndex(uint64(idx), validator); err != nil {
+				return nil, err
+			}
 		}
 
 		// Process the validators for ejection.
@@ -96,14 +102,13 @@ func ProcessRegistryUpdates(state *pb.BeaconState) (*pb.BeaconState, error) {
 
 	// Queue validators eligible for activation and not yet dequeued for activation.
 	var activationQ []uint64
-	for idx, validator := range state.Validators {
+	for idx, validator := range vals {
 		if helpers.IsEligibleForActivation(state, validator) {
 			activationQ = append(activationQ, uint64(idx))
 		}
 	}
 
-	epochState = state
-	sort.Sort(sortableIndices(activationQ))
+	sort.Sort(sortableIndices{indices: activationQ, validators: vals})
 
 	// Only activate just enough validators according to the activation churn limit.
 	limit := len(activationQ)
@@ -123,10 +128,15 @@ func ProcessRegistryUpdates(state *pb.BeaconState) (*pb.BeaconState, error) {
 	}
 
 	for _, index := range activationQ[:limit] {
-		validator := state.Validators[index]
-		validator.ActivationEpoch = helpers.DelayedActivationExitEpoch(currentEpoch)
+		validator, err := state.ValidatorAtIndex(index)
+		if err != nil {
+			return nil, err
+		}
+		validator.ActivationEpoch = helpers.ActivationExitEpoch(currentEpoch)
+		if err := state.UpdateValidatorAtIndex(index, validator); err != nil {
+			return nil, err
+		}
 	}
-
 	return state, nil
 }
 
@@ -141,7 +151,7 @@ func ProcessRegistryUpdates(state *pb.BeaconState) (*pb.BeaconState, error) {
 //			  penalty_numerator = validator.effective_balance // increment * min(sum(state.slashings) * 3, total_balance)
 //            penalty = penalty_numerator // total_balance * increment
 //            decrease_balance(state, ValidatorIndex(index), penalty)
-func ProcessSlashings(state *pb.BeaconState) (*pb.BeaconState, error) {
+func ProcessSlashings(state *stateTrie.BeaconState) (*stateTrie.BeaconState, error) {
 	currentEpoch := helpers.CurrentEpoch(state)
 	totalBalance, err := helpers.TotalActiveBalance(state)
 	if err != nil {
@@ -152,22 +162,27 @@ func ProcessSlashings(state *pb.BeaconState) (*pb.BeaconState, error) {
 	exitLength := params.BeaconConfig().EpochsPerSlashingsVector
 
 	// Compute the sum of state slashings
+	slashings := state.Slashings()
 	totalSlashing := uint64(0)
-	for _, slashing := range state.Slashings {
+	for _, slashing := range slashings {
 		totalSlashing += slashing
 	}
 
-	// Compute slashing for each validator.
-	for index, validator := range state.Validators {
-		correctEpoch := (currentEpoch + exitLength/2) == validator.WithdrawableEpoch
-		if validator.Slashed && correctEpoch {
+	// a callback is used here to apply the following actions  to all validators
+	// below equally.
+	err = state.ApplyToEveryValidator(func(idx int, val *ethpb.Validator) error {
+		correctEpoch := (currentEpoch + exitLength/2) == val.WithdrawableEpoch
+		if val.Slashed && correctEpoch {
 			minSlashing := mathutil.Min(totalSlashing*3, totalBalance)
 			increment := params.BeaconConfig().EffectiveBalanceIncrement
-			penaltyNumerator := validator.EffectiveBalance / increment * minSlashing
+			penaltyNumerator := val.EffectiveBalance / increment * minSlashing
 			penalty := penaltyNumerator / totalBalance * increment
-			state = helpers.DecreaseBalance(state, uint64(index), penalty)
+			if err := helpers.DecreaseBalance(state, uint64(idx), penalty); err != nil {
+				return err
+			}
 		}
-	}
+		return nil
+	})
 	return state, err
 }
 
@@ -207,67 +222,96 @@ func ProcessSlashings(state *pb.BeaconState) (*pb.BeaconState, error) {
 //    # Rotate current/previous epoch attestations
 //    state.previous_epoch_attestations = state.current_epoch_attestations
 //    state.current_epoch_attestations = []
-func ProcessFinalUpdates(state *pb.BeaconState) (*pb.BeaconState, error) {
+func ProcessFinalUpdates(state *stateTrie.BeaconState) (*stateTrie.BeaconState, error) {
 	currentEpoch := helpers.CurrentEpoch(state)
 	nextEpoch := currentEpoch + 1
 
 	// Reset ETH1 data votes.
-	if (state.Slot+1)%params.BeaconConfig().SlotsPerEth1VotingPeriod == 0 {
-		state.Eth1DataVotes = []*ethpb.Eth1Data{}
+	if (state.Slot()+1)%params.BeaconConfig().SlotsPerEth1VotingPeriod == 0 {
+		if err := state.SetEth1DataVotes([]*ethpb.Eth1Data{}); err != nil {
+			return nil, err
+		}
 	}
 
+	bals := state.Balances()
 	// Update effective balances with hysteresis.
-	for i, v := range state.Validators {
-		if v == nil {
-			return nil, fmt.Errorf("validator %d is nil in state", i)
+	validatorFunc := func(idx int, val *ethpb.Validator) error {
+		if val == nil {
+			return fmt.Errorf("validator %d is nil in state", idx)
 		}
-		if i >= len(state.Balances) {
-			return nil, fmt.Errorf("validator index exceeds validator length in state %d >= %d", i, len(state.Balances))
+		if idx >= len(bals) {
+			return fmt.Errorf("validator index exceeds validator length in state %d >= %d", idx, len(state.Balances()))
 		}
-		balance := state.Balances[i]
+		balance := bals[idx]
 		halfInc := params.BeaconConfig().EffectiveBalanceIncrement / 2
-		if balance < v.EffectiveBalance || v.EffectiveBalance+3*halfInc < balance {
-			v.EffectiveBalance = params.BeaconConfig().MaxEffectiveBalance
-			if v.EffectiveBalance > balance-balance%params.BeaconConfig().EffectiveBalanceIncrement {
-				v.EffectiveBalance = balance - balance%params.BeaconConfig().EffectiveBalanceIncrement
+		if balance < val.EffectiveBalance || val.EffectiveBalance+3*halfInc < balance {
+			val.EffectiveBalance = params.BeaconConfig().MaxEffectiveBalance
+			if val.EffectiveBalance > balance-balance%params.BeaconConfig().EffectiveBalanceIncrement {
+				val.EffectiveBalance = balance - balance%params.BeaconConfig().EffectiveBalanceIncrement
 			}
 		}
+		return nil
+	}
+
+	if err := state.ApplyToEveryValidator(validatorFunc); err != nil {
+		return nil, err
 	}
 
 	// Set total slashed balances.
 	slashedExitLength := params.BeaconConfig().EpochsPerSlashingsVector
 	slashedEpoch := int(nextEpoch % slashedExitLength)
-	if len(state.Slashings) != int(slashedExitLength) {
-		return nil, fmt.Errorf("state slashing length %d different than EpochsPerHistoricalVector %d", len(state.Slashings), slashedExitLength)
+	slashings := state.Slashings()
+	if len(slashings) != int(slashedExitLength) {
+		return nil, fmt.Errorf(
+			"state slashing length %d different than EpochsPerHistoricalVector %d",
+			len(slashings),
+			slashedExitLength,
+		)
+	}
+	if err := state.UpdateSlashingsAtIndex(uint64(slashedEpoch) /* index */, 0 /* value */); err != nil {
+		return nil, err
 	}
-	state.Slashings[slashedEpoch] = 0
 
 	// Set RANDAO mix.
 	randaoMixLength := params.BeaconConfig().EpochsPerHistoricalVector
-	if len(state.RandaoMixes) != int(randaoMixLength) {
-		return nil, fmt.Errorf("state randao length %d different than EpochsPerHistoricalVector %d", len(state.RandaoMixes), randaoMixLength)
+	if state.RandaoMixesLength() != int(randaoMixLength) {
+		return nil, fmt.Errorf(
+			"state randao length %d different than EpochsPerHistoricalVector %d",
+			state.RandaoMixesLength(),
+			randaoMixLength,
+		)
+	}
+	mix, err := helpers.RandaoMix(state, currentEpoch)
+	if err != nil {
+		return nil, err
+	}
+	if err := state.UpdateRandaoMixesAtIndex(mix, nextEpoch%randaoMixLength); err != nil {
+		return nil, err
 	}
-	mix := helpers.RandaoMix(state, currentEpoch)
-	state.RandaoMixes[nextEpoch%randaoMixLength] = mix
 
 	// Set historical root accumulator.
 	epochsPerHistoricalRoot := params.BeaconConfig().SlotsPerHistoricalRoot / params.BeaconConfig().SlotsPerEpoch
 	if nextEpoch%epochsPerHistoricalRoot == 0 {
 		historicalBatch := &pb.HistoricalBatch{
-			BlockRoots: state.BlockRoots,
-			StateRoots: state.StateRoots,
+			BlockRoots: state.BlockRoots(),
+			StateRoots: state.StateRoots(),
 		}
 		batchRoot, err := ssz.HashTreeRoot(historicalBatch)
 		if err != nil {
 			return nil, errors.Wrap(err, "could not hash historical batch")
 		}
-		state.HistoricalRoots = append(state.HistoricalRoots, batchRoot[:])
+		if err := state.AppendHistoricalRoots(batchRoot); err != nil {
+			return nil, err
+		}
 	}
 
 	// Rotate current and previous epoch attestations.
-	state.PreviousEpochAttestations = state.CurrentEpochAttestations
-	state.CurrentEpochAttestations = []*pb.PendingAttestation{}
-
+	if err := state.SetPreviousEpochAttestations(state.CurrentEpochAttestations()); err != nil {
+		return nil, err
+	}
+	if err := state.SetCurrentEpochAttestations([]*pb.PendingAttestation{}); err != nil {
+		return nil, err
+	}
 	return state, nil
 }
 
@@ -281,7 +325,7 @@ func ProcessFinalUpdates(state *pb.BeaconState) (*pb.BeaconState, error) {
 //    for a in attestations:
 //        output = output.union(get_attesting_indices(state, a.data, a.aggregation_bits))
 //    return set(filter(lambda index: not state.validators[index].slashed, output))
-func unslashedAttestingIndices(state *pb.BeaconState, atts []*pb.PendingAttestation) ([]uint64, error) {
+func unslashedAttestingIndices(state *stateTrie.BeaconState, atts []*pb.PendingAttestation) ([]uint64, error) {
 	var setIndices []uint64
 	seen := make(map[uint64]bool)
 
@@ -290,7 +334,7 @@ func unslashedAttestingIndices(state *pb.BeaconState, atts []*pb.PendingAttestat
 		if err != nil {
 			return nil, err
 		}
-		attestingIndices, err := helpers.AttestingIndices(att.AggregationBits, committee)
+		attestingIndices, err := attestationutil.AttestingIndices(att.AggregationBits, committee)
 		if err != nil {
 			return nil, errors.Wrap(err, "could not get attester indices")
 		}
@@ -308,7 +352,7 @@ func unslashedAttestingIndices(state *pb.BeaconState, atts []*pb.PendingAttestat
 	sort.Slice(setIndices, func(i, j int) bool { return setIndices[i] < setIndices[j] })
 	// Remove the slashed validator indices.
 	for i := 0; i < len(setIndices); i++ {
-		if state.Validators[setIndices[i]].Slashed {
+		if v, _ := state.ValidatorAtIndex(setIndices[i]); v != nil && v.Slashed {
 			setIndices = append(setIndices[:i], setIndices[i+1:]...)
 		}
 	}
@@ -327,12 +371,16 @@ func unslashedAttestingIndices(state *pb.BeaconState, atts []*pb.PendingAttestat
 //      total_balance = get_total_active_balance(state)
 //	    effective_balance = state.validator_registry[index].effective_balance
 //	    return effective_balance * BASE_REWARD_FACTOR // integer_squareroot(total_balance) // BASE_REWARDS_PER_EPOCH
-func BaseReward(state *pb.BeaconState, index uint64) (uint64, error) {
+func BaseReward(state *stateTrie.BeaconState, index uint64) (uint64, error) {
 	totalBalance, err := helpers.TotalActiveBalance(state)
 	if err != nil {
 		return 0, errors.Wrap(err, "could not calculate active balance")
 	}
-	effectiveBalance := state.Validators[index].EffectiveBalance
+	val, err := state.ValidatorAtIndex(index)
+	if err != nil {
+		return 0, err
+	}
+	effectiveBalance := val.EffectiveBalance
 	baseReward := effectiveBalance * params.BeaconConfig().BaseRewardFactor /
 		mathutil.IntegerSquareRoot(totalBalance) / params.BeaconConfig().BaseRewardsPerEpoch
 	return baseReward, nil

beacon-chain/core/epoch/epoch_processing_fuzz_test.go

@@ -4,15 +4,20 @@ import (
 	"testing"
 
 	fuzz "github.com/google/gofuzz"
+	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	ethereum_beacon_p2p_v1 "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 )
 
 func TestFuzzFinalUpdates_10000(t *testing.T) {
 	fuzzer := fuzz.NewWithSeed(0)
-	state := &ethereum_beacon_p2p_v1.BeaconState{}
+	base := &ethereum_beacon_p2p_v1.BeaconState{}
 
 	for i := 0; i < 10000; i++ {
-		fuzzer.Fuzz(state)
-		_, _ = ProcessFinalUpdates(state)
+		fuzzer.Fuzz(base)
+		s, err := beaconstate.InitializeFromProtoUnsafe(base)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_, _ = ProcessFinalUpdates(s)
 	}
 }

beacon-chain/core/epoch/epoch_processing_test.go

@@ -8,6 +8,7 @@ import (
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/go-bitfield"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
@@ -39,10 +40,14 @@ func TestUnslashedAttestingIndices_CanSortAndFilter(t *testing.T) {
 			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
 		}
 	}
-	state := &pb.BeaconState{
+	base := &pb.BeaconState{
 		Validators:  validators,
 		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
 	}
+	state, err := state.InitializeFromProto(base)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	indices, err := unslashedAttestingIndices(state, atts)
 	if err != nil {
@@ -56,7 +61,9 @@ func TestUnslashedAttestingIndices_CanSortAndFilter(t *testing.T) {
 
 	// Verify the slashed validator is filtered.
 	slashedValidator := indices[0]
-	state.Validators[slashedValidator].Slashed = true
+	validators = state.Validators()
+	validators[slashedValidator].Slashed = true
+	state.SetValidators(validators)
 	indices, err = unslashedAttestingIndices(state, atts)
 	if err != nil {
 		t.Fatal(err)
@@ -87,10 +94,14 @@ func TestUnslashedAttestingIndices_DuplicatedAttestations(t *testing.T) {
 			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
 		}
 	}
-	state := &pb.BeaconState{
+	base := &pb.BeaconState{
 		Validators:  validators,
 		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
 	}
+	state, err := state.InitializeFromProto(base)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	indices, err := unslashedAttestingIndices(state, atts)
 	if err != nil {
@@ -105,6 +116,7 @@ func TestUnslashedAttestingIndices_DuplicatedAttestations(t *testing.T) {
 }
 
 func TestAttestingBalance_CorrectBalance(t *testing.T) {
+	helpers.ClearCache()
 	// Generate 2 attestations.
 	atts := make([]*pb.PendingAttestation, 2)
 	for i := 0; i < len(atts); i++ {
@@ -129,13 +141,17 @@ func TestAttestingBalance_CorrectBalance(t *testing.T) {
 		}
 		balances[i] = params.BeaconConfig().MaxEffectiveBalance
 	}
-	state := &pb.BeaconState{
+	base := &pb.BeaconState{
 		Slot:        2,
 		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
 
 		Validators: validators,
 		Balances:   balances,
 	}
+	state, err := state.InitializeFromProto(base)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	balance, err := AttestingBalance(state, atts)
 	if err != nil {
@@ -159,11 +175,15 @@ func TestBaseReward_AccurateRewards(t *testing.T) {
 		{40 * 1e9, params.BeaconConfig().MaxEffectiveBalance, 2862174},
 	}
 	for _, tt := range tests {
-		state := &pb.BeaconState{
+		base := &pb.BeaconState{
 			Validators: []*ethpb.Validator{
 				{ExitEpoch: params.BeaconConfig().FarFutureEpoch, EffectiveBalance: tt.b}},
 			Balances: []uint64{tt.a},
 		}
+		state, err := state.InitializeFromProto(base)
+		if err != nil {
+			t.Fatal(err)
+		}
 		c, err := BaseReward(state, 0)
 		if err != nil {
 			t.Fatal(err)
@@ -176,19 +196,23 @@ func TestBaseReward_AccurateRewards(t *testing.T) {
 }
 
 func TestProcessSlashings_NotSlashed(t *testing.T) {
-	s := &pb.BeaconState{
+	base := &pb.BeaconState{
 		Slot:       0,
 		Validators: []*ethpb.Validator{{Slashed: true}},
 		Balances:   []uint64{params.BeaconConfig().MaxEffectiveBalance},
 		Slashings:  []uint64{0, 1e9},
 	}
+	s, err := state.InitializeFromProto(base)
+	if err != nil {
+		t.Fatal(err)
+	}
 	newState, err := ProcessSlashings(s)
 	if err != nil {
 		t.Fatal(err)
 	}
 	wanted := params.BeaconConfig().MaxEffectiveBalance
-	if newState.Balances[0] != wanted {
-		t.Errorf("Wanted slashed balance: %d, got: %d", wanted, newState.Balances[0])
+	if newState.Balances()[0] != wanted {
+		t.Errorf("Wanted slashed balance: %d, got: %d", wanted, newState.Balances()[0])
 	}
 }
 
@@ -262,16 +286,20 @@ func TestProcessSlashings_SlashedLess(t *testing.T) {
 	for i, tt := range tests {
 		t.Run(string(i), func(t *testing.T) {
 			original := proto.Clone(tt.state)
-			newState, err := ProcessSlashings(tt.state)
+			s, err := state.InitializeFromProto(tt.state)
+			if err != nil {
+				t.Fatal(err)
+			}
+			newState, err := ProcessSlashings(s)
 			if err != nil {
 				t.Fatal(err)
 			}
 
-			if newState.Balances[0] != tt.want {
+			if newState.Balances()[0] != tt.want {
 				t.Errorf(
 					"ProcessSlashings({%v}) = newState; newState.Balances[0] = %d; wanted %d",
 					original,
-					newState.Balances[0],
+					newState.Balances()[0],
 					tt.want,
 				)
 			}
@@ -283,48 +311,54 @@ func TestProcessFinalUpdates_CanProcess(t *testing.T) {
 	s := buildState(params.BeaconConfig().SlotsPerHistoricalRoot-1, params.BeaconConfig().SlotsPerEpoch)
 	ce := helpers.CurrentEpoch(s)
 	ne := ce + 1
-	s.Eth1DataVotes = []*ethpb.Eth1Data{}
-	s.Balances[0] = 29 * 1e9
-	s.Slashings[ce] = 0
-	s.RandaoMixes[ce] = []byte{'A'}
+	s.SetEth1DataVotes([]*ethpb.Eth1Data{})
+	balances := s.Balances()
+	balances[0] = 29 * 1e9
+	s.SetBalances(balances)
+	slashings := s.Slashings()
+	slashings[ce] = 0
+	s.SetSlashings(slashings)
+	mixes := s.RandaoMixes()
+	mixes[ce] = []byte{'A'}
+	s.SetRandaoMixes(mixes)
 	newS, err := ProcessFinalUpdates(s)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Verify effective balance is correctly updated.
-	if newS.Validators[0].EffectiveBalance != 29*1e9 {
-		t.Errorf("effective balance incorrectly updated, got %d", s.Validators[0].EffectiveBalance)
+	if newS.Validators()[0].EffectiveBalance != 29*1e9 {
+		t.Errorf("effective balance incorrectly updated, got %d", s.Validators()[0].EffectiveBalance)
 	}
 
 	// Verify slashed balances correctly updated.
-	if newS.Slashings[ce] != newS.Slashings[ne] {
+	if newS.Slashings()[ce] != newS.Slashings()[ne] {
 		t.Errorf("wanted slashed balance %d, got %d",
-			newS.Slashings[ce],
-			newS.Slashings[ne])
+			newS.Slashings()[ce],
+			newS.Slashings()[ne])
 	}
 
 	// Verify randao is correctly updated in the right position.
-	if bytes.Equal(newS.RandaoMixes[ne], params.BeaconConfig().ZeroHash[:]) {
+	if mix, _ := newS.RandaoMixAtIndex(ne); bytes.Equal(mix, params.BeaconConfig().ZeroHash[:]) {
 		t.Error("latest RANDAO still zero hashes")
 	}
 
 	// Verify historical root accumulator was appended.
-	if len(newS.HistoricalRoots) != 1 {
-		t.Errorf("wanted slashed balance %d, got %d", 1, len(newS.HistoricalRoots[ce]))
+	if len(newS.HistoricalRoots()) != 1 {
+		t.Errorf("wanted slashed balance %d, got %d", 1, len(newS.HistoricalRoots()[ce]))
 	}
 
-	if newS.CurrentEpochAttestations == nil {
+	if newS.CurrentEpochAttestations() == nil {
 		t.Error("nil value stored in current epoch attestations instead of empty slice")
 	}
 }
 
 func TestProcessRegistryUpdates_NoRotation(t *testing.T) {
-	state := &pb.BeaconState{
+	base := &pb.BeaconState{
 		Slot: 5 * params.BeaconConfig().SlotsPerEpoch,
 		Validators: []*ethpb.Validator{
-			{ExitEpoch: params.BeaconConfig().MaxSeedLookhead},
-			{ExitEpoch: params.BeaconConfig().MaxSeedLookhead},
+			{ExitEpoch: params.BeaconConfig().MaxSeedLookahead},
+			{ExitEpoch: params.BeaconConfig().MaxSeedLookahead},
 		},
 		Balances: []uint64{
 			params.BeaconConfig().MaxEffectiveBalance,
@@ -332,20 +366,24 @@ func TestProcessRegistryUpdates_NoRotation(t *testing.T) {
 		},
 		FinalizedCheckpoint: &ethpb.Checkpoint{},
 	}
+	state, err := state.InitializeFromProto(base)
+	if err != nil {
+		t.Fatal(err)
+	}
 	newState, err := ProcessRegistryUpdates(state)
 	if err != nil {
 		t.Fatal(err)
 	}
-	for i, validator := range newState.Validators {
-		if validator.ExitEpoch != params.BeaconConfig().MaxSeedLookhead {
+	for i, validator := range newState.Validators() {
+		if validator.ExitEpoch != params.BeaconConfig().MaxSeedLookahead {
 			t.Errorf("Could not update registry %d, wanted exit slot %d got %d",
-				i, params.BeaconConfig().MaxSeedLookhead, validator.ExitEpoch)
+				i, params.BeaconConfig().MaxSeedLookahead, validator.ExitEpoch)
 		}
 	}
 }
 
 func TestProcessRegistryUpdates_EligibleToActivate(t *testing.T) {
-	state := &pb.BeaconState{
+	base := &pb.BeaconState{
 		Slot:                5 * params.BeaconConfig().SlotsPerEpoch,
 		FinalizedCheckpoint: &ethpb.Checkpoint{Epoch: 6},
 	}
@@ -354,25 +392,26 @@ func TestProcessRegistryUpdates_EligibleToActivate(t *testing.T) {
 		t.Error(err)
 	}
 	for i := 0; i < int(limit)+10; i++ {
-		state.Validators = append(state.Validators, &ethpb.Validator{
+		base.Validators = append(base.Validators, &ethpb.Validator{
 			ActivationEligibilityEpoch: params.BeaconConfig().FarFutureEpoch,
 			EffectiveBalance:           params.BeaconConfig().MaxEffectiveBalance,
 			ActivationEpoch:            params.BeaconConfig().FarFutureEpoch,
 		})
 	}
+	state, err := state.InitializeFromProto(base)
 	currentEpoch := helpers.CurrentEpoch(state)
 	newState, err := ProcessRegistryUpdates(state)
 	if err != nil {
 		t.Error(err)
 	}
-	for i, validator := range newState.Validators {
+	for i, validator := range newState.Validators() {
 		if validator.ActivationEligibilityEpoch != currentEpoch+1 {
 			t.Errorf("Could not update registry %d, wanted activation eligibility epoch %d got %d",
 				i, currentEpoch, validator.ActivationEligibilityEpoch)
 		}
-		if i < int(limit) && validator.ActivationEpoch != helpers.DelayedActivationExitEpoch(currentEpoch) {
+		if i < int(limit) && validator.ActivationEpoch != helpers.ActivationExitEpoch(currentEpoch) {
 			t.Errorf("Could not update registry %d, validators failed to activate: wanted activation epoch %d, got %d",
-				i, helpers.DelayedActivationExitEpoch(currentEpoch), validator.ActivationEpoch)
+				i, helpers.ActivationExitEpoch(currentEpoch), validator.ActivationEpoch)
 		}
 		if i >= int(limit) && validator.ActivationEpoch != params.BeaconConfig().FarFutureEpoch {
 			t.Errorf("Could not update registry %d, validators should not have been activated, wanted activation epoch: %d, got %d",
@@ -382,30 +421,34 @@ func TestProcessRegistryUpdates_EligibleToActivate(t *testing.T) {
 }
 
 func TestProcessRegistryUpdates_ActivationCompletes(t *testing.T) {
-	state := &pb.BeaconState{
+	base := &pb.BeaconState{
 		Slot: 5 * params.BeaconConfig().SlotsPerEpoch,
 		Validators: []*ethpb.Validator{
-			{ExitEpoch: params.BeaconConfig().MaxSeedLookhead,
-				ActivationEpoch: 5 + params.BeaconConfig().MaxSeedLookhead + 1},
-			{ExitEpoch: params.BeaconConfig().MaxSeedLookhead,
-				ActivationEpoch: 5 + params.BeaconConfig().MaxSeedLookhead + 1},
+			{ExitEpoch: params.BeaconConfig().MaxSeedLookahead,
+				ActivationEpoch: 5 + params.BeaconConfig().MaxSeedLookahead + 1},
+			{ExitEpoch: params.BeaconConfig().MaxSeedLookahead,
+				ActivationEpoch: 5 + params.BeaconConfig().MaxSeedLookahead + 1},
 		},
 		FinalizedCheckpoint: &ethpb.Checkpoint{},
 	}
+	state, err := state.InitializeFromProto(base)
+	if err != nil {
+		t.Fatal(err)
+	}
 	newState, err := ProcessRegistryUpdates(state)
 	if err != nil {
 		t.Error(err)
 	}
-	for i, validator := range newState.Validators {
-		if validator.ExitEpoch != params.BeaconConfig().MaxSeedLookhead {
+	for i, validator := range newState.Validators() {
+		if validator.ExitEpoch != params.BeaconConfig().MaxSeedLookahead {
 			t.Errorf("Could not update registry %d, wanted exit slot %d got %d",
-				i, params.BeaconConfig().MaxSeedLookhead, validator.ExitEpoch)
+				i, params.BeaconConfig().MaxSeedLookahead, validator.ExitEpoch)
 		}
 	}
 }
 
 func TestProcessRegistryUpdates_ValidatorsEjected(t *testing.T) {
-	state := &pb.BeaconState{
+	base := &pb.BeaconState{
 		Slot: 0,
 		Validators: []*ethpb.Validator{
 			{
@@ -419,23 +462,27 @@ func TestProcessRegistryUpdates_ValidatorsEjected(t *testing.T) {
 		},
 		FinalizedCheckpoint: &ethpb.Checkpoint{},
 	}
+	state, err := state.InitializeFromProto(base)
+	if err != nil {
+		t.Fatal(err)
+	}
 	newState, err := ProcessRegistryUpdates(state)
 	if err != nil {
 		t.Error(err)
 	}
-	for i, validator := range newState.Validators {
-		if validator.ExitEpoch != params.BeaconConfig().MaxSeedLookhead+1 {
+	for i, validator := range newState.Validators() {
+		if validator.ExitEpoch != params.BeaconConfig().MaxSeedLookahead+1 {
 			t.Errorf("Could not update registry %d, wanted exit slot %d got %d",
-				i, params.BeaconConfig().MaxSeedLookhead+1, validator.ExitEpoch)
+				i, params.BeaconConfig().MaxSeedLookahead+1, validator.ExitEpoch)
 		}
 	}
 }
 
 func TestProcessRegistryUpdates_CanExits(t *testing.T) {
 	epoch := uint64(5)
-	exitEpoch := helpers.DelayedActivationExitEpoch(epoch)
+	exitEpoch := helpers.ActivationExitEpoch(epoch)
 	minWithdrawalDelay := params.BeaconConfig().MinValidatorWithdrawabilityDelay
-	state := &pb.BeaconState{
+	base := &pb.BeaconState{
 		Slot: epoch * params.BeaconConfig().SlotsPerEpoch,
 		Validators: []*ethpb.Validator{
 			{
@@ -447,11 +494,15 @@ func TestProcessRegistryUpdates_CanExits(t *testing.T) {
 		},
 		FinalizedCheckpoint: &ethpb.Checkpoint{},
 	}
+	state, err := state.InitializeFromProto(base)
+	if err != nil {
+		t.Fatal(err)
+	}
 	newState, err := ProcessRegistryUpdates(state)
 	if err != nil {
 		t.Fatal(err)
 	}
-	for i, validator := range newState.Validators {
+	for i, validator := range newState.Validators() {
 		if validator.ExitEpoch != exitEpoch {
 			t.Errorf("Could not update registry %d, wanted exit slot %d got %d",
 				i,
@@ -462,7 +513,7 @@ func TestProcessRegistryUpdates_CanExits(t *testing.T) {
 	}
 }
 
-func buildState(slot uint64, validatorCount uint64) *pb.BeaconState {
+func buildState(slot uint64, validatorCount uint64) *state.BeaconState {
 	validators := make([]*ethpb.Validator, validatorCount)
 	for i := 0; i < len(validators); i++ {
 		validators[i] = &ethpb.Validator{
@@ -488,7 +539,7 @@ func buildState(slot uint64, validatorCount uint64) *pb.BeaconState {
 	for i := 0; i < len(latestRandaoMixes); i++ {
 		latestRandaoMixes[i] = params.BeaconConfig().ZeroHash[:]
 	}
-	return &pb.BeaconState{
+	s, err := state.InitializeFromProto(&pb.BeaconState{
 		Slot:                        slot,
 		Balances:                    validatorBalances,
 		Validators:                  validators,
@@ -498,5 +549,9 @@ func buildState(slot uint64, validatorCount uint64) *pb.BeaconState {
 		FinalizedCheckpoint:         &ethpb.Checkpoint{},
 		PreviousJustifiedCheckpoint: &ethpb.Checkpoint{},
 		CurrentJustifiedCheckpoint:  &ethpb.Checkpoint{},
+	})
+	if err != nil {
+		panic(err)
 	}
+	return s
 }

beacon-chain/core/epoch/precompute/BUILD.bazel

@@ -14,7 +14,9 @@ go_library(
     visibility = ["//beacon-chain:__subpackages__"],
     deps = [
         "//beacon-chain/core/helpers:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
+        "//shared/attestationutil:go_default_library",
         "//shared/mathutil:go_default_library",
         "//shared/params:go_default_library",
         "//shared/traceutil:go_default_library",
@@ -37,7 +39,9 @@ go_test(
     deps = [
         "//beacon-chain/core/epoch:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
+        "//shared/attestationutil:go_default_library",
         "//shared/params:go_default_library",
         "//shared/testutil:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",

beacon-chain/core/epoch/precompute/attestation.go

@@ -6,7 +6,9 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/attestationutil"
 	"github.com/prysmaticlabs/prysm/shared/traceutil"
 	"go.opencensus.io/trace"
 )
@@ -19,16 +21,17 @@ var Balances *Balance
 // it also tracks and updates epoch attesting balances.
 func ProcessAttestations(
 	ctx context.Context,
-	state *pb.BeaconState,
+	state *stateTrie.BeaconState,
 	vp []*Validator,
-	bp *Balance) ([]*Validator, *Balance, error) {
+	bp *Balance,
+) ([]*Validator, *Balance, error) {
 	ctx, span := trace.StartSpan(ctx, "precomputeEpoch.ProcessAttestations")
 	defer span.End()
 
 	v := &Validator{}
 	var err error
 
-	for _, a := range append(state.PreviousEpochAttestations, state.CurrentEpochAttestations...) {
+	for _, a := range append(state.PreviousEpochAttestations(), state.CurrentEpochAttestations()...) {
 		v.IsCurrentEpochAttester, v.IsCurrentEpochTargetAttester, err = AttestedCurrentEpoch(state, a)
 		if err != nil {
 			traceutil.AnnotateError(span, err)
@@ -44,7 +47,7 @@ func ProcessAttestations(
 		if err != nil {
 			return nil, nil, err
 		}
-		indices, err := helpers.AttestingIndices(a.AggregationBits, committee)
+		indices, err := attestationutil.AttestingIndices(a.AggregationBits, committee)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -58,7 +61,7 @@ func ProcessAttestations(
 }
 
 // AttestedCurrentEpoch returns true if attestation `a` attested once in current epoch and/or epoch boundary block.
-func AttestedCurrentEpoch(s *pb.BeaconState, a *pb.PendingAttestation) (bool, bool, error) {
+func AttestedCurrentEpoch(s *stateTrie.BeaconState, a *pb.PendingAttestation) (bool, bool, error) {
 	currentEpoch := helpers.CurrentEpoch(s)
 	var votedCurrentEpoch, votedTarget bool
 	// Did validator vote current epoch.
@@ -76,7 +79,7 @@ func AttestedCurrentEpoch(s *pb.BeaconState, a *pb.PendingAttestation) (bool, bo
 }
 
 // AttestedPrevEpoch returns true if attestation `a` attested once in previous epoch and epoch boundary block and/or the same head.
-func AttestedPrevEpoch(s *pb.BeaconState, a *pb.PendingAttestation) (bool, bool, bool, error) {
+func AttestedPrevEpoch(s *stateTrie.BeaconState, a *pb.PendingAttestation) (bool, bool, bool, error) {
 	prevEpoch := helpers.PrevEpoch(s)
 	var votedPrevEpoch, votedTarget, votedHead bool
 	// Did validator vote previous epoch.
@@ -102,7 +105,7 @@ func AttestedPrevEpoch(s *pb.BeaconState, a *pb.PendingAttestation) (bool, bool,
 }
 
 // SameTarget returns true if attestation `a` attested to the same target block in state.
-func SameTarget(state *pb.BeaconState, a *pb.PendingAttestation, e uint64) (bool, error) {
+func SameTarget(state *stateTrie.BeaconState, a *pb.PendingAttestation, e uint64) (bool, error) {
 	r, err := helpers.BlockRoot(state, e)
 	if err != nil {
 		return false, err
@@ -114,7 +117,7 @@ func SameTarget(state *pb.BeaconState, a *pb.PendingAttestation, e uint64) (bool
 }
 
 // SameHead returns true if attestation `a` attested to the same block by attestation slot in state.
-func SameHead(state *pb.BeaconState, a *pb.PendingAttestation) (bool, error) {
+func SameHead(state *stateTrie.BeaconState, a *pb.PendingAttestation) (bool, error) {
 	r, err := helpers.BlockRootAtSlot(state, a.Data.Slot)
 	if err != nil {
 		return false, err

beacon-chain/core/epoch/precompute/attestation_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/attestationutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
@@ -73,12 +74,14 @@ func TestUpdateBalance(t *testing.T) {
 
 func TestSameHead(t *testing.T) {
 	beaconState, _ := testutil.DeterministicGenesisState(t, 100)
-	beaconState.Slot = 1
+	beaconState.SetSlot(1)
 	att := &ethpb.Attestation{Data: &ethpb.AttestationData{
 		Target: &ethpb.Checkpoint{Epoch: 0}}}
-	r := []byte{'A'}
-	beaconState.BlockRoots[0] = r
-	att.Data.BeaconBlockRoot = r
+	r := [32]byte{'A'}
+	br := beaconState.BlockRoots()
+	br[0] = r[:]
+	beaconState.SetBlockRoots(br)
+	att.Data.BeaconBlockRoot = r[:]
 	same, err := precompute.SameHead(beaconState, &pb.PendingAttestation{Data: att.Data})
 	if err != nil {
 		t.Fatal(err)
@@ -86,7 +89,8 @@ func TestSameHead(t *testing.T) {
 	if !same {
 		t.Error("head in state does not match head in attestation")
 	}
-	att.Data.BeaconBlockRoot = []byte{'B'}
+	newRoot := [32]byte{'B'}
+	att.Data.BeaconBlockRoot = newRoot[:]
 	same, err = precompute.SameHead(beaconState, &pb.PendingAttestation{Data: att.Data})
 	if err != nil {
 		t.Fatal(err)
@@ -98,12 +102,14 @@ func TestSameHead(t *testing.T) {
 
 func TestSameTarget(t *testing.T) {
 	beaconState, _ := testutil.DeterministicGenesisState(t, 100)
-	beaconState.Slot = 1
+	beaconState.SetSlot(1)
 	att := &ethpb.Attestation{Data: &ethpb.AttestationData{
 		Target: &ethpb.Checkpoint{Epoch: 0}}}
-	r := []byte{'A'}
-	beaconState.BlockRoots[0] = r
-	att.Data.Target.Root = r
+	r := [32]byte{'A'}
+	br := beaconState.BlockRoots()
+	br[0] = r[:]
+	beaconState.SetBlockRoots(br)
+	att.Data.Target.Root = r[:]
 	same, err := precompute.SameTarget(beaconState, &pb.PendingAttestation{Data: att.Data}, 0)
 	if err != nil {
 		t.Fatal(err)
@@ -111,7 +117,8 @@ func TestSameTarget(t *testing.T) {
 	if !same {
 		t.Error("head in state does not match head in attestation")
 	}
-	att.Data.Target.Root = []byte{'B'}
+	newRoot := [32]byte{'B'}
+	att.Data.Target.Root = newRoot[:]
 	same, err = precompute.SameTarget(beaconState, &pb.PendingAttestation{Data: att.Data}, 0)
 	if err != nil {
 		t.Fatal(err)
@@ -123,13 +130,15 @@ func TestSameTarget(t *testing.T) {
 
 func TestAttestedPrevEpoch(t *testing.T) {
 	beaconState, _ := testutil.DeterministicGenesisState(t, 100)
-	beaconState.Slot = params.BeaconConfig().SlotsPerEpoch
+	beaconState.SetSlot(params.BeaconConfig().SlotsPerEpoch)
 	att := &ethpb.Attestation{Data: &ethpb.AttestationData{
 		Target: &ethpb.Checkpoint{Epoch: 0}}}
-	r := []byte{'A'}
-	beaconState.BlockRoots[0] = r
-	att.Data.Target.Root = r
-	att.Data.BeaconBlockRoot = r
+	r := [32]byte{'A'}
+	br := beaconState.BlockRoots()
+	br[0] = r[:]
+	beaconState.SetBlockRoots(br)
+	att.Data.Target.Root = r[:]
+	att.Data.BeaconBlockRoot = r[:]
 	votedEpoch, votedTarget, votedHead, err := precompute.AttestedPrevEpoch(beaconState, &pb.PendingAttestation{Data: att.Data})
 	if err != nil {
 		t.Fatal(err)
@@ -147,13 +156,16 @@ func TestAttestedPrevEpoch(t *testing.T) {
 
 func TestAttestedCurrentEpoch(t *testing.T) {
 	beaconState, _ := testutil.DeterministicGenesisState(t, 100)
-	beaconState.Slot = params.BeaconConfig().SlotsPerEpoch + 1
+	beaconState.SetSlot(params.BeaconConfig().SlotsPerEpoch + 1)
 	att := &ethpb.Attestation{Data: &ethpb.AttestationData{
 		Target: &ethpb.Checkpoint{Epoch: 1}}}
-	r := []byte{'A'}
-	beaconState.BlockRoots[params.BeaconConfig().SlotsPerEpoch] = r
-	att.Data.Target.Root = r
-	att.Data.BeaconBlockRoot = r
+	r := [32]byte{'A'}
+
+	br := beaconState.BlockRoots()
+	br[params.BeaconConfig().SlotsPerEpoch] = r[:]
+	beaconState.SetBlockRoots(br)
+	att.Data.Target.Root = r[:]
+	att.Data.BeaconBlockRoot = r[:]
 	votedEpoch, votedTarget, err := precompute.AttestedCurrentEpoch(beaconState, &pb.PendingAttestation{Data: att.Data})
 	if err != nil {
 		t.Fatal(err)
@@ -172,7 +184,7 @@ func TestProcessAttestations(t *testing.T) {
 
 	validators := uint64(64)
 	beaconState, _ := testutil.DeterministicGenesisState(t, validators)
-	beaconState.Slot = params.BeaconConfig().SlotsPerEpoch
+	beaconState.SetSlot(params.BeaconConfig().SlotsPerEpoch)
 
 	bf := []byte{0xff}
 	att1 := &ethpb.Attestation{Data: &ethpb.AttestationData{
@@ -181,14 +193,17 @@ func TestProcessAttestations(t *testing.T) {
 	att2 := &ethpb.Attestation{Data: &ethpb.AttestationData{
 		Target: &ethpb.Checkpoint{Epoch: 0}},
 		AggregationBits: bf}
-	beaconState.BlockRoots[0] = []byte{'A'}
-	att1.Data.Target.Root = []byte{'A'}
-	att1.Data.BeaconBlockRoot = []byte{'A'}
-	beaconState.BlockRoots[0] = []byte{'B'}
-	att2.Data.Target.Root = []byte{'A'}
-	att2.Data.BeaconBlockRoot = []byte{'B'}
-	beaconState.PreviousEpochAttestations = []*pb.PendingAttestation{{Data: att1.Data, AggregationBits: bf}}
-	beaconState.CurrentEpochAttestations = []*pb.PendingAttestation{{Data: att2.Data, AggregationBits: bf}}
+	rt := [32]byte{'A'}
+	att1.Data.Target.Root = rt[:]
+	att1.Data.BeaconBlockRoot = rt[:]
+	br := beaconState.BlockRoots()
+	newRt := [32]byte{'B'}
+	br[0] = newRt[:]
+	beaconState.SetBlockRoots(br)
+	att2.Data.Target.Root = rt[:]
+	att2.Data.BeaconBlockRoot = newRt[:]
+	beaconState.SetPreviousEpochAttestations([]*pb.PendingAttestation{{Data: att1.Data, AggregationBits: bf}})
+	beaconState.SetCurrentEpochAttestations([]*pb.PendingAttestation{{Data: att2.Data, AggregationBits: bf}})
 
 	vp := make([]*precompute.Validator, validators)
 	for i := 0; i < len(vp); i++ {
@@ -204,7 +219,7 @@ func TestProcessAttestations(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	indices, _ := helpers.AttestingIndices(att1.AggregationBits, committee)
+	indices, _ := attestationutil.AttestingIndices(att1.AggregationBits, committee)
 	for _, i := range indices {
 		if !vp[i].IsPrevEpochAttester {
 			t.Error("Not a prev epoch attester")
@@ -214,7 +229,7 @@ func TestProcessAttestations(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	indices, _ = helpers.AttestingIndices(att2.AggregationBits, committee)
+	indices, _ = attestationutil.AttestingIndices(att2.AggregationBits, committee)
 	for _, i := range indices {
 		if !vp[i].IsPrevEpochAttester {
 			t.Error("Not a prev epoch attester")

beacon-chain/core/epoch/precompute/justification_finalization.go

@@ -4,25 +4,31 @@ import (
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 )
 
 // ProcessJustificationAndFinalizationPreCompute processes justification and finalization during
 // epoch processing. This is where a beacon node can justify and finalize a new epoch.
 // Note: this is an optimized version by passing in precomputed total and attesting balances.
-func ProcessJustificationAndFinalizationPreCompute(state *pb.BeaconState, p *Balance) (*pb.BeaconState, error) {
-	if state.Slot <= helpers.StartSlot(2) {
+func ProcessJustificationAndFinalizationPreCompute(state *stateTrie.BeaconState, p *Balance) (*stateTrie.BeaconState, error) {
+	if state.Slot() <= helpers.StartSlot(2) {
 		return state, nil
 	}
 
 	prevEpoch := helpers.PrevEpoch(state)
 	currentEpoch := helpers.CurrentEpoch(state)
-	oldPrevJustifiedCheckpoint := state.PreviousJustifiedCheckpoint
-	oldCurrJustifiedCheckpoint := state.CurrentJustifiedCheckpoint
+	oldPrevJustifiedCheckpoint := state.PreviousJustifiedCheckpoint()
+	oldCurrJustifiedCheckpoint := state.CurrentJustifiedCheckpoint()
 
 	// Process justifications
-	state.PreviousJustifiedCheckpoint = state.CurrentJustifiedCheckpoint
-	state.JustificationBits.Shift(1)
+	if err := state.SetPreviousJustifiedCheckpoint(state.CurrentJustifiedCheckpoint()); err != nil {
+		return nil, err
+	}
+	newBits := state.JustificationBits()
+	newBits.Shift(1)
+	if err := state.SetJustificationBits(newBits); err != nil {
+		return nil, err
+	}
 
 	// Note: the spec refers to the bit index position starting at 1 instead of starting at zero.
 	// We will use that paradigm here for consistency with the godoc spec definition.
@@ -33,8 +39,14 @@ func ProcessJustificationAndFinalizationPreCompute(state *pb.BeaconState, p *Bal
 		if err != nil {
 			return nil, errors.Wrapf(err, "could not get block root for previous epoch %d", prevEpoch)
 		}
-		state.CurrentJustifiedCheckpoint = &ethpb.Checkpoint{Epoch: prevEpoch, Root: blockRoot}
-		state.JustificationBits.SetBitAt(1, true)
+		if err := state.SetCurrentJustifiedCheckpoint(&ethpb.Checkpoint{Epoch: prevEpoch, Root: blockRoot}); err != nil {
+			return nil, err
+		}
+		newBits := state.JustificationBits()
+		newBits.SetBitAt(1, true)
+		if err := state.SetJustificationBits(newBits); err != nil {
+			return nil, err
+		}
 	}
 
 	// If 2/3 or more of the total balance attested in the current epoch.
@@ -43,31 +55,45 @@ func ProcessJustificationAndFinalizationPreCompute(state *pb.BeaconState, p *Bal
 		if err != nil {
 			return nil, errors.Wrapf(err, "could not get block root for current epoch %d", prevEpoch)
 		}
-		state.CurrentJustifiedCheckpoint = &ethpb.Checkpoint{Epoch: currentEpoch, Root: blockRoot}
-		state.JustificationBits.SetBitAt(0, true)
+		if err := state.SetCurrentJustifiedCheckpoint(&ethpb.Checkpoint{Epoch: currentEpoch, Root: blockRoot}); err != nil {
+			return nil, err
+		}
+		newBits := state.JustificationBits()
+		newBits.SetBitAt(0, true)
+		if err := state.SetJustificationBits(newBits); err != nil {
+			return nil, err
+		}
 	}
 
 	// Process finalization according to ETH2.0 specifications.
-	justification := state.JustificationBits.Bytes()[0]
+	justification := state.JustificationBits().Bytes()[0]
 
 	// 2nd/3rd/4th (0b1110) most recent epochs are justified, the 2nd using the 4th as source.
 	if justification&0x0E == 0x0E && (oldPrevJustifiedCheckpoint.Epoch+3) == currentEpoch {
-		state.FinalizedCheckpoint = oldPrevJustifiedCheckpoint
+		if err := state.SetFinalizedCheckpoint(oldPrevJustifiedCheckpoint); err != nil {
+			return nil, err
+		}
 	}
 
 	// 2nd/3rd (0b0110) most recent epochs are justified, the 2nd using the 3rd as source.
 	if justification&0x06 == 0x06 && (oldPrevJustifiedCheckpoint.Epoch+2) == currentEpoch {
-		state.FinalizedCheckpoint = oldPrevJustifiedCheckpoint
+		if err := state.SetFinalizedCheckpoint(oldPrevJustifiedCheckpoint); err != nil {
+			return nil, err
+		}
 	}
 
 	// 1st/2nd/3rd (0b0111) most recent epochs are justified, the 1st using the 3rd as source.
 	if justification&0x07 == 0x07 && (oldCurrJustifiedCheckpoint.Epoch+2) == currentEpoch {
-		state.FinalizedCheckpoint = oldCurrJustifiedCheckpoint
+		if err := state.SetFinalizedCheckpoint(oldCurrJustifiedCheckpoint); err != nil {
+			return nil, err
+		}
 	}
 
 	// The 1st/2nd (0b0011) most recent epochs are justified, the 1st using the 2nd as source
 	if justification&0x03 == 0x03 && (oldCurrJustifiedCheckpoint.Epoch+1) == currentEpoch {
-		state.FinalizedCheckpoint = oldCurrJustifiedCheckpoint
+		if err := state.SetFinalizedCheckpoint(oldCurrJustifiedCheckpoint); err != nil {
+			return nil, err
+		}
 	}
 
 	return state, nil

beacon-chain/core/epoch/precompute/justification_finalization_test.go

@@ -7,6 +7,7 @@ import (
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/go-bitfield"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
+	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
@@ -18,7 +19,7 @@ func TestProcessJustificationAndFinalizationPreCompute_ConsecutiveEpochs(t *test
 	for i := 0; i < len(blockRoots); i++ {
 		blockRoots[i] = []byte{byte(i)}
 	}
-	state := &pb.BeaconState{
+	base := &pb.BeaconState{
 		Slot: params.BeaconConfig().SlotsPerEpoch*2 + 1,
 		PreviousJustifiedCheckpoint: &ethpb.Checkpoint{
 			Epoch: 0,
@@ -34,30 +35,35 @@ func TestProcessJustificationAndFinalizationPreCompute_ConsecutiveEpochs(t *test
 		Balances:            []uint64{a, a, a, a}, // validator total balance should be 128000000000
 		BlockRoots:          blockRoots,
 	}
+	state, err := beaconstate.InitializeFromProto(base)
+	if err != nil {
+		t.Fatal(err)
+	}
 	attestedBalance := 4 * e * 3 / 2
 	b := &precompute.Balance{PrevEpochTargetAttesters: attestedBalance}
 	newState, err := precompute.ProcessJustificationAndFinalizationPreCompute(state, b)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !bytes.Equal(newState.CurrentJustifiedCheckpoint.Root, []byte{byte(64)}) {
+	rt := [32]byte{byte(64)}
+	if !bytes.Equal(newState.CurrentJustifiedCheckpoint().Root, rt[:]) {
 		t.Errorf("Wanted current justified root: %v, got: %v",
-			[]byte{byte(64)}, newState.CurrentJustifiedCheckpoint.Root)
+			[]byte{byte(64)}, newState.CurrentJustifiedCheckpoint().Root)
 	}
-	if newState.CurrentJustifiedCheckpoint.Epoch != 2 {
+	if newState.CurrentJustifiedCheckpoint().Epoch != 2 {
 		t.Errorf("Wanted justified epoch: %d, got: %d",
-			2, newState.CurrentJustifiedCheckpoint.Epoch)
+			2, newState.CurrentJustifiedCheckpoint().Epoch)
 	}
-	if newState.PreviousJustifiedCheckpoint.Epoch != 0 {
+	if newState.PreviousJustifiedCheckpoint().Epoch != 0 {
 		t.Errorf("Wanted previous justified epoch: %d, got: %d",
-			0, newState.PreviousJustifiedCheckpoint.Epoch)
+			0, newState.PreviousJustifiedCheckpoint().Epoch)
 	}
-	if !bytes.Equal(newState.FinalizedCheckpoint.Root, params.BeaconConfig().ZeroHash[:]) {
+	if !bytes.Equal(newState.FinalizedCheckpoint().Root, params.BeaconConfig().ZeroHash[:]) {
 		t.Errorf("Wanted current finalized root: %v, got: %v",
-			params.BeaconConfig().ZeroHash, newState.FinalizedCheckpoint.Root)
+			params.BeaconConfig().ZeroHash, newState.FinalizedCheckpoint().Root)
 	}
-	if newState.FinalizedCheckpoint.Epoch != 0 {
-		t.Errorf("Wanted finalized epoch: 0, got: %d", newState.FinalizedCheckpoint.Epoch)
+	if newState.FinalizedCheckpointEpoch() != 0 {
+		t.Errorf("Wanted finalized epoch: 0, got: %d", newState.FinalizedCheckpointEpoch())
 	}
 }
 
@@ -68,7 +74,7 @@ func TestProcessJustificationAndFinalizationPreCompute_JustifyCurrentEpoch(t *te
 	for i := 0; i < len(blockRoots); i++ {
 		blockRoots[i] = []byte{byte(i)}
 	}
-	state := &pb.BeaconState{
+	base := &pb.BeaconState{
 		Slot: params.BeaconConfig().SlotsPerEpoch*2 + 1,
 		PreviousJustifiedCheckpoint: &ethpb.Checkpoint{
 			Epoch: 0,
@@ -84,30 +90,35 @@ func TestProcessJustificationAndFinalizationPreCompute_JustifyCurrentEpoch(t *te
 		Balances:            []uint64{a, a, a, a}, // validator total balance should be 128000000000
 		BlockRoots:          blockRoots,
 	}
+	state, err := beaconstate.InitializeFromProto(base)
+	if err != nil {
+		t.Fatal(err)
+	}
 	attestedBalance := 4 * e * 3 / 2
 	b := &precompute.Balance{PrevEpochTargetAttesters: attestedBalance}
 	newState, err := precompute.ProcessJustificationAndFinalizationPreCompute(state, b)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !bytes.Equal(newState.CurrentJustifiedCheckpoint.Root, []byte{byte(64)}) {
+	rt := [32]byte{byte(64)}
+	if !bytes.Equal(newState.CurrentJustifiedCheckpoint().Root, rt[:]) {
 		t.Errorf("Wanted current justified root: %v, got: %v",
-			[]byte{byte(64)}, newState.CurrentJustifiedCheckpoint.Root)
+			[]byte{byte(64)}, newState.CurrentJustifiedCheckpoint().Root)
 	}
-	if newState.CurrentJustifiedCheckpoint.Epoch != 2 {
+	if newState.CurrentJustifiedCheckpoint().Epoch != 2 {
 		t.Errorf("Wanted justified epoch: %d, got: %d",
-			2, newState.CurrentJustifiedCheckpoint.Epoch)
+			2, newState.CurrentJustifiedCheckpoint().Epoch)
 	}
-	if newState.PreviousJustifiedCheckpoint.Epoch != 0 {
+	if newState.PreviousJustifiedCheckpoint().Epoch != 0 {
 		t.Errorf("Wanted previous justified epoch: %d, got: %d",
-			0, newState.PreviousJustifiedCheckpoint.Epoch)
+			0, newState.PreviousJustifiedCheckpoint().Epoch)
 	}
-	if !bytes.Equal(newState.FinalizedCheckpoint.Root, params.BeaconConfig().ZeroHash[:]) {
+	if !bytes.Equal(newState.FinalizedCheckpoint().Root, params.BeaconConfig().ZeroHash[:]) {
 		t.Errorf("Wanted current finalized root: %v, got: %v",
-			params.BeaconConfig().ZeroHash, newState.FinalizedCheckpoint.Root)
+			params.BeaconConfig().ZeroHash, newState.FinalizedCheckpoint().Root)
 	}
-	if newState.FinalizedCheckpoint.Epoch != 0 {
-		t.Errorf("Wanted finalized epoch: 0, got: %d", newState.FinalizedCheckpoint.Epoch)
+	if newState.FinalizedCheckpointEpoch() != 0 {
+		t.Errorf("Wanted finalized epoch: 0, got: %d", newState.FinalizedCheckpointEpoch())
 	}
 }
 
@@ -118,7 +129,7 @@ func TestProcessJustificationAndFinalizationPreCompute_JustifyPrevEpoch(t *testi
 	for i := 0; i < len(blockRoots); i++ {
 		blockRoots[i] = []byte{byte(i)}
 	}
-	state := &pb.BeaconState{
+	base := &pb.BeaconState{
 		Slot: params.BeaconConfig().SlotsPerEpoch*2 + 1,
 		PreviousJustifiedCheckpoint: &ethpb.Checkpoint{
 			Epoch: 0,
@@ -133,29 +144,34 @@ func TestProcessJustificationAndFinalizationPreCompute_JustifyPrevEpoch(t *testi
 		Balances:          []uint64{a, a, a, a}, // validator total balance should be 128000000000
 		BlockRoots:        blockRoots, FinalizedCheckpoint: &ethpb.Checkpoint{},
 	}
+	state, err := beaconstate.InitializeFromProto(base)
+	if err != nil {
+		t.Fatal(err)
+	}
 	attestedBalance := 4 * e * 3 / 2
 	b := &precompute.Balance{PrevEpochTargetAttesters: attestedBalance}
 	newState, err := precompute.ProcessJustificationAndFinalizationPreCompute(state, b)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !bytes.Equal(newState.CurrentJustifiedCheckpoint.Root, []byte{byte(64)}) {
+	rt := [32]byte{byte(64)}
+	if !bytes.Equal(newState.CurrentJustifiedCheckpoint().Root, rt[:]) {
 		t.Errorf("Wanted current justified root: %v, got: %v",
-			[]byte{byte(64)}, newState.CurrentJustifiedCheckpoint.Root)
+			[]byte{byte(64)}, newState.CurrentJustifiedCheckpoint().Root)
 	}
-	if newState.PreviousJustifiedCheckpoint.Epoch != 0 {
+	if newState.PreviousJustifiedCheckpoint().Epoch != 0 {
 		t.Errorf("Wanted previous justified epoch: %d, got: %d",
-			0, newState.PreviousJustifiedCheckpoint.Epoch)
+			0, newState.PreviousJustifiedCheckpoint().Epoch)
 	}
-	if newState.CurrentJustifiedCheckpoint.Epoch != 2 {
+	if newState.CurrentJustifiedCheckpoint().Epoch != 2 {
 		t.Errorf("Wanted justified epoch: %d, got: %d",
-			2, newState.CurrentJustifiedCheckpoint.Epoch)
+			2, newState.CurrentJustifiedCheckpoint().Epoch)
 	}
-	if !bytes.Equal(newState.FinalizedCheckpoint.Root, params.BeaconConfig().ZeroHash[:]) {
+	if !bytes.Equal(newState.FinalizedCheckpoint().Root, params.BeaconConfig().ZeroHash[:]) {
 		t.Errorf("Wanted current finalized root: %v, got: %v",
-			params.BeaconConfig().ZeroHash, newState.FinalizedCheckpoint.Root)
+			params.BeaconConfig().ZeroHash, newState.FinalizedCheckpoint().Root)
 	}
-	if newState.FinalizedCheckpoint.Epoch != 0 {
-		t.Errorf("Wanted finalized epoch: 0, got: %d", newState.FinalizedCheckpoint.Epoch)
+	if newState.FinalizedCheckpointEpoch() != 0 {
+		t.Errorf("Wanted finalized epoch: 0, got: %d", newState.FinalizedCheckpointEpoch())
 	}
 }

beacon-chain/core/epoch/precompute/new.go

@@ -4,7 +4,7 @@ import (
 	"context"
 
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"go.opencensus.io/trace"
 )
@@ -12,40 +12,40 @@ import (
 // New gets called at the beginning of process epoch cycle to return
 // pre computed instances of validators attesting records and total
 // balances attested in an epoch.
-func New(ctx context.Context, state *pb.BeaconState) ([]*Validator, *Balance) {
+func New(ctx context.Context, state *stateTrie.BeaconState) ([]*Validator, *Balance) {
 	ctx, span := trace.StartSpan(ctx, "precomputeEpoch.New")
 	defer span.End()
-
-	vp := make([]*Validator, len(state.Validators))
+	vp := make([]*Validator, state.NumValidators())
 	bp := &Balance{}
 
 	currentEpoch := helpers.CurrentEpoch(state)
 	prevEpoch := helpers.PrevEpoch(state)
 
-	for i, v := range state.Validators {
+	state.ReadFromEveryValidator(func(idx int, val *stateTrie.ReadOnlyValidator) error {
 		// Was validator withdrawable or slashed
-		withdrawable := currentEpoch >= v.WithdrawableEpoch
+		withdrawable := currentEpoch >= val.WithdrawableEpoch()
 		p := &Validator{
-			IsSlashed:                    v.Slashed,
+			IsSlashed:                    val.Slashed(),
 			IsWithdrawableCurrentEpoch:   withdrawable,
-			CurrentEpochEffectiveBalance: v.EffectiveBalance,
+			CurrentEpochEffectiveBalance: val.EffectiveBalance(),
 		}
 		// Was validator active current epoch
-		if helpers.IsActiveValidator(v, currentEpoch) {
+		if helpers.IsActiveValidatorUsingTrie(val, currentEpoch) {
 			p.IsActiveCurrentEpoch = true
-			bp.CurrentEpoch += v.EffectiveBalance
+			bp.CurrentEpoch += val.EffectiveBalance()
 		}
 		// Was validator active previous epoch
-		if helpers.IsActiveValidator(v, prevEpoch) {
+		if helpers.IsActiveValidatorUsingTrie(val, prevEpoch) {
 			p.IsActivePrevEpoch = true
-			bp.PrevEpoch += v.EffectiveBalance
+			bp.PrevEpoch += val.EffectiveBalance()
 		}
 		// Set inclusion slot and inclusion distance to be max, they will be compared and replaced
 		// with the lower values
 		p.InclusionSlot = params.BeaconConfig().FarFutureEpoch
 		p.InclusionDistance = params.BeaconConfig().FarFutureEpoch
 
-		vp[i] = p
-	}
+		vp[idx] = p
+		return nil
+	})
 	return vp, bp
 }

beacon-chain/core/epoch/precompute/new_test.go

@@ -7,13 +7,14 @@ import (
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
+	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
 func TestNew(t *testing.T) {
 	ffe := params.BeaconConfig().FarFutureEpoch
-	s := &pb.BeaconState{
+	s, err := beaconstate.InitializeFromProto(&pb.BeaconState{
 		Slot: params.BeaconConfig().SlotsPerEpoch,
 		// Validator 0 is slashed
 		// Validator 1 is withdrawable
@@ -25,6 +26,9 @@ func TestNew(t *testing.T) {
 			{WithdrawableEpoch: ffe, ExitEpoch: ffe, EffectiveBalance: 100},
 			{WithdrawableEpoch: ffe, ExitEpoch: 1, EffectiveBalance: 100},
 		},
+	})
+	if err != nil {
+		t.Fatal(err)
 	}
 	e := params.BeaconConfig().FarFutureEpoch
 	v, b := precompute.New(context.Background(), s)

beacon-chain/core/epoch/precompute/reward_penalty.go

@@ -2,22 +2,28 @@ package precompute
 
 import (
 	"github.com/pkg/errors"
+
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/mathutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
 // ProcessRewardsAndPenaltiesPrecompute processes the rewards and penalties of individual validator.
 // This is an optimized version by passing in precomputed validator attesting records and and total epoch balances.
-func ProcessRewardsAndPenaltiesPrecompute(state *pb.BeaconState, bp *Balance, vp []*Validator) (*pb.BeaconState, error) {
+func ProcessRewardsAndPenaltiesPrecompute(
+	state *stateTrie.BeaconState,
+	bp *Balance,
+	vp []*Validator,
+) (*stateTrie.BeaconState, error) {
 	// Can't process rewards and penalties in genesis epoch.
 	if helpers.CurrentEpoch(state) == 0 {
 		return state, nil
 	}
 
+	numOfVals := state.NumValidators()
 	// Guard against an out-of-bounds using validator balance precompute.
-	if len(vp) != len(state.Validators) || len(vp) != len(state.Balances) {
+	if len(vp) != numOfVals || len(vp) != state.BalancesLength() {
 		return state, errors.New("precomputed registries not the same length as state registries")
 	}
 
@@ -29,18 +35,34 @@ func ProcessRewardsAndPenaltiesPrecompute(state *pb.BeaconState, bp *Balance, vp
 	if err != nil {
 		return nil, errors.Wrap(err, "could not get attestation delta")
 	}
-	for i := 0; i < len(state.Validators); i++ {
-		state = helpers.IncreaseBalance(state, uint64(i), attsRewards[i]+proposerRewards[i])
-		state = helpers.DecreaseBalance(state, uint64(i), attsPenalties[i])
+	for i := 0; i < numOfVals; i++ {
+		vp[i].BeforeEpochTransitionBalance, err = state.BalanceAtIndex(uint64(i))
+		if err != nil {
+			return nil, errors.Wrap(err, "could not get validator balance before epoch")
+		}
+
+		if err := helpers.IncreaseBalance(state, uint64(i), attsRewards[i]+proposerRewards[i]); err != nil {
+			return nil, err
+		}
+		if err := helpers.DecreaseBalance(state, uint64(i), attsPenalties[i]); err != nil {
+			return nil, err
+		}
+
+		vp[i].AfterEpochTransitionBalance, err = state.BalanceAtIndex(uint64(i))
+		if err != nil {
+			return nil, errors.Wrap(err, "could not get validator balance after epoch")
+		}
 	}
+
 	return state, nil
 }
 
 // This computes the rewards and penalties differences for individual validators based on the
 // voting records.
-func attestationDeltas(state *pb.BeaconState, bp *Balance, vp []*Validator) ([]uint64, []uint64, error) {
-	rewards := make([]uint64, len(state.Validators))
-	penalties := make([]uint64, len(state.Validators))
+func attestationDeltas(state *stateTrie.BeaconState, bp *Balance, vp []*Validator) ([]uint64, []uint64, error) {
+	numOfVals := state.NumValidators()
+	rewards := make([]uint64, numOfVals)
+	penalties := make([]uint64, numOfVals)
 
 	for i, v := range vp {
 		rewards[i], penalties[i] = attestationDelta(state, bp, v)
@@ -48,7 +70,7 @@ func attestationDeltas(state *pb.BeaconState, bp *Balance, vp []*Validator) ([]u
 	return rewards, penalties, nil
 }
 
-func attestationDelta(state *pb.BeaconState, bp *Balance, v *Validator) (uint64, uint64) {
+func attestationDelta(state *stateTrie.BeaconState, bp *Balance, v *Validator) (uint64, uint64) {
 	eligible := v.IsActivePrevEpoch || (v.IsSlashed && !v.IsWithdrawableCurrentEpoch)
 	if !eligible {
 		return 0, 0
@@ -84,7 +106,8 @@ func attestationDelta(state *pb.BeaconState, bp *Balance, v *Validator) (uint64,
 	}
 
 	// Process finality delay penalty
-	finalityDelay := e - state.FinalizedCheckpoint.Epoch
+	finalizedEpoch := state.FinalizedCheckpointEpoch()
+	finalityDelay := e - finalizedEpoch
 	if finalityDelay > params.BeaconConfig().MinEpochsToInactivityPenalty {
 		p += params.BeaconConfig().BaseRewardsPerEpoch * br
 		if !v.IsPrevEpochTargetAttester {
@@ -96,8 +119,9 @@ func attestationDelta(state *pb.BeaconState, bp *Balance, v *Validator) (uint64,
 
 // This computes the rewards and penalties differences for individual validators based on the
 // proposer inclusion records.
-func proposerDeltaPrecompute(state *pb.BeaconState, bp *Balance, vp []*Validator) ([]uint64, error) {
-	rewards := make([]uint64, len(state.Validators))
+func proposerDeltaPrecompute(state *stateTrie.BeaconState, bp *Balance, vp []*Validator) ([]uint64, error) {
+	numofVals := state.NumValidators()
+	rewards := make([]uint64, numofVals)
 
 	totalBalance := bp.CurrentEpoch
 

beacon-chain/core/epoch/precompute/reward_penalty_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/prysmaticlabs/go-bitfield"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
@@ -15,7 +16,7 @@ import (
 func TestProcessRewardsAndPenaltiesPrecompute(t *testing.T) {
 	e := params.BeaconConfig().SlotsPerEpoch
 	validatorCount := uint64(2048)
-	state := buildState(e+3, validatorCount)
+	base := buildState(e+3, validatorCount)
 	atts := make([]*pb.PendingAttestation, 3)
 	for i := 0; i < len(atts); i++ {
 		atts[i] = &pb.PendingAttestation{
@@ -27,10 +28,15 @@ func TestProcessRewardsAndPenaltiesPrecompute(t *testing.T) {
 			InclusionDelay:  1,
 		}
 	}
-	state.PreviousEpochAttestations = atts
+	base.PreviousEpochAttestations = atts
+
+	state, err := state.InitializeFromProto(base)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	vp, bp := New(context.Background(), state)
-	vp, bp, err := ProcessAttestations(context.Background(), state, vp, bp)
+	vp, bp, err = ProcessAttestations(context.Background(), state, vp, bp)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -42,38 +48,48 @@ func TestProcessRewardsAndPenaltiesPrecompute(t *testing.T) {
 
 	// Indices that voted everything except for head, lost a bit money
 	wanted := uint64(31999810265)
-	if state.Balances[4] != wanted {
+	if state.Balances()[4] != wanted {
 		t.Errorf("wanted balance: %d, got: %d",
-			wanted, state.Balances[4])
+			wanted, state.Balances()[4])
 	}
 
 	// Indices that did not vote, lost more money
 	wanted = uint64(31999873505)
-	if state.Balances[0] != wanted {
+	if state.Balances()[0] != wanted {
 		t.Errorf("wanted balance: %d, got: %d",
-			wanted, state.Balances[0])
+			wanted, state.Balances()[0])
 	}
 }
 
 func TestAttestationDeltaPrecompute(t *testing.T) {
 	e := params.BeaconConfig().SlotsPerEpoch
 	validatorCount := uint64(2048)
-	state := buildState(e+2, validatorCount)
+	base := buildState(e+2, validatorCount)
 	atts := make([]*pb.PendingAttestation, 3)
+	var emptyRoot [32]byte
 	for i := 0; i < len(atts); i++ {
 		atts[i] = &pb.PendingAttestation{
 			Data: &ethpb.AttestationData{
-				Target: &ethpb.Checkpoint{},
-				Source: &ethpb.Checkpoint{},
+				Target: &ethpb.Checkpoint{
+					Root: emptyRoot[:],
+				},
+				Source: &ethpb.Checkpoint{
+					Root: emptyRoot[:],
+				},
+				BeaconBlockRoot: emptyRoot[:],
 			},
 			AggregationBits: bitfield.Bitlist{0xC0, 0xC0, 0xC0, 0xC0, 0x01},
 			InclusionDelay:  1,
 		}
 	}
-	state.PreviousEpochAttestations = atts
+	base.PreviousEpochAttestations = atts
+	state, err := state.InitializeFromProto(base)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	vp, bp := New(context.Background(), state)
-	vp, bp, err := ProcessAttestations(context.Background(), state, vp, bp)
+	vp, bp, err = ProcessAttestations(context.Background(), state, vp, bp)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -91,7 +107,7 @@ func TestAttestationDeltaPrecompute(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	attestedIndices := []uint64{100, 106, 196, 641, 654, 1606}
+	attestedIndices := []uint64{55, 1339, 1746, 1811, 1569, 1413}
 	for _, i := range attestedIndices {
 		base, err := epoch.BaseReward(state, i)
 		if err != nil {
@@ -104,7 +120,7 @@ func TestAttestationDeltaPrecompute(t *testing.T) {
 		proposerReward := base / params.BeaconConfig().ProposerRewardQuotient
 		wanted += (base - proposerReward) * params.BeaconConfig().MinAttestationInclusionDelay
 		if rewards[i] != wanted {
-			t.Errorf("Wanted reward balance %d, got %d", wanted, rewards[i])
+			t.Errorf("Wanted reward balance %d, got %d for validator with index %d", wanted, rewards[i], i)
 		}
 		// Since all these validators attested, they shouldn't get penalized.
 		if penalties[i] != 0 {
@@ -112,7 +128,7 @@ func TestAttestationDeltaPrecompute(t *testing.T) {
 		}
 	}
 
-	nonAttestedIndices := []uint64{12, 23, 45, 79}
+	nonAttestedIndices := []uint64{434, 677, 872, 791}
 	for _, i := range nonAttestedIndices {
 		base, err := epoch.BaseReward(state, i)
 		if err != nil {

beacon-chain/core/epoch/precompute/slashing.go

@@ -1,34 +1,39 @@
 package precompute
 
 import (
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/mathutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
 // ProcessSlashingsPrecompute processes the slashed validators during epoch processing.
 // This is an optimized version by passing in precomputed total epoch balances.
-func ProcessSlashingsPrecompute(state *pb.BeaconState, p *Balance) *pb.BeaconState {
+func ProcessSlashingsPrecompute(state *stateTrie.BeaconState, p *Balance) error {
 	currentEpoch := helpers.CurrentEpoch(state)
 	exitLength := params.BeaconConfig().EpochsPerSlashingsVector
 
 	// Compute the sum of state slashings
+	slashings := state.Slashings()
 	totalSlashing := uint64(0)
-	for _, slashing := range state.Slashings {
+	for _, slashing := range slashings {
 		totalSlashing += slashing
 	}
 
-	// Compute slashing for each validator.
-	for index, validator := range state.Validators {
-		correctEpoch := (currentEpoch + exitLength/2) == validator.WithdrawableEpoch
-		if validator.Slashed && correctEpoch {
+	validatorFunc := func(idx int, val *ethpb.Validator) error {
+		correctEpoch := (currentEpoch + exitLength/2) == val.WithdrawableEpoch
+		if val.Slashed && correctEpoch {
 			minSlashing := mathutil.Min(totalSlashing*3, p.CurrentEpoch)
 			increment := params.BeaconConfig().EffectiveBalanceIncrement
-			penaltyNumerator := validator.EffectiveBalance / increment * minSlashing
+			penaltyNumerator := val.EffectiveBalance / increment * minSlashing
 			penalty := penaltyNumerator / p.CurrentEpoch * increment
-			state = helpers.DecreaseBalance(state, uint64(index), penalty)
+			if err := helpers.DecreaseBalance(state, uint64(idx), penalty); err != nil {
+				return err
+			}
 		}
+		return nil
 	}
-	return state
+
+	return state.ApplyToEveryValidator(validatorFunc)
 }

beacon-chain/core/epoch/precompute/slashing_test.go

@@ -6,23 +6,29 @@ import (
 	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
+	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
 func TestProcessSlashingsPrecompute_NotSlashed(t *testing.T) {
-	s := &pb.BeaconState{
+	s, err := beaconstate.InitializeFromProto(&pb.BeaconState{
 		Slot:       0,
 		Validators: []*ethpb.Validator{{Slashed: true}},
 		Balances:   []uint64{params.BeaconConfig().MaxEffectiveBalance},
 		Slashings:  []uint64{0, 1e9},
+	})
+	if err != nil {
+		t.Fatal(err)
 	}
 	bp := &precompute.Balance{CurrentEpoch: params.BeaconConfig().MaxEffectiveBalance}
-	newState := precompute.ProcessSlashingsPrecompute(s, bp)
+	if err := precompute.ProcessSlashingsPrecompute(s, bp); err != nil {
+		t.Fatal(err)
+	}
 
 	wanted := params.BeaconConfig().MaxEffectiveBalance
-	if newState.Balances[0] != wanted {
-		t.Errorf("Wanted slashed balance: %d, got: %d", wanted, newState.Balances[0])
+	if s.Balances()[0] != wanted {
+		t.Errorf("Wanted slashed balance: %d, got: %d", wanted, s.Balances()[0])
 	}
 }
 
@@ -107,13 +113,19 @@ func TestProcessSlashingsPrecompute_SlashedLess(t *testing.T) {
 			bp := &precompute.Balance{CurrentEpoch: ab}
 
 			original := proto.Clone(tt.state)
-			newState := precompute.ProcessSlashingsPrecompute(tt.state, bp)
+			state, err := beaconstate.InitializeFromProto(tt.state)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if err := precompute.ProcessSlashingsPrecompute(state, bp); err != nil {
+				t.Fatal(err)
+			}
 
-			if newState.Balances[0] != tt.want {
+			if state.Balances()[0] != tt.want {
 				t.Errorf(
 					"ProcessSlashings({%v}) = newState; newState.Balances[0] = %d; wanted %d",
 					original,
-					newState.Balances[0],
+					state.Balances()[0],
 					tt.want,
 				)
 			}

beacon-chain/core/epoch/precompute/type.go

@@ -31,6 +31,10 @@ type Validator struct {
 	InclusionDistance uint64
 	// ProposerIndex is the index of proposer at slot where this validator's attestation was included.
 	ProposerIndex uint64
+	// BeforeEpochTransitionBalance is the validator balance prior to epoch transition.
+	BeforeEpochTransitionBalance uint64
+	// AfterEpochTransitionBalance is the validator balance after epoch transition.
+	AfterEpochTransitionBalance uint64
 }
 
 // Balance stores the pre computation of the total participated balances for a given epoch

beacon-chain/core/epoch/spectest/BUILD.bazel

@@ -28,6 +28,7 @@ go_test(
         "//beacon-chain/core/epoch:go_default_library",
         "//beacon-chain/core/epoch/precompute:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/params/spectest:go_default_library",
         "//shared/testutil:go_default_library",
@@ -58,6 +59,7 @@ go_test(
         "//beacon-chain/core/epoch:go_default_library",
         "//beacon-chain/core/epoch/precompute:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
+        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/params/spectest:go_default_library",
         "//shared/testutil:go_default_library",

beacon-chain/core/epoch/spectest/final_updates_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/params/spectest"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
@@ -24,7 +24,7 @@ func runFinalUpdatesTests(t *testing.T, config string) {
 	}
 }
 
-func processFinalUpdatesWrapper(t *testing.T, state *pb.BeaconState) (*pb.BeaconState, error) {
+func processFinalUpdatesWrapper(t *testing.T, state *beaconstate.BeaconState) (*beaconstate.BeaconState, error) {
 	state, err := epoch.ProcessFinalUpdates(state)
 	if err != nil {
 		t.Fatalf("could not process final updates: %v", err)

beacon-chain/core/epoch/spectest/justification_and_finalization_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/params/spectest"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
@@ -26,7 +26,7 @@ func runJustificationAndFinalizationTests(t *testing.T, config string) {
 	}
 }
 
-func processJustificationAndFinalizationPrecomputeWrapper(t *testing.T, state *pb.BeaconState) (*pb.BeaconState, error) {
+func processJustificationAndFinalizationPrecomputeWrapper(t *testing.T, state *state.BeaconState) (*state.BeaconState, error) {
 	ctx := context.Background()
 	vp, bp := precompute.New(ctx, state)
 	_, bp, err := precompute.ProcessAttestations(ctx, state, vp, bp)

beacon-chain/core/epoch/spectest/registry_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/params/spectest"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
@@ -24,7 +24,7 @@ func runRegistryUpdatesTests(t *testing.T, config string) {
 	}
 }
 
-func processRegistryUpdatesWrapper(t *testing.T, state *pb.BeaconState) (*pb.BeaconState, error) {
+func processRegistryUpdatesWrapper(t *testing.T, state *beaconstate.BeaconState) (*beaconstate.BeaconState, error) {
 	state, err := epoch.ProcessRegistryUpdates(state)
 	if err != nil {
 		t.Fatalf("could not process registry updates: %v", err)

beacon-chain/core/epoch/spectest/slashings_test.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/params/spectest"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
@@ -27,7 +27,7 @@ func runSlashingsTests(t *testing.T, config string) {
 	}
 }
 
-func processSlashingsWrapper(t *testing.T, state *pb.BeaconState) (*pb.BeaconState, error) {
+func processSlashingsWrapper(t *testing.T, state *beaconstate.BeaconState) (*beaconstate.BeaconState, error) {
 	state, err := epoch.ProcessSlashings(state)
 	if err != nil {
 		t.Fatalf("could not process slashings: %v", err)
@@ -35,7 +35,7 @@ func processSlashingsWrapper(t *testing.T, state *pb.BeaconState) (*pb.BeaconSta
 	return state, nil
 }
 
-func processSlashingsPrecomputeWrapper(t *testing.T, state *pb.BeaconState) (*pb.BeaconState, error) {
+func processSlashingsPrecomputeWrapper(t *testing.T, state *beaconstate.BeaconState) (*beaconstate.BeaconState, error) {
 	ctx := context.Background()
 	vp, bp := precompute.New(ctx, state)
 	_, bp, err := precompute.ProcessAttestations(ctx, state, vp, bp)
@@ -43,6 +43,5 @@ func processSlashingsPrecomputeWrapper(t *testing.T, state *pb.BeaconState) (*pb
 		t.Fatal(err)
 	}
 
-	state = precompute.ProcessSlashingsPrecompute(state, bp)
-	return state, nil
+	return state, precompute.ProcessSlashingsPrecompute(state, bp)
 }

beacon-chain/core/exit/BUILD.bazel

@@ -1,37 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
-
-go_library(
-    name = "go_default_library",
-    srcs = ["validation.go"],
-    importpath = "github.com/prysmaticlabs/prysm/beacon-chain/core/exit",
-    visibility = [
-        "//beacon-chain:__subpackages__",
-    ],
-    deps = [
-        "//beacon-chain/core/helpers:go_default_library",
-        "//proto/beacon/p2p/v1:go_default_library",
-        "//shared/bls:go_default_library",
-        "//shared/mathutil:go_default_library",
-        "//shared/params:go_default_library",
-        "//shared/roughtime:go_default_library",
-        "@com_github_pkg_errors//:go_default_library",
-        "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
-        "@com_github_prysmaticlabs_go_ssz//:go_default_library",
-    ],
-)
-
-go_test(
-    name = "go_default_test",
-    srcs = ["validation_test.go"],
-    embed = [":go_default_library"],
-    deps = [
-        "//beacon-chain/blockchain/testing:go_default_library",
-        "//beacon-chain/core/blocks:go_default_library",
-        "//beacon-chain/core/state:go_default_library",
-        "//beacon-chain/db/testing:go_default_library",
-        "//shared/params:go_default_library",
-        "//shared/testutil:go_default_library",
-        "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
-        "@com_github_prysmaticlabs_go_ssz//:go_default_library",
-    ],
-)

beacon-chain/core/exit/validation.go

@@ -1,66 +0,0 @@
-package exit
-
-import (
-	"fmt"
-	"time"
-
-	"github.com/pkg/errors"
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bls"
-	"github.com/prysmaticlabs/prysm/shared/mathutil"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/prysmaticlabs/prysm/shared/roughtime"
-)
-
-// ValidateVoluntaryExit validates the voluntary exit.
-// If it is invalid for some reason an error, if valid it will return no error.
-func ValidateVoluntaryExit(state *pb.BeaconState, genesisTime time.Time, signed *ethpb.SignedVoluntaryExit) error {
-	if signed == nil || signed.Exit == nil {
-		return errors.New("nil signed voluntary exit")
-	}
-	ve := signed.Exit
-	if ve.ValidatorIndex >= uint64(len(state.Validators)) {
-		return fmt.Errorf("unknown validator index %d", ve.ValidatorIndex)
-	}
-	validator := state.Validators[ve.ValidatorIndex]
-
-	if !helpers.IsActiveValidator(validator, ve.Epoch) {
-		return fmt.Errorf("validator %d not active at epoch %d", ve.ValidatorIndex, ve.Epoch)
-	}
-	if validator.ExitEpoch != params.BeaconConfig().FarFutureEpoch {
-		return fmt.Errorf("validator %d already exiting or exited", ve.ValidatorIndex)
-	}
-
-	secondsPerEpoch := params.BeaconConfig().SecondsPerSlot * params.BeaconConfig().SlotsPerEpoch
-	currentEpoch := uint64(roughtime.Now().Unix()-genesisTime.Unix()) / secondsPerEpoch
-	earliestRequestedExitEpoch := mathutil.Max(ve.Epoch, currentEpoch)
-	earliestExitEpoch := validator.ActivationEpoch + params.BeaconConfig().PersistentCommitteePeriod
-	if earliestRequestedExitEpoch < earliestExitEpoch {
-		return fmt.Errorf("validator %d cannot exit before epoch %d", ve.ValidatorIndex, earliestExitEpoch)
-	}
-
-	// Confirm signature is valid
-	root, err := ssz.HashTreeRoot(ve)
-	if err != nil {
-		return errors.Wrap(err, "cannot confirm signature")
-	}
-	sig, err := bls.SignatureFromBytes(signed.Signature)
-	if err != nil {
-		return errors.Wrap(err, "malformed signature")
-	}
-	validatorPubKey, err := bls.PublicKeyFromBytes(validator.PublicKey)
-	if err != nil {
-		return errors.Wrap(err, "invalid validator public key")
-	}
-	domain := bls.ComputeDomain(params.BeaconConfig().DomainVoluntaryExit)
-	verified := sig.Verify(root[:], validatorPubKey, domain)
-	if !verified {
-		return errors.New("incorrect signature")
-	}
-
-	// Parameters are valid.
-	return nil
-}

beacon-chain/core/exit/validation_test.go

@@ -1,125 +0,0 @@
-package exit_test
-
-import (
-	"context"
-	"errors"
-	"testing"
-	"time"
-
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	mockChain "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
-	blk "github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/exit"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
-	dbutil "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/prysmaticlabs/prysm/shared/testutil"
-)
-
-// Set genesis to a small set for faster test processing.
-func init() {
-	p := params.BeaconConfig()
-	p.MinGenesisActiveValidatorCount = 8
-	params.OverrideBeaconConfig(p)
-}
-
-func TestValidation(t *testing.T) {
-	tests := []struct {
-		name           string
-		epoch          uint64
-		validatorIndex uint64
-		signature      []byte
-		err            error
-	}{
-		{
-			name:           "MissingValidator",
-			epoch:          2048,
-			validatorIndex: 16,
-			err:            errors.New("unknown validator index 16"),
-		},
-		{
-			name:           "EarlyExit",
-			epoch:          2047,
-			validatorIndex: 0,
-			err:            errors.New("validator 0 cannot exit before epoch 2048"),
-		},
-		{
-			name:           "NoSignature",
-			epoch:          2048,
-			validatorIndex: 0,
-			err:            errors.New("malformed signature: signature must be 96 bytes"),
-		},
-		{
-			name:           "InvalidSignature",
-			epoch:          2048,
-			validatorIndex: 0,
-			signature:      []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
-			err:            errors.New("malformed signature: could not unmarshal bytes into signature: err blsSignatureDeserialize 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"),
-		},
-		{
-			name:           "IncorrectSignature",
-			epoch:          2048,
-			validatorIndex: 0,
-			signature:      []byte{0xab, 0xb0, 0x12, 0x4c, 0x75, 0x74, 0xf2, 0x81, 0xa2, 0x93, 0xf4, 0x18, 0x5c, 0xad, 0x3c, 0xb2, 0x26, 0x81, 0xd5, 0x20, 0x91, 0x7c, 0xe4, 0x66, 0x65, 0x24, 0x3e, 0xac, 0xb0, 0x51, 0x00, 0x0d, 0x8b, 0xac, 0xf7, 0x5e, 0x14, 0x51, 0x87, 0x0c, 0xa6, 0xb3, 0xb9, 0xe6, 0xc9, 0xd4, 0x1a, 0x7b, 0x02, 0xea, 0xd2, 0x68, 0x5a, 0x84, 0x18, 0x8a, 0x4f, 0xaf, 0xd3, 0x82, 0x5d, 0xaf, 0x6a, 0x98, 0x96, 0x25, 0xd7, 0x19, 0xcc, 0xd2, 0xd8, 0x3a, 0x40, 0x10, 0x1f, 0x4a, 0x45, 0x3f, 0xca, 0x62, 0x87, 0x8c, 0x89, 0x0e, 0xca, 0x62, 0x23, 0x63, 0xf9, 0xdd, 0xb8, 0xf3, 0x67, 0xa9, 0x1e, 0x84},
-			err:            errors.New("incorrect signature"),
-		},
-		{
-			name:           "Good",
-			epoch:          2048,
-			validatorIndex: 0,
-			signature:      []byte{0xb3, 0xe1, 0x9d, 0xc6, 0x7c, 0x78, 0x6c, 0xcf, 0x33, 0x1d, 0xb9, 0x6f, 0x59, 0x64, 0x44, 0xe1, 0x29, 0xd0, 0x87, 0x03, 0x26, 0x6e, 0x49, 0x1c, 0x05, 0xae, 0x16, 0x7b, 0x04, 0x0f, 0x3f, 0xf8, 0x82, 0x77, 0x60, 0xfc, 0xcf, 0x2f, 0x59, 0xc7, 0x40, 0x0b, 0x2c, 0xa9, 0x23, 0x8a, 0x6c, 0x8d, 0x01, 0x21, 0x5e, 0xa8, 0xac, 0x36, 0x70, 0x31, 0xb0, 0xe1, 0xa8, 0xb8, 0x8f, 0x93, 0x8c, 0x1c, 0xa2, 0x86, 0xe7, 0x22, 0x00, 0x6a, 0x7d, 0x36, 0xc0, 0x2b, 0x86, 0x2c, 0xf5, 0xf9, 0x10, 0xb9, 0xf2, 0xbd, 0x5e, 0xa6, 0x5f, 0x12, 0x86, 0x43, 0x20, 0x4d, 0xa2, 0x9d, 0x8b, 0xe6, 0x6f, 0x09},
-		},
-	}
-
-	db := dbutil.SetupDB(t)
-	defer dbutil.TeardownDB(t, db)
-	ctx := context.Background()
-	deposits, _, _ := testutil.DeterministicDepositsAndKeys(params.BeaconConfig().MinGenesisActiveValidatorCount)
-	beaconState, err := state.GenesisBeaconState(deposits, 0, &ethpb.Eth1Data{BlockHash: make([]byte, 32)})
-	if err != nil {
-		t.Fatal(err)
-	}
-	block := blk.NewGenesisBlock([]byte{})
-	if err := db.SaveBlock(ctx, block); err != nil {
-		t.Fatalf("Could not save genesis block: %v", err)
-	}
-	genesisRoot, err := ssz.HashTreeRoot(block.Block)
-	if err != nil {
-		t.Fatalf("Could not get signing root %v", err)
-	}
-
-	// Set genesis time to be 100 epochs ago
-	genesisTime := time.Now().Add(time.Duration(-100*int64(params.BeaconConfig().SecondsPerSlot*params.BeaconConfig().SlotsPerEpoch)) * time.Second)
-	mockChainService := &mockChain.ChainService{State: beaconState, Root: genesisRoot[:], Genesis: genesisTime}
-	headState, err := mockChainService.HeadState(context.Background())
-	if err != nil {
-		t.Fatal("Failed to obtain head state")
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			req := &ethpb.SignedVoluntaryExit{
-				Exit: &ethpb.VoluntaryExit{
-					Epoch:          test.epoch,
-					ValidatorIndex: test.validatorIndex,
-				},
-				Signature: test.signature,
-			}
-
-			err := exit.ValidateVoluntaryExit(headState, genesisTime, req)
-			if test.err == nil {
-				if err != nil {
-					t.Errorf("Unexpected error: received %v", err)
-				}
-			} else {
-				if err == nil {
-					t.Error("Failed to receive expected error")
-				}
-				if err.Error() != test.err.Error() {
-					t.Errorf("Unexpected error: expected %s, received %s", test.err.Error(), err.Error())
-				}
-			}
-		})
-	}
-}

beacon-chain/core/feed/block/BUILD.bazel

@@ -0,0 +1,15 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "events.go",
+        "notifier.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/block",
+    visibility = ["//beacon-chain:__subpackages__"],
+    deps = [
+        "//shared/event:go_default_library",
+        "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
+    ],
+)

beacon-chain/core/feed/block/events.go

@@ -0,0 +1,15 @@
+package block
+
+import (
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+)
+
+const (
+	// ReceivedBlock is sent after a block has been received by the beacon node via p2p or RPC.
+	ReceivedBlock = iota + 1
+)
+
+// ReceivedBlockData is the data sent with ReceivedBlock events.
+type ReceivedBlockData struct {
+	SignedBlock *ethpb.SignedBeaconBlock
+}

beacon-chain/core/feed/block/notifier.go

@@ -0,0 +1,8 @@
+package block
+
+import "github.com/prysmaticlabs/prysm/shared/event"
+
+// Notifier interface defines the methods of the service that provides block updates to consumers.
+type Notifier interface {
+	BlockFeed() *event.Feed
+}

beacon-chain/core/feed/state/events.go

@@ -13,7 +13,9 @@ const (
 
 // BlockProcessedData is the data sent with BlockProcessed events.
 type BlockProcessedData struct {
-	// BlockHash is the hash of the processed block.
+	// Slot is the slot of the processed block.
+	Slot uint64
+	// BlockRoot is the hash of the processed block.
 	BlockRoot [32]byte
 	// Verified is true if the block's BLS contents have been verified.
 	Verified bool