Make Follow Distance Lookup Simpler (#7884)

* faster eth1 search

* simplify it much more

* Update beacon-chain/powchain/block_reader.go

Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com>

.bazelrc

@@ -5,6 +5,9 @@ test --test_verbose_timeout_warnings
 test --build_tests_only
 test --test_output=errors
 
+# E2E run with debug gotag
+test:e2e --define gotags=debug
+
 # Clearly indicate that coverage is enabled to disable certain nogo checks.
 coverage --define=coverage_enabled=1
 

.bazelversion

@@ -1 +1 @@
-3.2.0
+3.7.0

.deepsource.toml

@@ -10,3 +10,7 @@ enabled = true
 [[analyzers]]
 name = "test-coverage"
 enabled = true
+
+[[analyzers]]
+name = "shell"
+enabled = true

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,8 +1,8 @@
 <!-- Thanks for sending a PR! Before submitting:
 
-1. If this is your first PR, please read CONTRIBUTING.md and sign the CLA
-   first. We cannot review code without a signed CLA.
-2. Please file an issue *first*. All features and most bug fixes should have
+1. If this is your first PR, check out our contribution guide here https://docs.prylabs.network/docs/contribute/contribution-guidelines
+   You will then need to sign our Contributor License Agreement (CLA), which will show up as a comment from a bot in this pull request after you open it. We cannot review code without a signed CLA.
+2. Please file an associated tracking issue if this pull request is non-trivial and requires context for our team to understand. All features and most bug fixes should have
    an associated issue with a design discussed and decided upon. Small bug
    fixes and documentation improvements don't need issues.
 3. New features and bug fixes must have tests. Documentation may need to

BUILD.bazel

@@ -112,6 +112,7 @@ nogo(
         "//tools/analyzers/nop:go_tool_library",
         "//tools/analyzers/slicedirect:go_tool_library",
         "//tools/analyzers/ineffassign:go_tool_library",
+        "//tools/analyzers/properpermissions:go_tool_library",
     ] + select({
         # nogo checks that fail with coverage enabled.
         ":coverage_enabled": [],

DEPENDENCIES.md

@@ -61,7 +61,7 @@ Example:
 
 ```bash
 go get github.com/prysmaticlabs/example@v1.2.3
-bazel run //:gazelle -- update-repos -from_file=go.mod -to_macro=deps.bzl%prysm_deps
+bazel run //:gazelle -- update-repos -from_file=go.mod -to_macro=deps.bzl%prysm_deps -prune=true
 ```
 
 The deps.bzl file should have been updated with the dependency and any transitive dependencies. 

README.md

@@ -1,17 +1,17 @@
 # Prysm: An Ethereum 2.0 Client Written in Go
 
 [![Build status](https://badge.buildkite.com/b555891daf3614bae4284dcf365b2340cefc0089839526f096.svg?branch=master)](https://buildkite.com/prysmatic-labs/prysm)
-[![ETH2.0_Spec_Version 0.12.3](https://img.shields.io/badge/ETH2.0%20Spec%20Version-v0.12.3-blue.svg)](https://github.com/ethereum/eth2.0-specs/tree/v0.12.3)
+[![Go Report Card](https://goreportcard.com/badge/github.com/prysmaticlabs/prysm)](https://goreportcard.com/report/github.com/prysmaticlabs/prysm)
+[![ETH2.0_Spec_Version 1.0.0](https://img.shields.io/badge/ETH2.0%20Spec%20Version-v1.0.0-blue.svg)](https://github.com/ethereum/eth2.0-specs/tree/v1.0.0)
 [![Discord](https://user-images.githubusercontent.com/7288322/34471967-1df7808a-efbb-11e7-9088-ed0b04151291.png)](https://discord.gg/KSA7rPr)
-[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/prysmaticlabs/geth-sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 
 This is the core repository for Prysm, a [Golang](https://golang.org/) implementation of the Ethereum 2.0 client specifications developed by [Prysmatic Labs](https://prysmaticlabs.com).
 
 ### Getting Started
-A detailed set of installation and usage instructions as well as breakdowns of each individual component are available in the [official documentation portal](https://docs.prylabs.network). If you still have questions, feel free to stop by either our [Discord](https://discord.gg/KSA7rPr) or [Gitter](https://gitter.im/prysmaticlabs/geth-sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) and a member of the team or our community will be happy to assist you.
+A detailed set of installation and usage instructions as well as breakdowns of each individual component are available in the [official documentation portal](https://docs.prylabs.network). If you still have questions, feel free to stop by our [Discord](https://discord.gg/KSA7rPr).
 
 ### Come join the testnet!
-Participation is now open to the public for our Ethereum 2.0 phase 0 testnet release. Visit [prylabs.net](https://prylabs.net) for more information on the project or to sign up as a validator on the network. You can visualize the nodes in the network on [eth2stats.io](https://eth2stats.io), explore validator rewards/penalties via Bitfly's block explorer: [beaconcha.in](https://beaconcha.in), and follow the latest blocks added to the chain on [Etherscan](https://beacon.etherscan.io).
+Participation is now open to the public for our Ethereum 2.0 phase 0 testnet release. Visit [prylabs.net](https://prylabs.net) for more information on the project or to sign up as a validator on the network. You can visualize the nodes in the network on [eth2stats.io](https://eth2stats.io), explore validator rewards/penalties via Bitfly's block explorer: [beaconcha.in](https://beaconcha.in), and follow the latest blocks added to the chain on [beaconscan](https://beaconscan.com).
 
 ## Contributing
 Want to get involved? Check out our [Contribution Guide](https://docs.prylabs.network/docs/contribute/contribution-guidelines/) to learn more!

WORKSPACE

@@ -5,19 +5,19 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 
 http_archive(
     name = "bazel_toolchains",
-    sha256 = "db48eed61552e25d36fe051a65d2a329cc0fb08442627e8f13960c5ab087a44e",
-    strip_prefix = "bazel-toolchains-3.2.0",
+    sha256 = "8e0633dfb59f704594f19ae996a35650747adc621ada5e8b9fb588f808c89cb0",
+    strip_prefix = "bazel-toolchains-3.7.0",
     urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/releases/download/3.2.0/bazel-toolchains-3.2.0.tar.gz",
-        "https://github.com/bazelbuild/bazel-toolchains/releases/download/3.2.0/bazel-toolchains-3.2.0.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/releases/download/3.7.0/bazel-toolchains-3.7.0.tar.gz",
+        "https://github.com/bazelbuild/bazel-toolchains/releases/download/3.7.0/bazel-toolchains-3.7.0.tar.gz",
     ],
 )
 
 http_archive(
     name = "com_grail_bazel_toolchain",
-    sha256 = "0bec89e35d8a141c87f28cfc506d6d344785c8eb2ff3a453140a1fe972ada79d",
-    strip_prefix = "bazel-toolchain-77a87103145f86f03f90475d19c2c8854398a444",
-    urls = ["https://github.com/grailbio/bazel-toolchain/archive/77a87103145f86f03f90475d19c2c8854398a444.tar.gz"],
+    sha256 = "b924b102adc0c3368d38a19bd971cb4fa75362a27bc363d0084b90ca6877d3f0",
+    strip_prefix = "bazel-toolchain-0.5.7",
+    urls = ["https://github.com/grailbio/bazel-toolchain/archive/0.5.7.tar.gz"],
 )
 
 load("@com_grail_bazel_toolchain//toolchain:deps.bzl", "bazel_toolchain_dependencies")
@@ -28,7 +28,7 @@ load("@com_grail_bazel_toolchain//toolchain:rules.bzl", "llvm_toolchain")
 
 llvm_toolchain(
     name = "llvm_toolchain",
-    llvm_version = "9.0.0",
+    llvm_version = "10.0.0",
 )
 
 load("@llvm_toolchain//:toolchains.bzl", "llvm_register_toolchains")
@@ -47,10 +47,10 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 http_archive(
     name = "bazel_skylib",
-    sha256 = "97e70364e9249702246c0e9444bccdc4b847bed1eb03c5a3ece4f83dfe6abc44",
+    sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c",
     urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz",
-        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz",
+        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz",
     ],
 )
 
@@ -60,10 +60,10 @@ bazel_skylib_workspace()
 
 http_archive(
     name = "bazel_gazelle",
-    sha256 = "d8c45ee70ec39a57e7a05e5027c32b1576cc7f16d9dd37135b0eddde45cf1b10",
+    sha256 = "1f4fc1d91826ec436ae04833430626f4cc02c20bb0a813c0c2f3c4c421307b1d",
+    strip_prefix = "bazel-gazelle-e368a11b76e92932122d824970dc0ce5feb9c349",
     urls = [
-        "https://storage.googleapis.com/bazel-mirror/github.com/bazelbuild/bazel-gazelle/releases/download/v0.20.0/bazel-gazelle-v0.20.0.tar.gz",
-        "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.20.0/bazel-gazelle-v0.20.0.tar.gz",
+        "https://github.com/bazelbuild/bazel-gazelle/archive/e368a11b76e92932122d824970dc0ce5feb9c349.tar.gz",
     ],
 )
 
@@ -76,9 +76,9 @@ http_archive(
 
 http_archive(
     name = "io_bazel_rules_docker",
-    sha256 = "dc97fccceacd4c6be14e800b2a00693d5e8d07f69ee187babfd04a80a9f8e250",
-    strip_prefix = "rules_docker-0.14.1",
-    url = "https://github.com/bazelbuild/rules_docker/archive/v0.14.1.tar.gz",
+    sha256 = "1698624e878b0607052ae6131aa216d45ebb63871ec497f26c67455b34119c80",
+    strip_prefix = "rules_docker-0.15.0",
+    urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.15.0/rules_docker-v0.15.0.tar.gz"],
 )
 
 http_archive(
@@ -89,10 +89,10 @@ http_archive(
         # nogo check fails for certain third_party dependencies.
         "//third_party:io_bazel_rules_go.patch",
     ],
-    sha256 = "7b9bbe3ea1fccb46dcfa6c3f3e29ba7ec740d8733370e21cdc8937467b4a4349",
+    sha256 = "207fad3e6689135c5d8713e5a17ba9d1290238f47b9ba545b63d9303406209c6",
     urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.22.4/rules_go-v0.22.4.tar.gz",
-        "https://github.com/bazelbuild/rules_go/releases/download/v0.22.4/rules_go-v0.22.4.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.24.7/rules_go-v0.24.7.tar.gz",
+        "https://github.com/bazelbuild/rules_go/releases/download/v0.24.7/rules_go-v0.24.7.tar.gz",
     ],
 )
 
@@ -155,7 +155,10 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_depe
 
 go_rules_dependencies()
 
-go_register_toolchains(nogo = "@//:nogo")
+go_register_toolchains(
+    go_version = "1.15.5",
+    nogo = "@//:nogo",
+)
 
 load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
 
@@ -219,8 +222,8 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "7e5f838e0f9110471ef8be9401ea687a8ed4d499664dc0eac34ecfdfd03c2ac3",
-    url = "https://github.com/ethereum/eth2.0-spec-tests/releases/download/v0.12.3/general.tar.gz",
+    sha256 = "ef5396e4b13995da9776eeb5ae346a2de90970c28da3c4f0dcaa4ab9f0ad1f93",
+    url = "https://github.com/ethereum/eth2.0-spec-tests/releases/download/v1.0.0/general.tar.gz",
 )
 
 http_archive(
@@ -235,8 +238,8 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "72c2f561db879ddcdf729fef93d10e0f9162b4cf3a697c513ef8935b93f6165a",
-    url = "https://github.com/ethereum/eth2.0-spec-tests/releases/download/v0.12.3/minimal.tar.gz",
+    sha256 = "170551b441e7d54b73248372ad9ce8cb6c148810b5f1364637117a63f4f1c085",
+    url = "https://github.com/ethereum/eth2.0-spec-tests/releases/download/v1.0.0/minimal.tar.gz",
 )
 
 http_archive(
@@ -251,8 +254,8 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "63eca02503692a0b6a2d7b70118e0dd62dff094153a3a542af6dbea721841b0d",
-    url = "https://github.com/ethereum/eth2.0-spec-tests/releases/download/v0.12.3/mainnet.tar.gz",
+    sha256 = "b541a9979b4703fa5ee5d2182b0b5313c38efc54ae7eaec2eef793230a52ec83",
+    url = "https://github.com/ethereum/eth2.0-spec-tests/releases/download/v1.0.0/mainnet.tar.gz",
 )
 
 http_archive(
@@ -268,9 +271,9 @@ buildifier_dependencies()
 
 git_repository(
     name = "com_google_protobuf",
-    commit = "4059c61f27eb1b06c4ee979546a238be792df0a4",
+    commit = "fde7cf7358ec7cd69e8db9be4f1fa6a5c431386a",  # v3.13.0
     remote = "https://github.com/protocolbuffers/protobuf",
-    shallow_since = "1558721209 -0700",
+    shallow_since = "1597443653 -0700",
 )
 
 load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")

beacon-chain/BUILD.bazel

@@ -1,7 +1,7 @@
 load("@prysm//tools/go:def.bzl", "go_library")
 load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_test")
 load("@io_bazel_rules_docker//go:image.bzl", "go_image")
-load("@io_bazel_rules_docker//container:container.bzl", "container_bundle")
+load("@io_bazel_rules_docker//container:container.bzl", "container_bundle", "container_image")
 load("//tools:go_image.bzl", "go_image_alpine", "go_image_debug")
 load("@io_bazel_rules_docker//contrib:push-all.bzl", "docker_push")
 
@@ -35,49 +35,29 @@ go_library(
 
 go_image(
     name = "image",
-    srcs = [
-        "main.go",
-        "usage.go",
-    ],
     base = select({
         "//tools:base_image_alpine": "//tools:alpine_cc_image",
         "//tools:base_image_cc": "//tools:cc_image",
         "//conditions:default": "//tools:cc_image",
     }),
-    goarch = "amd64",
-    goos = "linux",
-    importpath = "github.com/prysmaticlabs/prysm/beacon-chain",
-    race = "off",
-    static = "off",  # Static enabled binary seems to cause issues with DNS lookup with cgo.
+    binary = ":beacon-chain",
     tags = ["manual"],
     visibility = ["//visibility:private"],
-    deps = [
-        "//beacon-chain/flags:go_default_library",
-        "//beacon-chain/node:go_default_library",
-        "//shared/tos:go_default_library",
-        "//shared/cmd:go_default_library",
-        "//shared/debug:go_default_library",
-        "//shared/featureconfig:go_default_library",
-        "//shared/journald:go_default_library",
-        "//shared/logutil:go_default_library",
-        "//shared/maxprocs:go_default_library",
-        "//shared/version:go_default_library",
-        "@com_github_ethereum_go_ethereum//log:go_default_library",
-        "@com_github_ipfs_go_log_v2//:go_default_library",
-        "@com_github_joonix_log//:go_default_library",
-        "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_urfave_cli_v2//:go_default_library",
-        "@com_github_x_cray_logrus_prefixed_formatter//:go_default_library",
-    ],
+)
+
+container_image(
+    name = "image_with_creation_time",
+    base = "image",
+    stamp = True,
 )
 
 container_bundle(
     name = "image_bundle",
     images = {
-        "gcr.io/prysmaticlabs/prysm/beacon-chain:latest": ":image",
-        "gcr.io/prysmaticlabs/prysm/beacon-chain:{DOCKER_TAG}": ":image",
-        "index.docker.io/prysmaticlabs/prysm-beacon-chain:latest": ":image",
-        "index.docker.io/prysmaticlabs/prysm-beacon-chain:{DOCKER_TAG}": ":image",
+        "gcr.io/prysmaticlabs/prysm/beacon-chain:latest": ":image_with_creation_time",
+        "gcr.io/prysmaticlabs/prysm/beacon-chain:{DOCKER_TAG}": ":image_with_creation_time",
+        "index.docker.io/prysmaticlabs/prysm-beacon-chain:latest": ":image_with_creation_time",
+        "index.docker.io/prysmaticlabs/prysm-beacon-chain:{DOCKER_TAG}": ":image_with_creation_time",
     },
     tags = ["manual"],
 )

beacon-chain/blockchain/BUILD.bazel

@@ -5,7 +5,6 @@ go_library(
     name = "go_default_library",
     srcs = [
         "chain_info.go",
-        "checkpoint_info_cache.go",
         "head.go",
         "info.go",
         "init_sync_process_block.go",
@@ -51,13 +50,12 @@ go_library(
         "//shared/bls:go_default_library",
         "//shared/bytesutil:go_default_library",
         "//shared/featureconfig:go_default_library",
-        "//shared/hashutil:go_default_library",
+        "//shared/mputil:go_default_library",
         "//shared/params:go_default_library",
         "//shared/slotutil:go_default_library",
         "//shared/timeutils:go_default_library",
         "//shared/traceutil:go_default_library",
         "@com_github_emicklei_dot//:go_default_library",
-        "@com_github_hashicorp_golang_lru//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
@@ -79,8 +77,8 @@ go_test(
     name = "go_raceoff_test",
     size = "medium",
     srcs = [
+        "blockchain_test.go",
         "chain_info_test.go",
-        "checkpoint_info_cache_test.go",
         "head_test.go",
         "info_test.go",
         "metrics_test.go",

beacon-chain/blockchain/blockchain_test.go

@@ -0,0 +1,19 @@
+package blockchain
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/sirupsen/logrus"
+)
+
+func TestMain(m *testing.M) {
+	run := func() int {
+		logrus.SetLevel(logrus.DebugLevel)
+		logrus.SetOutput(ioutil.Discard)
+
+		return m.Run()
+	}
+	os.Exit(run())
+}

beacon-chain/blockchain/chain_info.go

@@ -163,7 +163,7 @@ func (s *Service) HeadState(ctx context.Context) (*state.BeaconState, error) {
 		return s.headState(ctx), nil
 	}
 
-	return s.beaconDB.HeadState(ctx)
+	return s.stateGen.StateByRoot(ctx, s.headRoot())
 }
 
 // HeadValidatorsIndices returns a list of active validator indices from the head view of a given epoch.

beacon-chain/blockchain/chain_info_test.go

@@ -7,7 +7,9 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
@@ -60,19 +62,19 @@ func TestFinalizedCheckpt_GenesisRootOk(t *testing.T) {
 func TestCurrentJustifiedCheckpt_CanRetrieve(t *testing.T) {
 	db, sc := testDB.SetupDB(t)
 
-	cp := &ethpb.Checkpoint{Epoch: 6, Root: bytesutil.PadTo([]byte("foo"), 32)}
 	c := setupBeaconChain(t, db, sc)
+	assert.Equal(t, params.BeaconConfig().ZeroHash, bytesutil.ToBytes32(c.CurrentJustifiedCheckpt().Root), "Unexpected justified epoch")
+	cp := &ethpb.Checkpoint{Epoch: 6, Root: bytesutil.PadTo([]byte("foo"), 32)}
 	c.justifiedCheckpt = cp
-
 	assert.Equal(t, cp.Epoch, c.CurrentJustifiedCheckpt().Epoch, "Unexpected justified epoch")
 }
 
 func TestJustifiedCheckpt_GenesisRootOk(t *testing.T) {
 	db, sc := testDB.SetupDB(t)
 
+	c := setupBeaconChain(t, db, sc)
 	genesisRoot := [32]byte{'B'}
 	cp := &ethpb.Checkpoint{Root: genesisRoot[:]}
-	c := setupBeaconChain(t, db, sc)
 	c.justifiedCheckpt = cp
 	c.genesisRoot = genesisRoot
 	assert.DeepEqual(t, c.genesisRoot[:], c.CurrentJustifiedCheckpt().Root)
@@ -83,6 +85,7 @@ func TestPreviousJustifiedCheckpt_CanRetrieve(t *testing.T) {
 
 	cp := &ethpb.Checkpoint{Epoch: 7, Root: bytesutil.PadTo([]byte("foo"), 32)}
 	c := setupBeaconChain(t, db, sc)
+	assert.Equal(t, params.BeaconConfig().ZeroHash, bytesutil.ToBytes32(c.CurrentJustifiedCheckpt().Root), "Unexpected justified epoch")
 	c.prevJustifiedCheckpt = cp
 	assert.Equal(t, cp.Epoch, c.PreviousJustifiedCheckpt().Epoch, "Unexpected previous justified epoch")
 }
@@ -114,6 +117,21 @@ func TestHeadRoot_CanRetrieve(t *testing.T) {
 	assert.Equal(t, [32]byte{'A'}, bytesutil.ToBytes32(r))
 }
 
+func TestHeadRoot_UseDB(t *testing.T) {
+	db, _ := testDB.SetupDB(t)
+	c := &Service{beaconDB: db}
+	c.head = &head{root: params.BeaconConfig().ZeroHash}
+	b := testutil.NewBeaconBlock()
+	br, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, db.SaveBlock(context.Background(), b))
+	require.NoError(t, db.SaveStateSummary(context.Background(), &pb.StateSummary{Root: br[:]}))
+	require.NoError(t, db.SaveHeadBlockRoot(context.Background(), br))
+	r, err := c.HeadRoot(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, br, bytesutil.ToBytes32(r))
+}
+
 func TestHeadBlock_CanRetrieve(t *testing.T) {
 	b := testutil.NewBeaconBlock()
 	b.Block.Slot = 1
@@ -154,6 +172,17 @@ func TestCurrentFork_CanRetrieve(t *testing.T) {
 	}
 }
 
+func TestCurrentFork_NilHeadSTate(t *testing.T) {
+	f := &pb.Fork{
+		PreviousVersion: params.BeaconConfig().GenesisForkVersion,
+		CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
+	}
+	c := &Service{}
+	if !proto.Equal(c.CurrentFork(), f) {
+		t.Error("Received incorrect fork version")
+	}
+}
+
 func TestGenesisValidatorRoot_CanRetrieve(t *testing.T) {
 	// Should not panic if head state is nil.
 	c := &Service{}
@@ -201,3 +230,54 @@ func TestIsCanonical_Ok(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, false, can)
 }
+
+func TestService_HeadValidatorsIndices(t *testing.T) {
+	s, _ := testutil.DeterministicGenesisState(t, 10)
+	c := &Service{}
+
+	c.head = &head{}
+	indices, err := c.HeadValidatorsIndices(context.Background(), 0)
+	require.NoError(t, err)
+	require.Equal(t, 0, len(indices))
+
+	c.head = &head{state: s}
+	indices, err = c.HeadValidatorsIndices(context.Background(), 0)
+	require.NoError(t, err)
+	require.Equal(t, 10, len(indices))
+}
+
+func TestService_HeadSeed(t *testing.T) {
+	s, _ := testutil.DeterministicGenesisState(t, 1)
+	c := &Service{}
+	seed, err := helpers.Seed(s, 0, params.BeaconConfig().DomainBeaconAttester)
+	require.NoError(t, err)
+
+	c.head = &head{}
+	root, err := c.HeadSeed(context.Background(), 0)
+	require.NoError(t, err)
+	require.Equal(t, [32]byte{}, root)
+
+	c.head = &head{state: s}
+	root, err = c.HeadSeed(context.Background(), 0)
+	require.NoError(t, err)
+	require.DeepEqual(t, seed, root)
+}
+
+func TestService_HeadGenesisValidatorRoot(t *testing.T) {
+	s, _ := testutil.DeterministicGenesisState(t, 1)
+	c := &Service{}
+
+	c.head = &head{}
+	root := c.HeadGenesisValidatorRoot()
+	require.Equal(t, [32]byte{}, root)
+
+	c.head = &head{state: s}
+	root = c.HeadGenesisValidatorRoot()
+	require.DeepEqual(t, root[:], s.GenesisValidatorRoot())
+}
+
+func TestService_ProtoArrayStore(t *testing.T) {
+	c := &Service{forkChoiceStore: protoarray.New(0, 0, [32]byte{})}
+	p := c.ProtoArrayStore()
+	require.Equal(t, 0, int(p.FinalizedEpoch()))
+}

beacon-chain/blockchain/checkpoint_info_cache.go

@@ -1,83 +0,0 @@
-package blockchain
-
-import (
-	"sync"
-
-	lru "github.com/hashicorp/golang-lru"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/hashutil"
-)
-
-var (
-	// This defines the max number of checkpoint info this cache can store.
-	// Each cache is calculated at 3MB(30K validators), the total cache size is around 100MB.
-	// Due to reorgs and long finality, it's good to keep the old cache around for quickly switch over.
-	maxInfoSize = 32
-
-	// This tracks the number of check point info requests that aren't present in the cache.
-	infoMiss = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "check_point_info_cache_miss",
-		Help: "The number of check point info requests that aren't present in the cache.",
-	})
-	// This tracks the number of check point info requests that are in the cache.
-	infoHit = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "check_point_info_cache_hit",
-		Help: "The number of check point info requests that are present in the cache.",
-	})
-)
-
-// checkPtInfoCache is a struct with 1 LRU cache for looking up check point info by checkpoint.
-type checkPtInfoCache struct {
-	cache *lru.Cache
-	lock  sync.RWMutex
-}
-
-// newCheckPointInfoCache creates a new checkpoint info cache for storing/accessing processed check point info object.
-func newCheckPointInfoCache() *checkPtInfoCache {
-	cache, err := lru.New(maxInfoSize)
-	if err != nil {
-		panic(err)
-	}
-	return &checkPtInfoCache{
-		cache: cache,
-	}
-}
-
-// get fetches check point info by check point. Returns the reference of the CheckPtInfo, nil if doesn't exist.
-func (c *checkPtInfoCache) get(cp *ethpb.Checkpoint) (*pb.CheckPtInfo, error) {
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-	h, err := hashutil.HashProto(cp)
-	if err != nil {
-		return nil, err
-	}
-
-	item, exists := c.cache.Get(h)
-
-	if exists && item != nil {
-		infoHit.Inc()
-		// Copy here is unnecessary since the returned check point info object
-		// will only be used to verify attestation signature.
-		return item.(*pb.CheckPtInfo), nil
-	}
-
-	infoMiss.Inc()
-	return nil, nil
-}
-
-// put adds CheckPtInfo object to the cache. This method also trims the least
-// recently added CheckPtInfo object if the cache size has ready the max cache size limit.
-func (c *checkPtInfoCache) put(cp *ethpb.Checkpoint, info *pb.CheckPtInfo) error {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-	h, err := hashutil.HashProto(cp)
-	if err != nil {
-		return err
-	}
-
-	c.cache.Add(h, info)
-	return nil
-}

beacon-chain/blockchain/checkpoint_info_cache_test.go

@@ -1,41 +0,0 @@
-package blockchain
-
-import (
-	"testing"
-
-	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/testutil/require"
-)
-
-func TestHotStateCache_RoundTrip(t *testing.T) {
-	c := newCheckPointInfoCache()
-	cp := &ethpb.Checkpoint{
-		Epoch: 1,
-		Root:  bytesutil.PadTo([]byte{'a'}, 32),
-	}
-	info, err := c.get(cp)
-	require.NoError(t, err)
-	require.Equal(t, (*pb.CheckPtInfo)(nil), info)
-
-	i := &pb.CheckPtInfo{
-		Seed:          bytesutil.PadTo([]byte{'c'}, 32),
-		GenesisRoot:   bytesutil.PadTo([]byte{'d'}, 32),
-		ActiveIndices: []uint64{0, 1, 2, 3},
-	}
-
-	require.NoError(t, c.put(cp, i))
-	info, err = c.get(cp)
-	require.NoError(t, err)
-	require.DeepEqual(t, info, i)
-}
-
-func TestHotStateCache_CanPrune(t *testing.T) {
-	c := newCheckPointInfoCache()
-	for i := 0; i < maxInfoSize+1; i++ {
-		cp := &ethpb.Checkpoint{Epoch: uint64(i), Root: make([]byte, 32)}
-		require.NoError(t, c.put(cp, &pb.CheckPtInfo{}))
-	}
-	require.Equal(t, len(c.cache.Keys()), maxInfoSize)
-}

beacon-chain/blockchain/head.go

@@ -276,7 +276,7 @@ func (s *Service) cacheJustifiedStateBalances(ctx context.Context, justifiedRoot
 	epoch := helpers.CurrentEpoch(justifiedState)
 
 	justifiedBalances := make([]uint64, justifiedState.NumValidators())
-	if err := justifiedState.ReadFromEveryValidator(func(idx int, val *stateTrie.ReadOnlyValidator) error {
+	if err := justifiedState.ReadFromEveryValidator(func(idx int, val stateTrie.ReadOnlyValidator) error {
 		if helpers.IsActiveValidatorUsingTrie(val, epoch) {
 			justifiedBalances[idx] = val.EffectiveBalance()
 		} else {

beacon-chain/blockchain/metrics.go

@@ -173,6 +173,10 @@ func reportEpochMetrics(ctx context.Context, postState, headState *stateTrie.Bea
 		activeBalance += bal
 		activeEffectiveBalance += validator.EffectiveBalance
 	}
+	activeInstances += exitingInstances + slashingInstances
+	activeBalance += exitingBalance + slashingBalance
+	activeEffectiveBalance += exitingEffectiveBalance + slashingEffectiveBalance
+
 	validatorsCount.WithLabelValues("Pending").Set(float64(pendingInstances))
 	validatorsCount.WithLabelValues("Active").Set(float64(activeInstances))
 	validatorsCount.WithLabelValues("Exiting").Set(float64(exitingInstances))

beacon-chain/blockchain/process_attestation.go

@@ -8,10 +8,7 @@ import (
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
-	"github.com/prysmaticlabs/prysm/shared/attestationutil"
-	"github.com/prysmaticlabs/prysm/shared/bls"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/timeutils"
 	"go.opencensus.io/trace"
@@ -69,53 +66,6 @@ func (s *Service) onAttestation(ctx context.Context, a *ethpb.Attestation) ([]ui
 		return nil, ErrTargetRootNotInDB
 	}
 
-	if featureconfig.Get().UseCheckPointInfoCache {
-		c, err := s.AttestationCheckPtInfo(ctx, a)
-		if err != nil {
-			return nil, err
-		}
-		if err := s.verifyAttTargetEpoch(ctx, uint64(s.genesisTime.Unix()), uint64(timeutils.Now().Unix()), tgt); err != nil {
-			return nil, err
-		}
-		if err := s.verifyBeaconBlock(ctx, a.Data); err != nil {
-			return nil, errors.Wrap(err, "could not verify attestation beacon block")
-		}
-		if err := s.verifyLMDFFGConsistent(ctx, a.Data.Target.Epoch, a.Data.Target.Root, a.Data.BeaconBlockRoot); err != nil {
-			return nil, errors.Wrap(err, "could not verify attestation beacon block")
-		}
-		if err := helpers.VerifySlotTime(uint64(s.genesisTime.Unix()), a.Data.Slot+1, params.BeaconNetworkConfig().MaximumGossipClockDisparity); err != nil {
-			return nil, err
-		}
-		committee, err := helpers.BeaconCommittee(c.ActiveIndices, bytesutil.ToBytes32(c.Seed), a.Data.Slot, a.Data.CommitteeIndex)
-		if err != nil {
-			return nil, err
-		}
-		indexedAtt := attestationutil.ConvertToIndexed(ctx, a, committee)
-		if err := attestationutil.IsValidAttestationIndices(ctx, indexedAtt); err != nil {
-			return nil, err
-		}
-		domain, err := helpers.Domain(c.Fork, indexedAtt.Data.Target.Epoch, params.BeaconConfig().DomainBeaconAttester, c.GenesisRoot)
-		if err != nil {
-			return nil, err
-		}
-		indices := indexedAtt.AttestingIndices
-		var pubkeys []bls.PublicKey
-		for i := 0; i < len(indices); i++ {
-			pubkeyAtIdx := c.PubKeys[indices[i]]
-			pk, err := bls.PublicKeyFromBytes(pubkeyAtIdx)
-			if err != nil {
-				return nil, err
-			}
-			pubkeys = append(pubkeys, pk)
-		}
-		if err := attestationutil.VerifyIndexedAttestationSig(ctx, indexedAtt, pubkeys, domain); err != nil {
-			return nil, err
-		}
-		s.forkChoiceStore.ProcessAttestation(ctx, indexedAtt.AttestingIndices, bytesutil.ToBytes32(a.Data.BeaconBlockRoot), a.Data.Target.Epoch)
-
-		return indexedAtt.AttestingIndices, nil
-	}
-
 	// Retrieve attestation's data beacon block pre state. Advance pre state to latest epoch if necessary and
 	// save it to the cache.
 	baseState, err := s.getAttPreState(ctx, tgt)

beacon-chain/blockchain/process_attestation_helpers.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"strconv"
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
@@ -11,18 +12,21 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/attestationutil"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/mputil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
 // getAttPreState retrieves the att pre state by either from the cache or the DB.
 func (s *Service) getAttPreState(ctx context.Context, c *ethpb.Checkpoint) (*stateTrie.BeaconState, error) {
-	s.checkpointStateLock.Lock()
-	defer s.checkpointStateLock.Unlock()
-
-	cachedState, err := s.checkpointState.StateByCheckpoint(c)
+	// Use a multilock to allow scoped holding of a mutex by a checkpoint root + epoch
+	// allowing us to behave smarter in terms of how this function is used concurrently.
+	epochKey := strconv.FormatUint(c.Epoch, 10 /* base 10 */)
+	lock := mputil.NewMultilock(string(c.Root) + epochKey)
+	lock.Lock()
+	defer lock.Unlock()
+	cachedState, err := s.checkpointStateCache.StateByCheckpoint(c)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not get cached checkpoint state")
 	}
@@ -45,18 +49,20 @@ func (s *Service) getAttPreState(ctx context.Context, c *ethpb.Checkpoint) (*sta
 		if err != nil {
 			return nil, errors.Wrapf(err, "could not process slots up to epoch %d", c.Epoch)
 		}
-		if err := s.checkpointState.AddCheckpointState(c, baseState); err != nil {
+		if err := s.checkpointStateCache.AddCheckpointState(c, baseState); err != nil {
 			return nil, errors.Wrap(err, "could not saved checkpoint state to cache")
 		}
 		return baseState, nil
 	}
 
-	has, err := s.stateGen.HasState(ctx, bytesutil.ToBytes32(c.Root))
+	// To avoid sharing the same state across checkpoint state cache and hot state cache,
+	// we don't add the state to check point cache.
+	has, err := s.stateGen.HasStateInCache(ctx, bytesutil.ToBytes32(c.Root))
 	if err != nil {
 		return nil, err
 	}
 	if !has {
-		if err := s.checkpointState.AddCheckpointState(c, baseState); err != nil {
+		if err := s.checkpointStateCache.AddCheckpointState(c, baseState); err != nil {
 			return nil, errors.Wrap(err, "could not saved checkpoint state to cache")
 		}
 	}
@@ -64,67 +70,6 @@ func (s *Service) getAttPreState(ctx context.Context, c *ethpb.Checkpoint) (*sta
 
 }
 
-// getAttCheckPtInfo retrieves the check point info given a check point. Check point info enables the node
-// to efficiently verify attestation signature without using beacon state. This function utilizes
-// the checkpoint info cache and will update the check point info cache on miss.
-func (s *Service) getAttCheckPtInfo(ctx context.Context, c *ethpb.Checkpoint, e uint64) (*pb.CheckPtInfo, error) {
-	// Return checkpoint info if exists in cache.
-	info, err := s.checkPtInfoCache.get(c)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not get cached checkpoint state")
-	}
-	if info != nil {
-		return info, nil
-	}
-
-	// Retrieve checkpoint state to compute checkpoint info.
-	baseState, err := s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(c.Root))
-	if err != nil {
-		return nil, errors.Wrapf(err, "could not get pre state for epoch %d", c.Epoch)
-	}
-	epochStartSlot, err := helpers.StartSlot(c.Epoch)
-	if err != nil {
-		return nil, err
-	}
-	if epochStartSlot > baseState.Slot() {
-		baseState = baseState.Copy()
-		baseState, err = state.ProcessSlots(ctx, baseState, epochStartSlot)
-		if err != nil {
-			return nil, errors.Wrapf(err, "could not process slots up to epoch %d", c.Epoch)
-		}
-	}
-	f := baseState.Fork()
-	g := bytesutil.ToBytes32(baseState.GenesisValidatorRoot())
-	seed, err := helpers.Seed(baseState, e, params.BeaconConfig().DomainBeaconAttester)
-	if err != nil {
-		return nil, err
-	}
-	indices, err := helpers.ActiveValidatorIndices(baseState, e)
-	if err != nil {
-		return nil, err
-	}
-	validators := baseState.ValidatorsReadOnly()
-	pks := make([][]byte, len(validators))
-	for i := 0; i < len(pks); i++ {
-		pk := validators[i].PublicKey()
-		pks[i] = pk[:]
-	}
-
-	// Cache and return the checkpoint info.
-	info = &pb.CheckPtInfo{
-		Fork:          f,
-		GenesisRoot:   g[:],
-		Seed:          seed[:],
-		ActiveIndices: indices,
-		PubKeys:       pks,
-	}
-	if err := s.checkPtInfoCache.put(c, info); err != nil {
-		return nil, err
-	}
-
-	return info, nil
-}
-
 // verifyAttTargetEpoch validates attestation is from the current or previous epoch.
 func (s *Service) verifyAttTargetEpoch(_ context.Context, genesisTime, nowTime uint64, c *ethpb.Checkpoint) error {
 	currentSlot := (nowTime - genesisTime) / params.BeaconConfig().SecondsPerSlot

beacon-chain/blockchain/process_attestation_test.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
@@ -14,7 +13,6 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
@@ -130,118 +128,6 @@ func TestStore_OnAttestation(t *testing.T) {
 	}
 }
 
-func TestStore_OnAttestationUsingCheckptCache(t *testing.T) {
-	resetCfg := featureconfig.InitWithReset(&featureconfig.Flags{UseCheckPointInfoCache: true})
-	defer resetCfg()
-
-	ctx := context.Background()
-	db, sc := testDB.SetupDB(t)
-
-	cfg := &Config{
-		BeaconDB:        db,
-		ForkChoiceStore: protoarray.New(0, 0, [32]byte{}),
-		StateGen:        stategen.New(db, sc),
-	}
-	service, err := NewService(ctx, cfg)
-	require.NoError(t, err)
-
-	_, err = blockTree1(db, []byte{'g'})
-	require.NoError(t, err)
-
-	BlkWithOutState := testutil.NewBeaconBlock()
-	BlkWithOutState.Block.Slot = 0
-	require.NoError(t, db.SaveBlock(ctx, BlkWithOutState))
-	BlkWithOutStateRoot, err := BlkWithOutState.Block.HashTreeRoot()
-	require.NoError(t, err)
-
-	BlkWithStateBadAtt := testutil.NewBeaconBlock()
-	BlkWithStateBadAtt.Block.Slot = 1
-	require.NoError(t, db.SaveBlock(ctx, BlkWithStateBadAtt))
-	BlkWithStateBadAttRoot, err := BlkWithStateBadAtt.Block.HashTreeRoot()
-	require.NoError(t, err)
-
-	s := testutil.NewBeaconState()
-	require.NoError(t, s.SetSlot(100*params.BeaconConfig().SlotsPerEpoch))
-	require.NoError(t, service.beaconDB.SaveState(ctx, s, BlkWithStateBadAttRoot))
-
-	BlkWithValidState := testutil.NewBeaconBlock()
-	BlkWithValidState.Block.Slot = 2
-	require.NoError(t, db.SaveBlock(ctx, BlkWithValidState))
-
-	BlkWithValidStateRoot, err := BlkWithValidState.Block.HashTreeRoot()
-	require.NoError(t, err)
-	s = testutil.NewBeaconState()
-	err = s.SetFork(&pb.Fork{
-		Epoch:           0,
-		CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
-		PreviousVersion: params.BeaconConfig().GenesisForkVersion,
-	})
-	require.NoError(t, err)
-	require.NoError(t, service.beaconDB.SaveState(ctx, s, BlkWithValidStateRoot))
-
-	tests := []struct {
-		name      string
-		a         *ethpb.Attestation
-		wantedErr string
-	}{
-		{
-			name:      "attestation's data slot not aligned with target vote",
-			a:         &ethpb.Attestation{Data: &ethpb.AttestationData{Slot: params.BeaconConfig().SlotsPerEpoch, Target: &ethpb.Checkpoint{Root: make([]byte, 32)}}},
-			wantedErr: "data slot is not in the same epoch as target 1 != 0",
-		},
-		{
-			name:      "attestation's target root not in db",
-			a:         &ethpb.Attestation{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{Root: bytesutil.PadTo([]byte{'A'}, 32)}}},
-			wantedErr: "target root does not exist in db",
-		},
-		{
-			name:      "no pre state for attestations's target block",
-			a:         &ethpb.Attestation{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{Root: BlkWithOutStateRoot[:]}}},
-			wantedErr: "could not get pre state for epoch 0",
-		},
-		{
-			name: "process attestation doesn't match current epoch",
-			a: &ethpb.Attestation{Data: &ethpb.AttestationData{Slot: 100 * params.BeaconConfig().SlotsPerEpoch, Target: &ethpb.Checkpoint{Epoch: 100,
-				Root: BlkWithStateBadAttRoot[:]}}},
-			wantedErr: "target epoch 100 does not match current epoch",
-		},
-		{
-			name:      "process nil attestation",
-			a:         nil,
-			wantedErr: "nil attestation",
-		},
-		{
-			name:      "process nil field (a.Data) in attestation",
-			a:         &ethpb.Attestation{},
-			wantedErr: "nil attestation.Data field",
-		},
-		{
-			name: "process nil field (a.Target) in attestation",
-			a: &ethpb.Attestation{
-				Data: &ethpb.AttestationData{
-					BeaconBlockRoot: make([]byte, 32),
-					Target:          nil,
-					Source:          &ethpb.Checkpoint{Root: make([]byte, 32)},
-				},
-				AggregationBits: make([]byte, 1),
-				Signature:       make([]byte, 96),
-			},
-			wantedErr: "nil attestation.Data.Target field",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			_, err := service.onAttestation(ctx, tt.a)
-			if tt.wantedErr != "" {
-				assert.ErrorContains(t, tt.wantedErr, err)
-			} else {
-				assert.NoError(t, err)
-			}
-		})
-	}
-}
-
 func TestStore_SaveCheckpointState(t *testing.T) {
 	ctx := context.Background()
 	db, sc := testDB.SetupDB(t)
@@ -292,11 +178,11 @@ func TestStore_SaveCheckpointState(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot(), "Unexpected state slot")
 
-	s1, err = service.checkpointState.StateByCheckpoint(cp1)
+	s1, err = service.checkpointStateCache.StateByCheckpoint(cp1)
 	require.NoError(t, err)
 	assert.Equal(t, 1*params.BeaconConfig().SlotsPerEpoch, s1.Slot(), "Unexpected state slot")
 
-	s2, err = service.checkpointState.StateByCheckpoint(cp2)
+	s2, err = service.checkpointStateCache.StateByCheckpoint(cp2)
 	require.NoError(t, err)
 	assert.Equal(t, 2*params.BeaconConfig().SlotsPerEpoch, s2.Slot(), "Unexpected state slot")
 
@@ -332,7 +218,7 @@ func TestStore_UpdateCheckpointState(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, returned.Slot(), checkpoint.Epoch*params.BeaconConfig().SlotsPerEpoch, "Incorrectly returned base state")
 
-	cached, err := service.checkpointState.StateByCheckpoint(checkpoint)
+	cached, err := service.checkpointStateCache.StateByCheckpoint(checkpoint)
 	require.NoError(t, err)
 	assert.Equal(t, returned.Slot(), cached.Slot(), "State should have been cached")
 
@@ -347,7 +233,7 @@ func TestStore_UpdateCheckpointState(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, returned.Slot(), baseState.Slot(), "Incorrectly returned base state")
 
-	cached, err = service.checkpointState.StateByCheckpoint(newCheckpoint)
+	cached, err = service.checkpointStateCache.StateByCheckpoint(newCheckpoint)
 	require.NoError(t, err)
 	if !proto.Equal(returned.InnerStateUnsafe(), cached.InnerStateUnsafe()) {
 		t.Error("Incorrectly cached base state")
@@ -574,47 +460,3 @@ func TestVerifyFinalizedConsistency_IsCanonical(t *testing.T) {
 	err = service.VerifyFinalizedConsistency(context.Background(), r33[:])
 	require.NoError(t, err)
 }
-
-func TestGetAttCheckptInfo(t *testing.T) {
-	ctx := context.Background()
-	db, _ := testDB.SetupDB(t)
-	cfg := &Config{BeaconDB: db, StateGen: stategen.New(db, cache.NewStateSummaryCache())}
-	service, err := NewService(ctx, cfg)
-	require.NoError(t, err)
-
-	baseState, _ := testutil.DeterministicGenesisState(t, 128)
-	b := testutil.NewBeaconBlock()
-	r, err := b.Block.HashTreeRoot()
-	require.NoError(t, err)
-	require.NoError(t, service.beaconDB.SaveState(ctx, baseState, r))
-	require.NoError(t, service.beaconDB.SaveBlock(ctx, b))
-	require.NoError(t, service.beaconDB.SaveGenesisBlockRoot(ctx, r))
-	checkpoint := &ethpb.Checkpoint{Root: r[:]}
-
-	returned, err := service.getAttCheckPtInfo(ctx, checkpoint, 0)
-	require.NoError(t, err)
-
-	seed, err := helpers.Seed(baseState, 0, params.BeaconConfig().DomainBeaconAttester)
-	require.NoError(t, err)
-	indices, err := helpers.ActiveValidatorIndices(baseState, 0)
-	require.NoError(t, err)
-	validators := baseState.ValidatorsReadOnly()
-	pks := make([][]byte, len(validators))
-	for i := 0; i < len(pks); i++ {
-		pk := validators[i].PublicKey()
-		pks[i] = pk[:]
-	}
-
-	wanted := &pb.CheckPtInfo{
-		Fork:          baseState.Fork(),
-		GenesisRoot:   baseState.GenesisValidatorRoot(),
-		Seed:          seed[:],
-		ActiveIndices: indices,
-		PubKeys:       pks,
-	}
-	require.DeepEqual(t, wanted, returned)
-
-	cached, err := service.checkPtInfoCache.get(checkpoint)
-	require.NoError(t, err)
-	require.DeepEqual(t, wanted, cached)
-}

beacon-chain/blockchain/process_block.go

@@ -84,10 +84,17 @@ func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock,
 		return err
 	}
 
-	postState, err := state.ExecuteStateTransition(ctx, preState, signed)
+	set, postState, err := state.ExecuteStateTransitionNoVerifyAnySig(ctx, preState, signed)
 	if err != nil {
 		return errors.Wrap(err, "could not execute state transition")
 	}
+	valid, err := set.Verify()
+	if err != nil {
+		return errors.Wrap(err, "could not batch verify signature")
+	}
+	if !valid {
+		return errors.New("signature in block failed to verify")
+	}
 
 	if err := s.savePostStateInfo(ctx, blockRoot, signed, postState, false /* reg sync */); err != nil {
 		return err

beacon-chain/blockchain/process_block_helpers.go

@@ -363,6 +363,7 @@ func (s *Service) finalizedImpliesNewJustified(ctx context.Context, state *state
 func (s *Service) fillInForkChoiceMissingBlocks(ctx context.Context, blk *ethpb.BeaconBlock,
 	fCheckpoint, jCheckpoint *ethpb.Checkpoint) error {
 	pendingNodes := make([]*ethpb.BeaconBlock, 0)
+	pendingRoots := make([][32]byte, 0)
 
 	parentRoot := bytesutil.ToBytes32(blk.ParentRoot)
 	slot := blk.Slot
@@ -380,6 +381,8 @@ func (s *Service) fillInForkChoiceMissingBlocks(ctx context.Context, blk *ethpb.
 		}
 
 		pendingNodes = append(pendingNodes, b.Block)
+		copiedRoot := parentRoot
+		pendingRoots = append(pendingRoots, copiedRoot)
 		parentRoot = bytesutil.ToBytes32(b.Block.ParentRoot)
 		slot = b.Block.Slot
 		higherThanFinalized = slot > fSlot
@@ -389,11 +392,7 @@ func (s *Service) fillInForkChoiceMissingBlocks(ctx context.Context, blk *ethpb.
 	// Lower slots should be at the end of the list.
 	for i := len(pendingNodes) - 1; i >= 0; i-- {
 		b := pendingNodes[i]
-		r, err := b.HashTreeRoot()
-		if err != nil {
-			return err
-		}
-
+		r := pendingRoots[i]
 		if err := s.forkChoiceStore.ProcessBlock(ctx,
 			b.Slot, r, bytesutil.ToBytes32(b.ParentRoot), bytesutil.ToBytes32(b.Body.Graffiti),
 			jCheckpoint.Epoch,

beacon-chain/blockchain/process_block_test.go

@@ -2,6 +2,7 @@ package blockchain
 
 import (
 	"context"
+	"fmt"
 	"testing"
 	"time"
 
@@ -319,7 +320,6 @@ func TestUpdateJustified_CouldUpdateBest(t *testing.T) {
 	service.justifiedCheckpt = &ethpb.Checkpoint{Root: []byte{'A'}}
 	service.bestJustifiedCheckpt = &ethpb.Checkpoint{Root: []byte{'A'}}
 	st := testutil.NewBeaconState()
-	service.initSyncState[r] = st.Copy()
 	require.NoError(t, db.SaveState(ctx, st.Copy(), r))
 
 	// Could update
@@ -373,6 +373,46 @@ func TestFillForkChoiceMissingBlocks_CanSave(t *testing.T) {
 	assert.Equal(t, true, service.forkChoiceStore.HasNode(bytesutil.ToBytes32(roots[8])), "Didn't save node")
 }
 
+func TestFillForkChoiceMissingBlocks_RootsMatch(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	require.NoError(t, err)
+	service.forkChoiceStore = protoarray.New(0, 0, [32]byte{'A'})
+	service.finalizedCheckpt = &ethpb.Checkpoint{Root: make([]byte, 32)}
+
+	genesisStateRoot := [32]byte{}
+	genesis := blocks.NewGenesisBlock(genesisStateRoot[:])
+	require.NoError(t, db.SaveBlock(ctx, genesis))
+	validGenesisRoot, err := genesis.Block.HashTreeRoot()
+	require.NoError(t, err)
+	st := testutil.NewBeaconState()
+
+	require.NoError(t, service.beaconDB.SaveState(ctx, st.Copy(), validGenesisRoot))
+	roots, err := blockTree1(db, validGenesisRoot[:])
+	require.NoError(t, err)
+
+	beaconState, _ := testutil.DeterministicGenesisState(t, 32)
+	block := testutil.NewBeaconBlock()
+	block.Block.Slot = 9
+	block.Block.ParentRoot = roots[8]
+
+	err = service.fillInForkChoiceMissingBlocks(
+		context.Background(), block.Block, beaconState.FinalizedCheckpoint(), beaconState.CurrentJustifiedCheckpoint())
+	require.NoError(t, err)
+
+	// 5 nodes from the block tree 1. B0 - B3 - B4 - B6 - B8
+	assert.Equal(t, 5, len(service.forkChoiceStore.Nodes()), "Miss match nodes")
+	// Ensure all roots and their respective blocks exist.
+	wantedRoots := [][]byte{roots[0], roots[3], roots[4], roots[6], roots[8]}
+	for i, rt := range wantedRoots {
+		assert.Equal(t, true, service.forkChoiceStore.HasNode(bytesutil.ToBytes32(rt)), fmt.Sprintf("Didn't save node: %d", i))
+		assert.Equal(t, true, service.beaconDB.HasBlock(context.Background(), bytesutil.ToBytes32(rt)))
+	}
+}
+
 func TestFillForkChoiceMissingBlocks_FilterFinalized(t *testing.T) {
 	ctx := context.Background()
 	db, _ := testDB.SetupDB(t)

beacon-chain/blockchain/receive_attestation.go

@@ -7,15 +7,12 @@ import (
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/slotutil"
-	"github.com/prysmaticlabs/prysm/shared/timeutils"
 	"github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
 )
@@ -23,9 +20,7 @@ import (
 // AttestationReceiver interface defines the methods of chain service receive and processing new attestations.
 type AttestationReceiver interface {
 	ReceiveAttestationNoPubsub(ctx context.Context, att *ethpb.Attestation) error
-	IsValidAttestation(ctx context.Context, att *ethpb.Attestation) bool
 	AttestationPreState(ctx context.Context, att *ethpb.Attestation) (*state.BeaconState, error)
-	AttestationCheckPtInfo(ctx context.Context, att *ethpb.Attestation) (*pb.CheckPtInfo, error)
 	VerifyLmdFfgConsistency(ctx context.Context, att *ethpb.Attestation) error
 	VerifyFinalizedConsistency(ctx context.Context, root []byte) error
 }
@@ -52,22 +47,6 @@ func (s *Service) ReceiveAttestationNoPubsub(ctx context.Context, att *ethpb.Att
 	return nil
 }
 
-// IsValidAttestation returns true if the attestation can be verified against its pre-state.
-func (s *Service) IsValidAttestation(ctx context.Context, att *ethpb.Attestation) bool {
-	baseState, err := s.AttestationPreState(ctx, att)
-	if err != nil {
-		log.WithError(err).Error("Failed to get attestation pre state")
-		return false
-	}
-
-	if err := blocks.VerifyAttestationSignature(ctx, baseState, att); err != nil {
-		log.WithError(err).Error("Failed to validate attestation")
-		return false
-	}
-
-	return true
-}
-
 // AttestationPreState returns the pre state of attestation.
 func (s *Service) AttestationPreState(ctx context.Context, att *ethpb.Attestation) (*state.BeaconState, error) {
 	ss, err := helpers.StartSlot(att.Data.Target.Epoch)
@@ -80,19 +59,6 @@ func (s *Service) AttestationPreState(ctx context.Context, att *ethpb.Attestatio
 	return s.getAttPreState(ctx, att.Data.Target)
 }
 
-// AttestationCheckPtInfo returns the check point info of attestation that can be used to verify the attestation
-// contents and signatures.
-func (s *Service) AttestationCheckPtInfo(ctx context.Context, att *ethpb.Attestation) (*pb.CheckPtInfo, error) {
-	ss, err := helpers.StartSlot(att.Data.Target.Epoch)
-	if err != nil {
-		return nil, err
-	}
-	if err := helpers.ValidateSlotClock(ss, uint64(s.genesisTime.Unix())); err != nil {
-		return nil, err
-	}
-	return s.getAttCheckPtInfo(ctx, att.Data.Target, helpers.SlotToEpoch(att.Data.Slot))
-}
-
 // VerifyLmdFfgConsistency verifies that attestation's LMD and FFG votes are consistency to each other.
 func (s *Service) VerifyLmdFfgConsistency(ctx context.Context, a *ethpb.Attestation) error {
 	return s.verifyLMDFFGConsistent(ctx, a.Data.Target.Epoch, a.Data.Target.Root, a.Data.BeaconBlockRoot)
@@ -160,7 +126,7 @@ func (s *Service) processAttestation(subscribedToStateEvents chan struct{}) {
 					log.WithError(err).Error("Could not delete fork choice attestation in pool")
 				}
 
-				if !s.verifyCheckpointEpoch(a.Data.Target) {
+				if !helpers.VerifyCheckpointEpoch(a.Data.Target, s.genesisTime) {
 					continue
 				}
 
@@ -177,23 +143,3 @@ func (s *Service) processAttestation(subscribedToStateEvents chan struct{}) {
 		}
 	}
 }
-
-// This verifies the epoch of input checkpoint is within current epoch and previous epoch
-// with respect to current time. Returns true if it's within, false if it's not.
-func (s *Service) verifyCheckpointEpoch(c *ethpb.Checkpoint) bool {
-	now := uint64(timeutils.Now().Unix())
-	genesisTime := uint64(s.genesisTime.Unix())
-	currentSlot := (now - genesisTime) / params.BeaconConfig().SecondsPerSlot
-	currentEpoch := helpers.SlotToEpoch(currentSlot)
-
-	var prevEpoch uint64
-	if currentEpoch > 1 {
-		prevEpoch = currentEpoch - 1
-	}
-
-	if c.Epoch != prevEpoch && c.Epoch != currentEpoch {
-		return false
-	}
-
-	return true
-}

beacon-chain/blockchain/receive_attestation_test.go

@@ -9,34 +9,10 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
 	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 )
 
-func TestVerifyCheckpointEpoch_Ok(t *testing.T) {
-	helpers.ClearCache()
-	db, sc := testDB.SetupDB(t)
-
-	chainService := setupBeaconChain(t, db, sc)
-	chainService.genesisTime = time.Now()
-
-	assert.Equal(t, true, chainService.verifyCheckpointEpoch(&ethpb.Checkpoint{Root: make([]byte, 32)}))
-	assert.Equal(t, false, chainService.verifyCheckpointEpoch(&ethpb.Checkpoint{Epoch: 1}))
-}
-
-func TestAttestationPreState_FarFutureSlot(t *testing.T) {
-	helpers.ClearCache()
-	db, sc := testDB.SetupDB(t)
-
-	chainService := setupBeaconChain(t, db, sc)
-	chainService.genesisTime = time.Now()
-
-	e := helpers.MaxSlotBuffer/params.BeaconConfig().SlotsPerEpoch + 1
-	_, err := chainService.AttestationCheckPtInfo(context.Background(), &ethpb.Attestation{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{Epoch: e}}})
-	require.ErrorContains(t, "exceeds max allowed value relative to the local clock", err)
-}
-
-func TestAttestationCheckPtInfo_FarFutureSlot(t *testing.T) {
+func TestAttestationCheckPtState_FarFutureSlot(t *testing.T) {
 	helpers.ClearCache()
 	db, sc := testDB.SetupDB(t)
 

beacon-chain/blockchain/receive_block_test.go

@@ -78,7 +78,6 @@ func TestService_ReceiveBlock(t *testing.T) {
 				}
 			},
 		},
-
 		{
 			name: "updates exit pool",
 			args: args{
@@ -93,14 +92,13 @@ func TestService_ReceiveBlock(t *testing.T) {
 				),
 			},
 			check: func(t *testing.T, s *Service) {
-				var n int
-				for i := uint64(0); int(i) < genesis.NumValidators(); i++ {
-					if s.exitPool.HasBeenIncluded(i) {
-						n++
-					}
-				}
-				if n != 3 {
-					t.Errorf("Did not mark the correct number of exits. Got %d but wanted %d", n, 3)
+				pending := s.exitPool.PendingExits(genesis, 1, true /* no limit */)
+				if len(pending) != 0 {
+					t.Errorf(
+						"Did not mark the correct number of exits. Got %d pending but wanted %d",
+						len(pending),
+						0,
+					)
 				}
 			},
 		},

beacon-chain/blockchain/service.go

@@ -34,9 +34,14 @@ import (
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/slotutil"
+	"github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
 )
 
+// headSyncMinEpochsAfterCheckpoint defines how many epochs should elapse after known finalization
+// checkpoint for head sync to be triggered.
+const headSyncMinEpochsAfterCheckpoint = 128
+
 // Service represents a service that handles the internal
 // logic of managing the full PoS beacon chain.
 type Service struct {
@@ -62,17 +67,14 @@ type Service struct {
 	finalizedCheckpt      *ethpb.Checkpoint
 	prevFinalizedCheckpt  *ethpb.Checkpoint
 	nextEpochBoundarySlot uint64
-	initSyncState         map[[32]byte]*stateTrie.BeaconState
 	boundaryRoots         [][32]byte
-	checkpointState       *cache.CheckpointStateCache
-	checkpointStateLock   sync.Mutex
+	checkpointStateCache  *cache.CheckpointStateCache
 	stateGen              *stategen.State
 	opsService            *attestations.Service
 	initSyncBlocks        map[[32]byte]*ethpb.SignedBeaconBlock
 	initSyncBlocksLock    sync.RWMutex
 	justifiedBalances     []uint64
 	justifiedBalancesLock sync.RWMutex
-	checkPtInfoCache      *checkPtInfoCache
 	wsEpoch               uint64
 	wsRoot                []byte
 	wsVerified            bool
@@ -102,38 +104,31 @@ type Config struct {
 func NewService(ctx context.Context, cfg *Config) (*Service, error) {
 	ctx, cancel := context.WithCancel(ctx)
 	return &Service{
-		ctx:               ctx,
-		cancel:            cancel,
-		beaconDB:          cfg.BeaconDB,
-		depositCache:      cfg.DepositCache,
-		chainStartFetcher: cfg.ChainStartFetcher,
-		attPool:           cfg.AttPool,
-		exitPool:          cfg.ExitPool,
-		slashingPool:      cfg.SlashingPool,
-		p2p:               cfg.P2p,
-		maxRoutines:       cfg.MaxRoutines,
-		stateNotifier:     cfg.StateNotifier,
-		forkChoiceStore:   cfg.ForkChoiceStore,
-		initSyncState:     make(map[[32]byte]*stateTrie.BeaconState),
-		boundaryRoots:     [][32]byte{},
-		checkpointState:   cache.NewCheckpointStateCache(),
-		opsService:        cfg.OpsService,
-		stateGen:          cfg.StateGen,
-		initSyncBlocks:    make(map[[32]byte]*ethpb.SignedBeaconBlock),
-		justifiedBalances: make([]uint64, 0),
-		checkPtInfoCache:  newCheckPointInfoCache(),
-		wsEpoch:           cfg.WspEpoch,
-		wsRoot:            cfg.WspBlockRoot,
+		ctx:                  ctx,
+		cancel:               cancel,
+		beaconDB:             cfg.BeaconDB,
+		depositCache:         cfg.DepositCache,
+		chainStartFetcher:    cfg.ChainStartFetcher,
+		attPool:              cfg.AttPool,
+		exitPool:             cfg.ExitPool,
+		slashingPool:         cfg.SlashingPool,
+		p2p:                  cfg.P2p,
+		maxRoutines:          cfg.MaxRoutines,
+		stateNotifier:        cfg.StateNotifier,
+		forkChoiceStore:      cfg.ForkChoiceStore,
+		boundaryRoots:        [][32]byte{},
+		checkpointStateCache: cache.NewCheckpointStateCache(),
+		opsService:           cfg.OpsService,
+		stateGen:             cfg.StateGen,
+		initSyncBlocks:       make(map[[32]byte]*ethpb.SignedBeaconBlock),
+		justifiedBalances:    make([]uint64, 0),
+		wsEpoch:              cfg.WspEpoch,
+		wsRoot:               cfg.WspBlockRoot,
 	}, nil
 }
 
 // Start a blockchain service's main event loop.
 func (s *Service) Start() {
-	beaconState, err := s.beaconDB.HeadState(s.ctx)
-	if err != nil {
-		log.Fatalf("Could not fetch beacon state: %v", err)
-	}
-
 	// For running initial sync with state cache, in an event of restart, we use
 	// last finalized check point as start point to sync instead of head
 	// state. This is because we no longer save state every slot during sync.
@@ -142,27 +137,25 @@ func (s *Service) Start() {
 		log.Fatalf("Could not fetch finalized cp: %v", err)
 	}
 
-	if beaconState == nil {
-		r := bytesutil.ToBytes32(cp.Root)
-		// Before the first finalized epoch, in the current epoch,
-		// the finalized root is defined as zero hashes instead of genesis root hash.
-		// We want to use genesis root to retrieve for state.
-		if r == params.BeaconConfig().ZeroHash {
-			genesisBlock, err := s.beaconDB.GenesisBlock(s.ctx)
-			if err != nil {
-				log.Fatalf("Could not fetch finalized cp: %v", err)
-			}
-			if genesisBlock != nil {
-				r, err = genesisBlock.Block.HashTreeRoot()
-				if err != nil {
-					log.Fatalf("Could not tree hash genesis block: %v", err)
-				}
-			}
-		}
-		beaconState, err = s.stateGen.StateByRoot(s.ctx, r)
+	r := bytesutil.ToBytes32(cp.Root)
+	// Before the first finalized epoch, in the current epoch,
+	// the finalized root is defined as zero hashes instead of genesis root hash.
+	// We want to use genesis root to retrieve for state.
+	if r == params.BeaconConfig().ZeroHash {
+		genesisBlock, err := s.beaconDB.GenesisBlock(s.ctx)
 		if err != nil {
-			log.Fatalf("Could not fetch beacon state by root: %v", err)
+			log.Fatalf("Could not fetch finalized cp: %v", err)
 		}
+		if genesisBlock != nil {
+			r, err = genesisBlock.Block.HashTreeRoot()
+			if err != nil {
+				log.Fatalf("Could not tree hash genesis block: %v", err)
+			}
+		}
+	}
+	beaconState, err := s.stateGen.StateByRoot(s.ctx, r)
+	if err != nil {
+		log.Fatalf("Could not fetch beacon state by root: %v", err)
 	}
 
 	// Make sure that attestation processor is subscribed and ready for state initializing event.
@@ -204,6 +197,19 @@ func (s *Service) Start() {
 		s.prevFinalizedCheckpt = stateTrie.CopyCheckpoint(finalizedCheckpoint)
 		s.resumeForkChoice(justifiedCheckpoint, finalizedCheckpoint)
 
+		ss, err := helpers.StartSlot(s.finalizedCheckpt.Epoch)
+		if err != nil {
+			log.Fatalf("Could not get start slot of finalized epoch: %v", err)
+		}
+		h := s.headBlock().Block
+		log.WithFields(logrus.Fields{
+			"startSlot": ss,
+			"endSlot":   h.Slot,
+		}).Info("Loading blocks to fork choice store, this may take a while.")
+		if err := s.fillInForkChoiceMissingBlocks(s.ctx, h, s.finalizedCheckpt, s.justifiedCheckpt); err != nil {
+			log.Fatalf("Could not fill in fork choice store missing blocks: %v", err)
+		}
+
 		if err := s.VerifyWeakSubjectivityRoot(s.ctx); err != nil {
 			// Exit run time if the node failed to verify weak subjectivity checkpoint.
 			log.Fatalf("Could not verify weak subjectivity checkpoint: %v", err)
@@ -333,18 +339,15 @@ func (s *Service) Stop() error {
 // Status always returns nil unless there is an error condition that causes
 // this service to be unhealthy.
 func (s *Service) Status() error {
+	if s.genesisRoot == params.BeaconConfig().ZeroHash {
+		return errors.New("genesis state has not been created")
+	}
 	if runtime.NumGoroutine() > s.maxRoutines {
 		return fmt.Errorf("too many goroutines %d", runtime.NumGoroutine())
 	}
 	return nil
 }
 
-// ClearCachedStates removes all stored caches states. This is done after the node
-// is synced.
-func (s *Service) ClearCachedStates() {
-	s.initSyncState = map[[32]byte]*stateTrie.BeaconState{}
-}
-
 // This gets called when beacon chain is first initialized to save genesis data (state, block, and more) in db.
 func (s *Service) saveGenesisData(ctx context.Context, genesisState *stateTrie.BeaconState) error {
 	stateRoot, err := genesisState.HashTreeRoot(ctx)
@@ -422,29 +425,6 @@ func (s *Service) initializeChainInfo(ctx context.Context) error {
 	}
 	s.genesisRoot = genesisBlkRoot
 
-	if flags.Get().HeadSync {
-		headBlock, err := s.beaconDB.HeadBlock(ctx)
-		if err != nil {
-			return errors.Wrap(err, "could not retrieve head block")
-		}
-		headRoot, err := headBlock.Block.HashTreeRoot()
-		if err != nil {
-			return errors.Wrap(err, "could not hash head block")
-		}
-		finalizedState, err := s.stateGen.Resume(ctx)
-		if err != nil {
-			return errors.Wrap(err, "could not get finalized state from db")
-		}
-		log.Infof("Regenerating state from the last checkpoint at slot %d to current head slot of %d."+
-			"This process may take a while, please wait.", finalizedState.Slot(), headBlock.Block.Slot)
-		headState, err := s.stateGen.StateByRoot(ctx, headRoot)
-		if err != nil {
-			return errors.Wrap(err, "could not retrieve head state")
-		}
-		s.setHead(headRoot, headBlock, headState)
-		return nil
-	}
-
 	finalized, err := s.beaconDB.FinalizedCheckpoint(ctx)
 	if err != nil {
 		return errors.Wrap(err, "could not get finalized checkpoint from db")
@@ -462,6 +442,42 @@ func (s *Service) initializeChainInfo(ctx context.Context) error {
 		return errors.Wrap(err, "could not get finalized state from db")
 	}
 
+	if flags.Get().HeadSync {
+		headBlock, err := s.beaconDB.HeadBlock(ctx)
+		if err != nil {
+			return errors.Wrap(err, "could not retrieve head block")
+		}
+		headEpoch := helpers.SlotToEpoch(headBlock.Block.Slot)
+		var epochsSinceFinality uint64
+		if headEpoch > finalized.Epoch {
+			epochsSinceFinality = headEpoch - finalized.Epoch
+		}
+		// Head sync when node is far enough beyond known finalized epoch,
+		// this becomes really useful during long period of non-finality.
+		if epochsSinceFinality >= headSyncMinEpochsAfterCheckpoint {
+			headRoot, err := headBlock.Block.HashTreeRoot()
+			if err != nil {
+				return errors.Wrap(err, "could not hash head block")
+			}
+			finalizedState, err := s.stateGen.Resume(ctx)
+			if err != nil {
+				return errors.Wrap(err, "could not get finalized state from db")
+			}
+			log.Infof("Regenerating state from the last checkpoint at slot %d to current head slot of %d."+
+				"This process may take a while, please wait.", finalizedState.Slot(), headBlock.Block.Slot)
+			headState, err := s.stateGen.StateByRoot(ctx, headRoot)
+			if err != nil {
+				return errors.Wrap(err, "could not retrieve head state")
+			}
+			s.setHead(headRoot, headBlock, headState)
+			return nil
+		} else {
+			log.Warnf("Finalized checkpoint at slot %d is too close to the current head slot, "+
+				"resetting head from the checkpoint ('--%s' flag is ignored).",
+				finalizedState.Slot(), flags.HeadSync.Name)
+		}
+	}
+
 	finalizedBlock, err := s.beaconDB.Block(ctx, finalizedRoot)
 	if err != nil {
 		return errors.Wrap(err, "could not get finalized block from db")

beacon-chain/blockchain/service_test.go

@@ -3,7 +3,6 @@ package blockchain
 import (
 	"bytes"
 	"context"
-	"io/ioutil"
 	"reflect"
 	"testing"
 	"time"
@@ -18,6 +17,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/flags"
 	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
 	"github.com/prysmaticlabs/prysm/beacon-chain/operations/attestations"
 	"github.com/prysmaticlabs/prysm/beacon-chain/p2p"
@@ -32,15 +32,9 @@ import (
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
 	"github.com/prysmaticlabs/prysm/shared/testutil/require"
-	"github.com/sirupsen/logrus"
 	logTest "github.com/sirupsen/logrus/hooks/test"
 )
 
-func init() {
-	logrus.SetLevel(logrus.DebugLevel)
-	logrus.SetOutput(ioutil.Discard)
-}
-
 type mockBeaconNode struct {
 	stateFeed *event.Feed
 }
@@ -323,6 +317,87 @@ func TestChainService_InitializeChainInfo_SetHeadAtGenesis(t *testing.T) {
 	assert.DeepEqual(t, genesis, c.head.block)
 }
 
+func TestChainService_InitializeChainInfo_HeadSync(t *testing.T) {
+	resetFlags := flags.Get()
+	flags.Init(&flags.GlobalFlags{
+		HeadSync: true,
+	})
+	defer func() {
+		flags.Init(resetFlags)
+	}()
+
+	hook := logTest.NewGlobal()
+	finalizedSlot := params.BeaconConfig().SlotsPerEpoch*2 + 1
+	db, sc := testDB.SetupDB(t)
+	ctx := context.Background()
+
+	genesisBlock := testutil.NewBeaconBlock()
+	genesisRoot, err := genesisBlock.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, db.SaveGenesisBlockRoot(ctx, genesisRoot))
+	require.NoError(t, db.SaveBlock(ctx, genesisBlock))
+
+	finalizedBlock := testutil.NewBeaconBlock()
+	finalizedBlock.Block.Slot = finalizedSlot
+	finalizedBlock.Block.ParentRoot = genesisRoot[:]
+	finalizedRoot, err := finalizedBlock.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, db.SaveBlock(ctx, finalizedBlock))
+
+	// Set head slot close to the finalization point, no head sync is triggered.
+	headBlock := testutil.NewBeaconBlock()
+	headBlock.Block.Slot = finalizedSlot + params.BeaconConfig().SlotsPerEpoch*5
+	headBlock.Block.ParentRoot = finalizedRoot[:]
+	headRoot, err := headBlock.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, db.SaveBlock(ctx, headBlock))
+
+	headState := testutil.NewBeaconState()
+	require.NoError(t, headState.SetSlot(headBlock.Block.Slot))
+	require.NoError(t, headState.SetGenesisValidatorRoot(params.BeaconConfig().ZeroHash[:]))
+	require.NoError(t, db.SaveState(ctx, headState, genesisRoot))
+	require.NoError(t, db.SaveState(ctx, headState, finalizedRoot))
+	require.NoError(t, db.SaveState(ctx, headState, headRoot))
+	require.NoError(t, db.SaveHeadBlockRoot(ctx, headRoot))
+	require.NoError(t, db.SaveFinalizedCheckpoint(ctx, &ethpb.Checkpoint{
+		Epoch: helpers.SlotToEpoch(finalizedBlock.Block.Slot),
+		Root:  finalizedRoot[:],
+	}))
+
+	c := &Service{beaconDB: db, stateGen: stategen.New(db, sc)}
+
+	require.NoError(t, c.initializeChainInfo(ctx))
+	s, err := c.HeadState(ctx)
+	require.NoError(t, err)
+	assert.DeepEqual(t, headState.InnerStateUnsafe(), s.InnerStateUnsafe(), "Head state incorrect")
+	assert.Equal(t, genesisRoot, c.genesisRoot, "Genesis block root incorrect")
+	// Since head sync is not triggered, chain is initialized to the last finalization checkpoint.
+	assert.DeepEqual(t, finalizedBlock, c.head.block)
+	assert.LogsContain(t, hook, "resetting head from the checkpoint ('--head-sync' flag is ignored)")
+	assert.LogsDoNotContain(t, hook, "Regenerating state from the last checkpoint at slot")
+
+	// Set head slot far beyond the finalization point, head sync should be triggered.
+	headBlock = testutil.NewBeaconBlock()
+	headBlock.Block.Slot = finalizedSlot + params.BeaconConfig().SlotsPerEpoch*headSyncMinEpochsAfterCheckpoint
+	headBlock.Block.ParentRoot = finalizedRoot[:]
+	headRoot, err = headBlock.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, db.SaveBlock(ctx, headBlock))
+	require.NoError(t, db.SaveState(ctx, headState, headRoot))
+	require.NoError(t, db.SaveHeadBlockRoot(ctx, headRoot))
+
+	hook.Reset()
+	require.NoError(t, c.initializeChainInfo(ctx))
+	s, err = c.HeadState(ctx)
+	require.NoError(t, err)
+	assert.DeepEqual(t, headState.InnerStateUnsafe(), s.InnerStateUnsafe(), "Head state incorrect")
+	assert.Equal(t, genesisRoot, c.genesisRoot, "Genesis block root incorrect")
+	// Head slot is far beyond the latest finalized checkpoint, head sync is triggered.
+	assert.DeepEqual(t, headBlock, c.head.block)
+	assert.LogsContain(t, hook, "Regenerating state from the last checkpoint at slot 225")
+	assert.LogsDoNotContain(t, hook, "resetting head from the checkpoint ('--head-sync' flag is ignored)")
+}
+
 func TestChainService_SaveHeadNoDB(t *testing.T) {
 	db, sc := testDB.SetupDB(t)
 	ctx := context.Background()

beacon-chain/blockchain/testing/BUILD.bazel

@@ -20,7 +20,6 @@ go_library(
         "//beacon-chain/forkchoice/protoarray:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
-        "//shared/bytesutil:go_default_library",
         "//shared/event:go_default_library",
         "//shared/params:go_default_library",
         "@com_github_pkg_errors//:go_default_library",

beacon-chain/blockchain/testing/mock.go

@@ -20,7 +20,6 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/event"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/sirupsen/logrus"
@@ -347,9 +346,6 @@ func (ms *ChainService) IsCanonical(_ context.Context, r [32]byte) (bool, error)
 	return true, nil
 }
 
-// ClearCachedStates does nothing.
-func (ms *ChainService) ClearCachedStates() {}
-
 // HasInitSyncBlock mocks the same method in the chain service.
 func (ms *ChainService) HasInitSyncBlock(_ [32]byte) bool {
 	return false
@@ -380,33 +376,3 @@ func (ms *ChainService) VerifyFinalizedConsistency(_ context.Context, r []byte)
 	}
 	return nil
 }
-
-// AttestationCheckPtInfo mocks AttestationCheckPtInfo and always returns nil.
-func (ms *ChainService) AttestationCheckPtInfo(_ context.Context, att *ethpb.Attestation) (*pb.CheckPtInfo, error) {
-	f := ms.State.Fork()
-	g := bytesutil.ToBytes32(ms.State.GenesisValidatorRoot())
-	seed, err := helpers.Seed(ms.State, helpers.SlotToEpoch(att.Data.Slot), params.BeaconConfig().DomainBeaconAttester)
-	if err != nil {
-		return nil, err
-	}
-	indices, err := helpers.ActiveValidatorIndices(ms.State, helpers.SlotToEpoch(att.Data.Slot))
-	if err != nil {
-		return nil, err
-	}
-	validators := ms.State.ValidatorsReadOnly()
-	pks := make([][]byte, len(validators))
-	for i := 0; i < len(pks); i++ {
-		pk := validators[i].PublicKey()
-		pks[i] = pk[:]
-	}
-
-	info := &pb.CheckPtInfo{
-		Fork:          f,
-		GenesisRoot:   g[:],
-		Seed:          seed[:],
-		ActiveIndices: indices,
-		PubKeys:       pks,
-	}
-
-	return info, nil
-}

beacon-chain/cache/BUILD.bazel

@@ -57,7 +57,7 @@ go_test(
         "checkpoint_state_test.go",
         "committee_fuzz_test.go",
         "committee_test.go",
-        "feature_flag_test.go",
+        "cache_test.go",
         "hot_state_cache_test.go",
         "skip_slot_cache_test.go",
         "subnet_ids_test.go",

beacon-chain/cache/cache_test.go

@@ -0,0 +1,17 @@
+package cache
+
+import (
+	"os"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+)
+
+func TestMain(m *testing.M) {
+	run := func() int {
+		resetCfg := featureconfig.InitWithReset(&featureconfig.Flags{EnableEth1DataVoteCache: true})
+		defer resetCfg()
+		return m.Run()
+	}
+	os.Exit(run())
+}

beacon-chain/cache/depositcache/deposits_cache.go

@@ -50,12 +50,10 @@ type FinalizedDeposits struct {
 // stores all the deposit related data that is required by the beacon-node.
 type DepositCache struct {
 	// Beacon chain deposits in memory.
-	pendingDeposits    []*dbpb.DepositContainer
-	deposits           []*dbpb.DepositContainer
-	finalizedDeposits  *FinalizedDeposits
-	depositsLock       sync.RWMutex
-	chainStartDeposits []*ethpb.Deposit
-	chainStartPubkeys  map[string]bool
+	pendingDeposits   []*dbpb.DepositContainer
+	deposits          []*dbpb.DepositContainer
+	finalizedDeposits *FinalizedDeposits
+	depositsLock      sync.RWMutex
 }
 
 // New instantiates a new deposit cache
@@ -68,11 +66,9 @@ func New() (*DepositCache, error) {
 	// finalizedDeposits.MerkleTrieIndex is initialized to -1 because it represents the index of the last trie item.
 	// Inserting the first item into the trie will set the value of the index to 0.
 	return &DepositCache{
-		pendingDeposits:    []*dbpb.DepositContainer{},
-		deposits:           []*dbpb.DepositContainer{},
-		finalizedDeposits:  &FinalizedDeposits{Deposits: finalizedDepositsTrie, MerkleTrieIndex: -1},
-		chainStartPubkeys:  make(map[string]bool),
-		chainStartDeposits: make([]*ethpb.Deposit, 0),
+		pendingDeposits:   []*dbpb.DepositContainer{},
+		deposits:          []*dbpb.DepositContainer{},
+		finalizedDeposits: &FinalizedDeposits{Deposits: finalizedDepositsTrie, MerkleTrieIndex: -1},
 	}, nil
 }
 
@@ -152,24 +148,6 @@ func (dc *DepositCache) AllDepositContainers(ctx context.Context) []*dbpb.Deposi
 	return dc.deposits
 }
 
-// MarkPubkeyForChainstart sets the pubkey deposit status to true.
-func (dc *DepositCache) MarkPubkeyForChainstart(ctx context.Context, pubkey string) {
-	ctx, span := trace.StartSpan(ctx, "DepositsCache.MarkPubkeyForChainstart")
-	defer span.End()
-	dc.chainStartPubkeys[pubkey] = true
-}
-
-// PubkeyInChainstart returns bool for whether the pubkey passed in has deposited.
-func (dc *DepositCache) PubkeyInChainstart(ctx context.Context, pubkey string) bool {
-	ctx, span := trace.StartSpan(ctx, "DepositsCache.PubkeyInChainstart")
-	defer span.End()
-	if dc.chainStartPubkeys != nil {
-		return dc.chainStartPubkeys[pubkey]
-	}
-	dc.chainStartPubkeys = make(map[string]bool)
-	return false
-}
-
 // AllDeposits returns a list of historical deposits until the given block number
 // (inclusive). If no block is specified then this method returns all historical deposits.
 func (dc *DepositCache) AllDeposits(ctx context.Context, untilBlk *big.Int) []*ethpb.Deposit {
@@ -266,7 +244,7 @@ func (dc *DepositCache) PruneProofs(ctx context.Context, untilDepositIndex int64
 	dc.depositsLock.Lock()
 	defer dc.depositsLock.Unlock()
 
-	if untilDepositIndex > int64(len(dc.deposits)) {
+	if untilDepositIndex >= int64(len(dc.deposits)) {
 		untilDepositIndex = int64(len(dc.deposits) - 1)
 	}
 

beacon-chain/cache/depositcache/deposits_cache_test.go

@@ -700,6 +700,49 @@ func TestPruneProofs_PruneAllWhenDepositIndexTooBig(t *testing.T) {
 	assert.DeepEqual(t, ([][]byte)(nil), dc.deposits[3].Deposit.Proof)
 }
 
+func TestPruneProofs_CorrectlyHandleLastIndex(t *testing.T) {
+	dc, err := New()
+	require.NoError(t, err)
+
+	deposits := []struct {
+		blkNum  uint64
+		deposit *ethpb.Deposit
+		index   int64
+	}{
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{Proof: makeDepositProof()},
+			index:   0,
+		},
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{Proof: makeDepositProof()},
+			index:   1,
+		},
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{Proof: makeDepositProof()},
+			index:   2,
+		},
+		{
+			blkNum:  0,
+			deposit: &ethpb.Deposit{Proof: makeDepositProof()},
+			index:   3,
+		},
+	}
+
+	for _, ins := range deposits {
+		dc.InsertDeposit(context.Background(), ins.deposit, ins.blkNum, ins.index, [32]byte{})
+	}
+
+	require.NoError(t, dc.PruneProofs(context.Background(), 4))
+
+	assert.DeepEqual(t, ([][]byte)(nil), dc.deposits[0].Deposit.Proof)
+	assert.DeepEqual(t, ([][]byte)(nil), dc.deposits[1].Deposit.Proof)
+	assert.DeepEqual(t, ([][]byte)(nil), dc.deposits[2].Deposit.Proof)
+	assert.DeepEqual(t, ([][]byte)(nil), dc.deposits[3].Deposit.Proof)
+}
+
 func makeDepositProof() [][]byte {
 	proof := make([][]byte, int(params.BeaconConfig().DepositContractTreeDepth)+1)
 	for i := range proof {

beacon-chain/cache/feature_flag_test.go

@@ -1,17 +0,0 @@
-package cache
-
-import (
-	"os"
-	"testing"
-
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-)
-
-func TestMain(m *testing.M) {
-	resetCfg := featureconfig.InitWithReset(&featureconfig.Flags{EnableEth1DataVoteCache: true})
-	defer resetCfg()
-	code := m.Run()
-	// os.Exit will prevent defer from being called
-	resetCfg()
-	os.Exit(code)
-}

beacon-chain/cache/skip_slot_cache.go

@@ -30,7 +30,7 @@ type SkipSlotCache struct {
 	cache      *lru.Cache
 	lock       sync.RWMutex
 	disabled   bool // Allow for programmatic toggling of the cache, useful during initial sync.
-	inProgress map[uint64]bool
+	inProgress map[[32]byte]bool
 }
 
 // NewSkipSlotCache initializes the map and underlying cache.
@@ -41,7 +41,7 @@ func NewSkipSlotCache() *SkipSlotCache {
 	}
 	return &SkipSlotCache{
 		cache:      cache,
-		inProgress: make(map[uint64]bool),
+		inProgress: make(map[[32]byte]bool),
 	}
 }
 
@@ -57,7 +57,7 @@ func (c *SkipSlotCache) Disable() {
 
 // Get waits for any in progress calculation to complete before returning a
 // cached response, if any.
-func (c *SkipSlotCache) Get(ctx context.Context, slot uint64) (*stateTrie.BeaconState, error) {
+func (c *SkipSlotCache) Get(ctx context.Context, r [32]byte) (*stateTrie.BeaconState, error) {
 	ctx, span := trace.StartSpan(ctx, "skipSlotCache.Get")
 	defer span.End()
 	if c.disabled {
@@ -77,7 +77,7 @@ func (c *SkipSlotCache) Get(ctx context.Context, slot uint64) (*stateTrie.Beacon
 		}
 
 		c.lock.RLock()
-		if !c.inProgress[slot] {
+		if !c.inProgress[r] {
 			c.lock.RUnlock()
 			break
 		}
@@ -92,7 +92,7 @@ func (c *SkipSlotCache) Get(ctx context.Context, slot uint64) (*stateTrie.Beacon
 	}
 	span.AddAttributes(trace.BoolAttribute("inProgress", inProgress))
 
-	item, exists := c.cache.Get(slot)
+	item, exists := c.cache.Get(r)
 
 	if exists && item != nil {
 		skipSlotCacheHit.Inc()
@@ -106,7 +106,7 @@ func (c *SkipSlotCache) Get(ctx context.Context, slot uint64) (*stateTrie.Beacon
 
 // MarkInProgress a request so that any other similar requests will block on
 // Get until MarkNotInProgress is called.
-func (c *SkipSlotCache) MarkInProgress(slot uint64) error {
+func (c *SkipSlotCache) MarkInProgress(r [32]byte) error {
 	if c.disabled {
 		return nil
 	}
@@ -114,16 +114,16 @@ func (c *SkipSlotCache) MarkInProgress(slot uint64) error {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
-	if c.inProgress[slot] {
+	if c.inProgress[r] {
 		return ErrAlreadyInProgress
 	}
-	c.inProgress[slot] = true
+	c.inProgress[r] = true
 	return nil
 }
 
 // MarkNotInProgress will release the lock on a given request. This should be
 // called after put.
-func (c *SkipSlotCache) MarkNotInProgress(slot uint64) error {
+func (c *SkipSlotCache) MarkNotInProgress(r [32]byte) error {
 	if c.disabled {
 		return nil
 	}
@@ -131,18 +131,18 @@ func (c *SkipSlotCache) MarkNotInProgress(slot uint64) error {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
-	delete(c.inProgress, slot)
+	delete(c.inProgress, r)
 	return nil
 }
 
 // Put the response in the cache.
-func (c *SkipSlotCache) Put(_ context.Context, slot uint64, state *stateTrie.BeaconState) error {
+func (c *SkipSlotCache) Put(_ context.Context, r [32]byte, state *stateTrie.BeaconState) error {
 	if c.disabled {
 		return nil
 	}
 
 	// Copy state so cached value is not mutated.
-	c.cache.Add(slot, state.Copy())
+	c.cache.Add(r, state.Copy())
 
 	return nil
 }

beacon-chain/cache/skip_slot_cache_test.go

@@ -15,21 +15,22 @@ func TestSkipSlotCache_RoundTrip(t *testing.T) {
 	ctx := context.Background()
 	c := cache.NewSkipSlotCache()
 
-	state, err := c.Get(ctx, 5)
+	r := [32]byte{'a'}
+	state, err := c.Get(ctx, r)
 	require.NoError(t, err)
 	assert.Equal(t, (*stateTrie.BeaconState)(nil), state, "Empty cache returned an object")
 
-	require.NoError(t, c.MarkInProgress(5))
+	require.NoError(t, c.MarkInProgress(r))
 
 	state, err = stateTrie.InitializeFromProto(&pb.BeaconState{
 		Slot: 10,
 	})
 	require.NoError(t, err)
 
-	require.NoError(t, c.Put(ctx, 5, state))
-	require.NoError(t, c.MarkNotInProgress(5))
+	require.NoError(t, c.Put(ctx, r, state))
+	require.NoError(t, c.MarkNotInProgress(r))
 
-	res, err := c.Get(ctx, 5)
+	res, err := c.Get(ctx, r)
 	require.NoError(t, err)
 	assert.DeepEqual(t, res.CloneInnerState(), state.CloneInnerState(), "Expected equal protos to return from cache")
 }

beacon-chain/cache/state_summary.go

@@ -50,11 +50,11 @@ func (s *StateSummaryCache) GetAll() []*pb.StateSummary {
 	s.initSyncStateSummariesLock.RLock()
 	defer s.initSyncStateSummariesLock.RUnlock()
 
-	blks := make([]*pb.StateSummary, 0, len(s.initSyncStateSummaries))
+	summaries := make([]*pb.StateSummary, 0, len(s.initSyncStateSummaries))
 	for _, b := range s.initSyncStateSummaries {
-		blks = append(blks, b)
+		summaries = append(summaries, b)
 	}
-	return blks
+	return summaries
 }
 
 // Clear clears out the initial sync state summaries cache.

beacon-chain/core/blocks/BUILD.bazel

@@ -50,6 +50,7 @@ go_test(
     name = "go_default_test",
     size = "medium",
     srcs = [
+        "attestation_regression_test.go",
         "attestation_test.go",
         "attester_slashing_test.go",
         "block_operations_fuzz_test.go",
@@ -59,9 +60,11 @@ go_test(
         "exit_test.go",
         "genesis_test.go",
         "header_test.go",
+        "proposer_slashing_regression_test.go",
         "proposer_slashing_test.go",
         "randao_test.go",
     ],
+    data = glob(["testdata/**"]),
     embed = [":go_default_library"],
     shard_count = 2,
     deps = [

beacon-chain/core/blocks/attestation.go

@@ -158,7 +158,7 @@ func ProcessAttestationNoVerifySignature(
 		return nil, err
 	}
 	c := helpers.SlotCommitteeCount(activeValidatorCount)
-	if att.Data.CommitteeIndex > c {
+	if att.Data.CommitteeIndex >= c {
 		return nil, fmt.Errorf("committee index %d >= committee count %d", att.Data.CommitteeIndex, c)
 	}
 

beacon-chain/core/blocks/attestation_regression_test.go

@@ -0,0 +1,44 @@
+package blocks_test
+
+import (
+	"context"
+	"io/ioutil"
+	"testing"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+)
+
+// Beaconfuzz discovered an off by one issue where an attestation could be produced which would pass
+// validation when att.Data.CommitteeIndex is 1 and the committee count per slot is also 1. The only
+// valid att.Data.Committee index would be 0, so this is an off by one error.
+// See: https://github.com/sigp/beacon-fuzz/issues/78
+func TestProcessAttestationNoVerifySignature_BeaconFuzzIssue78(t *testing.T) {
+	attData, err := ioutil.ReadFile("testdata/beaconfuzz_78_attestation.ssz")
+	if err != nil {
+		t.Fatal(err)
+	}
+	att := &ethpb.Attestation{}
+	if err := att.UnmarshalSSZ(attData); err != nil {
+		t.Fatal(err)
+	}
+	stateData, err := ioutil.ReadFile("testdata/beaconfuzz_78_beacon.ssz")
+	if err != nil {
+		t.Fatal(err)
+	}
+	spb := &pb.BeaconState{}
+	if err := spb.UnmarshalSSZ(stateData); err != nil {
+		t.Fatal(err)
+	}
+	st, err := state.InitializeFromProtoUnsafe(spb)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ctx := context.Background()
+	_, err = blocks.ProcessAttestationNoVerifySignature(ctx, st, att)
+	require.ErrorContains(t, "committee index 1 >= committee count 1", err)
+}

beacon-chain/core/blocks/attester_slashing.go

@@ -53,7 +53,7 @@ func ProcessAttesterSlashings(
 		currentEpoch := helpers.SlotToEpoch(beaconState.Slot())
 		var err error
 		var slashedAny bool
-		var val *stateTrie.ReadOnlyValidator
+		var val stateTrie.ReadOnlyValidator
 		for _, validatorIndex := range slashableIndices {
 			val, err = beaconState.ValidatorAtIndexReadOnly(validatorIndex)
 			if err != nil {

beacon-chain/core/blocks/block_operations_fuzz_test.go

@@ -429,13 +429,13 @@ func TestFuzzProcessVoluntaryExitsNoVerify_10000(t *testing.T) {
 func TestFuzzVerifyExit_10000(t *testing.T) {
 	fuzzer := fuzz.NewWithSeed(0)
 	ve := &eth.SignedVoluntaryExit{}
-	val := &stateTrie.ReadOnlyValidator{}
+	val := stateTrie.ReadOnlyValidator{}
 	fork := &pb.Fork{}
 	var slot uint64
 
 	for i := 0; i < 10000; i++ {
 		fuzzer.Fuzz(ve)
-		fuzzer.Fuzz(val)
+		fuzzer.Fuzz(&val)
 		fuzzer.Fuzz(fork)
 		fuzzer.Fuzz(&slot)
 		err := VerifyExitAndSignature(val, slot, fork, ve, params.BeaconConfig().ZeroHash[:])

beacon-chain/core/blocks/deposit.go

@@ -33,8 +33,10 @@ func ProcessPreGenesisDeposits(
 	for _, deposit := range deposits {
 		pubkey := deposit.Data.PublicKey
 		index, ok := beaconState.ValidatorIndexByPubkey(bytesutil.ToBytes48(pubkey))
+		// In the event of the pubkey not existing, we continue processing the other
+		// deposits.
 		if !ok {
-			return beaconState, nil
+			continue
 		}
 		balance, err := beaconState.BalanceAtIndex(index)
 		if err != nil {

beacon-chain/core/blocks/deposit_test.go

@@ -132,7 +132,8 @@ func TestProcessDeposits_AddsNewValidatorDeposit(t *testing.T) {
 }
 
 func TestProcessDeposits_RepeatedDeposit_IncreasesValidatorBalance(t *testing.T) {
-	sk := bls.RandKey()
+	sk, err := bls.RandKey()
+	require.NoError(t, err)
 	deposit := &ethpb.Deposit{
 		Data: &ethpb.Deposit_Data{
 			PublicKey:             sk.PublicKey().Marshal(),
@@ -271,3 +272,71 @@ func TestProcessDeposit_SkipsInvalidDeposit(t *testing.T) {
 		t.Errorf("Expected validator balance at index 0 to stay 0, received: %v", newState.Balances()[0])
 	}
 }
+
+func TestPreGenesisDeposits_SkipInvalidDeposit(t *testing.T) {
+	testutil.ResetCache()
+	dep, _, err := testutil.DeterministicDepositsAndKeys(100)
+	require.NoError(t, err)
+	defer func() {
+		testutil.ResetCache()
+	}()
+	dep[0].Data.Signature = make([]byte, 96)
+	trie, _, err := testutil.DepositTrieFromDeposits(dep)
+	require.NoError(t, err)
+
+	for i := range dep {
+		proof, err := trie.MerkleProof(i)
+		require.NoError(t, err)
+		dep[i].Proof = proof
+	}
+	root := trie.Root()
+	eth1Data := &ethpb.Eth1Data{
+		DepositRoot:  root[:],
+		DepositCount: 1,
+	}
+	registry := []*ethpb.Validator{
+		{
+			PublicKey:             []byte{1},
+			WithdrawalCredentials: []byte{1, 2, 3},
+		},
+	}
+	balances := []uint64{0}
+	beaconState, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Validators: registry,
+		Balances:   balances,
+		Eth1Data:   eth1Data,
+		Fork: &pb.Fork{
+			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
+			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
+		},
+	})
+	require.NoError(t, err)
+	newState, err := blocks.ProcessPreGenesisDeposits(context.Background(), beaconState, dep)
+	require.NoError(t, err, "Expected invalid block deposit to be ignored without error")
+
+	_, ok := newState.ValidatorIndexByPubkey(bytesutil.ToBytes48(dep[0].Data.PublicKey))
+	require.Equal(t, false, ok, "bad pubkey should not exist in state")
+
+	for i := 1; i < newState.NumValidators(); i++ {
+		val, err := newState.ValidatorAtIndex(uint64(i))
+		require.NoError(t, err)
+		require.Equal(t, params.BeaconConfig().MaxEffectiveBalance, val.EffectiveBalance, "unequal effective balance")
+		require.Equal(t, uint64(0), val.ActivationEpoch)
+		require.Equal(t, uint64(0), val.ActivationEligibilityEpoch)
+	}
+	if newState.Eth1DepositIndex() != 100 {
+		t.Errorf(
+			"Expected Eth1DepositIndex to be increased by 99 after processing an invalid deposit, received change: %v",
+			newState.Eth1DepositIndex(),
+		)
+	}
+	if len(newState.Validators()) != 100 {
+		t.Errorf("Expected validator list to have length 100, received: %v", len(newState.Validators()))
+	}
+	if len(newState.Balances()) != 100 {
+		t.Errorf("Expected validator balances list to have length 100, received: %v", len(newState.Balances()))
+	}
+	if newState.Balances()[0] != 0 {
+		t.Errorf("Expected validator balance at index 0 to stay 0, received: %v", newState.Balances()[0])
+	}
+}

beacon-chain/core/blocks/exit.go

@@ -14,7 +14,7 @@ import (
 )
 
 // ValidatorAlreadyExitedMsg defines a message saying that a validator has already exited.
-var ValidatorAlreadyExitedMsg = "validator has already submitted an exit, which will take place at epoch"
+var ValidatorAlreadyExitedMsg = "has already submitted an exit, which will take place at epoch"
 
 // ValidatorCannotExitYetMsg defines a message saying that a validator cannot exit
 // because it has not been active long enough.
@@ -125,7 +125,7 @@ func ProcessVoluntaryExitsNoVerifySignature(
 //    # Verify signature
 //    domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, exit.epoch)
 //    assert bls_verify(validator.pubkey, signing_root(exit), exit.signature, domain)
-func VerifyExitAndSignature(validator *stateTrie.ReadOnlyValidator, currentSlot uint64, fork *pb.Fork, signed *ethpb.SignedVoluntaryExit, genesisRoot []byte) error {
+func VerifyExitAndSignature(validator stateTrie.ReadOnlyValidator, currentSlot uint64, fork *pb.Fork, signed *ethpb.SignedVoluntaryExit, genesisRoot []byte) error {
 	if signed == nil || signed.Exit == nil {
 		return errors.New("nil exit")
 	}
@@ -161,7 +161,7 @@ func VerifyExitAndSignature(validator *stateTrie.ReadOnlyValidator, currentSlot
 //    assert get_current_epoch(state) >= exit.epoch
 //    # Verify the validator has been active long enough
 //    assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD
-func verifyExitConditions(validator *stateTrie.ReadOnlyValidator, currentSlot uint64, exit *ethpb.VoluntaryExit) error {
+func verifyExitConditions(validator stateTrie.ReadOnlyValidator, currentSlot uint64, exit *ethpb.VoluntaryExit) error {
 	currentEpoch := helpers.SlotToEpoch(currentSlot)
 	// Verify the validator is active.
 	if !helpers.IsActiveValidatorUsingTrie(validator, currentEpoch) {
@@ -169,7 +169,7 @@ func verifyExitConditions(validator *stateTrie.ReadOnlyValidator, currentSlot ui
 	}
 	// Verify the validator has not yet submitted an exit.
 	if validator.ExitEpoch() != params.BeaconConfig().FarFutureEpoch {
-		return fmt.Errorf("%s: %v", ValidatorAlreadyExitedMsg, validator.ExitEpoch())
+		return fmt.Errorf("validator with index %d %s: %v", exit.ValidatorIndex, ValidatorAlreadyExitedMsg, validator.ExitEpoch())
 	}
 	// Exits must specify an epoch when they become valid; they are not valid before then.
 	if currentEpoch < exit.Epoch {

beacon-chain/core/blocks/exit_test.go

@@ -139,7 +139,7 @@ func TestProcessVoluntaryExits_ExitAlreadySubmitted(t *testing.T) {
 		},
 	}
 
-	want := "validator has already submitted an exit, which will take place at epoch: 10"
+	want := "validator with index 0 has already submitted an exit, which will take place at epoch: 10"
 	_, err = blocks.ProcessVoluntaryExits(context.Background(), state, b)
 	assert.ErrorContains(t, want, err)
 }
@@ -171,7 +171,9 @@ func TestProcessVoluntaryExits_AppliesCorrectStatus(t *testing.T) {
 	err = state.SetSlot(state.Slot() + (params.BeaconConfig().ShardCommitteePeriod * params.BeaconConfig().SlotsPerEpoch))
 	require.NoError(t, err)
 
-	priv := bls.RandKey()
+	priv, err := bls.RandKey()
+	require.NoError(t, err)
+
 	val, err := state.ValidatorAtIndex(0)
 	require.NoError(t, err)
 	val.PublicKey = priv.PublicKey().Marshal()

beacon-chain/core/blocks/header_test.go

@@ -48,7 +48,8 @@ func TestProcessBlockHeader_ImproperBlockSlot(t *testing.T) {
 	require.NoError(t, err)
 
 	currentEpoch := helpers.CurrentEpoch(state)
-	priv := bls.RandKey()
+	priv, err := bls.RandKey()
+	require.NoError(t, err)
 	pID, err := helpers.BeaconProposerIndex(state)
 	require.NoError(t, err)
 	block := testutil.NewBeaconBlock()
@@ -126,7 +127,8 @@ func TestProcessBlockHeader_DifferentSlots(t *testing.T) {
 	require.NoError(t, err)
 	currentEpoch := helpers.CurrentEpoch(state)
 
-	priv := bls.RandKey()
+	priv, err := bls.RandKey()
+	require.NoError(t, err)
 	blockSig, err := helpers.ComputeDomainAndSign(state, currentEpoch, []byte("hello"), params.BeaconConfig().DomainBeaconProposer, priv)
 	require.NoError(t, err)
 	validators[5896].PublicKey = priv.PublicKey().Marshal()
@@ -164,7 +166,8 @@ func TestProcessBlockHeader_PreviousBlockRootNotSignedRoot(t *testing.T) {
 	bh.Slot = 9
 	require.NoError(t, state.SetLatestBlockHeader(bh))
 	currentEpoch := helpers.CurrentEpoch(state)
-	priv := bls.RandKey()
+	priv, err := bls.RandKey()
+	require.NoError(t, err)
 	blockSig, err := helpers.ComputeDomainAndSign(state, currentEpoch, []byte("hello"), params.BeaconConfig().DomainBeaconProposer, priv)
 	require.NoError(t, err)
 	validators[5896].PublicKey = priv.PublicKey().Marshal()
@@ -202,7 +205,8 @@ func TestProcessBlockHeader_SlashedProposer(t *testing.T) {
 	parentRoot, err := state.LatestBlockHeader().HashTreeRoot()
 	require.NoError(t, err)
 	currentEpoch := helpers.CurrentEpoch(state)
-	priv := bls.RandKey()
+	priv, err := bls.RandKey()
+	require.NoError(t, err)
 	blockSig, err := helpers.ComputeDomainAndSign(state, currentEpoch, []byte("hello"), params.BeaconConfig().DomainBeaconProposer, priv)
 	require.NoError(t, err)
 
@@ -247,7 +251,8 @@ func TestProcessBlockHeader_OK(t *testing.T) {
 	require.NoError(t, err)
 
 	currentEpoch := helpers.CurrentEpoch(state)
-	priv := bls.RandKey()
+	priv, err := bls.RandKey()
+	require.NoError(t, err)
 	pID, err := helpers.BeaconProposerIndex(state)
 	require.NoError(t, err)
 	block := testutil.NewBeaconBlock()
@@ -307,7 +312,8 @@ func TestBlockSignatureSet_OK(t *testing.T) {
 	require.NoError(t, err)
 
 	currentEpoch := helpers.CurrentEpoch(state)
-	priv := bls.RandKey()
+	priv, err := bls.RandKey()
+	require.NoError(t, err)
 	pID, err := helpers.BeaconProposerIndex(state)
 	require.NoError(t, err)
 	block := testutil.NewBeaconBlock()

beacon-chain/core/blocks/proposer_slashing.go

@@ -4,13 +4,14 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/prysmaticlabs/prysm/shared/params"
+
 	"github.com/gogo/protobuf/proto"
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	v "github.com/prysmaticlabs/prysm/beacon-chain/core/validators"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
-	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
 // ProcessProposerSlashings is one of the operations performed
@@ -86,7 +87,7 @@ func VerifyProposerSlashing(
 	if err != nil {
 		return err
 	}
-	if !helpers.IsSlashableValidatorUsingTrie(proposer, helpers.SlotToEpoch(hSlot)) {
+	if !helpers.IsSlashableValidatorUsingTrie(proposer, helpers.CurrentEpoch(beaconState)) {
 		return fmt.Errorf("validator with key %#x is not slashable", proposer.PublicKey())
 	}
 	headers := []*ethpb.SignedBeaconBlockHeader{slashing.Header_1, slashing.Header_2}
@@ -96,6 +97,5 @@ func VerifyProposerSlashing(
 			return errors.Wrap(err, "could not verify beacon block header")
 		}
 	}
-
 	return nil
 }

beacon-chain/core/blocks/proposer_slashing_regression_test.go

@@ -0,0 +1,38 @@
+package blocks_test
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+
+	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/testutil/require"
+)
+
+// Beaconfuzz discovered an issue where a proposer slashing could be produced which would pass
+// validation where we use the slashing's slot instead of the current epoch of our state for validation.
+// This would lead to us accepting an invalid slashing by marking the respective validator as 'slashable'
+// when it was not in actuality.
+// See: https://github.com/sigp/beacon-fuzz/issues/91
+func TestVerifyProposerSlashing_BeaconFuzzIssue91(t *testing.T) {
+	file, err := ioutil.ReadFile("testdata/beaconfuzz_91_beacon.ssz")
+	require.NoError(t, err)
+	rawState := &pb.BeaconState{}
+	err = rawState.UnmarshalSSZ(file)
+	require.NoError(t, err)
+
+	st, err := stateTrie.InitializeFromProtoUnsafe(rawState)
+	require.NoError(t, err)
+
+	file, err = ioutil.ReadFile("testdata/beaconfuzz_91_proposer_slashing.ssz")
+	require.NoError(t, err)
+	slashing := &ethpb.ProposerSlashing{}
+	err = slashing.UnmarshalSSZ(file)
+	require.NoError(t, err)
+
+	err = blocks.VerifyProposerSlashing(st, slashing)
+	require.ErrorContains(t, "validator with key 0x97f1d3a73197d7942695638c4fa9ac0fc3688c4f9774b905a14e3a3f171bac586c55e83ff97a1aeffb3af00adb22c6bb is not slashable", err)
+}

beacon-chain/core/blocks/proposer_slashing_test.go

@@ -194,6 +194,13 @@ func TestVerifyProposerSlashing(t *testing.T) {
 	beaconState, sks := testutil.DeterministicGenesisState(t, 2)
 	currentSlot := uint64(0)
 	require.NoError(t, beaconState.SetSlot(currentSlot))
+	rand1, err := bls.RandKey()
+	require.NoError(t, err)
+	sig1 := rand1.Sign([]byte("foo")).Marshal()
+
+	rand2, err := bls.RandKey()
+	require.NoError(t, err)
+	sig2 := rand2.Sign([]byte("bar")).Marshal()
 
 	tests := []struct {
 		name    string
@@ -239,7 +246,7 @@ func TestVerifyProposerSlashing(t *testing.T) {
 							BodyRoot:      bytesutil.PadTo([]byte{}, 32),
 							ParentRoot:    bytesutil.PadTo([]byte{}, 32),
 						},
-						Signature: bls.RandKey().Sign([]byte("foo")).Marshal(),
+						Signature: sig1,
 					},
 					Header_2: &ethpb.SignedBeaconBlockHeader{
 						Header: &ethpb.BeaconBlockHeader{
@@ -249,7 +256,7 @@ func TestVerifyProposerSlashing(t *testing.T) {
 							BodyRoot:      bytesutil.PadTo([]byte{}, 32),
 							ParentRoot:    bytesutil.PadTo([]byte{}, 32),
 						},
-						Signature: bls.RandKey().Sign([]byte("bar")).Marshal(),
+						Signature: sig2,
 					},
 				},
 				beaconState: beaconState,

beacon-chain/core/blocks/randao_test.go

@@ -86,7 +86,7 @@ func TestRandaoSignatureSet_OK(t *testing.T) {
 		},
 	}
 
-	set, _, err := blocks.RandaoSignatureSet(beaconState, block.Body)
+	set, err := blocks.RandaoSignatureSet(beaconState, block.Body)
 	require.NoError(t, err)
 	verified, err := set.Verify()
 	require.NoError(t, err)

beacon-chain/core/blocks/signature.go

@@ -92,16 +92,16 @@ func BlockSignatureSet(beaconState *stateTrie.BeaconState, block *ethpb.SignedBe
 // from a block and its corresponding state.
 func RandaoSignatureSet(beaconState *stateTrie.BeaconState,
 	body *ethpb.BeaconBlockBody,
-) (*bls.SignatureSet, *stateTrie.BeaconState, error) {
+) (*bls.SignatureSet, error) {
 	buf, proposerPub, domain, err := randaoSigningData(beaconState)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 	set, err := retrieveSignatureSet(buf, proposerPub, body.RandaoReveal, domain)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
-	return set, beaconState, nil
+	return set, nil
 }
 
 // retrieves the randao related signing data from the state.

beacon-chain/core/epoch/epoch_processing.go

@@ -366,7 +366,7 @@ func UnslashedAttestingIndices(state *stateTrie.BeaconState, atts []*pb.PendingA
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to look up validator")
 		}
-		if v != nil && v.Slashed() {
+		if !v.IsNil() && v.Slashed() {
 			setIndices = append(setIndices[:i], setIndices[i+1:]...)
 		}
 	}

beacon-chain/core/epoch/epoch_processing_test.go

@@ -196,9 +196,9 @@ func TestProcessSlashings_SlashedLess(t *testing.T) {
 				Balances:  []uint64{params.BeaconConfig().MaxEffectiveBalance, params.BeaconConfig().MaxEffectiveBalance},
 				Slashings: []uint64{0, 1e9},
 			},
-			// penalty    = validator balance / increment * (3*total_penalties) / total_balance * increment
-			// 3000000000 = (32 * 1e9)        / (1 * 1e9) * (3*1e9)             / (32*1e9)      * (1 * 1e9)
-			want: uint64(29000000000), // 32 * 1e9 - 3000000000
+			// penalty    = validator balance / increment * (2*total_penalties) / total_balance * increment
+			// 1000000000 = (32 * 1e9)        / (1 * 1e9) * (1*1e9)             / (32*1e9)      * (1 * 1e9)
+			want: uint64(31000000000), // 32 * 1e9 - 1000000000
 		},
 		{
 			state: &pb.BeaconState{
@@ -212,9 +212,9 @@ func TestProcessSlashings_SlashedLess(t *testing.T) {
 				Balances:  []uint64{params.BeaconConfig().MaxEffectiveBalance, params.BeaconConfig().MaxEffectiveBalance},
 				Slashings: []uint64{0, 1e9},
 			},
-			// penalty    = validator balance / increment * (3*total_penalties) / total_balance * increment
-			// 1000000000 = (32 * 1e9)        / (1 * 1e9) * (3*1e9)             / (64*1e9)      * (1 * 1e9)
-			want: uint64(31000000000), // 32 * 1e9 - 1000000000
+			// penalty    = validator balance / increment * (2*total_penalties) / total_balance * increment
+			// 500000000 = (32 * 1e9)        / (1 * 1e9) * (1*1e9)             / (32*1e9)      * (1 * 1e9)
+			want: uint64(32000000000), // 32 * 1e9 - 500000000
 		},
 		{
 			state: &pb.BeaconState{
@@ -229,8 +229,8 @@ func TestProcessSlashings_SlashedLess(t *testing.T) {
 				Slashings: []uint64{0, 2 * 1e9},
 			},
 			// penalty    = validator balance / increment * (3*total_penalties) / total_balance * increment
-			// 3000000000 = (32 * 1e9)        / (1 * 1e9) * (3*2e9)             / (64*1e9)      * (1 * 1e9)
-			want: uint64(29000000000), // 32 * 1e9 - 3000000000
+			// 1000000000 = (32 * 1e9)        / (1 * 1e9) * (1*2e9)             / (64*1e9)      * (1 * 1e9)
+			want: uint64(31000000000), // 32 * 1e9 - 1000000000
 		},
 		{
 			state: &pb.BeaconState{
@@ -243,8 +243,8 @@ func TestProcessSlashings_SlashedLess(t *testing.T) {
 				Slashings: []uint64{0, 1e9},
 			},
 			// penalty    = validator balance           / increment * (3*total_penalties) / total_balance        * increment
-			// 3000000000 = (32  * 1e9 - 1*1e9)         / (1 * 1e9) * (3*1e9)             / (31*1e9)             * (1 * 1e9)
-			want: uint64(28000000000), // 31 * 1e9 - 3000000000
+			// 2000000000 = (32  * 1e9 - 1*1e9)         / (1 * 1e9) * (2*1e9)             / (31*1e9)             * (1 * 1e9)
+			want: uint64(30000000000), // 32 * 1e9 - 2000000000
 		},
 	}
 

beacon-chain/core/epoch/precompute/new.go

@@ -25,7 +25,7 @@ func New(ctx context.Context, state *stateTrie.BeaconState) ([]*Validator, *Bala
 	currentEpoch := helpers.CurrentEpoch(state)
 	prevEpoch := helpers.PrevEpoch(state)
 
-	if err := state.ReadFromEveryValidator(func(idx int, val *stateTrie.ReadOnlyValidator) error {
+	if err := state.ReadFromEveryValidator(func(idx int, val stateTrie.ReadOnlyValidator) error {
 		// Was validator withdrawable or slashed
 		withdrawable := prevEpoch+1 >= val.WithdrawableEpoch()
 		pVal := &Validator{

beacon-chain/core/epoch/precompute/slashing.go

@@ -1,8 +1,6 @@
 package precompute
 
 import (
-	"errors"
-
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
@@ -26,18 +24,17 @@ func ProcessSlashingsPrecompute(state *stateTrie.BeaconState, pBal *Balance) err
 	minSlashing := mathutil.Min(totalSlashing*params.BeaconConfig().ProportionalSlashingMultiplier, pBal.ActiveCurrentEpoch)
 	epochToWithdraw := currentEpoch + exitLength/2
 
-	vs := state.ValidatorsReadOnly()
 	var hasSlashing bool
 	// Iterate through validator list in state, stop until a validator satisfies slashing condition of current epoch.
-	for _, v := range vs {
-		if v == nil {
-			return errors.New("nil validator in state")
-		}
-		correctEpoch := epochToWithdraw == v.WithdrawableEpoch()
-		if v.Slashed() && correctEpoch {
+	err := state.ReadFromEveryValidator(func(idx int, val stateTrie.ReadOnlyValidator) error {
+		correctEpoch := epochToWithdraw == val.WithdrawableEpoch()
+		if val.Slashed() && correctEpoch {
 			hasSlashing = true
-			break
 		}
+		return nil
+	})
+	if err != nil {
+		return err
 	}
 	// Exit early if there's no meaningful slashing to process.
 	if !hasSlashing {

beacon-chain/core/epoch/precompute/slashing_test.go

@@ -58,9 +58,9 @@ func TestProcessSlashingsPrecompute_SlashedLess(t *testing.T) {
 				Balances:  []uint64{params.BeaconConfig().MaxEffectiveBalance, params.BeaconConfig().MaxEffectiveBalance},
 				Slashings: []uint64{0, 1e9},
 			},
-			// penalty    = validator balance / increment * (3*total_penalties) / total_balance * increment
-			// 3000000000 = (32 * 1e9)        / (1 * 1e9) * (3*1e9)             / (32*1e9)      * (1 * 1e9)
-			want: uint64(29000000000), // 32 * 1e9 - 3000000000
+			// penalty    = validator balance / increment * (2*total_penalties) / total_balance * increment
+			// 1000000000 = (32 * 1e9)        / (1 * 1e9) * (1*1e9)             / (32*1e9)      * (1 * 1e9)
+			want: uint64(31000000000), // 32 * 1e9 - 1000000000
 		},
 		{
 			state: &pb.BeaconState{
@@ -74,9 +74,9 @@ func TestProcessSlashingsPrecompute_SlashedLess(t *testing.T) {
 				Balances:  []uint64{params.BeaconConfig().MaxEffectiveBalance, params.BeaconConfig().MaxEffectiveBalance, params.BeaconConfig().MaxEffectiveBalance},
 				Slashings: []uint64{0, 1e9},
 			},
-			// penalty    = validator balance / increment * (3*total_penalties) / total_balance * increment
-			// 1000000000 = (32 * 1e9)        / (1 * 1e9) * (3*1e9)             / (64*1e9)      * (1 * 1e9)
-			want: uint64(31000000000), // 32 * 1e9 - 1000000000
+			// penalty    = validator balance / increment * (2*total_penalties) / total_balance * increment
+			// 500000000 = (32 * 1e9)        / (1 * 1e9) * (1*1e9)             / (32*1e9)      * (1 * 1e9)
+			want: uint64(32000000000), // 32 * 1e9 - 500000000
 		},
 		{
 			state: &pb.BeaconState{
@@ -91,8 +91,8 @@ func TestProcessSlashingsPrecompute_SlashedLess(t *testing.T) {
 				Slashings: []uint64{0, 2 * 1e9},
 			},
 			// penalty    = validator balance / increment * (3*total_penalties) / total_balance * increment
-			// 3000000000 = (32 * 1e9)        / (1 * 1e9) * (3*2e9)             / (64*1e9)      * (1 * 1e9)
-			want: uint64(29000000000), // 32 * 1e9 - 3000000000
+			// 1000000000 = (32 * 1e9)        / (1 * 1e9) * (1*2e9)             / (64*1e9)      * (1 * 1e9)
+			want: uint64(31000000000), // 32 * 1e9 - 1000000000
 		},
 		{
 			state: &pb.BeaconState{
@@ -105,8 +105,8 @@ func TestProcessSlashingsPrecompute_SlashedLess(t *testing.T) {
 				Slashings: []uint64{0, 1e9},
 			},
 			// penalty    = validator balance           / increment * (3*total_penalties) / total_balance        * increment
-			// 3000000000 = (32  * 1e9 - 1*1e9)         / (1 * 1e9) * (3*1e9)             / (31*1e9)             * (1 * 1e9)
-			want: uint64(28000000000), // 31 * 1e9 - 3000000000
+			// 2000000000 = (32  * 1e9 - 1*1e9)         / (1 * 1e9) * (2*1e9)             / (31*1e9)             * (1 * 1e9)
+			want: uint64(30000000000), // 32 * 1e9 - 2000000000
 		},
 	}
 

beacon-chain/core/epoch/precompute/spectest/BUILD.bazel

@@ -31,15 +31,15 @@ go_test(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
-        "//shared/params/spectest:go_default_library",
         "//shared/params:go_default_library",
+        "//shared/params/spectest:go_default_library",
         "//shared/testutil:go_default_library",
         "//shared/testutil/require:go_default_library",
+        "@com_github_ghodss_yaml//:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
         "@com_github_prysmaticlabs_go_ssz//:go_default_library",
         "@in_gopkg_d4l3k_messagediff_v1//:go_default_library",
         "@io_bazel_rules_go//go/tools/bazel:go_default_library",
-        "@com_github_ghodss_yaml//:go_default_library",
     ],
 )
 
@@ -65,14 +65,14 @@ go_test(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
-        "//shared/params/spectest:go_default_library",
         "//shared/params:go_default_library",
+        "//shared/params/spectest:go_default_library",
         "//shared/testutil:go_default_library",
         "//shared/testutil/require:go_default_library",
+        "@com_github_ghodss_yaml//:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
+        "@com_github_prysmaticlabs_go_ssz//:go_default_library",
         "@in_gopkg_d4l3k_messagediff_v1//:go_default_library",
         "@io_bazel_rules_go//go/tools/bazel:go_default_library",
-        "@com_github_ghodss_yaml//:go_default_library",
-        "@com_github_prysmaticlabs_go_ssz//:go_default_library",
     ],
 )

beacon-chain/core/epoch/spectest/BUILD.bazel

@@ -30,8 +30,8 @@ go_test(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
-        "//shared/params/spectest:go_default_library",
         "//shared/params:go_default_library",
+        "//shared/params/spectest:go_default_library",
         "//shared/testutil:go_default_library",
         "//shared/testutil/require:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
@@ -63,8 +63,8 @@ go_test(
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
-        "//shared/params/spectest:go_default_library",
         "//shared/params:go_default_library",
+        "//shared/params/spectest:go_default_library",
         "//shared/testutil:go_default_library",
         "//shared/testutil/require:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",

beacon-chain/core/epoch/spectest/spectest_test.go

@@ -8,12 +8,14 @@ import (
 )
 
 func TestMain(m *testing.M) {
-	prevConfig := params.BeaconConfig().Copy()
-	c := params.BeaconConfig()
-	c.MinGenesisActiveValidatorCount = 16384
-	params.OverrideBeaconConfig(c)
+	run := func() int {
+		prevConfig := params.BeaconConfig().Copy()
+		defer params.OverrideBeaconConfig(prevConfig)
+		c := params.BeaconConfig()
+		c.MinGenesisActiveValidatorCount = 16384
+		params.OverrideBeaconConfig(c)
 
-	retVal := m.Run()
-	params.OverrideBeaconConfig(prevConfig)
-	os.Exit(retVal)
+		return m.Run()
+	}
+	os.Exit(run())
 }

beacon-chain/core/helpers/BUILD.bazel

@@ -36,7 +36,6 @@ go_library(
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/bls:go_default_library",
         "//shared/bytesutil:go_default_library",
-        "//shared/featureconfig:go_default_library",
         "//shared/hashutil:go_default_library",
         "//shared/mathutil:go_default_library",
         "//shared/params:go_default_library",

beacon-chain/core/helpers/attestation.go

@@ -144,3 +144,23 @@ func ValidateAttestationTime(attSlot uint64, genesisTime time.Time) error {
 	}
 	return nil
 }
+
+// VerifyCheckpointEpoch is within current epoch and previous epoch
+// with respect to current time. Returns true if it's within, false if it's not.
+func VerifyCheckpointEpoch(c *ethpb.Checkpoint, genesis time.Time) bool {
+	now := uint64(timeutils.Now().Unix())
+	genesisTime := uint64(genesis.Unix())
+	currentSlot := (now - genesisTime) / params.BeaconConfig().SecondsPerSlot
+	currentEpoch := SlotToEpoch(currentSlot)
+
+	var prevEpoch uint64
+	if currentEpoch > 1 {
+		prevEpoch = currentEpoch - 1
+	}
+
+	if c.Epoch != prevEpoch && c.Epoch != currentEpoch {
+		return false
+	}
+
+	return true
+}

beacon-chain/core/helpers/attestation_test.go

@@ -49,7 +49,8 @@ func TestAttestation_AggregateSignature(t *testing.T) {
 		atts := make([]*ethpb.Attestation, 0, 100)
 		msg := bytesutil.ToBytes32([]byte("hello"))
 		for i := 0; i < 100; i++ {
-			priv := bls.RandKey()
+			priv, err := bls.RandKey()
+			require.NoError(t, err)
 			pub := priv.PublicKey()
 			sig := priv.Sign(msg[:])
 			pubkeys = append(pubkeys, pub)
@@ -66,7 +67,8 @@ func TestAttestation_AggregateSignature(t *testing.T) {
 		atts := make([]*ethpb.Attestation, 0, 100)
 		msg := []byte("hello")
 		for i := 0; i < 100; i++ {
-			priv := bls.RandKey()
+			priv, err := bls.RandKey()
+			require.NoError(t, err)
 			pub := priv.PublicKey()
 			sig := priv.Sign(msg)
 			pubkeys = append(pubkeys, pub)
@@ -214,3 +216,12 @@ func Test_ValidateAttestationTime(t *testing.T) {
 		})
 	}
 }
+
+func TestVerifyCheckpointEpoch_Ok(t *testing.T) {
+	// Genesis was 6 epochs ago exactly.
+	genesis := time.Now().Add(-1 * time.Second * time.Duration(params.BeaconConfig().SecondsPerSlot*params.BeaconConfig().SlotsPerEpoch*6))
+	assert.Equal(t, true, helpers.VerifyCheckpointEpoch(&ethpb.Checkpoint{Epoch: 6}, genesis))
+	assert.Equal(t, true, helpers.VerifyCheckpointEpoch(&ethpb.Checkpoint{Epoch: 5}, genesis))
+	assert.Equal(t, false, helpers.VerifyCheckpointEpoch(&ethpb.Checkpoint{Epoch: 4}, genesis))
+	assert.Equal(t, false, helpers.VerifyCheckpointEpoch(&ethpb.Checkpoint{Epoch: 2}, genesis))
+}

beacon-chain/core/helpers/block.go

@@ -1,6 +1,8 @@
 package helpers
 
 import (
+	"math"
+
 	"github.com/pkg/errors"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/params"
@@ -17,6 +19,9 @@ import (
 //    assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT
 //    return state.block_roots[slot % SLOTS_PER_HISTORICAL_ROOT]
 func BlockRootAtSlot(state *stateTrie.BeaconState, slot uint64) ([]byte, error) {
+	if math.MaxUint64-slot < params.BeaconConfig().SlotsPerHistoricalRoot {
+		return []byte{}, errors.New("slot overflows uint64")
+	}
 	if slot >= state.Slot() || state.Slot() > slot+params.BeaconConfig().SlotsPerHistoricalRoot {
 		return []byte{}, errors.Errorf("slot %d out of bounds", slot)
 	}

beacon-chain/core/helpers/block_test.go

@@ -2,6 +2,7 @@ package helpers_test
 
 import (
 	"fmt"
+	"math"
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
@@ -101,6 +102,11 @@ func TestBlockRootAtSlot_OutOfBounds(t *testing.T) {
 			stateSlot:   params.BeaconConfig().SlotsPerHistoricalRoot + 2,
 			expectedErr: "slot 1 out of bounds",
 		},
+		{
+			slot:        math.MaxUint64 - 5,
+			stateSlot:   0, // Doesn't matter
+			expectedErr: "slot overflows uint64",
+		},
 	}
 	for _, tt := range tests {
 		state.Slot = tt.stateSlot

beacon-chain/core/helpers/committee.go

@@ -13,7 +13,6 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/hashutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/sliceutil"
@@ -149,25 +148,6 @@ func ComputeCommittee(
 		return nil, err
 	}
 
-	// This updates the cache on a miss.
-	if featureconfig.Get().UseCheckPointInfoCache {
-		sortedIndices := make([]uint64, len(indices))
-		copy(sortedIndices, indices)
-		sort.Slice(sortedIndices, func(i, j int) bool {
-			return sortedIndices[i] < sortedIndices[j]
-		})
-
-		count = SlotCommitteeCount(uint64(len(shuffledIndices)))
-		if err := committeeCache.AddCommitteeShuffledList(&cache.Committees{
-			ShuffledIndices: shuffledList,
-			CommitteeCount:  count * params.BeaconConfig().SlotsPerEpoch,
-			Seed:            seed,
-			SortedIndices:   sortedIndices,
-		}); err != nil {
-			return nil, err
-		}
-	}
-
 	return shuffledList[start:end], nil
 }
 
@@ -206,7 +186,9 @@ func CommitteeAssignments(
 		return nil, nil, err
 	}
 	proposerIndexToSlots := make(map[uint64][]uint64, params.BeaconConfig().SlotsPerEpoch)
-	for slot := startSlot; slot < startSlot+params.BeaconConfig().SlotsPerEpoch; slot++ {
+	// Proposal epochs do not have a look ahead, so we skip them over here.
+	validProposalEpoch := epoch < nextEpoch
+	for slot := startSlot; slot < startSlot+params.BeaconConfig().SlotsPerEpoch && validProposalEpoch; slot++ {
 		// Skip proposer assignment for genesis slot.
 		if slot == 0 {
 			continue
@@ -292,7 +274,7 @@ func ShuffledIndices(state *stateTrie.BeaconState, epoch uint64) ([]uint64, erro
 	}
 
 	indices := make([]uint64, 0, state.NumValidators())
-	if err := state.ReadFromEveryValidator(func(idx int, val *stateTrie.ReadOnlyValidator) error {
+	if err := state.ReadFromEveryValidator(func(idx int, val stateTrie.ReadOnlyValidator) error {
 		if IsActiveValidatorUsingTrie(val, epoch) {
 			indices = append(indices, uint64(idx))
 		}

beacon-chain/core/helpers/committee_test.go

@@ -203,6 +203,36 @@ func TestCommitteeAssignments_CanRetrieve(t *testing.T) {
 	}
 }
 
+func TestCommitteeAssignments_CannotRetrieveFuture(t *testing.T) {
+	// Initialize test with 256 validators, each slot and each index gets 4 validators.
+	validators := make([]*ethpb.Validator, 4*params.BeaconConfig().SlotsPerEpoch)
+	for i := 0; i < len(validators); i++ {
+		// First 2 epochs only half validators are activated.
+		var activationEpoch uint64
+		if i >= len(validators)/2 {
+			activationEpoch = 3
+		}
+		validators[i] = &ethpb.Validator{
+			ActivationEpoch: activationEpoch,
+			ExitEpoch:       params.BeaconConfig().FarFutureEpoch,
+		}
+	}
+
+	state, err := beaconstate.InitializeFromProto(&pb.BeaconState{
+		Validators:  validators,
+		Slot:        2 * params.BeaconConfig().SlotsPerEpoch, // epoch 2
+		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+	})
+	require.NoError(t, err)
+	_, proposerIndxs, err := CommitteeAssignments(state, CurrentEpoch(state))
+	require.NoError(t, err)
+	require.NotEqual(t, 0, len(proposerIndxs), "wanted non-zero proposer index set")
+
+	_, proposerIndxs, err = CommitteeAssignments(state, CurrentEpoch(state)+1)
+	require.NoError(t, err)
+	require.Equal(t, 0, len(proposerIndxs), "wanted empty proposer index set")
+}
+
 func TestCommitteeAssignments_EverySlotHasMin1Proposer(t *testing.T) {
 	// Initialize test with 256 validators, each slot and each index gets 4 validators.
 	validators := make([]*ethpb.Validator, 4*params.BeaconConfig().SlotsPerEpoch)

beacon-chain/core/helpers/rewards_penalties.go

@@ -45,7 +45,7 @@ func TotalBalance(state *stateTrie.BeaconState, indices []uint64) uint64 {
 //    return get_total_balance(state, set(get_active_validator_indices(state, get_current_epoch(state))))
 func TotalActiveBalance(state *stateTrie.BeaconState) (uint64, error) {
 	total := uint64(0)
-	if err := state.ReadFromEveryValidator(func(idx int, val *stateTrie.ReadOnlyValidator) error {
+	if err := state.ReadFromEveryValidator(func(idx int, val stateTrie.ReadOnlyValidator) error {
 		if IsActiveValidatorUsingTrie(val, SlotToEpoch(state.Slot())) {
 			total += val.EffectiveBalance()
 		}

beacon-chain/core/helpers/validators.go

@@ -27,7 +27,7 @@ func IsActiveValidator(validator *ethpb.Validator, epoch uint64) bool {
 }
 
 // IsActiveValidatorUsingTrie checks if a read only validator is active.
-func IsActiveValidatorUsingTrie(validator *stateTrie.ReadOnlyValidator, epoch uint64) bool {
+func IsActiveValidatorUsingTrie(validator stateTrie.ReadOnlyValidator, epoch uint64) bool {
 	return checkValidatorActiveStatus(validator.ActivationEpoch(), validator.ExitEpoch(), epoch)
 }
 
@@ -49,7 +49,7 @@ func IsSlashableValidator(activationEpoch, withdrawableEpoch uint64, slashed boo
 }
 
 // IsSlashableValidatorUsingTrie checks if a read only validator is slashable.
-func IsSlashableValidatorUsingTrie(val *stateTrie.ReadOnlyValidator, epoch uint64) bool {
+func IsSlashableValidatorUsingTrie(val stateTrie.ReadOnlyValidator, epoch uint64) bool {
 	return checkValidatorSlashable(val.ActivationEpoch(), val.WithdrawableEpoch(), val.Slashed(), epoch)
 }
 
@@ -85,7 +85,7 @@ func ActiveValidatorIndices(state *stateTrie.BeaconState, epoch uint64) ([]uint6
 		return activeIndices, nil
 	}
 	var indices []uint64
-	if err := state.ReadFromEveryValidator(func(idx int, val *stateTrie.ReadOnlyValidator) error {
+	if err := state.ReadFromEveryValidator(func(idx int, val stateTrie.ReadOnlyValidator) error {
 		if IsActiveValidatorUsingTrie(val, epoch) {
 			indices = append(indices, uint64(idx))
 		}
@@ -117,7 +117,7 @@ func ActiveValidatorCount(state *stateTrie.BeaconState, epoch uint64) (uint64, e
 	}
 
 	count := uint64(0)
-	if err := state.ReadFromEveryValidator(func(idx int, val *stateTrie.ReadOnlyValidator) error {
+	if err := state.ReadFromEveryValidator(func(idx int, val stateTrie.ReadOnlyValidator) error {
 		if IsActiveValidatorUsingTrie(val, epoch) {
 			count++
 		}
@@ -319,7 +319,7 @@ func IsEligibleForActivationQueue(validator *ethpb.Validator) bool {
 
 // IsEligibleForActivationQueueUsingTrie checks if the read-only validator is eligible to
 // be placed into the activation queue.
-func IsEligibleForActivationQueueUsingTrie(validator *stateTrie.ReadOnlyValidator) bool {
+func IsEligibleForActivationQueueUsingTrie(validator stateTrie.ReadOnlyValidator) bool {
 	return isEligibileForActivationQueue(validator.ActivationEligibilityEpoch(), validator.EffectiveBalance())
 }
 
@@ -348,7 +348,7 @@ func IsEligibleForActivation(state *stateTrie.BeaconState, validator *ethpb.Vali
 }
 
 // IsEligibleForActivationUsingTrie checks if the validator is eligible for activation.
-func IsEligibleForActivationUsingTrie(state *stateTrie.BeaconState, validator *stateTrie.ReadOnlyValidator) bool {
+func IsEligibleForActivationUsingTrie(state *stateTrie.BeaconState, validator stateTrie.ReadOnlyValidator) bool {
 	cpt := state.FinalizedCheckpoint()
 	if cpt == nil {
 		return false

beacon-chain/core/helpers/validators_test.go

@@ -275,6 +275,39 @@ func TestBeaconProposerIndex_OK(t *testing.T) {
 	}
 }
 
+func TestBeaconProposerIndex_BadState(t *testing.T) {
+	params.SetupTestConfigCleanup(t)
+	ClearCache()
+	c := params.BeaconConfig()
+	c.MinGenesisActiveValidatorCount = 16384
+	params.OverrideBeaconConfig(c)
+	validators := make([]*ethpb.Validator, params.BeaconConfig().MinGenesisActiveValidatorCount/8)
+	for i := 0; i < len(validators); i++ {
+		validators[i] = &ethpb.Validator{
+			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
+		}
+	}
+	roots := make([][]byte, params.BeaconConfig().SlotsPerHistoricalRoot)
+	for i := uint64(0); i < params.BeaconConfig().SlotsPerHistoricalRoot; i++ {
+		roots[i] = make([]byte, 32)
+	}
+
+	state, err := beaconstate.InitializeFromProto(&pb.BeaconState{
+		Validators:  validators,
+		Slot:        0,
+		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+		BlockRoots:  roots,
+		StateRoots:  roots,
+	})
+	require.NoError(t, err)
+	// Set a very high slot, so that retrieved block root will be
+	// non existent for the proposer cache.
+	require.NoError(t, state.SetSlot(100))
+	_, err = BeaconProposerIndex(state)
+	require.NoError(t, err)
+	assert.Equal(t, 0, len(proposerIndicesCache.ProposerIndicesCache.ListKeys()))
+}
+
 func TestComputeProposerIndex_Compatibility(t *testing.T) {
 	validators := make([]*ethpb.Validator, params.BeaconConfig().MinGenesisActiveValidatorCount)
 	for i := 0; i < len(validators); i++ {

beacon-chain/core/state/BUILD.bazel

@@ -32,6 +32,8 @@ go_library(
         "//beacon-chain/state/stateutil:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/bls:go_default_library",
+        "//shared/bytesutil:go_default_library",
+        "//shared/hashutil:go_default_library",
         "//shared/mathutil:go_default_library",
         "//shared/params:go_default_library",
         "//shared/traceutil:go_default_library",

beacon-chain/core/state/benchmarks_test.go

@@ -16,19 +16,6 @@ import (
 
 var runAmount = 25
 
-func TestExecuteStateTransition_FullBlock(t *testing.T) {
-	benchutil.SetBenchmarkConfig()
-	beaconState, err := benchutil.PreGenState1Epoch()
-	require.NoError(t, err)
-	block, err := benchutil.PreGenFullBlock()
-	require.NoError(t, err)
-
-	oldSlot := beaconState.Slot()
-	beaconState, err = state.ExecuteStateTransition(context.Background(), beaconState, block)
-	require.NoError(t, err, "Failed to process block, benchmarks will fail")
-	require.NotEqual(t, oldSlot, beaconState.Slot(), "Expected slots to be different")
-}
-
 func BenchmarkExecuteStateTransition_FullBlock(b *testing.B) {
 	benchutil.SetBenchmarkConfig()
 	beaconState, err := benchutil.PreGenState1Epoch()

beacon-chain/core/state/interop/BUILD.bazel

@@ -15,6 +15,7 @@ go_library(
     deps = [
         "//beacon-chain/state:go_default_library",
         "//shared/featureconfig:go_default_library",
+        "//shared/fileutil:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
     ],

beacon-chain/core/state/interop/write_block_to_disk.go

@@ -2,12 +2,12 @@ package interop
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+	"github.com/prysmaticlabs/prysm/shared/fileutil"
 )
 
 // WriteBlockToDisk as a block ssz. Writes to temp directory. Debug!
@@ -27,7 +27,7 @@ func WriteBlockToDisk(block *ethpb.SignedBeaconBlock, failed bool) {
 		log.WithError(err).Error("Failed to ssz encode block")
 		return
 	}
-	if err := ioutil.WriteFile(fp, enc, 0664); err != nil {
+	if err := fileutil.WriteFile(fp, enc); err != nil {
 		log.WithError(err).Error("Failed to write to disk")
 	}
 }

beacon-chain/core/state/interop/write_state_to_disk.go

@@ -2,12 +2,12 @@ package interop
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path"
 
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+	"github.com/prysmaticlabs/prysm/shared/fileutil"
 )
 
 // WriteStateToDisk as a state ssz. Writes to temp directory. Debug!
@@ -22,7 +22,7 @@ func WriteStateToDisk(state *stateTrie.BeaconState) {
 		log.WithError(err).Error("Failed to ssz encode state")
 		return
 	}
-	if err := ioutil.WriteFile(fp, enc, 0664); err != nil {
+	if err := fileutil.WriteFile(fp, enc); err != nil {
 		log.WithError(err).Error("Failed to write to disk")
 	}
 }

beacon-chain/core/state/skip_slot_cache.go

@@ -1,7 +1,12 @@
 package state
 
 import (
+	"context"
+
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
+	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/hashutil"
 )
 
 // SkipSlotCache exists for the unlikely scenario that is a large gap between the head state and
@@ -9,3 +14,14 @@ import (
 // difficult or impossible to compute the appropriate beacon state for assignments within a
 // reasonable amount of time.
 var SkipSlotCache = cache.NewSkipSlotCache()
+
+// The key for skip slot cache is mixed between state root and state slot.
+// state root is in the mix to defend against different forks with same skip slots
+// to hit the same cache. We don't want beacon states mixed up between different chains.
+func cacheKey(ctx context.Context, state *beaconstate.BeaconState) ([32]byte, error) {
+	r, err := state.HashTreeRoot(ctx)
+	if err != nil {
+		return [32]byte{}, err
+	}
+	return hashutil.Hash(append(bytesutil.Bytes32(state.Slot()), r[:]...)), nil
+}

beacon-chain/core/state/skip_slot_cache_test.go

@@ -1,7 +1,9 @@
 package state_test
 
 import (
+	"bytes"
 	"context"
+	"sync"
 	"testing"
 
 	"github.com/prysmaticlabs/go-ssz"
@@ -36,3 +38,123 @@ func TestSkipSlotCache_OK(t *testing.T) {
 		t.Fatal("Skipped slots cache leads to different states")
 	}
 }
+
+func TestSkipSlotCache_ConcurrentMixup(t *testing.T) {
+	bState, privs := testutil.DeterministicGenesisState(t, params.MinimalSpecConfig().MinGenesisActiveValidatorCount)
+	originalState, err := beaconstate.InitializeFromProto(bState.CloneInnerState())
+	require.NoError(t, err)
+
+	blkCfg := testutil.DefaultBlockGenConfig()
+	blkCfg.NumAttestations = 1
+
+	state.SkipSlotCache.Disable()
+
+	// First transition will be with an empty cache, so the cache becomes populated
+	// with the state
+	blk, err := testutil.GenerateFullBlock(bState, privs, blkCfg, originalState.Slot()+10)
+	require.NoError(t, err)
+	originalState, err = state.ExecuteStateTransition(context.Background(), originalState, blk)
+	require.NoError(t, err, "Could not run state transition")
+
+	// Create two shallow but different forks
+	var state1, state2 *beaconstate.BeaconState
+	{
+		blk, err := testutil.GenerateFullBlock(originalState.Copy(), privs, blkCfg, originalState.Slot()+10)
+		require.NoError(t, err)
+		copy(blk.Block.Body.Graffiti, "block 1")
+		signature, err := testutil.BlockSignature(originalState, blk.Block, privs)
+		require.NoError(t, err)
+		blk.Signature = signature.Marshal()
+		state1, err = state.ExecuteStateTransition(context.Background(), originalState.Copy(), blk)
+		require.NoError(t, err, "Could not run state transition")
+	}
+
+	{
+		blk, err := testutil.GenerateFullBlock(originalState.Copy(), privs, blkCfg, originalState.Slot()+10)
+		require.NoError(t, err)
+		copy(blk.Block.Body.Graffiti, "block 2")
+		signature, err := testutil.BlockSignature(originalState, blk.Block, privs)
+		require.NoError(t, err)
+		blk.Signature = signature.Marshal()
+		state2, err = state.ExecuteStateTransition(context.Background(), originalState.Copy(), blk)
+		require.NoError(t, err, "Could not run state transition")
+	}
+
+	r1, err := state1.HashTreeRoot(context.Background())
+	require.NoError(t, err)
+	r2, err := state2.HashTreeRoot(context.Background())
+	require.NoError(t, err)
+	if r1 == r2 {
+		t.Fatalf("need different starting states, got: %x", r1)
+	}
+
+	if state1.Slot() != state2.Slot() {
+		t.Fatalf("expecting different chains, but states at same slot")
+	}
+
+	// prepare copies for both states
+	var setups []*beaconstate.BeaconState
+	for i := uint64(0); i < 300; i++ {
+		var st *beaconstate.BeaconState
+		if i%2 == 0 {
+			st = state1
+		} else {
+			st = state2
+		}
+		setups = append(setups, st.Copy())
+	}
+
+	problemSlot := state1.Slot() + 2
+	expected1, err := state.ProcessSlots(context.Background(), state1.Copy(), problemSlot)
+	require.NoError(t, err)
+	expectedRoot1, err := expected1.HashTreeRoot(context.Background())
+	require.NoError(t, err)
+	t.Logf("chain 1 (even i) expected root %x at slot %d", expectedRoot1[:], problemSlot)
+
+	tmp1, err := state.ProcessSlots(context.Background(), expected1.Copy(), problemSlot+1)
+	require.NoError(t, err)
+	if gotRoot := tmp1.StateRoots()[problemSlot]; !bytes.Equal(gotRoot, expectedRoot1[:]) {
+		t.Fatalf("state roots for chain 1 are bad, expected root doesn't match: %x <> %x", gotRoot, expectedRoot1[:])
+	}
+
+	expected2, err := state.ProcessSlots(context.Background(), state2.Copy(), problemSlot)
+	require.NoError(t, err)
+	expectedRoot2, err := expected2.HashTreeRoot(context.Background())
+	require.NoError(t, err)
+	t.Logf("chain 2 (odd i) expected root %x at slot %d", expectedRoot2[:], problemSlot)
+
+	tmp2, err := state.ProcessSlots(context.Background(), expected2.Copy(), problemSlot+1)
+	require.NoError(t, err)
+	if gotRoot := tmp2.StateRoots()[problemSlot]; !bytes.Equal(gotRoot, expectedRoot2[:]) {
+		t.Fatalf("state roots for chain 2 are bad, expected root doesn't match %x <> %x", gotRoot, expectedRoot2[:])
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(len(setups))
+
+	step := func(i int, setup *beaconstate.BeaconState) {
+		// go at least 1 past problemSlot, to ensure problem slot state root is available
+		outState, err := state.ProcessSlots(context.Background(), setup, problemSlot+1+uint64(i)) // keep increasing, to hit and extend the cache
+		require.NoError(t, err, "Could not process state transition")
+		roots := outState.StateRoots()
+		gotRoot := roots[problemSlot]
+		if i%2 == 0 {
+			if !bytes.Equal(gotRoot, expectedRoot1[:]) {
+				t.Errorf("unexpected root on chain 1, item %3d: %x", i, gotRoot)
+			}
+		} else {
+			if !bytes.Equal(gotRoot, expectedRoot2[:]) {
+				t.Errorf("unexpected root on chain 2, item %3d: %x", i, gotRoot)
+			}
+		}
+		wg.Done()
+	}
+
+	state.SkipSlotCache.Enable()
+	// now concurrently apply the blocks (alternating between states, and increasing skip slots)
+	for i, setup := range setups {
+		go step(i, setup)
+	}
+	// Wait for all transitions to finish
+	wg.Wait()
+}

beacon-chain/core/state/stateutils/BUILD.bazel

@@ -19,8 +19,8 @@ go_test(
     name = "go_default_test",
     size = "small",
     srcs = ["validator_index_map_test.go"],
-    embed = [":go_default_library"],
     deps = [
+        ":go_default_library",
         "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/bytesutil:go_default_library",

beacon-chain/core/state/transition.go

@@ -282,7 +282,10 @@ func ProcessSlots(ctx context.Context, state *stateTrie.BeaconState, slot uint64
 	}
 
 	highestSlot := state.Slot()
-	key := state.Slot()
+	key, err := cacheKey(ctx, state)
+	if err != nil {
+		return nil, err
+	}
 
 	// Restart from cached value, if one exists.
 	cachedState, err := SkipSlotCache.Get(ctx, key)
@@ -412,7 +415,7 @@ func ProcessBlockNoVerifyAnySig(
 		traceutil.AnnotateError(span, err)
 		return nil, nil, errors.Wrap(err, "could not retrieve block signature set")
 	}
-	rSet, state, err := b.RandaoSignatureSet(state, signed.Block.Body)
+	rSet, err := b.RandaoSignatureSet(state, signed.Block.Body)
 	if err != nil {
 		traceutil.AnnotateError(span, err)
 		return nil, nil, errors.Wrap(err, "could not retrieve randao signature set")

beacon-chain/core/state/transition_test.go

@@ -201,7 +201,8 @@ func TestProcessBlock_IncorrectProposerSlashing(t *testing.T) {
 
 func TestProcessBlock_IncorrectProcessBlockAttestations(t *testing.T) {
 	beaconState, privKeys := testutil.DeterministicGenesisState(t, 100)
-
+	priv, err := bls.RandKey()
+	require.NoError(t, err)
 	att := &ethpb.Attestation{
 		Data: &ethpb.AttestationData{
 			Target:          &ethpb.Checkpoint{Epoch: 0, Root: make([]byte, 32)},
@@ -209,7 +210,7 @@ func TestProcessBlock_IncorrectProcessBlockAttestations(t *testing.T) {
 			BeaconBlockRoot: make([]byte, 32),
 		},
 		AggregationBits: bitfield.NewBitlist(3),
-		Signature:       bls.RandKey().Sign([]byte("foo")).Marshal(),
+		Signature:       priv.Sign([]byte("foo")).Marshal(),
 	}
 
 	block, err := testutil.GenerateFullBlock(beaconState, privKeys, nil, 1)
@@ -675,7 +676,8 @@ func BenchmarkProcessBlk_65536Validators_FullBlock(b *testing.B) {
 	// Set up randao reveal object for block
 	proposerIdx, err := helpers.BeaconProposerIndex(s)
 	require.NoError(b, err)
-	priv := bls.RandKey()
+	priv, err := bls.RandKey()
+	require.NoError(b, err)
 	v := s.Validators()
 	v[proposerIdx].PublicKey = priv.PublicKey().Marshal()
 	buf := make([]byte, 32)

beacon-chain/core/validators/validator.go

@@ -5,8 +5,6 @@
 package validators
 
 import (
-	"fmt"
-
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
@@ -39,24 +37,22 @@ import (
 //    validator.exit_epoch = exit_queue_epoch
 //    validator.withdrawable_epoch = Epoch(validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY)
 func InitiateValidatorExit(state *stateTrie.BeaconState, idx uint64) (*stateTrie.BeaconState, error) {
-	readOnlyVals := state.ValidatorsReadOnly()
-	if idx >= uint64(len(readOnlyVals)) {
-		return nil, fmt.Errorf("validator idx %d is higher then validator count %d", idx, len(readOnlyVals))
-	}
-
 	validator, err := state.ValidatorAtIndex(idx)
 	if err != nil {
 		return nil, err
 	}
-
 	if validator.ExitEpoch != params.BeaconConfig().FarFutureEpoch {
 		return state, nil
 	}
 	var exitEpochs []uint64
-	for _, val := range readOnlyVals {
+	err = state.ReadFromEveryValidator(func(idx int, val stateTrie.ReadOnlyValidator) error {
 		if val.ExitEpoch() != params.BeaconConfig().FarFutureEpoch {
 			exitEpochs = append(exitEpochs, val.ExitEpoch())
 		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
 	}
 	exitEpochs = append(exitEpochs, helpers.ActivationExitEpoch(helpers.CurrentEpoch(state)))
 
@@ -70,10 +66,14 @@ func InitiateValidatorExit(state *stateTrie.BeaconState, idx uint64) (*stateTrie
 
 	// We use the exit queue churn to determine if we have passed a churn limit.
 	exitQueueChurn := uint64(0)
-	for _, val := range readOnlyVals {
+	err = state.ReadFromEveryValidator(func(idx int, val stateTrie.ReadOnlyValidator) error {
 		if val.ExitEpoch() == exitQueueEpoch {
 			exitQueueChurn++
 		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
 	}
 	activeValidatorCount, err := helpers.ActiveValidatorCount(state, helpers.CurrentEpoch(state))
 	if err != nil {

beacon-chain/core/validators/validator_test.go

@@ -155,7 +155,6 @@ func TestActivatedValidatorIndices(t *testing.T) {
 	}{
 		{
 			state: &pb.BeaconState{
-				Slot: 0,
 				Validators: []*ethpb.Validator{
 					{
 						ActivationEpoch: 0,
@@ -178,7 +177,6 @@ func TestActivatedValidatorIndices(t *testing.T) {
 		},
 		{
 			state: &pb.BeaconState{
-				Slot: 0,
 				Validators: []*ethpb.Validator{
 					{
 						ActivationEpoch: helpers.ActivationExitEpoch(10),
@@ -189,7 +187,6 @@ func TestActivatedValidatorIndices(t *testing.T) {
 		},
 		{
 			state: &pb.BeaconState{
-				Slot: 0,
 				Validators: []*ethpb.Validator{
 					{
 						ActivationEpoch: 0,
@@ -215,7 +212,6 @@ func TestSlashedValidatorIndices(t *testing.T) {
 	}{
 		{
 			state: &pb.BeaconState{
-				Slot: 0,
 				Validators: []*ethpb.Validator{
 					{
 						WithdrawableEpoch: params.BeaconConfig().EpochsPerSlashingsVector,
@@ -235,7 +231,6 @@ func TestSlashedValidatorIndices(t *testing.T) {
 		},
 		{
 			state: &pb.BeaconState{
-				Slot: 0,
 				Validators: []*ethpb.Validator{
 					{
 						WithdrawableEpoch: params.BeaconConfig().EpochsPerSlashingsVector,
@@ -246,7 +241,6 @@ func TestSlashedValidatorIndices(t *testing.T) {
 		},
 		{
 			state: &pb.BeaconState{
-				Slot: 0,
 				Validators: []*ethpb.Validator{
 					{
 						WithdrawableEpoch: params.BeaconConfig().EpochsPerSlashingsVector,
@@ -272,7 +266,6 @@ func TestExitedValidatorIndices(t *testing.T) {
 	}{
 		{
 			state: &pb.BeaconState{
-				Slot: helpers.SlotToEpoch(1),
 				Validators: []*ethpb.Validator{
 					{
 						EffectiveBalance:  params.BeaconConfig().MaxEffectiveBalance,
@@ -295,7 +288,6 @@ func TestExitedValidatorIndices(t *testing.T) {
 		},
 		{
 			state: &pb.BeaconState{
-				Slot: helpers.SlotToEpoch(1),
 				Validators: []*ethpb.Validator{
 					{
 						EffectiveBalance:  params.BeaconConfig().MaxEffectiveBalance,
@@ -308,7 +300,6 @@ func TestExitedValidatorIndices(t *testing.T) {
 		},
 		{
 			state: &pb.BeaconState{
-				Slot: helpers.SlotToEpoch(1),
 				Validators: []*ethpb.Validator{
 					{
 						EffectiveBalance:  params.BeaconConfig().MaxEffectiveBalance,

beacon-chain/db/http_backup_handler.go

@@ -9,13 +9,13 @@ import (
 )
 
 // BackupHandler for accepting requests to initiate a new database backup.
-func BackupHandler(db Database) func(http.ResponseWriter, *http.Request) {
+func BackupHandler(db Database, outputDir string) func(http.ResponseWriter, *http.Request) {
 	log := logrus.WithField("prefix", "db")
 
 	return func(w http.ResponseWriter, _ *http.Request) {
 		log.Debug("Creating database backup from HTTP webhook.")
 
-		if err := db.Backup(context.Background()); err != nil {
+		if err := db.Backup(context.Background(), outputDir); err != nil {
 			log.WithError(err).Error("Failed to create backup")
 			w.WriteHeader(http.StatusInternalServerError)
 			return

beacon-chain/db/iface/interface.go

@@ -84,6 +84,8 @@ type NoHeadAccessDatabase interface {
 
 	// Run any required database migrations.
 	RunMigrations(ctx context.Context) error
+
+	CleanUpDirtyStates(ctx context.Context, slotsPerArchivedPoint uint64) error
 }
 
 // HeadAccessDatabase defines a struct with access to reading chain head data.
@@ -93,8 +95,6 @@ type HeadAccessDatabase interface {
 	// Block related methods.
 	HeadBlock(ctx context.Context) (*eth.SignedBeaconBlock, error)
 	SaveHeadBlockRoot(ctx context.Context, blockRoot [32]byte) error
-	// State related methods.
-	HeadState(ctx context.Context) (*state.BeaconState, error)
 }
 
 // Database interface with full access.
@@ -106,5 +106,5 @@ type Database interface {
 	ClearDB() error
 
 	// Backup and restore methods
-	Backup(ctx context.Context) error
+	Backup(ctx context.Context, outputDir string) error
 }

beacon-chain/db/kafka/passthrough.go

@@ -22,8 +22,8 @@ func (e Exporter) ClearDB() error {
 }
 
 // Backup -- passthrough.
-func (e Exporter) Backup(ctx context.Context) error {
-	return e.db.Backup(ctx)
+func (e Exporter) Backup(ctx context.Context, outputDir string) error {
+	return e.db.Backup(ctx, outputDir)
 }
 
 // Block -- passthrough.
@@ -61,11 +61,6 @@ func (e Exporter) StateSummary(ctx context.Context, blockRoot [32]byte) (*pb.Sta
 	return e.db.StateSummary(ctx, blockRoot)
 }
 
-// HeadState -- passthrough.
-func (e Exporter) HeadState(ctx context.Context) (*state.BeaconState, error) {
-	return e.db.HeadState(ctx)
-}
-
 // GenesisState -- passthrough.
 func (e Exporter) GenesisState(ctx context.Context) (*state.BeaconState, error) {
 	return e.db.GenesisState(ctx)
@@ -255,3 +250,8 @@ func (e Exporter) LastArchivedSlot(ctx context.Context) (uint64, error) {
 func (e Exporter) RunMigrations(ctx context.Context) error {
 	return e.db.RunMigrations(ctx)
 }
+
+// CleanUpDirtyStates -- passthrough
+func (e Exporter) CleanUpDirtyStates(ctx context.Context, slotsPerArchivedPoint uint64) error {
+	return e.db.RunMigrations(ctx)
+}

beacon-chain/db/kv/BUILD.bazel

@@ -34,6 +34,7 @@ go_library(
         "//proto/beacon/db:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/bytesutil:go_default_library",
+        "//shared/fileutil:go_default_library",
         "//shared/params:go_default_library",
         "//shared/sliceutil:go_default_library",
         "//shared/traceutil:go_default_library",

beacon-chain/db/kv/backup.go

@@ -3,10 +3,10 @@ package kv
 import (
 	"context"
 	"fmt"
-	"os"
 	"path"
 
 	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/shared/fileutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/sirupsen/logrus"
 	bolt "go.etcd.io/bbolt"
@@ -17,11 +17,20 @@ const backupsDirectoryName = "backups"
 
 // Backup the database to the datadir backup directory.
 // Example for backup at slot 345: $DATADIR/backups/prysm_beacondb_at_slot_0000345.backup
-func (s *Store) Backup(ctx context.Context) error {
+func (s *Store) Backup(ctx context.Context, outputDir string) error {
 	ctx, span := trace.StartSpan(ctx, "BeaconDB.Backup")
 	defer span.End()
 
-	backupsDir := path.Join(s.databasePath, backupsDirectoryName)
+	var backupsDir string
+	var err error
+	if outputDir != "" {
+		backupsDir, err = fileutil.ExpandPath(outputDir)
+		if err != nil {
+			return err
+		}
+	} else {
+		backupsDir = path.Join(s.databasePath, backupsDirectoryName)
+	}
 	head, err := s.HeadBlock(ctx)
 	if err != nil {
 		return err
@@ -30,13 +39,17 @@ func (s *Store) Backup(ctx context.Context) error {
 		return errors.New("no head block")
 	}
 	// Ensure the backups directory exists.
-	if err := os.MkdirAll(backupsDir, params.BeaconIoConfig().ReadWriteExecutePermissions); err != nil {
+	if err := fileutil.MkdirAll(backupsDir); err != nil {
 		return err
 	}
 	backupPath := path.Join(backupsDir, fmt.Sprintf("prysm_beacondb_at_slot_%07d.backup", head.Block.Slot))
 	logrus.WithField("prefix", "db").WithField("backup", backupPath).Info("Writing backup database.")
 
-	copyDB, err := bolt.Open(backupPath, params.BeaconIoConfig().ReadWritePermissions, &bolt.Options{Timeout: params.BeaconIoConfig().BoltTimeout})
+	copyDB, err := bolt.Open(
+		backupPath,
+		params.BeaconIoConfig().ReadWritePermissions,
+		&bolt.Options{Timeout: params.BeaconIoConfig().BoltTimeout},
+	)
 	if err != nil {
 		panic(err)
 	}

beacon-chain/db/kv/backup_test.go

@@ -24,7 +24,7 @@ func TestStore_Backup(t *testing.T) {
 	require.NoError(t, db.SaveState(ctx, st, root))
 	require.NoError(t, db.SaveHeadBlockRoot(ctx, root))
 
-	require.NoError(t, db.Backup(ctx))
+	require.NoError(t, db.Backup(ctx, ""))
 
 	files, err := ioutil.ReadDir(path.Join(db.databasePath, backupsDirectoryName))
 	require.NoError(t, err)

beacon-chain/db/kv/blocks_test.go

@@ -106,15 +106,13 @@ func TestStore_BlocksHandleZeroCase(t *testing.T) {
 	ctx := context.Background()
 	numBlocks := 10
 	totalBlocks := make([]*ethpb.SignedBeaconBlock, numBlocks)
-	blockRoots := make([][32]byte, 0)
 	for i := 0; i < len(totalBlocks); i++ {
 		b := testutil.NewBeaconBlock()
 		b.Block.Slot = uint64(i)
 		b.Block.ParentRoot = bytesutil.PadTo([]byte("parent"), 32)
 		totalBlocks[i] = b
-		r, err := totalBlocks[i].Block.HashTreeRoot()
+		_, err := totalBlocks[i].Block.HashTreeRoot()
 		require.NoError(t, err)
-		blockRoots = append(blockRoots, r)
 	}
 	require.NoError(t, db.SaveBlocks(ctx, totalBlocks))
 	zeroFilter := filters.NewFilter().SetStartSlot(0).SetEndSlot(0)
@@ -128,16 +126,14 @@ func TestStore_BlocksHandleInvalidEndSlot(t *testing.T) {
 	ctx := context.Background()
 	numBlocks := 10
 	totalBlocks := make([]*ethpb.SignedBeaconBlock, numBlocks)
-	blockRoots := make([][32]byte, 0)
 	// Save blocks from slot 1 onwards.
 	for i := 0; i < len(totalBlocks); i++ {
 		b := testutil.NewBeaconBlock()
 		b.Block.Slot = uint64(i) + 1
 		b.Block.ParentRoot = bytesutil.PadTo([]byte("parent"), 32)
 		totalBlocks[i] = b
-		r, err := totalBlocks[i].Block.HashTreeRoot()
+		_, err := totalBlocks[i].Block.HashTreeRoot()
 		require.NoError(t, err)
-		blockRoots = append(blockRoots, r)
 	}
 	require.NoError(t, db.SaveBlocks(ctx, totalBlocks))
 	badFilter := filters.NewFilter().SetStartSlot(5).SetEndSlot(1)

beacon-chain/db/kv/kv.go

@@ -13,6 +13,7 @@ import (
 	prombolt "github.com/prysmaticlabs/prombbolt"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db/iface"
+	"github.com/prysmaticlabs/prysm/shared/fileutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	bolt "go.etcd.io/bbolt"
 )
@@ -46,9 +47,15 @@ type Store struct {
 // path specified, creates the kv-buckets based on the schema, and stores
 // an open connection db object as a property of the Store struct.
 func NewKVStore(dirPath string, stateSummaryCache *cache.StateSummaryCache) (*Store, error) {
-	if err := os.MkdirAll(dirPath, params.BeaconIoConfig().ReadWriteExecutePermissions); err != nil {
+	hasDir, err := fileutil.HasDir(dirPath)
+	if err != nil {
 		return nil, err
 	}
+	if !hasDir {
+		if err := fileutil.MkdirAll(dirPath); err != nil {
+			return nil, err
+		}
+	}
 	datafile := path.Join(dirPath, databaseFileName)
 	boltDB, err := bolt.Open(datafile, params.BeaconIoConfig().ReadWritePermissions, &bolt.Options{Timeout: 1 * time.Second, InitialMmapSize: 10e6})
 	if err != nil {

beacon-chain/db/kv/kv_test.go

@@ -1,29 +1,18 @@
 package kv
 
 import (
-	"crypto/rand"
-	"fmt"
-	"math/big"
-	"os"
-	"path"
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	"github.com/prysmaticlabs/prysm/shared/testutil"
 	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 )
 
 // setupDB instantiates and returns a Store instance.
 func setupDB(t testing.TB) *Store {
-	randPath, err := rand.Int(rand.Reader, big.NewInt(1000000))
-	require.NoError(t, err, "Could not generate random file path")
-	p := path.Join(testutil.TempDir(), fmt.Sprintf("/%d", randPath))
-	require.NoError(t, os.RemoveAll(p), "Failed to remove directory")
-	db, err := NewKVStore(p, cache.NewStateSummaryCache())
+	db, err := NewKVStore(t.TempDir(), cache.NewStateSummaryCache())
 	require.NoError(t, err, "Failed to instantiate DB")
 	t.Cleanup(func() {
 		require.NoError(t, db.Close(), "Failed to close database")
-		require.NoError(t, os.RemoveAll(db.DatabasePath()), "Failed to remove directory")
 	})
 	return db
 }

beacon-chain/db/kv/state.go

@@ -6,9 +6,11 @@ import (
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	log "github.com/sirupsen/logrus"
 	bolt "go.etcd.io/bbolt"
 	"go.opencensus.io/trace"
 )
@@ -35,40 +37,6 @@ func (s *Store) State(ctx context.Context, blockRoot [32]byte) (*state.BeaconSta
 	return state.InitializeFromProtoUnsafe(st)
 }
 
-// HeadState returns the latest canonical state in beacon chain.
-func (s *Store) HeadState(ctx context.Context) (*state.BeaconState, error) {
-	ctx, span := trace.StartSpan(ctx, "BeaconDB.HeadState")
-	defer span.End()
-	var st *pb.BeaconState
-	err := s.db.View(func(tx *bolt.Tx) error {
-		// Retrieve head block's signing root from blocks bucket,
-		// to look up what the head state is.
-		bucket := tx.Bucket(blocksBucket)
-		headBlkRoot := bucket.Get(headBlockRootKey)
-
-		bucket = tx.Bucket(stateBucket)
-		enc := bucket.Get(headBlkRoot)
-		if enc == nil {
-			return nil
-		}
-
-		var err error
-		st, err = createState(ctx, enc)
-		return err
-	})
-	if err != nil {
-		return nil, err
-	}
-	if st == nil {
-		return nil, nil
-	}
-	span.AddAttributes(trace.BoolAttribute("exists", s != nil))
-	if st != nil {
-		span.AddAttributes(trace.Int64Attribute("slot", int64(st.Slot)))
-	}
-	return state.InitializeFromProtoUnsafe(st)
-}
-
 // GenesisState returns the genesis state in beacon chain.
 func (s *Store) GenesisState(ctx context.Context) (*state.BeaconState, error) {
 	ctx, span := trace.StartSpan(ctx, "BeaconDB.GenesisState")
@@ -153,23 +121,6 @@ func (s *Store) HasState(ctx context.Context, blockRoot [32]byte) bool {
 func (s *Store) DeleteState(ctx context.Context, blockRoot [32]byte) error {
 	ctx, span := trace.StartSpan(ctx, "BeaconDB.DeleteState")
 	defer span.End()
-	return s.DeleteStates(ctx, [][32]byte{blockRoot})
-}
-
-// DeleteStates by block roots.
-//
-// Note: bkt.Delete(key) uses a binary search to find the item in the database. Iterating with a
-// cursor is faster when there are a large set of keys to delete. This method is O(n) deletion where
-// n is the number of keys in the database. The alternative of calling  bkt.Delete on each key to
-// delete would be O(m*log(n)) which would be much slower given a large set of keys to delete.
-func (s *Store) DeleteStates(ctx context.Context, blockRoots [][32]byte) error {
-	ctx, span := trace.StartSpan(ctx, "BeaconDB.DeleteStates")
-	defer span.End()
-
-	rootMap := make(map[[32]byte]bool, len(blockRoots))
-	for _, blockRoot := range blockRoots {
-		rootMap[blockRoot] = true
-	}
 
 	return s.db.Update(func(tx *bolt.Tx) error {
 		bkt := tx.Bucket(blocksBucket)
@@ -187,34 +138,38 @@ func (s *Store) DeleteStates(ctx context.Context, blockRoots [][32]byte) error {
 		blockBkt := tx.Bucket(blocksBucket)
 		headBlkRoot := blockBkt.Get(headBlockRootKey)
 		bkt = tx.Bucket(stateBucket)
-		c := bkt.Cursor()
-
-		for blockRoot, _ := c.First(); blockRoot != nil; blockRoot, _ = c.Next() {
-			if !rootMap[bytesutil.ToBytes32(blockRoot)] {
-				continue
-			}
-			// Safe guard against deleting genesis, finalized, head state.
-			if bytes.Equal(blockRoot, checkpoint.Root) || bytes.Equal(blockRoot, genesisBlockRoot) || bytes.Equal(blockRoot, headBlkRoot) {
-				return errors.New("cannot delete genesis, finalized, or head state")
-			}
-
-			slot, err := slotByBlockRoot(ctx, tx, blockRoot)
-			if err != nil {
-				return err
-			}
-			indicesByBucket := createStateIndicesFromStateSlot(ctx, slot)
-			if err := deleteValueForIndices(ctx, indicesByBucket, blockRoot, tx); err != nil {
-				return errors.Wrap(err, "could not delete root for DB indices")
-			}
-
-			if err := c.Delete(); err != nil {
-				return err
-			}
+		// Safe guard against deleting genesis, finalized, head state.
+		if bytes.Equal(blockRoot[:], checkpoint.Root) || bytes.Equal(blockRoot[:], genesisBlockRoot) || bytes.Equal(blockRoot[:], headBlkRoot) {
+			return errors.New("cannot delete genesis, finalized, or head state")
 		}
-		return nil
+
+		slot, err := slotByBlockRoot(ctx, tx, blockRoot[:])
+		if err != nil {
+			return err
+		}
+		indicesByBucket := createStateIndicesFromStateSlot(ctx, slot)
+		if err := deleteValueForIndices(ctx, indicesByBucket, blockRoot[:], tx); err != nil {
+			return errors.Wrap(err, "could not delete root for DB indices")
+		}
+
+		return bkt.Delete(blockRoot[:])
 	})
 }
 
+// DeleteStates by block roots.
+func (s *Store) DeleteStates(ctx context.Context, blockRoots [][32]byte) error {
+	ctx, span := trace.StartSpan(ctx, "BeaconDB.DeleteStates")
+	defer span.End()
+
+	for _, r := range blockRoots {
+		if err := s.DeleteState(ctx, r); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // creates state from marshaled proto state bytes.
 func createState(ctx context.Context, enc []byte) (*pb.BeaconState, error) {
 	protoState := &pb.BeaconState{}
@@ -352,3 +307,61 @@ func createStateIndicesFromStateSlot(ctx context.Context, slot uint64) map[strin
 	}
 	return indicesByBucket
 }
+
+// CleanUpDirtyStates removes states in DB that falls to under archived point interval rules.
+// Only following states would be kept:
+// 1.) state_slot % archived_interval == 0. (e.g. archived_interval=2048, states with slot 2048, 4096... etc)
+// 2.) archived_interval - archived_interval/3 < state_slot % archived_interval
+//   (e.g. archived_interval=2048, states with slots after 1365).
+//   This is to tolerate skip slots. Not every state lays on the boundary.
+// 3.) state with current finalized root
+// 4.) unfinalized States
+func (s *Store) CleanUpDirtyStates(ctx context.Context, slotsPerArchivedPoint uint64) error {
+	ctx, span := trace.StartSpan(ctx, "BeaconDB. CleanUpDirtyStates")
+	defer span.End()
+
+	f, err := s.FinalizedCheckpoint(ctx)
+	if err != nil {
+		return err
+	}
+	finalizedSlot, err := helpers.StartSlot(f.Epoch)
+	if err != nil {
+		return err
+	}
+	deletedRoots := make([][32]byte, 0)
+
+	err = s.db.View(func(tx *bolt.Tx) error {
+		bkt := tx.Bucket(stateSlotIndicesBucket)
+		return bkt.ForEach(func(k, v []byte) error {
+			if ctx.Err() != nil {
+				return ctx.Err()
+			}
+
+			finalizedChkpt := bytesutil.ToBytes32(f.Root) == bytesutil.ToBytes32(v)
+			slot := bytesutil.BytesToUint64BigEndian(k)
+			mod := slot % slotsPerArchivedPoint
+			nonFinalized := slot > finalizedSlot
+
+			// The following conditions cover 1, 2, 3 and 4 above.
+			if mod != 0 && mod <= slotsPerArchivedPoint-slotsPerArchivedPoint/3 && !finalizedChkpt && !nonFinalized {
+				deletedRoots = append(deletedRoots, bytesutil.ToBytes32(v))
+			}
+			return nil
+		})
+	})
+	if err != nil {
+		return err
+	}
+
+	// Length of to be deleted roots is 0. Nothing to do.
+	if len(deletedRoots) == 0 {
+		return nil
+	}
+
+	log.WithField("count", len(deletedRoots)).Info("Cleaning up dirty states")
+	if err := s.DeleteStates(ctx, deletedRoots); err != nil {
+		return err
+	}
+
+	return err
+}

beacon-chain/db/kv/state_test.go

@@ -9,6 +9,7 @@ import (
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 	"github.com/prysmaticlabs/prysm/shared/testutil/assert"
 	"github.com/prysmaticlabs/prysm/shared/testutil/require"
@@ -41,21 +42,6 @@ func TestState_CanSaveRetrieve(t *testing.T) {
 	assert.Equal(t, (*state.BeaconState)(nil), savedS, "Unsaved state should've been nil")
 }
 
-func TestHeadState_CanSaveRetrieve(t *testing.T) {
-	db := setupDB(t)
-
-	headRoot := [32]byte{'A'}
-
-	st := testutil.NewBeaconState()
-	require.NoError(t, st.SetSlot(100))
-	require.NoError(t, db.SaveState(context.Background(), st, headRoot))
-	require.NoError(t, db.SaveHeadBlockRoot(context.Background(), headRoot))
-
-	savedHeadS, err := db.HeadState(context.Background())
-	require.NoError(t, err)
-	assert.DeepEqual(t, st.InnerStateUnsafe(), savedHeadS.InnerStateUnsafe(), "Did not retrieve saved state")
-}
-
 func TestGenesisState_CanSaveRetrieve(t *testing.T) {
 	db := setupDB(t)
 
@@ -70,10 +56,6 @@ func TestGenesisState_CanSaveRetrieve(t *testing.T) {
 	require.NoError(t, err)
 	assert.DeepEqual(t, st.InnerStateUnsafe(), savedGenesisS.InnerStateUnsafe(), "Did not retrieve saved state")
 	require.NoError(t, db.SaveGenesisBlockRoot(context.Background(), [32]byte{'C'}))
-
-	savedGenesisS, err = db.HeadState(context.Background())
-	require.NoError(t, err)
-	assert.Equal(t, (*state.BeaconState)(nil), savedGenesisS, "Unsaved genesis state should've been nil")
 }
 
 func TestStore_StatesBatchDelete(t *testing.T) {
@@ -245,3 +227,96 @@ func TestStore_GenesisState_CanGetHighestBelow(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, true, proto.Equal(highest[0].InnerStateUnsafe(), genesisState.InnerStateUnsafe()))
 }
+
+func TestStore_CleanUpDirtyStates_AboveThreshold(t *testing.T) {
+	db := setupDB(t)
+
+	genesisState := testutil.NewBeaconState()
+	genesisRoot := [32]byte{'a'}
+	require.NoError(t, db.SaveGenesisBlockRoot(context.Background(), genesisRoot))
+	require.NoError(t, db.SaveState(context.Background(), genesisState, genesisRoot))
+
+	bRoots := make([][32]byte, 0)
+	slotsPerArchivedPoint := uint64(128)
+	prevRoot := genesisRoot
+	for i := uint64(1); i <= slotsPerArchivedPoint; i++ {
+		b := testutil.NewBeaconBlock()
+		b.Block.Slot = i
+		b.Block.ParentRoot = prevRoot[:]
+		r, err := b.Block.HashTreeRoot()
+		require.NoError(t, err)
+		require.NoError(t, db.SaveBlock(context.Background(), b))
+		bRoots = append(bRoots, r)
+		prevRoot = r
+
+		st := testutil.NewBeaconState()
+		require.NoError(t, st.SetSlot(i))
+		require.NoError(t, db.SaveState(context.Background(), st, r))
+	}
+
+	require.NoError(t, db.SaveFinalizedCheckpoint(context.Background(), &ethpb.Checkpoint{Root: bRoots[len(bRoots)-1][:], Epoch: slotsPerArchivedPoint / params.BeaconConfig().SlotsPerEpoch}))
+	require.NoError(t, db.CleanUpDirtyStates(context.Background(), slotsPerArchivedPoint))
+
+	for i, root := range bRoots {
+		if uint64(i) >= slotsPerArchivedPoint-slotsPerArchivedPoint/3 {
+			require.Equal(t, true, db.HasState(context.Background(), root))
+		} else {
+			require.Equal(t, false, db.HasState(context.Background(), root))
+		}
+	}
+}
+
+func TestStore_CleanUpDirtyStates_Finalized(t *testing.T) {
+	db := setupDB(t)
+
+	genesisState := testutil.NewBeaconState()
+	genesisRoot := [32]byte{'a'}
+	require.NoError(t, db.SaveGenesisBlockRoot(context.Background(), genesisRoot))
+	require.NoError(t, db.SaveState(context.Background(), genesisState, genesisRoot))
+
+	for i := uint64(1); i <= params.BeaconConfig().SlotsPerEpoch; i++ {
+		b := testutil.NewBeaconBlock()
+		b.Block.Slot = i
+		r, err := b.Block.HashTreeRoot()
+		require.NoError(t, err)
+		require.NoError(t, db.SaveBlock(context.Background(), b))
+
+		st := testutil.NewBeaconState()
+		require.NoError(t, st.SetSlot(i))
+		require.NoError(t, db.SaveState(context.Background(), st, r))
+	}
+
+	require.NoError(t, db.SaveFinalizedCheckpoint(context.Background(), &ethpb.Checkpoint{Root: genesisRoot[:]}))
+	require.NoError(t, db.CleanUpDirtyStates(context.Background(), params.BeaconConfig().SlotsPerEpoch))
+	require.Equal(t, true, db.HasState(context.Background(), genesisRoot))
+}
+
+func TestStore_CleanUpDirtyStates_DontDeleteNonFinalized(t *testing.T) {
+	db := setupDB(t)
+
+	genesisState := testutil.NewBeaconState()
+	genesisRoot := [32]byte{'a'}
+	require.NoError(t, db.SaveGenesisBlockRoot(context.Background(), genesisRoot))
+	require.NoError(t, db.SaveState(context.Background(), genesisState, genesisRoot))
+
+	unfinalizedRoots := [][32]byte{}
+	for i := uint64(1); i <= params.BeaconConfig().SlotsPerEpoch; i++ {
+		b := testutil.NewBeaconBlock()
+		b.Block.Slot = i
+		r, err := b.Block.HashTreeRoot()
+		require.NoError(t, err)
+		require.NoError(t, db.SaveBlock(context.Background(), b))
+		unfinalizedRoots = append(unfinalizedRoots, r)
+
+		st := testutil.NewBeaconState()
+		require.NoError(t, st.SetSlot(i))
+		require.NoError(t, db.SaveState(context.Background(), st, r))
+	}
+
+	require.NoError(t, db.SaveFinalizedCheckpoint(context.Background(), &ethpb.Checkpoint{Root: genesisRoot[:]}))
+	require.NoError(t, db.CleanUpDirtyStates(context.Background(), params.BeaconConfig().SlotsPerEpoch))
+
+	for _, rt := range unfinalizedRoots {
+		require.Equal(t, true, db.HasState(context.Background(), rt))
+	}
+}

beacon-chain/db/testing/BUILD.bazel

@@ -10,7 +10,5 @@ go_library(
         "//beacon-chain/cache:go_default_library",
         "//beacon-chain/db:go_default_library",
         "//beacon-chain/db/kv:go_default_library",
-        "//shared/rand:go_default_library",
-        "//shared/testutil:go_default_library",
     ],
 )

beacon-chain/db/testing/setup_db.go

@@ -3,27 +3,17 @@
 package testing
 
 import (
-	"fmt"
-	"os"
-	"path"
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db/kv"
-	"github.com/prysmaticlabs/prysm/shared/rand"
-	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
 
 // SetupDB instantiates and returns database backed by key value store.
 func SetupDB(t testing.TB) (db.Database, *cache.StateSummaryCache) {
-	randPath := rand.NewDeterministicGenerator().Int()
-	p := path.Join(testutil.TempDir(), fmt.Sprintf("/%d", randPath))
-	if err := os.RemoveAll(p); err != nil {
-		t.Fatalf("failed to remove directory: %v", err)
-	}
 	sc := cache.NewStateSummaryCache()
-	s, err := kv.NewKVStore(p, sc)
+	s, err := kv.NewKVStore(t.TempDir(), sc)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -31,9 +21,6 @@ func SetupDB(t testing.TB) (db.Database, *cache.StateSummaryCache) {
 		if err := s.Close(); err != nil {
 			t.Fatalf("failed to close database: %v", err)
 		}
-		if err := os.RemoveAll(s.DatabasePath()); err != nil {
-			t.Fatalf("could not remove tmp db dir: %v", err)
-		}
 	})
 	return s, sc
 }