Refactor params/config to separate files for clarity (#6427)

* split mainnet/onyx/altona/e2e configs into different files
* set onyx as the default network
* lint and fix comment
* more commentary
* gofmt
* Merge branch 'master' of github.com:prysmaticlabs/prysm into onyx-config
* gofmt

.bazelrc

@@ -5,6 +5,9 @@ test --test_verbose_timeout_warnings
 test --build_tests_only
 test --test_output=errors
 
+# Clearly indicate that coverage is enabled to disable certain nogo checks.
+coverage --define=coverage_enabled=1
+
 # Fix for rules_docker. See: https://github.com/bazelbuild/rules_docker/issues/842
 build --host_force_python=PY2
 test --host_force_python=PY2
@@ -30,20 +33,60 @@ build --define kafka_enabled=false
 test --define kafka_enabled=false
 run --define kafka_enabled=false
 
+build:kafka_enabled --define kafka_enabled=true
+build:kafka_enabled --define gotags=kafka_enabled
+
 # Release flags
 build:release --workspace_status_command=./scripts/workspace_status.sh
 build:release --stamp
 build:release --compilation_mode=opt
+build:release --config=llvm
 
 # LLVM compiler for building C/C++ dependencies.
 build:llvm --crosstool_top=@llvm_toolchain//:toolchain
 build:llvm --define compiler=llvm
+build:llvm --copt -fno-sanitize=vptr,function
+build:llvm --linkopt -fno-sanitize=vptr,function
+
+build:asan --copt -fsanitize=address,undefined
+build:asan --copt -fno-omit-frame-pointer
+build:asan --linkopt -fsanitize=address,undefined
+build:asan --copt -fno-sanitize=vptr,function
+build:asan --linkopt -fno-sanitize=vptr,function
+build:asan --copt -DADDRESS_SANITIZER=1
+build:asan --copt -D__SANITIZE_ADDRESS__
+build:asan --linkopt -ldl
+
+build:llvm-asan --config=llvm
+build:llvm-asan --config=asan
+build:llvm-asan --linkopt -fuse-ld=ld.lld
+
+build:fuzz --define=gotags=libfuzzer
+build:fuzz --config=llvm-asan
+build:fuzz --copt=-fsanitize=fuzzer-no-link
+build:fuzz --linkopt=-fsanitize=fuzzer
+build:fuzz --copt=-fno-omit-frame-pointer
+build:fuzz --define=FUZZING_ENGINE=libfuzzer
+build:fuzz --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+build:fuzz --linkopt -Wl,--no-as-needed
+build:fuzz --define=gc_goopts=-d=libfuzzer
+build:fuzz --run_under=//tools:fuzz_wrapper
+build:fuzz --compilation_mode=opt
+
+test:fuzz --local_test_jobs="HOST_CPUS*.5"
+
+test:fuzzit --config=fuzz
+test:fuzzit --test_env=FUZZIT_API_KEY
+test:fuzzit --test_env=PRYSM_BUILD_IMAGE=gcr.io/prysmaticlabs/prysm-fuzzit:v0.11.0
+test:fuzzit --test_timeout=1200
+test:fuzzit --run_under=//tools:fuzzit_wrapper
 
 # Build binary with cgo symbolizer for debugging / profiling.
 build:cgo_symbolizer --config=llvm
 build:cgo_symbolizer --copt=-g
 build:cgo_symbolizer --define=USE_CGO_SYMBOLIZER=true
 build:cgo_symbolizer -c dbg
+build:cgo_symbolizer --define=gotags=cgosymbolizer_enabled
 
 # multi-arch cross-compiling toolchain configs:
 -----------------------------------------------

.bazelversion

@@ -1 +1 @@
-3.0.0
+3.2.0

.buildkite-bazelrc

@@ -45,7 +45,15 @@ build --flaky_test_attempts=5
 # build --features=race
 
 # Enable kafka for CI tests only.
-test --define kafka_enabled=true
+test --config=kafka_enabled
 
 build --bes_backend=grpcs://builds.prylabs.net:1985
 build --bes_results_url=https://builds.prylabs.net/invocation/
+
+# Disable flaky test detection for fuzzing.
+test:fuzz --flaky_test_attempts=1
+
+# Expose test environment variables in CI
+test:fuzzit --test_env=GITHUB_REF
+test:fuzzit --test_env=GITHUB_SHA
+test:fuzzit --test_env=DOCKER_HOST

.github/CODEOWNERS

@@ -3,3 +3,6 @@
 
 # Starlark code owners
 *.bzl @prestonvanloon
+
+# Anyone on prylabs team can approve dependency updates.
+deps.bzl @prysmaticlabs/core-team

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,59 @@
+---
+name: "\U0001F41EBug report"
+about: Report a bug or problem with running Prysm
+---
+<!--💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎
+
+Hellooo! 😄 
+
+To help us tend to your issue faster, please search our currently open issues before submitting a new one.
+Existing issues often contain information about workarounds, resolution, or progress updates.
+
+💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎-->
+
+# 🐞 Bug Report
+
+### Description
+
+<!-- ✍️--> A clear and concise description of the problem...
+
+### Has this worked before in a previous version?
+
+<!-- Did this behavior use to work in the previous version? -->
+<!-- ✍️--> Yes, the previous version in which this bug was not present was: ....
+
+## 🔬 Minimal Reproduction
+
+<!--
+Please let us know how we can reproduce this issue. Include the exact method you used to run Prysm along with any flags used in your beacon chain and/or validator. Make sure you don't upload any confidential files or private keys.
+-->
+
+## 🔥 Error
+
+<pre><code>
+<!-- If the issue is accompanied by an error, please share the error logs with us below. If you have a lot of logs, place make a paste bin with your logs and share the link with us here: -->
+<!-- ✍️-->
+
+</code></pre>
+
+
+## 🌍  Your Environment
+
+**Operating System:**
+
+<pre>
+  <code>
+
+  </code>
+</pre>
+
+**What version of Prysm are you running? (Which release)**
+
+<pre>
+  <code>
+
+  </code>
+</pre>
+
+**Anything else relevant (validator index / public key)?**
+

.github/ISSUE_TEMPLATE/discussion.md

@@ -0,0 +1,21 @@
+---
+name: "\U0001F48EGeneral issue / discussion"
+about: Any other type of general issue or discussion
+
+---
+<!--💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎
+
+Hellooo! 😄 
+
+To help us tend to your issue faster, please search our currently open issues before submitting a new one.
+Existing issues often contain information about workarounds, resolution, or progress updates.
+
+💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎-->
+
+# 💎 Issue
+
+### Background
+
+<!-- ✍️--> Context and background information on the discussion...
+
+### Description

.github/ISSUE_TEMPLATE/feature_flag.md

@@ -0,0 +1,38 @@
+---
+name: "\U0001F984Feature Flag Tracking"
+about: Track a new feature, in development, in Prysm. This issue template should only be used by developers or contributors!
+labels: Tracking
+---
+<!--💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎
+
+Hellooo! 😄 
+
+Thanks for taking the time to file a tracking issue for your new feature. These issues really help
+us track progress of features as they work their way through development. Be sure to review 
+shared/featureconfig/README.md for the latest documentation around feature flags.
+
+💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎-->
+
+# 🦄 Feature Tracking
+
+**Current status** (opt-in or opt-out): 
+
+**Opt-in feature flag name**:
+
+**Opt-out feature flag name**: 
+
+### Feature Description
+
+<!-- ✍️--> A clear and concise description of the new feature. Provide links to your technical
+design document or other supporting information.
+
+### Tasks
+
+- [ ] Feature flag created as opt-in
+- [ ] Feature implemented / code complete
+- [ ] Feature tested in production for adequate amount of time
+- [ ] Feature flag is inverted to be opt-out and the opt-in flag is deprecated
+- [ ] Feature has made it to a tagged production release as an opt-out flag
+- [ ] Opt-out feature flag is deprecated and old code paths are cleaned up
+
+This issue should be closed after all of the above tasks are complete.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,27 @@
+---
+name: "\U0001F680Feature request"
+about: Suggest a feature for Prysm
+
+---
+<!--💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎
+
+Hellooo! 😄 
+
+To help us tend to your issue faster, please search our currently open issues before submitting a new one.
+Existing issues often contain information about workarounds, resolution, or progress updates.
+
+💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎-->
+
+# 🚀 Feature Request
+
+### Description
+
+<!-- ✍️--> A clear and concise description of the problem or missing capability...
+
+### Describe the solution you'd like
+
+<!-- ✍️--> If you have a solution in mind, please describe it.
+
+### Describe alternatives you've considered
+
+<!-- ✍️--> Have you considered any alternative solutions or workarounds?

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,30 @@
+<!-- Thanks for sending a PR! Before submitting:
+
+1. If this is your first PR, please read CONTRIBUTING.md and sign the CLA
+   first. We cannot review code without a signed CLA.
+2. Please file an issue *first*. All features and most bug fixes should have
+   an associated issue with a design discussed and decided upon. Small bug
+   fixes and documentation improvements don't need issues.
+3. New features and bug fixes must have tests. Documentation may need to
+   be updated. If you're unsure what to update, send the PR, and we'll discuss
+   in review.
+4. Note that PRs updating dependencies and new Go versions are not accepted.
+   Please file an issue instead.
+-->
+
+**What type of PR is this?**
+
+> Uncomment one line below and remove others.
+>
+> Bug fix
+> Feature
+> Documentation
+> Other
+
+**What does this PR do? Why is it needed?**
+
+**Which issues(s) does this PR fix?**
+
+Fixes #
+
+**Other notes for review**

.github/actions/gofmt/Dockerfile

@@ -0,0 +1,5 @@
+FROM cytopia/gofmt
+
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

.github/actions/gofmt/action.yml

@@ -0,0 +1,12 @@
+name: 'Gofmt checker'
+description: 'Checks that all project files have been properly formatted.'
+inputs:
+  path:
+    description: 'Path to check'
+    required: true
+    default: './'
+runs:
+  using: 'docker'
+  image: 'Dockerfile'
+  args:
+    - ${{ inputs.path }}

.github/actions/gofmt/entrypoint.sh

@@ -0,0 +1,15 @@
+#!/bin/sh -l
+set -e
+
+cd $GITHUB_WORKSPACE
+
+# Check if any files are not formatted.
+nonformatted="$(gofmt -l $1 2>&1)"
+
+# Return if `go fmt` passes.
+[ -z "$nonformatted" ] && exit 0
+
+# Notify of issues with formatting.
+echo "Following files need to be properly formatted:"
+echo "$nonformatted"
+exit 1

.github/actions/gomodtidy/Dockerfile

@@ -0,0 +1,5 @@
+FROM golang:alpine
+
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

.github/actions/gomodtidy/action.yml

@@ -0,0 +1,5 @@
+name: 'Go mod tidy checker'
+description: 'Checks that `go mod tidy` has been applied.'
+runs:
+  using: 'docker'
+  image: 'Dockerfile'

.github/actions/gomodtidy/entrypoint.sh

@@ -0,0 +1,34 @@
+#!/bin/sh -l
+set -e
+export PATH=$PATH:/usr/local/go/bin
+
+cd $GITHUB_WORKSPACE
+
+cp go.mod go.mod.orig
+cp go.sum go.sum.orig
+
+go mod tidy
+
+echo "Checking go.mod and go.sum:"
+checks=0
+if [ "$(diff -s go.mod.orig go.mod | grep -c 'Files go.mod.orig and go.mod are identical')" = 1 ]; then
+  echo "- go.mod is up to date."
+  checks=$((checks + 1))
+else
+  echo "- go.mod is NOT up to date."
+fi
+
+if [ "$(diff -s go.sum.orig go.sum | grep -c 'Files go.sum.orig and go.sum are identical')" = 1 ]; then
+  echo "- go.sum is up to date."
+  checks=$((checks + 1))
+else
+  echo "- go.sum is NOT up to date."
+fi
+
+if [ $checks -eq 2 ]; then
+  exit 0
+fi
+
+# Notify of any issues.
+echo "Run 'go mod tidy' to update."
+exit 1

.github/workflows/dappnode-release-trigger.yml

@@ -0,0 +1,41 @@
+name: Update DAppNodePackages
+
+on:
+  push:
+    tags:
+    - '*'
+
+jobs:
+  dappnode-update-beacon-chain:
+    name: Trigger a beacon-chain release
+    runs-on: ubuntu-latest
+    steps:
+    - name: Get latest tag
+      id: get_tag
+      run: echo ::set-output name=TAG::${GITHUB_REF/refs\/tags\//}
+    - name: Send dispatch event to DAppNodePackage-prysm-beacon-chain
+      env: 
+        DISPATCH_REPO: dappnode/DAppNodePackage-prysm-beacon-chain
+      run: |
+        curl -v -X POST -u "${{ secrets.PAT_GITHUB }}" \
+        -H "Accept: application/vnd.github.everest-preview+json" \
+        -H "Content-Type: application/json" \
+        --data '{"event_type":"new_release", "client_payload": { "tag":"${{ steps.get_tag.outputs.TAG }}"}}' \
+        https://api.github.com/repos/$DISPATCH_REPO/dispatches
+        
+  dappnode-update-validator:
+    name: Trigger a validator release
+    runs-on: ubuntu-latest
+    steps:
+    - name: Get latest tag
+      id: get_tag
+      run: echo ::set-output name=TAG::${GITHUB_REF/refs\/tags\//}
+    - name: Send dispatch event to DAppNodePackage validator repository
+      env: 
+        DISPATCH_REPO: dappnode/DAppNodePackage-prysm-validator
+      run: |
+        curl -v -X POST -u "${{ secrets.PAT_GITHUB }}" \
+        -H "Accept: application/vnd.github.everest-preview+json" \
+        -H "Content-Type: application/json" \
+        --data '{"event_type":"new_release", "client_payload": { "tag":"${{ steps.get_tag.outputs.TAG }}"}}' \
+        https://api.github.com/repos/$DISPATCH_REPO/dispatches

.github/workflows/go.yml

@@ -0,0 +1,50 @@
+name: Go
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ '*' ]
+
+jobs:
+
+  check:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+
+      - name: Go mod tidy checker
+        id: gomodtidy
+        uses: ./.github/actions/gomodtidy
+
+      - name: Gofmt checker
+        id: gofmt
+        uses: ./.github/actions/gofmt
+        with:
+          path: ./
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.14
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Get dependencies
+        run: |
+          go get -v -t -d ./...
+
+      - name: Build
+        run: go build -v ./...
+
+# Tests run via Bazel for now...
+#      - name: Test
+#        run: go test -v ./...

.gitignore

@@ -27,9 +27,5 @@ yarn-error.log
 # Ignore password file
 password.txt
 
-# go dependancy
-/go.mod
-/go.sum
-
 # Dist files
 dist

BUILD.bazel

@@ -4,6 +4,7 @@ load("@com_github_atlassian_bazel_tools//goimports:def.bzl", "goimports")
 load("@io_kubernetes_build//defs:run_in_workspace.bzl", "workspace_binary")
 load("@io_bazel_rules_go//go:def.bzl", "nogo")
 load("@graknlabs_bazel_distribution//common:rules.bzl", "assemble_targz", "assemble_versioned")
+load("@bazel_skylib//rules:common_settings.bzl", "string_setting")
 
 prefix = "github.com/prysmaticlabs/prysm"
 
@@ -12,6 +13,8 @@ exports_files([
 ])
 
 # gazelle:prefix github.com/prysmaticlabs/prysm
+# gazelle:map_kind go_library go_library @prysm//tools/go:def.bzl
+# gazelle:map_kind go_repository go_repository @prysm//tools/go:def.bzl
 gazelle(
     name = "gazelle",
     prefix = prefix,
@@ -43,7 +46,7 @@ alias(
 # Protobuf gRPC gateway compiler
 alias(
     name = "grpc_gateway_proto_compiler",
-    actual = "@grpc_ecosystem_grpc_gateway//protoc-gen-grpc-gateway:go_gen_grpc_gateway",
+    actual = "@com_github_grpc_ecosystem_grpc_gateway//protoc-gen-grpc-gateway:go_gen_grpc_gateway",
     visibility = ["//visibility:public"],
 )
 
@@ -85,15 +88,12 @@ nogo(
         "@org_golang_x_tools//go/analysis/passes/pkgfact:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/nilness:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/nilfunc:go_tool_library",
-        # lost cancel ignore doesn't seem to work when running with coverage
-        #"@org_golang_x_tools//go/analysis/passes/lostcancel:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/loopclosure:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/httpresponse:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/findcall:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/deepequalerrors:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/ctrlflow:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/copylock:go_tool_library",
-        "@org_golang_x_tools//go/analysis/passes/composite:go_tool_library",
         #  "@org_golang_x_tools//go/analysis/passes/cgocall:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/buildtag:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/buildssa:go_tool_library",
@@ -105,8 +105,22 @@ nogo(
         "@org_golang_x_tools//go/analysis/passes/asmdecl:go_tool_library",
         "//tools/analyzers/maligned:go_tool_library",
         "//tools/analyzers/roughtime:go_tool_library",
+        "//tools/analyzers/cryptorand:go_tool_library",
         "//tools/analyzers/errcheck:go_tool_library",
-    ],
+        "//tools/analyzers/featureconfig:go_tool_library",
+    ] + select({
+        # nogo checks that fail with coverage enabled.
+        ":coverage_enabled": [],
+        "//conditions:default": [
+            "@org_golang_x_tools//go/analysis/passes/lostcancel:go_tool_library",
+            "@org_golang_x_tools//go/analysis/passes/composite:go_tool_library",
+        ],
+    }),
+)
+
+config_setting(
+    name = "coverage_enabled",
+    values = {"define": "coverage_enabled=1"},
 )
 
 common_files = {
@@ -119,3 +133,12 @@ toolchain(
     toolchain = "@rules_foreign_cc//tools/build_defs/native_tools:built_cmake",
     toolchain_type = "@rules_foreign_cc//tools/build_defs:cmake_toolchain",
 )
+
+string_setting(
+    name = "gotags",
+    build_setting_default = "",
+    values = [
+        "",
+        "libfuzzer",
+    ],
+)

CONTRIBUTING.md

@@ -1,6 +1,8 @@
 # Contribution Guidelines
 
-Excited by our work and want to get involved in building out our sharding releases? Or maybe you haven't learned as much about the Ethereum protocol but are a savvy developer? Our [READINGS.md](https://github.com/prysmaticlabs/prysm/blob/master/docs/READINGS.md) doc includes comprehensive information on Ethereum and sharding for both part-time and core contributors to the project.
+Note: The latest and most up to date documenation can be found on our [docs portal](https://docs.prylabs.network/docs/contribute/contribution-guidelines).
+
+Excited by our work and want to get involved in building out our sharding releases? Or maybe you haven't learned as much about the Ethereum protocol but are a savvy developer? 
 
 You can explore our [Open Issues](https://github.com/prysmaticlabs/prysm/issues) in-the works for our different releases. Feel free to fork our repo and start creating PR’s after assigning yourself to an issue of interest. We are always chatting on [Discord](https://discord.gg/che9auJ) or [Gitter](https://gitter.im/prysmaticlabs/geth-sharding) drop us a line there if you want to get more involved or have any questions on our implementation!
 

DEPENDENCIES.md

@@ -0,0 +1,69 @@
+# Dependency Managagement in Prysm
+
+Prysm is go project with many complicated dependencies, including some c++ based libraries. There
+are two parts to Prysm's dependency management. Go modules and bazel managed dependencies. Be sure 
+to read [Why Bazel?](https://github.com/prysmaticlabs/documentation/issues/138) to fully
+understand the reasoning behind an additional layer of build tooling via Bazel rather than a pure
+"go build" project.
+
+## Go Module support
+
+The Prysm project officially supports go modules with some caveats. 
+
+### Caveat 1: Some c++ libraries are precompiled archives
+
+Given some of Prysm's c++ dependencies have very complicated project structures which make building
+difficult or impossible with "go build" alone. Additionally, building c++ dependencies with certain
+compilers, like clang / LLVM, offer a significant performance improvement. To get around this 
+issue, c++ dependencies have been precompiled as linkable archives. While there isn't necessarily
+anything bad about precompiled archives, these files are a "blackbox" which a 3rd party author
+could have compiled anything for this archive and detecting undesired behavior would be nearly
+impossible. If your risk tolerance is low, always compile everything from source yourself, 
+including complicated c++ dependencies.
+
+*Recommendation: Use go build only for local development and use bazel build for production.*
+
+### Caveat 2: Generated gRPC protobuf libraries
+
+One key advantage of Bazel over vanilla `go build` is that Bazel automatically (re)builds generated
+pb.go files at build time when file changes are present in any protobuf definition file or after
+any updates to the protobuf compiler or other relevant dependencies. Vanilla go users should run
+the following scripts often to ensure their generated files are up to date. Further more, Prysm
+generates SSZ marshal related code based on defined data structures. These generated files must
+also be updated and checked in as frequently.
+
+```bash
+./scripts/update-go-pbs.sh
+./scripts/update-go-ssz.sh
+```
+
+*Recommendation: Use go build only for local development and use bazel build for production.*
+
+### Caveat 3: Compile-time optimizations 
+
+When Prysmatic Labs builds production binaries, they use the "release" configuration of bazel to
+compile with several compiler optimizations and recommended production build configurations.
+Additionally, the release build properly stamps the built binaries to include helpful metadata
+about how and when the binary was built. 
+
+*Recommendation: Use go build only for local development and use bazel build for production.*
+
+```bash
+bazel build //beacon-chain --config=release
+```
+ 
+## Adding / updating dependencies
+
+1. Add your dependency as you would with go modules. I.e. `go get ...`
+1. Run `gazelle update-repos -from_file=go.mod` to update the bazel managed dependencies.
+
+Example:
+
+```bash
+go get github.com/prysmaticlabs/example@v1.2.3
+bazel run //:gazelle -- update-repos -from_file=go.mod -to_macro=deps.bzl%prysm_deps
+```
+
+The deps.bzl file should have been updated with the dependency and any transitive dependencies. 
+
+Do NOT add new `go_repository` to the WORKSPACE file. All dependencies should live in deps.bzl.

PULL_REQUEST_TEMPLATE.md

@@ -1,13 +0,0 @@
-TODO(you): choose "part of" or "resolves" and the associated github issue #.
-
-[Part of|Resolves] #531 
-
----
-
-# Description
-
-**Write why you are making the changes in this pull request**
-
-**Write a summary of the changes you are making**
-
-**Link anything that would be helpful or relevant to the reviewers**

README.md

@@ -1,7 +1,8 @@
 # Prysm: An Ethereum 2.0 Client Written in Go
 
 [![Build status](https://badge.buildkite.com/b555891daf3614bae4284dcf365b2340cefc0089839526f096.svg?branch=master)](https://buildkite.com/prysmatic-labs/prysm)
-[![ETH2.0_Spec_Version 0.11.1](https://img.shields.io/badge/ETH2.0%20Spec%20Version-v0.11.1-blue.svg)](https://github.com/ethereum/eth2.0-specs/tree/v0.11.1)
+[![fuzzit](https://app.fuzzit.dev/badge?org_id=prysmaticlabs-gh)](https://app.fuzzit.dev/orgs/prysmaticlabs-gh/dashboard)
+[![ETH2.0_Spec_Version 0.12.1](https://img.shields.io/badge/ETH2.0%20Spec%20Version-v0.12.1-blue.svg)](https://github.com/ethereum/eth2.0-specs/tree/v0.12.1)
 [![Discord](https://user-images.githubusercontent.com/7288322/34471967-1df7808a-efbb-11e7-9088-ed0b04151291.png)](https://discord.gg/KSA7rPr)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/prysmaticlabs/geth-sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 

beacon-chain/BUILD.bazel

@@ -1,4 +1,5 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_test")
 load("@io_bazel_rules_docker//go:image.bzl", "go_image")
 load("@io_bazel_rules_docker//container:container.bzl", "container_bundle")
 load("//tools:go_image.bzl", "go_image_alpine", "go_image_debug")
@@ -20,13 +21,13 @@ go_library(
         "//shared/featureconfig:go_default_library",
         "//shared/logutil:go_default_library",
         "//shared/version:go_default_library",
-        "@com_github_ipfs_go_log//:go_default_library",
+        "@com_github_ethereum_go_ethereum//log:go_default_library",
+        "@com_github_ipfs_go_log_v2//:go_default_library",
         "@com_github_joonix_log//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_whyrusleeping_go_logging//:go_default_library",
+        "@com_github_urfave_cli_v2//:go_default_library",
+        "@com_github_urfave_cli_v2//altsrc:go_default_library",
         "@com_github_x_cray_logrus_prefixed_formatter//:go_default_library",
-        "@in_gopkg_urfave_cli_v2//:go_default_library",
-        "@in_gopkg_urfave_cli_v2//altsrc:go_default_library",
         "@org_uber_go_automaxprocs//:go_default_library",
     ],
 )
@@ -57,13 +58,13 @@ go_image(
         "//shared/featureconfig:go_default_library",
         "//shared/logutil:go_default_library",
         "//shared/version:go_default_library",
-        "@com_github_ipfs_go_log//:go_default_library",
+        "@com_github_ethereum_go_ethereum//log:go_default_library",
+        "@com_github_ipfs_go_log_v2//:go_default_library",
         "@com_github_joonix_log//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_whyrusleeping_go_logging//:go_default_library",
+        "@com_github_urfave_cli_v2//:go_default_library",
+        "@com_github_urfave_cli_v2//altsrc:go_default_library",
         "@com_github_x_cray_logrus_prefixed_formatter//:go_default_library",
-        "@in_gopkg_urfave_cli_v2//:go_default_library",
-        "@in_gopkg_urfave_cli_v2//altsrc:go_default_library",
         "@org_uber_go_automaxprocs//:go_default_library",
     ],
 )
@@ -73,6 +74,8 @@ container_bundle(
     images = {
         "gcr.io/prysmaticlabs/prysm/beacon-chain:latest": ":image",
         "gcr.io/prysmaticlabs/prysm/beacon-chain:{DOCKER_TAG}": ":image",
+        "index.docker.io/prysmaticlabs/prysm-beacon-chain:latest": ":image",
+        "index.docker.io/prysmaticlabs/prysm-beacon-chain:{DOCKER_TAG}": ":image",
     },
     tags = ["manual"],
 )
@@ -88,6 +91,8 @@ container_bundle(
     images = {
         "gcr.io/prysmaticlabs/prysm/beacon-chain:latest-debug": ":image_debug",
         "gcr.io/prysmaticlabs/prysm/beacon-chain:{DOCKER_TAG}-debug": ":image_debug",
+        "index.docker.io/prysmaticlabs/prysm-beacon-chain:latest-debug": ":image_debug",
+        "index.docker.io/prysmaticlabs/prysm-beacon-chain:{DOCKER_TAG}-debug": ":image_debug",
     },
     tags = ["manual"],
 )
@@ -103,6 +108,8 @@ container_bundle(
     images = {
         "gcr.io/prysmaticlabs/prysm/beacon-chain:latest-alpine": ":image_alpine",
         "gcr.io/prysmaticlabs/prysm/beacon-chain:{DOCKER_TAG}-alpine": ":image_alpine",
+        "index.docker.io/prysmaticlabs/prysm-beacon-chain:latest-alpine": ":image_alpine",
+        "index.docker.io/prysmaticlabs/prysm-beacon-chain:{DOCKER_TAG}-alpine": ":image_alpine",
     },
     tags = ["manual"],
 )
@@ -141,6 +148,6 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//shared/featureconfig:go_default_library",
-        "@in_gopkg_urfave_cli_v2//:go_default_library",
+        "@com_github_urfave_cli_v2//:go_default_library",
     ],
 )

beacon-chain/archiver/BUILD.bazel

@@ -1,4 +1,5 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_library(
     name = "go_default_library",

beacon-chain/archiver/service.go

@@ -119,13 +119,13 @@ func (s *Service) archiveActiveSetChanges(ctx context.Context, headState *state.
 // We compute participation metrics by first retrieving the head state and
 // matching validator attestations during the epoch.
 func (s *Service) archiveParticipation(ctx context.Context, epoch uint64) error {
-	p := s.participationFetcher.Participation(epoch)
+	pBal := s.participationFetcher.Participation(epoch)
 	participation := &ethpb.ValidatorParticipation{}
-	if p != nil {
+	if pBal != nil {
 		participation = &ethpb.ValidatorParticipation{
-			EligibleEther:           p.PrevEpoch,
-			VotedEther:              p.PrevEpochTargetAttesters,
-			GlobalParticipationRate: float32(p.PrevEpochTargetAttesters) / float32(p.PrevEpoch),
+			EligibleEther:           pBal.ActivePrevEpoch,
+			VotedEther:              pBal.PrevEpochTargetAttested,
+			GlobalParticipationRate: float32(pBal.PrevEpochTargetAttested) / float32(pBal.ActivePrevEpoch),
 		}
 	}
 	return s.beaconDB.SaveArchivedValidatorParticipation(ctx, epoch, participation)

beacon-chain/archiver/service_test.go

@@ -31,8 +31,7 @@ func init() {
 
 func TestArchiverService_ReceivesBlockProcessedEvent(t *testing.T) {
 	hook := logTest.NewGlobal()
-	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
+	svc, _ := setupService(t)
 	st := testutil.NewBeaconState()
 	if err := st.SetSlot(1); err != nil {
 		t.Fatal(err)
@@ -55,8 +54,7 @@ func TestArchiverService_ReceivesBlockProcessedEvent(t *testing.T) {
 
 func TestArchiverService_OnlyArchiveAtEpochEnd(t *testing.T) {
 	hook := logTest.NewGlobal()
-	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
+	svc, _ := setupService(t)
 	// The head state is NOT an epoch end.
 	st := testutil.NewBeaconState()
 	if err := st.SetSlot(params.BeaconConfig().SlotsPerEpoch - 2); err != nil {
@@ -86,13 +84,12 @@ func TestArchiverService_OnlyArchiveAtEpochEnd(t *testing.T) {
 
 func TestArchiverService_ArchivesEvenThroughSkipSlot(t *testing.T) {
 	hook := logTest.NewGlobal()
-	svc, beaconDB := setupService(t)
+	svc, _ := setupService(t)
 	validatorCount := uint64(100)
 	headState, err := setupState(validatorCount)
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer dbutil.TeardownDB(t, beaconDB)
 	event := &feed.Event{
 		Type: statefeed.BlockProcessed,
 		Data: &statefeed.BlockProcessedData{
@@ -146,8 +143,7 @@ func TestArchiverService_ComputesAndSavesParticipation(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
+	svc, _ := setupService(t)
 	svc.headFetcher = &mock.ChainService{
 		State: headState,
 	}
@@ -187,8 +183,7 @@ func TestArchiverService_SavesIndicesAndBalances(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
+	svc, _ := setupService(t)
 	svc.headFetcher = &mock.ChainService{
 		State: headState,
 	}
@@ -223,8 +218,7 @@ func TestArchiverService_SavesCommitteeInfo(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
+	svc, _ := setupService(t)
 	svc.headFetcher = &mock.ChainService{
 		State: headState,
 	}
@@ -274,7 +268,6 @@ func TestArchiverService_SavesActivatedValidatorChanges(t *testing.T) {
 		t.Fatal(err)
 	}
 	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
 	svc.headFetcher = &mock.ChainService{
 		State: headState,
 	}
@@ -326,7 +319,6 @@ func TestArchiverService_SavesSlashedValidatorChanges(t *testing.T) {
 		t.Fatal(err)
 	}
 	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
 	svc.headFetcher = &mock.ChainService{
 		State: headState,
 	}
@@ -377,7 +369,6 @@ func TestArchiverService_SavesExitedValidatorChanges(t *testing.T) {
 		t.Fatal(err)
 	}
 	svc, beaconDB := setupService(t)
-	defer dbutil.TeardownDB(t, beaconDB)
 	svc.headFetcher = &mock.ChainService{
 		State: headState,
 	}
@@ -445,7 +436,7 @@ func setupState(validatorCount uint64) (*stateTrie.BeaconState, error) {
 }
 
 func setupService(t *testing.T) (*Service, db.Database) {
-	beaconDB := dbutil.SetupDB(t)
+	beaconDB, _ := dbutil.SetupDB(t)
 	ctx, cancel := context.WithCancel(context.Background())
 	validatorCount := uint64(100)
 	totalBalance := validatorCount * params.BeaconConfig().MaxEffectiveBalance
@@ -456,7 +447,7 @@ func setupService(t *testing.T) (*Service, db.Database) {
 		cancel:        cancel,
 		stateNotifier: mockChainService.StateNotifier(),
 		participationFetcher: &mock.ChainService{
-			Balance: &precompute.Balance{PrevEpoch: totalBalance, PrevEpochTargetAttesters: 1}},
+			Balance: &precompute.Balance{ActivePrevEpoch: totalBalance, PrevEpochTargetAttested: 1}},
 	}, beaconDB
 }
 

beacon-chain/blockchain/BUILD.bazel

@@ -1,4 +1,5 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -54,7 +55,6 @@ go_library(
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
-        "@com_github_prysmaticlabs_go_ssz//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@io_opencensus_go//trace:go_default_library",
     ],

beacon-chain/blockchain/chain_info.go

@@ -1,17 +1,19 @@
 package blockchain
 
 import (
-	"bytes"
 	"context"
 	"time"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
+	"go.opencensus.io/trace"
 )
 
 // ChainInfoFetcher defines a common interface for methods in blockchain service which
@@ -19,6 +21,8 @@ import (
 type ChainInfoFetcher interface {
 	HeadFetcher
 	FinalizationFetcher
+	GenesisFetcher
+	CanonicalFetcher
 }
 
 // TimeFetcher retrieves the Eth2 data that's related to time.
@@ -27,6 +31,11 @@ type TimeFetcher interface {
 	CurrentSlot() uint64
 }
 
+// GenesisFetcher retrieves the eth2 data related to its genesis.
+type GenesisFetcher interface {
+	GenesisValidatorRoot() [32]byte
+}
+
 // HeadFetcher defines a common interface for methods in blockchain service which
 // directly retrieves head related data.
 type HeadFetcher interface {
@@ -34,8 +43,11 @@ type HeadFetcher interface {
 	HeadRoot(ctx context.Context) ([]byte, error)
 	HeadBlock(ctx context.Context) (*ethpb.SignedBeaconBlock, error)
 	HeadState(ctx context.Context) (*state.BeaconState, error)
-	HeadValidatorsIndices(epoch uint64) ([]uint64, error)
-	HeadSeed(epoch uint64) ([32]byte, error)
+	HeadValidatorsIndices(ctx context.Context, epoch uint64) ([]uint64, error)
+	HeadSeed(ctx context.Context, epoch uint64) ([32]byte, error)
+	HeadGenesisValidatorRoot() [32]byte
+	HeadETH1Data() *ethpb.Eth1Data
+	ProtoArrayStore() *protoarray.Store
 }
 
 // ForkFetcher retrieves the current fork information of the Ethereum beacon chain.
@@ -43,6 +55,11 @@ type ForkFetcher interface {
 	CurrentFork() *pb.Fork
 }
 
+// CanonicalFetcher retrieves the current chain's canonical information.
+type CanonicalFetcher interface {
+	IsCanonical(ctx context.Context, blockRoot [32]byte) (bool, error)
+}
+
 // FinalizationFetcher defines a common interface for methods in blockchain service which
 // directly retrieves finalization and justification related data.
 type FinalizationFetcher interface {
@@ -63,12 +80,6 @@ func (s *Service) FinalizedCheckpt() *ethpb.Checkpoint {
 		return &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
 	}
 
-	// If head state exists but there hasn't been a finalized check point,
-	// the check point's root should refer to genesis block root.
-	if bytes.Equal(s.finalizedCheckpt.Root, params.BeaconConfig().ZeroHash[:]) {
-		return &ethpb.Checkpoint{Root: s.genesisRoot[:]}
-	}
-
 	return state.CopyCheckpoint(s.finalizedCheckpt)
 }
 
@@ -78,12 +89,6 @@ func (s *Service) CurrentJustifiedCheckpt() *ethpb.Checkpoint {
 		return &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
 	}
 
-	// If head state exists but there hasn't been a justified check point,
-	// the check point root should refer to genesis block root.
-	if bytes.Equal(s.justifiedCheckpt.Root, params.BeaconConfig().ZeroHash[:]) {
-		return &ethpb.Checkpoint{Root: s.genesisRoot[:]}
-	}
-
 	return state.CopyCheckpoint(s.justifiedCheckpt)
 }
 
@@ -93,12 +98,6 @@ func (s *Service) PreviousJustifiedCheckpt() *ethpb.Checkpoint {
 		return &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
 	}
 
-	// If head state exists but there hasn't been a justified check point,
-	// the check point root should refer to genesis block root.
-	if bytes.Equal(s.prevJustifiedCheckpt.Root, params.BeaconConfig().ZeroHash[:]) {
-		return &ethpb.Checkpoint{Root: s.genesisRoot[:]}
-	}
-
 	return state.CopyCheckpoint(s.prevJustifiedCheckpt)
 }
 
@@ -126,7 +125,7 @@ func (s *Service) HeadRoot(ctx context.Context) ([]byte, error) {
 		return params.BeaconConfig().ZeroHash[:], nil
 	}
 
-	r, err := ssz.HashTreeRoot(b.Block)
+	r, err := stateutil.BlockRoot(b.Block)
 	if err != nil {
 		return nil, err
 	}
@@ -135,7 +134,7 @@ func (s *Service) HeadRoot(ctx context.Context) ([]byte, error) {
 }
 
 // HeadBlock returns the head block of the chain.
-// If the head state is nil from service struct,
+// If the head is nil from service struct,
 // it will attempt to get the head block from DB.
 func (s *Service) HeadBlock(ctx context.Context) (*ethpb.SignedBeaconBlock, error) {
 	if s.hasHeadState() {
@@ -146,31 +145,59 @@ func (s *Service) HeadBlock(ctx context.Context) (*ethpb.SignedBeaconBlock, erro
 }
 
 // HeadState returns the head state of the chain.
-// If the head state is nil from service struct,
+// If the head is nil from service struct,
 // it will attempt to get the head state from DB.
 func (s *Service) HeadState(ctx context.Context) (*state.BeaconState, error) {
-	if s.hasHeadState() {
-		return s.headState(), nil
+	ctx, span := trace.StartSpan(ctx, "blockchain.HeadState")
+	defer span.End()
+
+	ok := s.hasHeadState()
+	span.AddAttributes(trace.BoolAttribute("cache_hit", ok))
+
+	if ok {
+		return s.headState(ctx), nil
 	}
 
 	return s.beaconDB.HeadState(ctx)
 }
 
 // HeadValidatorsIndices returns a list of active validator indices from the head view of a given epoch.
-func (s *Service) HeadValidatorsIndices(epoch uint64) ([]uint64, error) {
+func (s *Service) HeadValidatorsIndices(ctx context.Context, epoch uint64) ([]uint64, error) {
 	if !s.hasHeadState() {
 		return []uint64{}, nil
 	}
-	return helpers.ActiveValidatorIndices(s.headState(), epoch)
+	return helpers.ActiveValidatorIndices(s.headState(ctx), epoch)
 }
 
 // HeadSeed returns the seed from the head view of a given epoch.
-func (s *Service) HeadSeed(epoch uint64) ([32]byte, error) {
+func (s *Service) HeadSeed(ctx context.Context, epoch uint64) ([32]byte, error) {
 	if !s.hasHeadState() {
 		return [32]byte{}, nil
 	}
 
-	return helpers.Seed(s.headState(), epoch, params.BeaconConfig().DomainBeaconAttester)
+	return helpers.Seed(s.headState(ctx), epoch, params.BeaconConfig().DomainBeaconAttester)
+}
+
+// HeadGenesisValidatorRoot returns genesis validator root of the head state.
+func (s *Service) HeadGenesisValidatorRoot() [32]byte {
+	if !s.hasHeadState() {
+		return [32]byte{}
+	}
+
+	return s.headGenesisValidatorRoot()
+}
+
+// HeadETH1Data returns the eth1data of the current head state.
+func (s *Service) HeadETH1Data() *ethpb.Eth1Data {
+	if !s.hasHeadState() {
+		return &ethpb.Eth1Data{}
+	}
+	return s.head.state.Eth1Data()
+}
+
+// ProtoArrayStore returns the proto array store object.
+func (s *Service) ProtoArrayStore() *protoarray.Store {
+	return s.forkChoiceStore.Store()
 }
 
 // GenesisTime returns the genesis time of beacon chain.
@@ -178,6 +205,15 @@ func (s *Service) GenesisTime() time.Time {
 	return s.genesisTime
 }
 
+// GenesisValidatorRoot returns the genesis validator
+// root of the chain.
+func (s *Service) GenesisValidatorRoot() [32]byte {
+	if !s.hasHeadState() {
+		return [32]byte{}
+	}
+	return bytesutil.ToBytes32(s.head.state.GenesisValidatorRoot())
+}
+
 // CurrentFork retrieves the latest fork information of the beacon chain.
 func (s *Service) CurrentFork() *pb.Fork {
 	if !s.hasHeadState() {
@@ -196,3 +232,17 @@ func (s *Service) Participation(epoch uint64) *precompute.Balance {
 
 	return s.epochParticipation[epoch]
 }
+
+// IsCanonical returns true if the input block root is part of the canonical chain.
+func (s *Service) IsCanonical(ctx context.Context, blockRoot [32]byte) (bool, error) {
+	// If the block has been finalized, the block will always be part of the canonical chain.
+	if s.beaconDB.IsFinalizedBlock(ctx, blockRoot) {
+		return true, nil
+	}
+
+	// If the block has not been finalized, the block must be recent. Check recent canonical roots
+	// mapping which uses proto array fork choice.
+	s.recentCanonicalBlocksLock.RLock()
+	defer s.recentCanonicalBlocksLock.RUnlock()
+	return s.recentCanonicalBlocks[blockRoot], nil
+}

beacon-chain/blockchain/chain_info_norace_test.go

@@ -5,19 +5,17 @@ import (
 	"testing"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 )
 
 func TestHeadSlot_DataRace(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 	s := &Service{
 		beaconDB: db,
 	}
 	go func() {
-		if err := s.saveHead(context.Background(), [32]byte{}, ); err != nil {
+		if err := s.saveHead(context.Background(), [32]byte{}); err != nil {
 			t.Fatal(err)
 		}
 	}()
@@ -25,15 +23,14 @@ func TestHeadSlot_DataRace(t *testing.T) {
 }
 
 func TestHeadRoot_DataRace(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 	s := &Service{
 		beaconDB: db,
 		head:     &head{root: [32]byte{'A'}},
-		stateGen: stategen.New(db, cache.NewStateSummaryCache()),
+		stateGen: stategen.New(db, sc),
 	}
 	go func() {
-		if err := s.saveHead(context.Background(), [32]byte{}, ); err != nil {
+		if err := s.saveHead(context.Background(), [32]byte{}); err != nil {
 			t.Fatal(err)
 		}
 	}()
@@ -43,15 +40,14 @@ func TestHeadRoot_DataRace(t *testing.T) {
 }
 
 func TestHeadBlock_DataRace(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 	s := &Service{
 		beaconDB: db,
 		head:     &head{block: &ethpb.SignedBeaconBlock{}},
-		stateGen: stategen.New(db, cache.NewStateSummaryCache()),
+		stateGen: stategen.New(db, sc),
 	}
 	go func() {
-		if err := s.saveHead(context.Background(), [32]byte{}, ); err != nil {
+		if err := s.saveHead(context.Background(), [32]byte{}); err != nil {
 			t.Fatal(err)
 		}
 	}()
@@ -61,14 +57,13 @@ func TestHeadBlock_DataRace(t *testing.T) {
 }
 
 func TestHeadState_DataRace(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 	s := &Service{
 		beaconDB: db,
-		stateGen: stategen.New(db, cache.NewStateSummaryCache()),
+		stateGen: stategen.New(db, sc),
 	}
 	go func() {
-		if err := s.saveHead(context.Background(), [32]byte{}, ); err != nil {
+		if err := s.saveHead(context.Background(), [32]byte{}); err != nil {
 			t.Fatal(err)
 		}
 	}()

beacon-chain/blockchain/chain_info_test.go

@@ -21,18 +21,16 @@ var _ = TimeFetcher(&Service{})
 var _ = ForkFetcher(&Service{})
 
 func TestFinalizedCheckpt_Nil(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	c := setupBeaconChain(t, db)
+	db, sc := testDB.SetupDB(t)
+	c := setupBeaconChain(t, db, sc)
 	if !bytes.Equal(c.FinalizedCheckpt().Root, params.BeaconConfig().ZeroHash[:]) {
 		t.Error("Incorrect pre chain start value")
 	}
 }
 
 func TestHeadRoot_Nil(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	c := setupBeaconChain(t, db)
+	db, sc := testDB.SetupDB(t)
+	c := setupBeaconChain(t, db, sc)
 	headRoot, err := c.HeadRoot(context.Background())
 	if err != nil {
 		t.Fatal(err)
@@ -43,11 +41,10 @@ func TestHeadRoot_Nil(t *testing.T) {
 }
 
 func TestFinalizedCheckpt_CanRetrieve(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	cp := &ethpb.Checkpoint{Epoch: 5, Root: []byte("foo")}
-	c := setupBeaconChain(t, db)
+	c := setupBeaconChain(t, db, sc)
 	c.finalizedCheckpt = cp
 
 	if c.FinalizedCheckpt().Epoch != cp.Epoch {
@@ -56,12 +53,11 @@ func TestFinalizedCheckpt_CanRetrieve(t *testing.T) {
 }
 
 func TestFinalizedCheckpt_GenesisRootOk(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	genesisRoot := [32]byte{'A'}
 	cp := &ethpb.Checkpoint{Root: genesisRoot[:]}
-	c := setupBeaconChain(t, db)
+	c := setupBeaconChain(t, db, sc)
 	c.finalizedCheckpt = cp
 	c.genesisRoot = genesisRoot
 
@@ -71,11 +67,10 @@ func TestFinalizedCheckpt_GenesisRootOk(t *testing.T) {
 }
 
 func TestCurrentJustifiedCheckpt_CanRetrieve(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	cp := &ethpb.Checkpoint{Epoch: 6, Root: []byte("foo")}
-	c := setupBeaconChain(t, db)
+	c := setupBeaconChain(t, db, sc)
 	c.justifiedCheckpt = cp
 
 	if c.CurrentJustifiedCheckpt().Epoch != cp.Epoch {
@@ -84,12 +79,11 @@ func TestCurrentJustifiedCheckpt_CanRetrieve(t *testing.T) {
 }
 
 func TestJustifiedCheckpt_GenesisRootOk(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	genesisRoot := [32]byte{'B'}
 	cp := &ethpb.Checkpoint{Root: genesisRoot[:]}
-	c := setupBeaconChain(t, db)
+	c := setupBeaconChain(t, db, sc)
 	c.justifiedCheckpt = cp
 	c.genesisRoot = genesisRoot
 
@@ -99,11 +93,10 @@ func TestJustifiedCheckpt_GenesisRootOk(t *testing.T) {
 }
 
 func TestPreviousJustifiedCheckpt_CanRetrieve(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	cp := &ethpb.Checkpoint{Epoch: 7, Root: []byte("foo")}
-	c := setupBeaconChain(t, db)
+	c := setupBeaconChain(t, db, sc)
 	c.prevJustifiedCheckpt = cp
 
 	if c.PreviousJustifiedCheckpt().Epoch != cp.Epoch {
@@ -112,12 +105,11 @@ func TestPreviousJustifiedCheckpt_CanRetrieve(t *testing.T) {
 }
 
 func TestPrevJustifiedCheckpt_GenesisRootOk(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	genesisRoot := [32]byte{'C'}
 	cp := &ethpb.Checkpoint{Root: genesisRoot[:]}
-	c := setupBeaconChain(t, db)
+	c := setupBeaconChain(t, db, sc)
 	c.prevJustifiedCheckpt = cp
 	c.genesisRoot = genesisRoot
 
@@ -201,3 +193,41 @@ func TestCurrentFork_CanRetrieve(t *testing.T) {
 		t.Error("Received incorrect fork version")
 	}
 }
+
+func TestGenesisValidatorRoot_CanRetrieve(t *testing.T) {
+	// Should not panic if head state is nil.
+	c := &Service{}
+	if c.GenesisValidatorRoot() != [32]byte{} {
+		t.Error("Did not get correct genesis validator root")
+	}
+
+	s, err := state.InitializeFromProto(&pb.BeaconState{GenesisValidatorsRoot: []byte{'a'}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	c.head = &head{state: s}
+	if c.GenesisValidatorRoot() != [32]byte{'a'} {
+		t.Error("Did not get correct genesis validator root")
+	}
+}
+
+func TestHeadETH1Data_Nil(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+	c := setupBeaconChain(t, db, sc)
+	if !reflect.DeepEqual(c.HeadETH1Data(), &ethpb.Eth1Data{}) {
+		t.Error("Incorrect pre chain start value")
+	}
+}
+
+func TestHeadETH1Data_CanRetrieve(t *testing.T) {
+	d := &ethpb.Eth1Data{DepositCount: 999}
+	s, err := state.InitializeFromProto(&pb.BeaconState{Eth1Data: d})
+	if err != nil {
+		t.Fatal(err)
+	}
+	c := &Service{}
+	c.head = &head{state: s}
+	if !proto.Equal(c.HeadETH1Data(), d) {
+		t.Error("Received incorrect eth1 data")
+	}
+}

beacon-chain/blockchain/head.go

@@ -2,14 +2,19 @@ package blockchain
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
+	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/sirupsen/logrus"
 	"go.opencensus.io/trace"
 )
 
@@ -21,8 +26,8 @@ type head struct {
 	state *state.BeaconState       // current head state.
 }
 
-// This gets head from the fork choice service and saves head related items
-// (ie root, block, state) to the local service cache.
+// Determined the head from the fork choice service and saves its new data
+// (head root, head block, and head state) to the local service cache.
 func (s *Service) updateHead(ctx context.Context, balances []uint64) error {
 	ctx, span := trace.StartSpan(ctx, "blockchain.updateHead")
 	defer span.End()
@@ -32,16 +37,29 @@ func (s *Service) updateHead(ctx context.Context, balances []uint64) error {
 	// ensure head gets its best justified info.
 	if s.bestJustifiedCheckpt.Epoch > s.justifiedCheckpt.Epoch {
 		s.justifiedCheckpt = s.bestJustifiedCheckpt
+		if err := s.cacheJustifiedStateBalances(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root)); err != nil {
+			return err
+		}
 	}
 
 	// Get head from the fork choice service.
 	f := s.finalizedCheckpt
 	j := s.justifiedCheckpt
-	headRoot, err := s.forkChoiceStore.Head(ctx, j.Epoch, bytesutil.ToBytes32(j.Root), balances, f.Epoch)
+	// To get head before the first justified epoch, the fork choice will start with genesis root
+	// instead of zero hashes.
+	headStartRoot := bytesutil.ToBytes32(j.Root)
+	if headStartRoot == params.BeaconConfig().ZeroHash {
+		headStartRoot = s.genesisRoot
+	}
+	headRoot, err := s.forkChoiceStore.Head(ctx, j.Epoch, headStartRoot, balances, f.Epoch)
 	if err != nil {
 		return err
 	}
 
+	if err := s.updateRecentCanonicalBlocks(ctx, headRoot); err != nil {
+		return err
+	}
+
 	// Save head to the local service cache.
 	return s.saveHead(ctx, headRoot)
 }
@@ -59,15 +77,8 @@ func (s *Service) saveHead(ctx context.Context, headRoot [32]byte) error {
 
 	// If the head state is not available, just return nil.
 	// There's nothing to cache
-	if !featureconfig.Get().DisableNewStateMgmt {
-		if !s.stateGen.StateSummaryExists(ctx, headRoot) {
-			return nil
-		}
-	} else {
-		_, cached := s.initSyncState[headRoot]
-		if !cached && !s.beaconDB.HasState(ctx, headRoot) {
-			return nil
-		}
+	if !s.stateGen.StateSummaryExists(ctx, headRoot) {
+		return nil
 	}
 
 	// Get the new head block from DB.
@@ -80,26 +91,31 @@ func (s *Service) saveHead(ctx context.Context, headRoot [32]byte) error {
 	}
 
 	// Get the new head state from cached state or DB.
-	var newHeadState *state.BeaconState
-	if !featureconfig.Get().DisableNewStateMgmt {
-		newHeadState, err = s.stateGen.StateByRoot(ctx, headRoot)
-		if err != nil {
-			return errors.Wrap(err, "could not retrieve head state in DB")
-		}
-	} else {
-		var exists bool
-		newHeadState, exists = s.initSyncState[headRoot]
-		if !exists {
-			newHeadState, err = s.beaconDB.State(ctx, headRoot)
-			if err != nil {
-				return errors.Wrap(err, "could not retrieve head state in DB")
-			}
-		}
+	newHeadState, err := s.stateGen.StateByRoot(ctx, headRoot)
+	if err != nil {
+		return errors.Wrap(err, "could not retrieve head state in DB")
 	}
 	if newHeadState == nil {
 		return errors.New("cannot save nil head state")
 	}
 
+	// A chain re-org occurred, so we fire an event notifying the rest of the services.
+	if bytesutil.ToBytes32(newHeadBlock.Block.ParentRoot) != s.headRoot() {
+		log.WithFields(logrus.Fields{
+			"newSlot": fmt.Sprintf("%d", newHeadBlock.Block.Slot),
+			"oldSlot": fmt.Sprintf("%d", s.headSlot()),
+		}).Debug("Chain reorg occurred")
+		s.stateNotifier.StateFeed().Send(&feed.Event{
+			Type: statefeed.Reorg,
+			Data: &statefeed.ReorgData{
+				NewSlot: newHeadBlock.Block.Slot,
+				OldSlot: s.headSlot(),
+			},
+		})
+
+		reorgCount.Inc()
+	}
+
 	// Cache the new head info.
 	s.setHead(headRoot, newHeadBlock, newHeadState)
 
@@ -112,39 +128,22 @@ func (s *Service) saveHead(ctx context.Context, headRoot [32]byte) error {
 }
 
 // This gets called to update canonical root mapping. It does not save head block
-// root in DB. With the inception of inital-sync-cache-state flag, it uses finalized
+// root in DB. With the inception of initial-sync-cache-state flag, it uses finalized
 // check point as anchors to resume sync therefore head is no longer needed to be saved on per slot basis.
 func (s *Service) saveHeadNoDB(ctx context.Context, b *ethpb.SignedBeaconBlock, r [32]byte) error {
 	if b == nil || b.Block == nil {
 		return errors.New("cannot save nil head block")
 	}
 
-	var headState *state.BeaconState
-	var err error
-	if !featureconfig.Get().DisableNewStateMgmt {
-		headState, err = s.stateGen.StateByRoot(ctx, r)
-		if err != nil {
-			return errors.Wrap(err, "could not retrieve head state in DB")
-		}
-	} else {
-		headState, err = s.beaconDB.State(ctx, r)
-		if err != nil {
-			return errors.Wrap(err, "could not retrieve head state in DB")
-		}
-		if headState == nil {
-			s.initSyncStateLock.RLock()
-			cachedHeadState, ok := s.initSyncState[r]
-			if ok {
-				headState = cachedHeadState
-			}
-			s.initSyncStateLock.RUnlock()
-		}
+	headState, err := s.stateGen.StateByRootInitialSync(ctx, r)
+	if err != nil {
+		return errors.Wrap(err, "could not retrieve head state in DB")
 	}
 	if headState == nil {
 		return errors.New("nil head state")
 	}
 
-	s.setHead(r, stateTrie.CopySignedBeaconBlock(b), headState)
+	s.setHeadInitialSync(r, stateTrie.CopySignedBeaconBlock(b), headState)
 
 	return nil
 }
@@ -163,6 +162,22 @@ func (s *Service) setHead(root [32]byte, block *ethpb.SignedBeaconBlock, state *
 	}
 }
 
+// This sets head view object which is used to track the head slot, root, block and state. The method
+// assumes that state being passed into the method will not be modified by any other alternate
+// caller which holds the state's reference.
+func (s *Service) setHeadInitialSync(root [32]byte, block *ethpb.SignedBeaconBlock, state *state.BeaconState) {
+	s.headLock.Lock()
+	defer s.headLock.Unlock()
+
+	// This does a full copy of the block only.
+	s.head = &head{
+		slot:  block.Block.Slot,
+		root:  root,
+		block: stateTrie.CopySignedBeaconBlock(block),
+		state: state,
+	}
+}
+
 // This returns the head slot.
 func (s *Service) headSlot() uint64 {
 	s.headLock.RLock()
@@ -195,13 +210,24 @@ func (s *Service) headBlock() *ethpb.SignedBeaconBlock {
 
 // This returns the head state.
 // It does a full copy on head state for immutability.
-func (s *Service) headState() *state.BeaconState {
+func (s *Service) headState(ctx context.Context) *stateTrie.BeaconState {
+	ctx, span := trace.StartSpan(ctx, "blockchain.headState")
+	defer span.End()
+
 	s.headLock.RLock()
 	defer s.headLock.RUnlock()
 
 	return s.head.state.Copy()
 }
 
+// This returns the genesis validator root of the head state.
+func (s *Service) headGenesisValidatorRoot() [32]byte {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	return bytesutil.ToBytes32(s.head.state.GenesisValidatorRoot())
+}
+
 // Returns true if head state exists.
 func (s *Service) hasHeadState() bool {
 	s.headLock.RLock()
@@ -209,3 +235,66 @@ func (s *Service) hasHeadState() bool {
 
 	return s.head != nil && s.head.state != nil
 }
+
+// This updates recent canonical block mapping. It uses input head root and retrieves
+// all the canonical block roots that are ancestor of the input head block root.
+func (s *Service) updateRecentCanonicalBlocks(ctx context.Context, headRoot [32]byte) error {
+	ctx, span := trace.StartSpan(ctx, "blockchain.updateRecentCanonicalBlocks")
+	defer span.End()
+
+	s.recentCanonicalBlocksLock.Lock()
+	defer s.recentCanonicalBlocksLock.Unlock()
+
+	s.recentCanonicalBlocks = make(map[[32]byte]bool)
+	s.recentCanonicalBlocks[headRoot] = true
+	nodes := s.forkChoiceStore.Nodes()
+	node := s.forkChoiceStore.Node(headRoot)
+	if node == nil {
+		return nil
+	}
+
+	for node.Parent != protoarray.NonExistentNode {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		node = nodes[node.Parent]
+		s.recentCanonicalBlocks[node.Root] = true
+	}
+
+	return nil
+}
+
+// This caches justified state balances to be used for fork choice.
+func (s *Service) cacheJustifiedStateBalances(ctx context.Context, justifiedRoot [32]byte) error {
+	if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+		return err
+	}
+
+	s.clearInitSyncBlocks()
+	justifiedState, err := s.stateGen.StateByRoot(ctx, justifiedRoot)
+	if err != nil {
+		return err
+	}
+
+	epoch := helpers.CurrentEpoch(justifiedState)
+	validators := justifiedState.Validators()
+	justifiedBalances := make([]uint64, len(validators))
+	for i, validator := range validators {
+		if helpers.IsActiveValidator(validator, epoch) {
+			justifiedBalances[i] = validator.EffectiveBalance
+		} else {
+			justifiedBalances[i] = 0
+		}
+	}
+
+	s.justifiedBalancesLock.Lock()
+	defer s.justifiedBalancesLock.Unlock()
+	s.justifiedBalances = justifiedBalances
+	return nil
+}
+
+func (s *Service) getJustifiedBalances() []uint64 {
+	s.justifiedBalancesLock.RLock()
+	defer s.justifiedBalancesLock.RUnlock()
+	return s.justifiedBalances
+}

beacon-chain/blockchain/head_test.go

@@ -7,16 +7,16 @@ import (
 	"testing"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
+	logTest "github.com/sirupsen/logrus/hooks/test"
 )
 
 func TestSaveHead_Same(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	service := setupBeaconChain(t, db)
+	db, sc := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db, sc)
 
 	r := [32]byte{'A'}
 	service.head = &head{slot: 0, root: r}
@@ -35,9 +35,9 @@ func TestSaveHead_Same(t *testing.T) {
 }
 
 func TestSaveHead_Different(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	service := setupBeaconChain(t, db)
+	ctx := context.Background()
+	db, sc := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db, sc)
 
 	oldRoot := [32]byte{'A'}
 	service.head = &head{slot: 0, root: oldRoot}
@@ -48,7 +48,7 @@ func TestSaveHead_Different(t *testing.T) {
 	if err := service.beaconDB.SaveBlock(context.Background(), newHeadSignedBlock); err != nil {
 		t.Fatal(err)
 	}
-	newRoot, err := ssz.HashTreeRoot(newHeadBlock)
+	newRoot, err := stateutil.BlockRoot(newHeadBlock)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -80,7 +80,143 @@ func TestSaveHead_Different(t *testing.T) {
 	if !reflect.DeepEqual(service.headBlock(), newHeadSignedBlock) {
 		t.Error("Head did not change")
 	}
-	if !reflect.DeepEqual(service.headState().CloneInnerState(), headState.CloneInnerState()) {
+	if !reflect.DeepEqual(service.headState(ctx).CloneInnerState(), headState.CloneInnerState()) {
 		t.Error("Head did not change")
 	}
 }
+
+func TestSaveHead_Different_Reorg(t *testing.T) {
+	ctx := context.Background()
+	hook := logTest.NewGlobal()
+	db, sc := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db, sc)
+
+	oldRoot := [32]byte{'A'}
+	service.head = &head{slot: 0, root: oldRoot}
+
+	reorgChainParent := [32]byte{'B'}
+	newHeadBlock := &ethpb.BeaconBlock{
+		Slot:       1,
+		ParentRoot: reorgChainParent[:],
+	}
+	newHeadSignedBlock := &ethpb.SignedBeaconBlock{Block: newHeadBlock}
+
+	if err := service.beaconDB.SaveBlock(context.Background(), newHeadSignedBlock); err != nil {
+		t.Fatal(err)
+	}
+	newRoot, err := stateutil.BlockRoot(newHeadBlock)
+	if err != nil {
+		t.Fatal(err)
+	}
+	headState := testutil.NewBeaconState()
+	if err := headState.SetSlot(1); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveStateSummary(context.Background(), &pb.StateSummary{Slot: 1, Root: newRoot[:]}); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveState(context.Background(), headState, newRoot); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.saveHead(context.Background(), newRoot); err != nil {
+		t.Fatal(err)
+	}
+
+	if service.HeadSlot() != 1 {
+		t.Error("Head did not change")
+	}
+
+	cachedRoot, err := service.HeadRoot(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(cachedRoot, newRoot[:]) {
+		t.Error("Head did not change")
+	}
+	if !reflect.DeepEqual(service.headBlock(), newHeadSignedBlock) {
+		t.Error("Head did not change")
+	}
+	if !reflect.DeepEqual(service.headState(ctx).CloneInnerState(), headState.CloneInnerState()) {
+		t.Error("Head did not change")
+	}
+	testutil.AssertLogsContain(t, hook, "Chain reorg occurred")
+}
+
+func TestUpdateRecentCanonicalBlocks_CanUpdateWithoutParent(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db, sc)
+
+	r := [32]byte{'a'}
+	if err := service.updateRecentCanonicalBlocks(context.Background(), r); err != nil {
+		t.Fatal(err)
+	}
+	canonical, err := service.IsCanonical(context.Background(), r)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !canonical {
+		t.Error("Block should be canonical")
+	}
+}
+
+func TestUpdateRecentCanonicalBlocks_CanUpdateWithParent(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db, sc)
+	oldHead := [32]byte{'a'}
+	if err := service.forkChoiceStore.ProcessBlock(context.Background(), 1, oldHead, [32]byte{'g'}, [32]byte{}, 0, 0); err != nil {
+		t.Fatal(err)
+	}
+	currentHead := [32]byte{'b'}
+	if err := service.forkChoiceStore.ProcessBlock(context.Background(), 3, currentHead, oldHead, [32]byte{}, 0, 0); err != nil {
+		t.Fatal(err)
+	}
+	forkedRoot := [32]byte{'c'}
+	if err := service.forkChoiceStore.ProcessBlock(context.Background(), 2, forkedRoot, oldHead, [32]byte{}, 0, 0); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := service.updateRecentCanonicalBlocks(context.Background(), currentHead); err != nil {
+		t.Fatal(err)
+	}
+	canonical, err := service.IsCanonical(context.Background(), currentHead)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !canonical {
+		t.Error("Block should be canonical")
+	}
+	canonical, err = service.IsCanonical(context.Background(), oldHead)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !canonical {
+		t.Error("Block should be canonical")
+	}
+	canonical, err = service.IsCanonical(context.Background(), forkedRoot)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if canonical {
+		t.Error("Block should not be canonical")
+	}
+}
+
+func TestCacheJustifiedStateBalances_CanCache(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+	service := setupBeaconChain(t, db, sc)
+
+	state, _ := testutil.DeterministicGenesisState(t, 100)
+	r := [32]byte{'a'}
+	if err := service.beaconDB.SaveStateSummary(context.Background(), &pb.StateSummary{Root: r[:]}); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.beaconDB.SaveState(context.Background(), state, r); err != nil {
+		t.Fatal(err)
+	}
+	if err := service.cacheJustifiedStateBalances(context.Background(), r); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(state.Balances(), service.getJustifiedBalances()) {
+		t.Fatal("Incorrect justified balances")
+	}
+}

beacon-chain/blockchain/info.go

@@ -1,9 +1,9 @@
 package blockchain
 
 import (
+	"encoding/hex"
 	"fmt"
 	"net/http"
-	"strconv"
 
 	"github.com/emicklei/dot"
 )
@@ -32,8 +32,8 @@ const template = `<html>
 </html>`
 
 // TreeHandler is a handler to serve /tree page in metrics.
-func (s *Service) TreeHandler(w http.ResponseWriter, _ *http.Request) {
-	if s.headState() == nil {
+func (s *Service) TreeHandler(w http.ResponseWriter, r *http.Request) {
+	if s.headState(r.Context()) == nil {
 		if _, err := w.Write([]byte("Unavailable during initial syncing")); err != nil {
 			log.WithError(err).Error("Failed to render p2p info page")
 		}
@@ -46,23 +46,24 @@ func (s *Service) TreeHandler(w http.ResponseWriter, _ *http.Request) {
 	graph.Attr("labeljust", "l")
 
 	dotNodes := make([]*dot.Node, len(nodes))
-	avgBalance := uint64(averageBalance(s.headState().Balances()))
+	avgBalance := uint64(averageBalance(s.headState(r.Context()).Balances()))
 
 	for i := len(nodes) - 1; i >= 0; i-- {
 		// Construct label for each node.
-		slot := strconv.Itoa(int(nodes[i].Slot))
-		weight := strconv.Itoa(int(nodes[i].Weight / 1e9)) // Convert unit Gwei to unit ETH.
-		votes := strconv.Itoa(int(nodes[i].Weight / 1e9 / avgBalance))
-		bestDescendent := strconv.Itoa(int(nodes[i].BestDescendent))
-		index := strconv.Itoa(int(i))
-		label := "slot: " + slot + "\n index: " + index + "\n bestDescendent: " + bestDescendent + "\n votes: " + votes + "\n weight: " + weight
+		slot := fmt.Sprintf("%d", nodes[i].Slot)
+		weight := fmt.Sprintf("%d", nodes[i].Weight/1e9) // Convert unit Gwei to unit ETH.
+		votes := fmt.Sprintf("%d", nodes[i].Weight/1e9/avgBalance)
+		index := fmt.Sprintf("%d", i)
+		g := nodes[i].Graffiti[:]
+		graffiti := hex.EncodeToString(g[:8])
+		label := "slot: " + slot + "\n votes: " + votes + "\n weight: " + weight + "\n graffiti: " + graffiti
 		var dotN dot.Node
 		if nodes[i].Parent != ^uint64(0) {
 			dotN = graph.Node(index).Box().Attr("label", label)
 		}
 
 		if nodes[i].Slot == s.headSlot() &&
-			nodes[i].BestDescendent == ^uint64(0) {
+			nodes[i].BestDescendant == ^uint64(0) {
 			dotN = dotN.Attr("color", "green")
 		}
 

beacon-chain/blockchain/init_sync_process_block.go

@@ -2,174 +2,24 @@ package blockchain
 
 import (
 	"context"
-	"sort"
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
-const maxCacheSize = 70
-const initialSyncCacheSize = 45
-const minimumCacheSize = initialSyncCacheSize / 3
-
-func (s *Service) persistCachedStates(ctx context.Context, numOfStates int) error {
-	oldStates := make([]*stateTrie.BeaconState, 0, numOfStates)
-
-	// Add slots to the map and add epoch boundary states to the slice.
-	for _, rt := range s.boundaryRoots[:numOfStates-minimumCacheSize] {
-		oldStates = append(oldStates, s.initSyncState[rt])
-	}
-
-	err := s.beaconDB.SaveStates(ctx, oldStates, s.boundaryRoots[:numOfStates-minimumCacheSize])
-	if err != nil {
-		return err
-	}
-	for _, rt := range s.boundaryRoots[:numOfStates-minimumCacheSize] {
-		delete(s.initSyncState, rt)
-	}
-	s.boundaryRoots = s.boundaryRoots[numOfStates-minimumCacheSize:]
-	return nil
-}
-
-// filter out boundary candidates from our currently processed batch of states.
-func (s *Service) filterBoundaryCandidates(ctx context.Context, root [32]byte, postState *stateTrie.BeaconState) {
-	// Only trigger on epoch start.
-	if !helpers.IsEpochStart(postState.Slot()) {
-		return
-	}
-
-	stateSlice := make([][32]byte, 0, len(s.initSyncState))
-	// Add epoch boundary roots to slice.
-	for rt := range s.initSyncState {
-		stateSlice = append(stateSlice, rt)
-	}
-
-	sort.Slice(stateSlice, func(i int, j int) bool {
-		return s.initSyncState[stateSlice[i]].Slot() < s.initSyncState[stateSlice[j]].Slot()
-	})
-	epochLength := params.BeaconConfig().SlotsPerEpoch
-
-	if len(s.boundaryRoots) > 0 {
-		// Retrieve previous boundary root.
-		previousBoundaryRoot := s.boundaryRoots[len(s.boundaryRoots)-1]
-		previousState, ok := s.initSyncState[previousBoundaryRoot]
-		if !ok {
-			// Remove the non-existent root and exit filtering.
-			s.boundaryRoots = s.boundaryRoots[:len(s.boundaryRoots)-1]
-			return
-		}
-		previousSlot := previousState.Slot()
-
-		// Round up slot number to account for skipped slots.
-		previousSlot = helpers.RoundUpToNearestEpoch(previousSlot)
-		if postState.Slot()-previousSlot >= epochLength {
-			targetSlot := postState.Slot()
-			tempRoots := s.loopThroughCandidates(stateSlice, previousBoundaryRoot, previousSlot, targetSlot)
-			s.boundaryRoots = append(s.boundaryRoots, tempRoots...)
-		}
-	}
-	s.boundaryRoots = append(s.boundaryRoots, root)
-	s.pruneOldStates()
-	s.pruneNonBoundaryStates()
-}
-
-// loop-through the provided candidate roots to filter out which would be appropriate boundary roots.
-func (s *Service) loopThroughCandidates(stateSlice [][32]byte, previousBoundaryRoot [32]byte,
-	previousSlot uint64, targetSlot uint64) [][32]byte {
-	tempRoots := [][32]byte{}
-	epochLength := params.BeaconConfig().SlotsPerEpoch
-
-	// Loop through current states to filter for valid boundary states.
-	for i := len(stateSlice) - 1; stateSlice[i] != previousBoundaryRoot && i >= 0; i-- {
-		currentSlot := s.initSyncState[stateSlice[i]].Slot()
-		// Skip if the current slot is larger than the previous epoch
-		// boundary.
-		if currentSlot > targetSlot-epochLength {
-			continue
-		}
-		tempRoots = append(tempRoots, stateSlice[i])
-
-		// Switch target slot if the current slot is greater than
-		// 1 epoch boundary from the previously saved boundary slot.
-		if currentSlot > previousSlot+epochLength {
-			currentSlot = helpers.RoundUpToNearestEpoch(currentSlot)
-			targetSlot = currentSlot
-			continue
-		}
-		break
-	}
-	// Reverse to append the roots in ascending order corresponding
-	// to the respective slots.
-	tempRoots = bytesutil.ReverseBytes32Slice(tempRoots)
-	return tempRoots
-}
-
-// prune for states past the current finalized checkpoint.
-func (s *Service) pruneOldStates() {
-	prunedBoundaryRoots := [][32]byte{}
-	for _, rt := range s.boundaryRoots {
-		st, ok := s.initSyncState[rt]
-		// Skip non-existent roots.
-		if !ok {
-			continue
-		}
-		if st.Slot() < helpers.StartSlot(s.FinalizedCheckpt().Epoch) {
-			delete(s.initSyncState, rt)
-			continue
-		}
-		prunedBoundaryRoots = append(prunedBoundaryRoots, rt)
-	}
-	s.boundaryRoots = prunedBoundaryRoots
-}
-
-// prune cache for non-boundary states.
-func (s *Service) pruneNonBoundaryStates() {
-	boundaryMap := make(map[[32]byte]bool)
-	for i := range s.boundaryRoots {
-		boundaryMap[s.boundaryRoots[i]] = true
-	}
-	for rt := range s.initSyncState {
-		if !boundaryMap[rt] {
-			delete(s.initSyncState, rt)
-		}
-	}
-}
-
-func (s *Service) pruneOldNonFinalizedStates() {
-	stateSlice := make([][32]byte, 0, len(s.initSyncState))
-	// Add epoch boundary roots to slice.
-	for rt := range s.initSyncState {
-		stateSlice = append(stateSlice, rt)
-	}
-
-	// Sort by slots.
-	sort.Slice(stateSlice, func(i int, j int) bool {
-		return s.initSyncState[stateSlice[i]].Slot() < s.initSyncState[stateSlice[j]].Slot()
-	})
-
-	boundaryMap := make(map[[32]byte]bool)
-	for i := range s.boundaryRoots {
-		boundaryMap[s.boundaryRoots[i]] = true
-	}
-	for _, rt := range stateSlice[:initialSyncCacheSize] {
-		if boundaryMap[rt] {
-			continue
-		}
-		delete(s.initSyncState, rt)
-	}
-}
-
 func (s *Service) generateState(ctx context.Context, startRoot [32]byte, endRoot [32]byte) (*stateTrie.BeaconState, error) {
 	preState, err := s.beaconDB.State(ctx, startRoot)
 	if err != nil {
 		return nil, err
 	}
 	if preState == nil {
+		if !s.stateGen.HasState(ctx, startRoot) {
+			if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+				return nil, errors.Wrap(err, "could not save initial sync blocks")
+			}
+			s.clearInitSyncBlocks()
+		}
 		preState, err = s.stateGen.StateByRoot(ctx, startRoot)
 		if err != nil {
 			return nil, err
@@ -183,7 +33,7 @@ func (s *Service) generateState(ctx context.Context, startRoot [32]byte, endRoot
 		return nil, err
 	}
 	var endBlock *ethpb.SignedBeaconBlock
-	if !featureconfig.Get().NoInitSyncBatchSaveBlocks && s.hasInitSyncBlock(endRoot) {
+	if s.hasInitSyncBlock(endRoot) {
 		endBlock = s.getInitSyncBlock(endRoot)
 		s.clearInitSyncBlocks()
 	} else {

beacon-chain/blockchain/init_sync_process_block_test.go

@@ -6,248 +6,27 @@ import (
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
-	messagediff "gopkg.in/d4l3k/messagediff.v1"
+	"gopkg.in/d4l3k/messagediff.v1"
 )
 
-func TestFilterBoundaryCandidates_FilterCorrect(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	cfg := &Config{BeaconDB: db}
-	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	st, err := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := uint64(0); i < 500; i++ {
-		if err := st.SetSlot(i); err != nil {
-			t.Fatal(err)
-		}
-		root := [32]byte{}
-		copy(root[:], bytesutil.Bytes32(i))
-		service.initSyncState[root] = st.Copy()
-		if helpers.IsEpochStart(i) {
-			service.boundaryRoots = append(service.boundaryRoots, root)
-		}
-	}
-	lastIndex := len(service.boundaryRoots) - 1
-	for i := uint64(500); i < 2000; i++ {
-		if err := st.SetSlot(i); err != nil {
-			t.Fatal(err)
-		}
-		root := [32]byte{}
-		copy(root[:], bytesutil.Bytes32(i))
-		service.initSyncState[root] = st.Copy()
-	}
-	// Set current state.
-	latestSlot := helpers.RoundUpToNearestEpoch(2000)
-	if err := st.SetSlot(latestSlot); err != nil {
-		t.Fatal(err)
-	}
-	lastRoot := [32]byte{}
-	copy(lastRoot[:], bytesutil.Bytes32(latestSlot))
-
-	service.initSyncState[lastRoot] = st.Copy()
-	service.finalizedCheckpt = &ethpb.Checkpoint{
-		Epoch: 0,
-		Root:  []byte{},
-	}
-	service.filterBoundaryCandidates(context.Background(), lastRoot, st)
-	if len(service.boundaryRoots[lastIndex+1:]) == 0 {
-		t.Fatal("Wanted non zero added boundary roots")
-	}
-	for _, rt := range service.boundaryRoots[lastIndex+1:] {
-		st, ok := service.initSyncState[rt]
-		if !ok {
-			t.Error("Root doen't exist in cache map")
-			continue
-		}
-		if !(helpers.IsEpochStart(st.Slot()) || helpers.IsEpochStart(st.Slot()-1) || helpers.IsEpochStart(st.Slot()+1)) {
-			t.Errorf("boundary roots not validly stored. They have slot %d", st.Slot())
-		}
-	}
-}
-
-func TestFilterBoundaryCandidates_HandleSkippedSlots(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	cfg := &Config{BeaconDB: db}
-	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	st, err := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := uint64(0); i < 500; i++ {
-		if err := st.SetSlot(i); err != nil {
-			t.Fatal(err)
-		}
-		root := [32]byte{}
-		copy(root[:], bytesutil.Bytes32(i))
-		service.initSyncState[root] = st.Copy()
-		if helpers.IsEpochStart(i) {
-			service.boundaryRoots = append(service.boundaryRoots, root)
-		}
-	}
-	lastIndex := len(service.boundaryRoots) - 1
-	for i := uint64(500); i < 2000; i++ {
-		if err := st.SetSlot(i); err != nil {
-			t.Fatal(err)
-		}
-		root := [32]byte{}
-		copy(root[:], bytesutil.Bytes32(i))
-		// save only for offsetted slots
-		if helpers.IsEpochStart(i + 10) {
-			service.initSyncState[root] = st.Copy()
-		}
-	}
-	// Set current state.
-	latestSlot := helpers.RoundUpToNearestEpoch(2000)
-	if err := st.SetSlot(latestSlot); err != nil {
-		t.Fatal(err)
-	}
-	lastRoot := [32]byte{}
-	copy(lastRoot[:], bytesutil.Bytes32(latestSlot))
-
-	service.initSyncState[lastRoot] = st.Copy()
-	service.finalizedCheckpt = &ethpb.Checkpoint{
-		Epoch: 0,
-		Root:  []byte{},
-	}
-	service.filterBoundaryCandidates(context.Background(), lastRoot, st)
-	if len(service.boundaryRoots[lastIndex+1:]) == 0 {
-		t.Fatal("Wanted non zero added boundary roots")
-	}
-	for _, rt := range service.boundaryRoots[lastIndex+1:] {
-		st, ok := service.initSyncState[rt]
-		if !ok {
-			t.Error("Root doen't exist in cache map")
-			continue
-		}
-		if st.Slot() >= 500 {
-			// Ignore head boundary root.
-			if st.Slot() == 2016 {
-				continue
-			}
-			if !helpers.IsEpochStart(st.Slot() + 10) {
-				t.Errorf("boundary roots not validly stored. They have slot %d "+
-					"instead of the offset from epoch start", st.Slot())
-			}
-		}
-	}
-}
-
-func TestPruneOldStates_AlreadyFinalized(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	cfg := &Config{BeaconDB: db}
-	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	st, err := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := uint64(100); i < 200; i++ {
-		if err := st.SetSlot(i); err != nil {
-			t.Fatal(err)
-		}
-		root := [32]byte{}
-		copy(root[:], bytesutil.Bytes32(i))
-		service.initSyncState[root] = st.Copy()
-		service.boundaryRoots = append(service.boundaryRoots, root)
-	}
-	finalizedEpoch := uint64(5)
-	service.finalizedCheckpt = &ethpb.Checkpoint{Epoch: finalizedEpoch}
-	service.pruneOldStates()
-	for _, rt := range service.boundaryRoots {
-		st, ok := service.initSyncState[rt]
-		if !ok {
-			t.Error("Root doen't exist in cache map")
-			continue
-		}
-		if st.Slot() < helpers.StartSlot(finalizedEpoch) {
-			t.Errorf("State with slot %d still exists and not pruned", st.Slot())
-		}
-	}
-}
-
-func TestPruneNonBoundary_CanPrune(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	cfg := &Config{BeaconDB: db}
-	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	st, err := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := uint64(0); i < 2000; i++ {
-		if err := st.SetSlot(i); err != nil {
-			t.Fatal(err)
-		}
-		root := [32]byte{}
-		copy(root[:], bytesutil.Bytes32(i))
-		service.initSyncState[root] = st.Copy()
-		if helpers.IsEpochStart(i) {
-			service.boundaryRoots = append(service.boundaryRoots, root)
-		}
-	}
-	service.pruneNonBoundaryStates()
-	for _, rt := range service.boundaryRoots {
-		st, ok := service.initSyncState[rt]
-		if !ok {
-			t.Error("Root doesn't exist in cache map")
-			continue
-		}
-		if !helpers.IsEpochStart(st.Slot()) {
-			t.Errorf("Non boundary state with slot %d still exists and not pruned", st.Slot())
-		}
-	}
-}
-
 func TestGenerateState_CorrectlyGenerated(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	cfg := &Config{BeaconDB: db, StateGen: stategen.New(db, cache.NewStateSummaryCache())}
+	db, sc := testDB.SetupDB(t)
+	cfg := &Config{BeaconDB: db, StateGen: stategen.New(db, sc)}
 	service, err := NewService(context.Background(), cfg)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	beaconState, privs := testutil.DeterministicGenesisState(t, 32)
-	genesisBlock := blocks.NewGenesisBlock([]byte{})
-	bodyRoot, err := ssz.HashTreeRoot(genesisBlock.Block)
+	genesisBlock := testutil.NewBeaconBlock()
+	bodyRoot, err := stateutil.BlockRoot(genesisBlock.Block)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -277,7 +56,7 @@ func TestGenerateState_CorrectlyGenerated(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	genRoot, err := ssz.HashTreeRoot(genesisBlock)
+	genRoot, err := stateutil.BlockRoot(genesisBlock.Block)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -302,7 +81,7 @@ func TestGenerateState_CorrectlyGenerated(t *testing.T) {
 		}
 		lastBlock = block
 	}
-	root, err := ssz.HashTreeRoot(lastBlock.Block)
+	root, err := stateutil.BlockRoot(lastBlock.Block)
 	if err != nil {
 		t.Fatal(err)
 	}

beacon-chain/blockchain/log.go

@@ -1,6 +1,7 @@
 package blockchain
 
 import (
+	"encoding/hex"
 	"fmt"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
@@ -15,7 +16,6 @@ var log = logrus.WithField("prefix", "blockchain")
 // logs state transition related data every slot.
 func logStateTransitionData(b *ethpb.BeaconBlock) {
 	log.WithFields(logrus.Fields{
-		"slot":              b.Slot,
 		"attestations":      len(b.Body.Attestations),
 		"deposits":          len(b.Body.Deposits),
 		"attesterSlashings": len(b.Body.AttesterSlashings),
@@ -24,6 +24,16 @@ func logStateTransitionData(b *ethpb.BeaconBlock) {
 	}).Info("Finished applying state transition")
 }
 
+func logBlockSyncStatus(block *ethpb.BeaconBlock, blockRoot [32]byte, finalized *ethpb.Checkpoint) {
+	log.WithFields(logrus.Fields{
+		"slot":           block.Slot,
+		"block":          fmt.Sprintf("0x%s...", hex.EncodeToString(blockRoot[:])[:8]),
+		"epoch":          helpers.SlotToEpoch(block.Slot),
+		"finalizedEpoch": finalized.Epoch,
+		"finalizedRoot":  fmt.Sprintf("0x%s...", hex.EncodeToString(finalized.Root[:])[:8]),
+	}).Info("Synced new block")
+}
+
 func logEpochData(beaconState *stateTrie.BeaconState) {
 	log.WithFields(logrus.Fields{
 		"epoch":                  helpers.CurrentEpoch(beaconState),

beacon-chain/blockchain/metrics.go

@@ -1,13 +1,17 @@
 package blockchain
 
 import (
+	"time"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/roughtime"
 )
 
 var (
@@ -19,10 +23,11 @@ var (
 		Name: "beacon_head_slot",
 		Help: "Slot of the head block of the beacon chain",
 	})
-	competingBlks = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "competing_blocks",
-		Help: "The # of blocks received and processed from a competing chain",
+	clockTimeSlot = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "beacon_clock_time_slot",
+		Help: "The current slot based on the genesis time and current clock",
 	})
+
 	headFinalizedEpoch = promauto.NewGauge(prometheus.GaugeOpts{
 		Name: "head_finalized_epoch",
 		Help: "Last finalized epoch of the head state",
@@ -73,17 +78,36 @@ var (
 	})
 	totalEligibleBalances = promauto.NewGauge(prometheus.GaugeOpts{
 		Name: "total_eligible_balances",
-		Help: "The total amount of ether, in gwei, that has been used in voting attestation target of previous epoch",
+		Help: "The total amount of ether, in gwei, that is eligible for voting of previous epoch",
 	})
 	totalVotedTargetBalances = promauto.NewGauge(prometheus.GaugeOpts{
 		Name: "total_voted_target_balances",
-		Help: "The total amount of ether, in gwei, that is eligible for voting of previous epoch",
+		Help: "The total amount of ether, in gwei, that has been used in voting attestation target of previous epoch",
 	})
+	reorgCount = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "beacon_reorg_total",
+		Help: "Count the number of times beacon chain has a reorg",
+	})
+	sentBlockPropagationHistogram = promauto.NewHistogram(
+		prometheus.HistogramOpts{
+			Name:    "block_sent_latency_milliseconds",
+			Help:    "Captures blocks broadcast time. Blocks sent in milliseconds distribution",
+			Buckets: []float64{1000, 2000, 3000, 4000, 5000, 6000},
+		},
+	)
+	attestationInclusionDelay = promauto.NewHistogram(
+		prometheus.HistogramOpts{
+			Name:    "attestation_inclusion_delay_slots",
+			Help:    "The number of slots between att.Slot and block.Slot",
+			Buckets: []float64{1, 2, 3, 4, 6, 32, 64},
+		},
+	)
 )
 
 // reportSlotMetrics reports slot related metrics.
-func reportSlotMetrics(currentSlot uint64, headSlot uint64, finalizedCheckpoint *ethpb.Checkpoint) {
-	beaconSlot.Set(float64(currentSlot))
+func reportSlotMetrics(stateSlot uint64, headSlot uint64, clockSlot uint64, finalizedCheckpoint *ethpb.Checkpoint) {
+	clockTimeSlot.Set(float64(clockSlot))
+	beaconSlot.Set(float64(stateSlot))
 	beaconHeadSlot.Set(float64(headSlot))
 	if finalizedCheckpoint != nil {
 		headFinalizedEpoch.Set(float64(finalizedCheckpoint.Epoch))
@@ -114,6 +138,7 @@ func reportEpochMetrics(state *stateTrie.BeaconState) {
 	for i, validator := range state.Validators() {
 		bal, err := state.BalanceAtIndex(uint64(i))
 		if err != nil {
+			log.Errorf("Could not load validator balance: %v", err)
 			continue
 		}
 		if validator.Slashed {
@@ -174,7 +199,25 @@ func reportEpochMetrics(state *stateTrie.BeaconState) {
 	currentEth1DataDepositCount.Set(float64(state.Eth1Data().DepositCount))
 
 	if precompute.Balances != nil {
-		totalEligibleBalances.Set(float64(precompute.Balances.PrevEpoch))
-		totalVotedTargetBalances.Set(float64(precompute.Balances.PrevEpochTargetAttesters))
+		totalEligibleBalances.Set(float64(precompute.Balances.ActivePrevEpoch))
+		totalVotedTargetBalances.Set(float64(precompute.Balances.PrevEpochTargetAttested))
+	}
+}
+
+// This captures metrics for block sent time by subtracts slot start time.
+func captureSentTimeMetric(genesisTime uint64, currentSlot uint64) error {
+	startTime, err := helpers.SlotToTime(genesisTime, currentSlot)
+	if err != nil {
+		return err
+	}
+	diffMs := roughtime.Now().Sub(startTime) / time.Millisecond
+	sentBlockPropagationHistogram.Observe(float64(diffMs))
+
+	return nil
+}
+
+func reportAttestationInclusion(blk *ethpb.BeaconBlock) {
+	for _, att := range blk.Body.Attestations {
+		attestationInclusionDelay.Observe(float64(blk.Slot - att.Data.Slot))
 	}
 }

beacon-chain/blockchain/process_attestation.go

@@ -10,6 +10,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/flags"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/roughtime"
 	"go.opencensus.io/trace"
 )
@@ -19,7 +20,8 @@ import (
 var ErrTargetRootNotInDB = errors.New("target root does not exist in db")
 
 // onAttestation is called whenever an attestation is received, verifies the attestation is valid and saves
-/// it to the DB.
+// it to the DB. As a stateless function, this does not hold nor delay attestation based on the spec descriptions.
+// The delay is handled by the caller in `processAttestation`.
 //
 // Spec pseudocode definition:
 //   def on_attestation(store: Service, attestation: Attestation) -> None:
@@ -35,14 +37,12 @@ var ErrTargetRootNotInDB = errors.New("target root does not exist in db")
 //    current_epoch = compute_epoch_at_slot(get_current_slot(store))
 //    # Use GENESIS_EPOCH for previous when genesis to avoid underflow
 //    previous_epoch = current_epoch - 1 if current_epoch > GENESIS_EPOCH else GENESIS_EPOCH
+//    # If attestation target is from a future epoch, delay consideration until the epoch arrives
 //    assert target.epoch in [current_epoch, previous_epoch]
 //    assert target.epoch == compute_epoch_at_slot(attestation.data.slot)
 //
 //    # Attestations target be for a known block. If target block is unknown, delay consideration until the block is found
 //    assert target.root in store.blocks
-//    # Attestations cannot be from future epochs. If they are, delay consideration until the epoch arrives
-//    base_state = store.block_states[target.root].copy()
-//    assert store.time >= base_state.genesis_time + compute_start_slot_at_epoch(target.epoch) * SECONDS_PER_SLOT
 //
 //    # Attestations must be for a known block. If block is unknown, delay consideration until the block is found
 //    assert attestation.data.beacon_block_root in store.blocks
@@ -67,12 +67,22 @@ var ErrTargetRootNotInDB = errors.New("target root does not exist in db")
 //    for i in indexed_attestation.attesting_indices:
 //        if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch:
 //            store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=attestation.data.beacon_block_root)
+// TODO(#6072): This code path is highly untested. Requires comprehensive tests and simpler refactoring.
 func (s *Service) onAttestation(ctx context.Context, a *ethpb.Attestation) ([]uint64, error) {
 	ctx, span := trace.StartSpan(ctx, "blockchain.onAttestation")
 	defer span.End()
 
+	if a == nil {
+		return nil, errors.New("nil attestation")
+	}
+	if a.Data == nil {
+		return nil, errors.New("nil attestation.Data field")
+	}
+	if a.Data.Target == nil {
+		return nil, errors.New("nil attestation.Data.Target field")
+	}
+
 	tgt := stateTrie.CopyCheckpoint(a.Data.Target)
-	tgtSlot := helpers.StartSlot(tgt.Epoch)
 
 	if helpers.SlotToEpoch(a.Data.Slot) != a.Data.Target.Epoch {
 		return nil, fmt.Errorf("data slot is not in the same epoch as target %d != %d", helpers.SlotToEpoch(a.Data.Slot), a.Data.Target.Epoch)
@@ -97,22 +107,22 @@ func (s *Service) onAttestation(ctx context.Context, a *ethpb.Attestation) ([]ui
 		return nil, err
 	}
 
-	// Verify Attestations cannot be from future epochs.
-	if err := helpers.VerifySlotTime(genesisTime, tgtSlot, helpers.TimeShiftTolerance); err != nil {
-		return nil, errors.Wrap(err, "could not verify attestation target slot")
-	}
-
 	// Verify attestation beacon block is known and not from the future.
 	if err := s.verifyBeaconBlock(ctx, a.Data); err != nil {
 		return nil, errors.Wrap(err, "could not verify attestation beacon block")
 	}
 
+	// Verify LMG GHOST and FFG votes are consistent with each other.
+	if err := s.verifyLMDFFGConsistent(ctx, a.Data.Target.Epoch, a.Data.Target.Root, a.Data.BeaconBlockRoot); err != nil {
+		return nil, errors.Wrap(err, "could not verify attestation beacon block")
+	}
+
 	// Verify attestations can only affect the fork choice of subsequent slots.
-	if err := helpers.VerifySlotTime(genesisTime, a.Data.Slot, helpers.TimeShiftTolerance); err != nil {
+	if err := helpers.VerifySlotTime(genesisTime, a.Data.Slot+1, params.BeaconNetworkConfig().MaximumGossipClockDisparity); err != nil {
 		return nil, err
 	}
 
-	// Use the target state to to validate attestation and calculate the committees.
+	// Use the target state to validate attestation and calculate the committees.
 	indexedAtt, err := s.verifyAttestation(ctx, baseState, a)
 	if err != nil {
 		return nil, err
@@ -128,12 +138,6 @@ func (s *Service) onAttestation(ctx context.Context, a *ethpb.Attestation) ([]ui
 	if indexedAtt.AttestingIndices == nil {
 		return nil, errors.New("nil attesting indices")
 	}
-	if a.Data == nil {
-		return nil, errors.New("nil att data")
-	}
-	if a.Data.Target == nil {
-		return nil, errors.New("nil att target")
-	}
 
 	// Update forkchoice store with the new attestation for updating weight.
 	s.forkChoiceStore.ProcessAttestation(ctx, indexedAtt.AttestingIndices, bytesutil.ToBytes32(a.Data.BeaconBlockRoot), a.Data.Target.Epoch)

beacon-chain/blockchain/process_attestation_helpers.go

@@ -3,7 +3,6 @@ package blockchain
 import (
 	"bytes"
 	"context"
-	"encoding/hex"
 	"fmt"
 
 	"github.com/pkg/errors"
@@ -15,7 +14,6 @@ import (
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/attestationutil"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
@@ -31,44 +29,20 @@ func (s *Service) getAttPreState(ctx context.Context, c *ethpb.Checkpoint) (*sta
 		return cachedState, nil
 	}
 
-	var baseState *stateTrie.BeaconState
-	if !featureconfig.Get().DisableNewStateMgmt {
-		baseState, err = s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(c.Root))
-		if err != nil {
-			return nil, errors.Wrapf(err, "could not get pre state for slot %d", helpers.StartSlot(c.Epoch))
-		}
-	} else {
-		if featureconfig.Get().CheckHeadState {
-			headRoot, err := s.HeadRoot(ctx)
-			if err != nil {
-				return nil, errors.Wrapf(err, "could not get head root")
-			}
-			if bytes.Equal(headRoot, c.Root) {
-				st, err := s.HeadState(ctx)
-				if err != nil {
-					return nil, errors.Wrapf(err, "could not get head state")
-				}
-				if err := s.checkpointState.AddCheckpointState(&cache.CheckpointState{
-					Checkpoint: c,
-					State:      st.Copy(),
-				}); err != nil {
-					return nil, errors.Wrap(err, "could not saved checkpoint state to cache")
-				}
-				return st, nil
-			}
-		}
-
-		baseState, err = s.beaconDB.State(ctx, bytesutil.ToBytes32(c.Root))
-		if err != nil {
-			return nil, errors.Wrapf(err, "could not get pre state for slot %d", helpers.StartSlot(c.Epoch))
+	if !s.stateGen.HasState(ctx, bytesutil.ToBytes32(c.Root)) {
+		if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+			return nil, errors.Wrap(err, "could not save initial sync blocks")
 		}
+		s.clearInitSyncBlocks()
 	}
 
-	if baseState == nil {
-		return nil, fmt.Errorf("pre state of target block %d does not exist", helpers.StartSlot(c.Epoch))
+	baseState, err := s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(c.Root))
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not get pre state for slot %d", helpers.StartSlot(c.Epoch))
 	}
 
 	if helpers.StartSlot(c.Epoch) > baseState.Slot() {
+		baseState = baseState.Copy()
 		baseState, err = state.ProcessSlots(ctx, baseState, helpers.StartSlot(c.Epoch))
 		if err != nil {
 			return nil, errors.Wrapf(err, "could not process slots up to %d", helpers.StartSlot(c.Epoch))
@@ -77,12 +51,13 @@ func (s *Service) getAttPreState(ctx context.Context, c *ethpb.Checkpoint) (*sta
 
 	if err := s.checkpointState.AddCheckpointState(&cache.CheckpointState{
 		Checkpoint: c,
-		State:      baseState.Copy(),
+		State:      baseState,
 	}); err != nil {
 		return nil, errors.Wrap(err, "could not saved checkpoint state to cache")
 	}
 
 	return baseState, nil
+
 }
 
 // verifyAttTargetEpoch validates attestation is from the current or previous epoch.
@@ -115,6 +90,20 @@ func (s *Service) verifyBeaconBlock(ctx context.Context, data *ethpb.Attestation
 	return nil
 }
 
+// verifyLMDFFGConsistent verifies LMD GHOST and FFG votes are consistent with each other.
+func (s *Service) verifyLMDFFGConsistent(ctx context.Context, ffgEpoch uint64, ffgRoot []byte, lmdRoot []byte) error {
+	ffgSlot := helpers.StartSlot(ffgEpoch)
+	r, err := s.ancestor(ctx, lmdRoot, ffgSlot)
+	if err != nil {
+		return err
+	}
+	if !bytes.Equal(ffgRoot, r) {
+		return errors.New("FFG and LMD votes are not consistent")
+	}
+
+	return nil
+}
+
 // verifyAttestation validates input attestation is valid.
 func (s *Service) verifyAttestation(ctx context.Context, baseState *stateTrie.BeaconState, a *ethpb.Attestation) (*ethpb.IndexedAttestation, error) {
 	committee, err := helpers.BeaconCommitteeFromState(baseState, a.Data.Slot, a.Data.CommitteeIndex)
@@ -123,42 +112,7 @@ func (s *Service) verifyAttestation(ctx context.Context, baseState *stateTrie.Be
 	}
 	indexedAtt := attestationutil.ConvertToIndexed(ctx, a, committee)
 	if err := blocks.VerifyIndexedAttestation(ctx, baseState, indexedAtt); err != nil {
-		if err == helpers.ErrSigFailedToVerify {
-			// When sig fails to verify, check if there's a differences in committees due to
-			// different seeds.
-			var aState *stateTrie.BeaconState
-			var err error
-			if !featureconfig.Get().DisableNewStateMgmt {
-				aState, err = s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
-				if err != nil {
-					return nil, err
-				}
-			} else {
-				aState, err = s.beaconDB.State(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
-				if err != nil {
-					return nil, err
-				}
-			}
-			if aState == nil {
-				return nil, fmt.Errorf("nil state for block root %#x", a.Data.BeaconBlockRoot)
-			}
-			epoch := helpers.SlotToEpoch(a.Data.Slot)
-			origSeed, err := helpers.Seed(baseState, epoch, params.BeaconConfig().DomainBeaconAttester)
-			if err != nil {
-				return nil, errors.Wrap(err, "could not get original seed")
-			}
-
-			aSeed, err := helpers.Seed(aState, epoch, params.BeaconConfig().DomainBeaconAttester)
-			if err != nil {
-				return nil, errors.Wrap(err, "could not get attester's seed")
-			}
-			if origSeed != aSeed {
-				return nil, fmt.Errorf("could not verify indexed attestation due to differences in seeds: %v != %v",
-					hex.EncodeToString(bytesutil.Trunc(origSeed[:])), hex.EncodeToString(bytesutil.Trunc(aSeed[:])))
-			}
-		}
 		return nil, errors.Wrap(err, "could not verify indexed attestation")
 	}
-
 	return indexedAtt, nil
 }

beacon-chain/blockchain/process_attestation_test.go

@@ -7,14 +7,13 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
@@ -23,13 +22,12 @@ import (
 
 func TestStore_OnAttestation(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	cfg := &Config{
 		BeaconDB:        db,
 		ForkChoiceStore: protoarray.New(0, 0, [32]byte{}),
-		StateGen:        stategen.New(db, cache.NewStateSummaryCache()),
+		StateGen:        stategen.New(db, sc),
 	}
 	service, err := NewService(ctx, cfg)
 	if err != nil {
@@ -45,7 +43,7 @@ func TestStore_OnAttestation(t *testing.T) {
 	if err := db.SaveBlock(ctx, BlkWithOutState); err != nil {
 		t.Fatal(err)
 	}
-	BlkWithOutStateRoot, err := ssz.HashTreeRoot(BlkWithOutState.Block)
+	BlkWithOutStateRoot, err := stateutil.BlockRoot(BlkWithOutState.Block)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -54,7 +52,7 @@ func TestStore_OnAttestation(t *testing.T) {
 	if err := db.SaveBlock(ctx, BlkWithStateBadAtt); err != nil {
 		t.Fatal(err)
 	}
-	BlkWithStateBadAttRoot, err := ssz.HashTreeRoot(BlkWithStateBadAtt.Block)
+	BlkWithStateBadAttRoot, err := stateutil.BlockRoot(BlkWithStateBadAtt.Block)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -72,7 +70,7 @@ func TestStore_OnAttestation(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	BlkWithValidStateRoot, err := ssz.HashTreeRoot(BlkWithValidState.Block)
+	BlkWithValidStateRoot, err := stateutil.BlockRoot(BlkWithValidState.Block)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -114,7 +112,7 @@ func TestStore_OnAttestation(t *testing.T) {
 			a:             &ethpb.Attestation{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{Root: BlkWithOutStateRoot[:]}}},
 			s:             &pb.BeaconState{},
 			wantErr:       true,
-			wantErrString: "could not get pre state for slot 0: unknown boundary state",
+			wantErrString: "could not get pre state for slot 0",
 		},
 		{
 			name: "process attestation doesn't match current epoch",
@@ -124,13 +122,34 @@ func TestStore_OnAttestation(t *testing.T) {
 			wantErr:       true,
 			wantErrString: "does not match current epoch",
 		},
+		{
+			name:          "process nil field (a.Target) in attestation",
+			a:             nil,
+			s:             &pb.BeaconState{},
+			wantErr:       true,
+			wantErrString: "nil attestation",
+		},
+		{
+			name:          "process nil field (a.Data) in attestation",
+			a:             &ethpb.Attestation{},
+			s:             &pb.BeaconState{},
+			wantErr:       true,
+			wantErrString: "nil attestation.Data field",
+		},
+		{
+			name:          "process nil field (a.Target) in attestation",
+			a:             &ethpb.Attestation{Data: &ethpb.AttestationData{}},
+			s:             &pb.BeaconState{},
+			wantErr:       true,
+			wantErrString: "nil attestation.Data.Target field",
+		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			_, err := service.onAttestation(ctx, tt.a)
 			if tt.wantErr {
-				if !strings.Contains(err.Error(), tt.wantErrString) {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErrString) {
 					t.Errorf("Store.onAttestation() error = %v, wantErr = %v", err, tt.wantErrString)
 				}
 			} else {
@@ -142,12 +161,11 @@ func TestStore_OnAttestation(t *testing.T) {
 
 func TestStore_SaveCheckpointState(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	cfg := &Config{
 		BeaconDB: db,
-		StateGen: stategen.New(db, cache.NewStateSummaryCache()),
+		StateGen: stategen.New(db, sc),
 	}
 	service, err := NewService(ctx, cfg)
 	if err != nil {
@@ -263,12 +281,11 @@ func TestStore_SaveCheckpointState(t *testing.T) {
 
 func TestStore_UpdateCheckpointState(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	cfg := &Config{
 		BeaconDB: db,
-		StateGen: stategen.New(db, cache.NewStateSummaryCache()),
+		StateGen: stategen.New(db, sc),
 	}
 	service, err := NewService(ctx, cfg)
 	if err != nil {
@@ -328,8 +345,7 @@ func TestStore_UpdateCheckpointState(t *testing.T) {
 
 func TestAttEpoch_MatchPrevEpoch(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 
 	cfg := &Config{BeaconDB: db}
 	service, err := NewService(ctx, cfg)
@@ -348,8 +364,7 @@ func TestAttEpoch_MatchPrevEpoch(t *testing.T) {
 
 func TestAttEpoch_MatchCurrentEpoch(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 
 	cfg := &Config{BeaconDB: db}
 	service, err := NewService(ctx, cfg)
@@ -368,8 +383,7 @@ func TestAttEpoch_MatchCurrentEpoch(t *testing.T) {
 
 func TestAttEpoch_NotMatch(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 
 	cfg := &Config{BeaconDB: db}
 	service, err := NewService(ctx, cfg)
@@ -389,8 +403,7 @@ func TestAttEpoch_NotMatch(t *testing.T) {
 
 func TestVerifyBeaconBlock_NoBlock(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 
 	cfg := &Config{BeaconDB: db}
 	service, err := NewService(ctx, cfg)
@@ -406,8 +419,7 @@ func TestVerifyBeaconBlock_NoBlock(t *testing.T) {
 
 func TestVerifyBeaconBlock_futureBlock(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 
 	cfg := &Config{BeaconDB: db}
 	service, err := NewService(ctx, cfg)
@@ -419,7 +431,7 @@ func TestVerifyBeaconBlock_futureBlock(t *testing.T) {
 	if err := service.beaconDB.SaveBlock(ctx, b); err != nil {
 		t.Fatal(err)
 	}
-	r, err := ssz.HashTreeRoot(b.Block)
+	r, err := stateutil.BlockRoot(b.Block)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -433,8 +445,7 @@ func TestVerifyBeaconBlock_futureBlock(t *testing.T) {
 
 func TestVerifyBeaconBlock_OK(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 
 	cfg := &Config{BeaconDB: db}
 	service, err := NewService(ctx, cfg)
@@ -446,7 +457,7 @@ func TestVerifyBeaconBlock_OK(t *testing.T) {
 	if err := service.beaconDB.SaveBlock(ctx, b); err != nil {
 		t.Fatal(err)
 	}
-	r, err := ssz.HashTreeRoot(b.Block)
+	r, err := stateutil.BlockRoot(b.Block)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -456,3 +467,68 @@ func TestVerifyBeaconBlock_OK(t *testing.T) {
 		t.Error("Did not receive the wanted error")
 	}
 }
+
+func TestVerifyLMDFFGConsistent_NotOK(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b32 := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 32}}
+	if err := service.beaconDB.SaveBlock(ctx, b32); err != nil {
+		t.Fatal(err)
+	}
+	r32, err := stateutil.BlockRoot(b32.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+	b33 := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 33, ParentRoot: r32[:]}}
+	if err := service.beaconDB.SaveBlock(ctx, b33); err != nil {
+		t.Fatal(err)
+	}
+	r33, err := stateutil.BlockRoot(b33.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	wanted := "FFG and LMD votes are not consistent"
+	if err := service.verifyLMDFFGConsistent(context.Background(), 1, []byte{'a'}, r33[:]); err.Error() != wanted {
+		t.Error("Did not get wanted error")
+	}
+}
+
+func TestVerifyLMDFFGConsistent_OK(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b32 := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 32}}
+	if err := service.beaconDB.SaveBlock(ctx, b32); err != nil {
+		t.Fatal(err)
+	}
+	r32, err := stateutil.BlockRoot(b32.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+	b33 := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 33, ParentRoot: r32[:]}}
+	if err := service.beaconDB.SaveBlock(ctx, b33); err != nil {
+		t.Fatal(err)
+	}
+	r33, err := stateutil.BlockRoot(b33.Block)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := service.verifyLMDFFGConsistent(context.Background(), 1, r32[:], r33[:]); err != nil {
+		t.Errorf("Could not verify LMD and FFG votes to be consistent: %v", err)
+	}
+}

beacon-chain/blockchain/process_block.go

@@ -11,7 +11,6 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/flags"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	"github.com/prysmaticlabs/prysm/shared/attestationutil"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/featureconfig"
@@ -24,6 +23,8 @@ import (
 var initialSyncBlockCacheSize = 2 * params.BeaconConfig().SlotsPerEpoch
 
 // onBlock is called when a gossip block is received. It runs regular state transition on the block.
+// The block's signing root should be computed before calling this method to avoid redundant
+// computation in this method and methods it calls into.
 //
 // Spec pseudocode definition:
 //   def on_block(store: Store, block: BeaconBlock) -> None:
@@ -40,7 +41,7 @@ var initialSyncBlockCacheSize = 2 * params.BeaconConfig().SlotsPerEpoch
 //        store.finalized_checkpoint.root
 //    )
 //    # Check that block is later than the finalized epoch slot
-//    assert block.slot > compute_start_slot_of_epoch(store.finalized_checkpoint.epoch)
+//    assert block.slot > compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)
 //    # Check the block is valid and compute the post-state
 //    state = state_transition(pre_state, block)
 //    # Add new state for this block to the store
@@ -54,7 +55,7 @@ var initialSyncBlockCacheSize = 2 * params.BeaconConfig().SlotsPerEpoch
 //    # Update finalized checkpoint
 //    if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch:
 //        store.finalized_checkpoint = state.finalized_checkpoint
-func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock) (*stateTrie.BeaconState, error) {
+func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock, blockRoot [32]byte) (*stateTrie.BeaconState, error) {
 	ctx, span := trace.StartSpan(ctx, "blockchain.onBlock")
 	defer span.End()
 
@@ -70,14 +71,10 @@ func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock)
 		return nil, err
 	}
 
-	root, err := stateutil.BlockRoot(b)
-	if err != nil {
-		return nil, errors.Wrapf(err, "could not get signing root of block %d", b.Slot)
-	}
 	log.WithFields(logrus.Fields{
 		"slot": b.Slot,
-		"root": fmt.Sprintf("0x%s...", hex.EncodeToString(root[:])[:8]),
-	}).Info("Executing state transition on block")
+		"root": fmt.Sprintf("0x%s...", hex.EncodeToString(blockRoot[:])[:8]),
+	}).Debug("Executing state transition on block")
 
 	postState, err := state.ExecuteStateTransition(ctx, preState, signed)
 	if err != nil {
@@ -88,18 +85,12 @@ func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock)
 		return nil, errors.Wrapf(err, "could not save block from slot %d", b.Slot)
 	}
 
-	if err := s.insertBlockToForkChoiceStore(ctx, b, root, postState); err != nil {
+	if err := s.insertBlockToForkChoiceStore(ctx, b, blockRoot, postState); err != nil {
 		return nil, errors.Wrapf(err, "could not insert block %d to fork choice store", b.Slot)
 	}
 
-	if !featureconfig.Get().DisableNewStateMgmt {
-		if err := s.stateGen.SaveState(ctx, root, postState); err != nil {
-			return nil, errors.Wrap(err, "could not save state")
-		}
-	} else {
-		if err := s.beaconDB.SaveState(ctx, postState, root); err != nil {
-			return nil, errors.Wrap(err, "could not save state")
-		}
+	if err := s.stateGen.SaveState(ctx, blockRoot, postState); err != nil {
+		return nil, errors.Wrap(err, "could not save state")
 	}
 
 	// Update justified check point.
@@ -111,27 +102,15 @@ func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock)
 
 	// Update finalized check point. Prune the block cache and helper caches on every new finalized epoch.
 	if postState.FinalizedCheckpointEpoch() > s.finalizedCheckpt.Epoch {
-		if !featureconfig.Get().NoInitSyncBatchSaveBlocks {
-			if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
-				return nil, err
-			}
-			s.clearInitSyncBlocks()
+		if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+			return nil, err
 		}
+		s.clearInitSyncBlocks()
 
 		if err := s.beaconDB.SaveFinalizedCheckpoint(ctx, postState.FinalizedCheckpoint()); err != nil {
 			return nil, errors.Wrap(err, "could not save finalized checkpoint")
 		}
 
-		if featureconfig.Get().DisableNewStateMgmt {
-			startSlot := helpers.StartSlot(s.prevFinalizedCheckpt.Epoch)
-			endSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
-			if endSlot > startSlot {
-				if err := s.rmStatesOlderThanLastFinalized(ctx, startSlot, endSlot); err != nil {
-					return nil, errors.Wrapf(err, "could not delete states prior to finalized check point, range: %d, %d",
-						startSlot, endSlot)
-				}
-			}
-		}
 		fRoot := bytesutil.ToBytes32(postState.FinalizedCheckpoint().Root)
 
 		// Prune proto array fork choice nodes, all nodes before finalized check point will
@@ -147,15 +126,12 @@ func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock)
 			return nil, errors.Wrap(err, "could not save new justified")
 		}
 
-		if !featureconfig.Get().DisableNewStateMgmt {
-			fRoot := bytesutil.ToBytes32(postState.FinalizedCheckpoint().Root)
-			fBlock, err := s.beaconDB.Block(ctx, fRoot)
-			if err != nil {
-				return nil, errors.Wrap(err, "could not get finalized block to migrate")
-			}
-			if err := s.stateGen.MigrateToCold(ctx, fBlock.Block.Slot, fRoot); err != nil {
-				return nil, errors.Wrap(err, "could not migrate to cold")
-			}
+		fBlock, err := s.beaconDB.Block(ctx, fRoot)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not get finalized block to migrate")
+		}
+		if err := s.stateGen.MigrateToCold(ctx, fBlock.Block.Slot, fRoot); err != nil {
+			return nil, errors.Wrap(err, "could not migrate to cold")
 		}
 	}
 
@@ -185,13 +161,17 @@ func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock)
 		s.slashingPool.MarkIncludedAttesterSlashing(b.Body.AttesterSlashings[i])
 	}
 
+	defer reportAttestationInclusion(b)
+
 	return postState, nil
 }
 
 // onBlockInitialSyncStateTransition is called when an initial sync block is received.
 // It runs state transition on the block and without any BLS verification. The excluded BLS verification
 // includes attestation's aggregated signature. It also does not save attestations.
-func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed *ethpb.SignedBeaconBlock) error {
+// The block's signing root should be computed before calling this method to avoid redundant
+// computation in this method and methods it calls into.
+func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
 	ctx, span := trace.StartSpan(ctx, "blockchain.onBlock")
 	defer span.End()
 
@@ -206,42 +186,32 @@ func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed
 	if err != nil {
 		return err
 	}
+	// To invalidate cache for parent root because pre state will get mutated.
+	s.stateGen.DeleteHotStateInCache(bytesutil.ToBytes32(b.ParentRoot))
 
 	// Exit early if the pre state slot is higher than incoming block's slot.
 	if preState.Slot() >= signed.Block.Slot {
 		return nil
 	}
 
-	postState, err := state.ExecuteStateTransitionNoVerifyAttSigs(ctx, preState, signed)
+	var postState *stateTrie.BeaconState
+	if featureconfig.Get().InitSyncNoVerify {
+		postState, err = state.ExecuteStateTransitionNoVerifyAttSigs(ctx, preState, signed)
+	} else {
+		postState, err = state.ExecuteStateTransition(ctx, preState, signed)
+	}
 	if err != nil {
 		return errors.Wrap(err, "could not execute state transition")
 	}
 
-	root, err := stateutil.BlockRoot(b)
-	if err != nil {
-		return errors.Wrapf(err, "could not get signing root of block %d", b.Slot)
-	}
-	if !featureconfig.Get().NoInitSyncBatchSaveBlocks {
-		s.saveInitSyncBlock(root, signed)
-	} else {
-		if err := s.beaconDB.SaveBlock(ctx, signed); err != nil {
-			return errors.Wrapf(err, "could not save block from slot %d", b.Slot)
-		}
-	}
+	s.saveInitSyncBlock(blockRoot, signed)
 
-	if err := s.insertBlockToForkChoiceStore(ctx, b, root, postState); err != nil {
+	if err := s.insertBlockToForkChoiceStore(ctx, b, blockRoot, postState); err != nil {
 		return errors.Wrapf(err, "could not insert block %d to fork choice store", b.Slot)
 	}
 
-	if !featureconfig.Get().DisableNewStateMgmt {
-		if err := s.stateGen.SaveState(ctx, root, postState); err != nil {
-			return errors.Wrap(err, "could not save state")
-		}
-	} else {
-		s.initSyncStateLock.Lock()
-		defer s.initSyncStateLock.Unlock()
-		s.initSyncState[root] = postState.Copy()
-		s.filterBoundaryCandidates(ctx, root, postState)
+	if err := s.stateGen.SaveState(ctx, blockRoot, postState); err != nil {
+		return errors.Wrap(err, "could not save state")
 	}
 
 	if flags.Get().EnableArchive {
@@ -251,15 +221,8 @@ func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed
 		}
 	}
 
-	// Update justified check point.
-	if postState.CurrentJustifiedCheckpoint().Epoch > s.justifiedCheckpt.Epoch {
-		if err := s.updateJustified(ctx, postState); err != nil {
-			return err
-		}
-	}
-
 	// Rate limit how many blocks (2 epochs worth of blocks) a node keeps in the memory.
-	if len(s.getInitSyncBlocks()) > int(initialSyncBlockCacheSize) {
+	if uint64(len(s.getInitSyncBlocks())) > initialSyncBlockCacheSize {
 		if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
 			return err
 		}
@@ -268,27 +231,10 @@ func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed
 
 	// Update finalized check point. Prune the block cache and helper caches on every new finalized epoch.
 	if postState.FinalizedCheckpointEpoch() > s.finalizedCheckpt.Epoch {
-		if featureconfig.Get().DisableNewStateMgmt {
-			startSlot := helpers.StartSlot(s.prevFinalizedCheckpt.Epoch)
-			endSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
-			if endSlot > startSlot {
-				if err := s.rmStatesOlderThanLastFinalized(ctx, startSlot, endSlot); err != nil {
-					return errors.Wrapf(err, "could not delete states prior to finalized check point, range: %d, %d",
-						startSlot, endSlot)
-				}
-			}
-
-			if err := s.saveInitState(ctx, postState); err != nil {
-				return errors.Wrap(err, "could not save init sync finalized state")
-			}
-		}
-
-		if !featureconfig.Get().NoInitSyncBatchSaveBlocks {
-			if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
-				return err
-			}
-			s.clearInitSyncBlocks()
+		if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+			return err
 		}
+		s.clearInitSyncBlocks()
 
 		if err := s.beaconDB.SaveFinalizedCheckpoint(ctx, postState.FinalizedCheckpoint()); err != nil {
 			return errors.Wrap(err, "could not save finalized checkpoint")
@@ -297,31 +243,13 @@ func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed
 		s.prevFinalizedCheckpt = s.finalizedCheckpt
 		s.finalizedCheckpt = postState.FinalizedCheckpoint()
 
-		if err := s.finalizedImpliesNewJustified(ctx, postState); err != nil {
-			return errors.Wrap(err, "could not save new justified")
+		fRoot := bytesutil.ToBytes32(postState.FinalizedCheckpoint().Root)
+		fBlock, err := s.beaconDB.Block(ctx, fRoot)
+		if err != nil {
+			return errors.Wrap(err, "could not get finalized block to migrate")
 		}
-
-		if !featureconfig.Get().DisableNewStateMgmt {
-			fRoot := bytesutil.ToBytes32(postState.FinalizedCheckpoint().Root)
-			fBlock, err := s.beaconDB.Block(ctx, fRoot)
-			if err != nil {
-				return errors.Wrap(err, "could not get finalized block to migrate")
-			}
-			if err := s.stateGen.MigrateToCold(ctx, fBlock.Block.Slot, fRoot); err != nil {
-				return errors.Wrap(err, "could not migrate to cold")
-			}
-		}
-	}
-
-	if featureconfig.Get().DisableNewStateMgmt {
-		numOfStates := len(s.boundaryRoots)
-		if numOfStates > initialSyncCacheSize {
-			if err = s.persistCachedStates(ctx, numOfStates); err != nil {
-				return err
-			}
-		}
-		if len(s.initSyncState) > maxCacheSize {
-			s.pruneOldNonFinalizedStates()
+		if err := s.stateGen.MigrateToCold(ctx, fBlock.Block.Slot, fRoot); err != nil {
+			return errors.Wrap(err, "could not migrate to cold")
 		}
 	}
 
@@ -337,12 +265,6 @@ func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed
 		if err := helpers.UpdateProposerIndicesInCache(postState, helpers.CurrentEpoch(postState)); err != nil {
 			return err
 		}
-
-		if featureconfig.Get().DisableNewStateMgmt && helpers.IsEpochStart(postState.Slot()) {
-			if err := s.beaconDB.SaveState(ctx, postState, root); err != nil {
-				return errors.Wrap(err, "could not save state")
-			}
-		}
 	}
 
 	return nil
@@ -357,7 +279,7 @@ func (s *Service) insertBlockToForkChoiceStore(ctx context.Context, blk *ethpb.B
 
 	// Feed in block to fork choice store.
 	if err := s.forkChoiceStore.ProcessBlock(ctx,
-		blk.Slot, root, bytesutil.ToBytes32(blk.ParentRoot),
+		blk.Slot, root, bytesutil.ToBytes32(blk.ParentRoot), bytesutil.ToBytes32(blk.Body.Graffiti),
 		state.CurrentJustifiedCheckpoint().Epoch,
 		state.FinalizedCheckpointEpoch()); err != nil {
 		return errors.Wrap(err, "could not process block for proto array fork choice")

beacon-chain/blockchain/process_block_helpers.go

@@ -7,12 +7,11 @@ import (
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db/filters"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
+	"github.com/prysmaticlabs/prysm/shared/attestationutil"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/roughtime"
 	"github.com/prysmaticlabs/prysm/shared/traceutil"
@@ -21,7 +20,12 @@ import (
 
 // CurrentSlot returns the current slot based on time.
 func (s *Service) CurrentSlot() uint64 {
-	return uint64(roughtime.Now().Unix()-s.genesisTime.Unix()) / params.BeaconConfig().SecondsPerSlot
+	now := roughtime.Now().Unix()
+	genesis := s.genesisTime.Unix()
+	if now < genesis {
+		return 0
+	}
+	return uint64(now-genesis) / params.BeaconConfig().SecondsPerSlot
 }
 
 // getBlockPreState returns the pre state of an incoming block. It uses the parent root of the block
@@ -37,8 +41,16 @@ func (s *Service) getBlockPreState(ctx context.Context, b *ethpb.BeaconBlock) (*
 		return nil, err
 	}
 
-	// Verify block slot time is not from the feature.
-	if err := helpers.VerifySlotTime(preState.GenesisTime(), b.Slot, helpers.TimeShiftTolerance); err != nil {
+	preState, err = s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(b.ParentRoot))
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not get pre state for slot %d", b.Slot)
+	}
+	if preState == nil {
+		return nil, errors.Wrapf(err, "nil pre state for slot %d", b.Slot)
+	}
+
+	// Verify block slot time is not from the future.
+	if err := helpers.VerifySlotTime(preState.GenesisTime(), b.Slot, params.BeaconNetworkConfig().MaximumGossipClockDisparity); err != nil {
 		return nil, err
 	}
 
@@ -60,49 +72,28 @@ func (s *Service) verifyBlkPreState(ctx context.Context, b *ethpb.BeaconBlock) (
 	ctx, span := trace.StartSpan(ctx, "chainService.verifyBlkPreState")
 	defer span.End()
 
-	if !featureconfig.Get().DisableNewStateMgmt {
-		parentRoot := bytesutil.ToBytes32(b.ParentRoot)
-		if !s.stateGen.StateSummaryExists(ctx, parentRoot) {
-			return nil, errors.New("provided block root does not have block saved in the db")
+	parentRoot := bytesutil.ToBytes32(b.ParentRoot)
+	// Loosen the check to HasBlock because state summary gets saved in batches
+	// during initial syncing. There's no risk given a state summary object is just a
+	// a subset of the block object.
+	if !s.stateGen.StateSummaryExists(ctx, parentRoot) && !s.beaconDB.HasBlock(ctx, parentRoot) {
+		return nil, errors.New("could not reconstruct parent state")
+	}
+	if !s.stateGen.HasState(ctx, parentRoot) {
+		if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+			return nil, errors.Wrap(err, "could not save initial sync blocks")
 		}
-		preState, err := s.stateGen.StateByRoot(ctx, parentRoot)
-		if err != nil {
-			return nil, errors.Wrapf(err, "could not get pre state for slot %d", b.Slot)
-		}
-		if preState == nil {
-			return nil, errors.Wrapf(err, "nil pre state for slot %d", b.Slot)
-		}
-		return preState, nil // No copy needed from newly hydrated state gen object.
+		s.clearInitSyncBlocks()
+	}
+	preState, err := s.stateGen.StateByRootInitialSync(ctx, parentRoot)
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not get pre state for slot %d", b.Slot)
+	}
+	if preState == nil {
+		return nil, errors.Wrapf(err, "nil pre state for slot %d", b.Slot)
 	}
 
-	preState := s.initSyncState[bytesutil.ToBytes32(b.ParentRoot)]
-	var err error
-	if preState == nil {
-		if featureconfig.Get().CheckHeadState {
-			headRoot, err := s.HeadRoot(ctx)
-			if err != nil {
-				return nil, errors.Wrapf(err, "could not get head root")
-			}
-			if bytes.Equal(headRoot, b.ParentRoot) {
-				return s.HeadState(ctx)
-			}
-		}
-		preState, err = s.beaconDB.State(ctx, bytesutil.ToBytes32(b.ParentRoot))
-		if err != nil {
-			return nil, errors.Wrapf(err, "could not get pre state for slot %d", b.Slot)
-		}
-		if preState == nil {
-			if bytes.Equal(s.finalizedCheckpt.Root, b.ParentRoot) {
-				return nil, fmt.Errorf("pre state of slot %d does not exist", b.Slot)
-			}
-			preState, err = s.generateState(ctx, bytesutil.ToBytes32(s.finalizedCheckpt.Root), bytesutil.ToBytes32(b.ParentRoot))
-			if err != nil {
-				return nil, err
-			}
-		}
-		return preState, nil // No copy needed from newly hydrated DB object.
-	}
-	return preState.Copy(), nil
+	return preState, nil // No copy needed from newly hydrated state gen object.
 }
 
 // verifyBlkDescendant validates input block root is a descendant of the
@@ -144,64 +135,6 @@ func (s *Service) verifyBlkFinalizedSlot(b *ethpb.BeaconBlock) error {
 	return nil
 }
 
-// rmStatesOlderThanLastFinalized deletes the states in db since last finalized check point.
-func (s *Service) rmStatesOlderThanLastFinalized(ctx context.Context, startSlot uint64, endSlot uint64) error {
-	ctx, span := trace.StartSpan(ctx, "forkchoice.rmStatesBySlots")
-	defer span.End()
-
-	// Make sure start slot is not a skipped slot
-	for i := startSlot; i > 0; i-- {
-		filter := filters.NewFilter().SetStartSlot(i).SetEndSlot(i)
-		b, err := s.beaconDB.Blocks(ctx, filter)
-		if err != nil {
-			return err
-		}
-		if len(b) > 0 {
-			startSlot = i
-			break
-		}
-	}
-
-	// Make sure finalized slot is not a skipped slot.
-	for i := endSlot; i > 0; i-- {
-		filter := filters.NewFilter().SetStartSlot(i).SetEndSlot(i)
-		b, err := s.beaconDB.Blocks(ctx, filter)
-		if err != nil {
-			return err
-		}
-		if len(b) > 0 {
-			endSlot = i - 1
-			break
-		}
-	}
-
-	// Do not remove genesis state
-	if startSlot == 0 {
-		startSlot++
-	}
-	// If end slot comes less than start slot
-	if endSlot < startSlot {
-		endSlot = startSlot
-	}
-
-	filter := filters.NewFilter().SetStartSlot(startSlot).SetEndSlot(endSlot)
-	roots, err := s.beaconDB.BlockRoots(ctx, filter)
-	if err != nil {
-		return err
-	}
-
-	roots, err = s.filterBlockRoots(ctx, roots)
-	if err != nil {
-		return err
-	}
-
-	if err := s.beaconDB.DeleteStates(ctx, roots); err != nil {
-		return err
-	}
-
-	return nil
-}
-
 // shouldUpdateCurrentJustified prevents bouncing attack, by only update conflicting justified
 // checkpoints in the fork choice if in the early slots of the epoch.
 // Otherwise, delay incorporation of new justified checkpoint until next epoch boundary.
@@ -211,9 +144,9 @@ func (s *Service) shouldUpdateCurrentJustified(ctx context.Context, newJustified
 		return true, nil
 	}
 	var newJustifiedBlockSigned *ethpb.SignedBeaconBlock
-	justifiedRoot := bytesutil.ToBytes32(newJustifiedCheckpt.Root)
+	justifiedRoot := s.ensureRootNotZeros(bytesutil.ToBytes32(newJustifiedCheckpt.Root))
 	var err error
-	if !featureconfig.Get().NoInitSyncBatchSaveBlocks && s.hasInitSyncBlock(justifiedRoot) {
+	if s.hasInitSyncBlock(justifiedRoot) {
 		newJustifiedBlockSigned = s.getInitSyncBlock(justifiedRoot)
 	} else {
 		newJustifiedBlockSigned, err = s.beaconDB.Block(ctx, justifiedRoot)
@@ -230,8 +163,8 @@ func (s *Service) shouldUpdateCurrentJustified(ctx context.Context, newJustified
 		return false, nil
 	}
 	var justifiedBlockSigned *ethpb.SignedBeaconBlock
-	cachedJustifiedRoot := bytesutil.ToBytes32(s.justifiedCheckpt.Root)
-	if !featureconfig.Get().NoInitSyncBatchSaveBlocks && s.hasInitSyncBlock(cachedJustifiedRoot) {
+	cachedJustifiedRoot := s.ensureRootNotZeros(bytesutil.ToBytes32(s.justifiedCheckpt.Root))
+	if s.hasInitSyncBlock(cachedJustifiedRoot) {
 		justifiedBlockSigned = s.getInitSyncBlock(cachedJustifiedRoot)
 	} else {
 		justifiedBlockSigned, err = s.beaconDB.Block(ctx, cachedJustifiedRoot)
@@ -244,7 +177,7 @@ func (s *Service) shouldUpdateCurrentJustified(ctx context.Context, newJustified
 		return false, errors.New("nil justified block")
 	}
 	justifiedBlock := justifiedBlockSigned.Block
-	b, err := s.ancestor(ctx, newJustifiedCheckpt.Root, justifiedBlock.Slot)
+	b, err := s.ancestor(ctx, justifiedRoot[:], justifiedBlock.Slot)
 	if err != nil {
 		return false, err
 	}
@@ -267,100 +200,26 @@ func (s *Service) updateJustified(ctx context.Context, state *stateTrie.BeaconSt
 	if canUpdate {
 		s.prevJustifiedCheckpt = s.justifiedCheckpt
 		s.justifiedCheckpt = cpt
-	}
-
-	if featureconfig.Get().DisableNewStateMgmt {
-		justifiedRoot := bytesutil.ToBytes32(cpt.Root)
-
-		justifiedState := s.initSyncState[justifiedRoot]
-		// If justified state is nil, resume back to normal syncing process and save
-		// justified check point.
-		var err error
-		if justifiedState == nil {
-			if s.beaconDB.HasState(ctx, justifiedRoot) {
-				return s.beaconDB.SaveJustifiedCheckpoint(ctx, cpt)
-			}
-			justifiedState, err = s.generateState(ctx, bytesutil.ToBytes32(s.finalizedCheckpt.Root), justifiedRoot)
-			if err != nil {
-				log.Error(err)
-				return s.beaconDB.SaveJustifiedCheckpoint(ctx, cpt)
-			}
-		}
-		if err := s.beaconDB.SaveState(ctx, justifiedState, justifiedRoot); err != nil {
-			return errors.Wrap(err, "could not save justified state")
+		if err := s.cacheJustifiedStateBalances(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root)); err != nil {
+			return err
 		}
 	}
 
 	return s.beaconDB.SaveJustifiedCheckpoint(ctx, cpt)
 }
 
-// This saves every finalized state in DB during initial sync, needed as part of optimization to
-// use cache state during initial sync in case of restart.
-func (s *Service) saveInitState(ctx context.Context, state *stateTrie.BeaconState) error {
-	cpt := state.FinalizedCheckpoint()
-	finalizedRoot := bytesutil.ToBytes32(cpt.Root)
-	fs := s.initSyncState[finalizedRoot]
-	if fs == nil {
-		var err error
-		fs, err = s.beaconDB.State(ctx, finalizedRoot)
-		if err != nil {
-			return err
-		}
-		if fs == nil {
-			fs, err = s.generateState(ctx, bytesutil.ToBytes32(s.prevFinalizedCheckpt.Root), finalizedRoot)
-			if err != nil {
-				// This might happen if the client was in sync and is now re-syncing for whatever reason.
-				log.Warn("Initial sync cache did not have finalized state root cached")
-				return err
-			}
-		}
-	}
-
-	if err := s.beaconDB.SaveState(ctx, fs, finalizedRoot); err != nil {
-		return errors.Wrap(err, "could not save state")
-	}
-	return nil
-}
-
-// This filters block roots that are not known as head root and finalized root in DB.
-// It serves as the last line of defence before we prune states.
-func (s *Service) filterBlockRoots(ctx context.Context, roots [][32]byte) ([][32]byte, error) {
-	f, err := s.beaconDB.FinalizedCheckpoint(ctx)
-	if err != nil {
-		return nil, err
-	}
-	fRoot := f.Root
-	h, err := s.beaconDB.HeadBlock(ctx)
-	if err != nil {
-		return nil, err
-	}
-	hRoot, err := ssz.HashTreeRoot(h.Block)
-	if err != nil {
-		return nil, err
-	}
-
-	filtered := make([][32]byte, 0, len(roots))
-	for _, root := range roots {
-		if bytes.Equal(root[:], fRoot[:]) || bytes.Equal(root[:], hRoot[:]) {
-			continue
-		}
-		filtered = append(filtered, root)
-	}
-
-	return filtered, nil
-}
-
 // ancestor returns the block root of an ancestry block from the input block root.
 //
 // Spec pseudocode definition:
-//   def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash:
+//   def get_ancestor(store: Store, root: Root, slot: Slot) -> Root:
 //    block = store.blocks[root]
 //    if block.slot > slot:
-//      return get_ancestor(store, block.parent_root, slot)
+//        return get_ancestor(store, block.parent_root, slot)
 //    elif block.slot == slot:
-//      return root
+//        return root
 //    else:
-//      return Bytes32()  # root is older than queried slot: no results.
+//        # root is older than queried slot, thus a skip slot. Return most recent root prior to slot
+//        return root
 func (s *Service) ancestor(ctx context.Context, root []byte, slot uint64) ([]byte, error) {
 	ctx, span := trace.StartSpan(ctx, "forkchoice.ancestor")
 	defer span.End()
@@ -375,7 +234,7 @@ func (s *Service) ancestor(ctx context.Context, root []byte, slot uint64) ([]byt
 		return nil, errors.Wrap(err, "could not get ancestor block")
 	}
 
-	if !featureconfig.Get().NoInitSyncBatchSaveBlocks && s.hasInitSyncBlock(bytesutil.ToBytes32(root)) {
+	if s.hasInitSyncBlock(bytesutil.ToBytes32(root)) {
 		signed = s.getInitSyncBlock(bytesutil.ToBytes32(root))
 	}
 
@@ -384,13 +243,7 @@ func (s *Service) ancestor(ctx context.Context, root []byte, slot uint64) ([]byt
 	}
 	b := signed.Block
 
-	// If we dont have the ancestor in the DB, simply return nil so rest of fork choice
-	// operation can proceed. This is not an error condition.
-	if b == nil || b.Slot < slot {
-		return nil, nil
-	}
-
-	if b.Slot == slot {
+	if b.Slot == slot || b.Slot < slot {
 		return root, nil
 	}
 
@@ -401,32 +254,46 @@ func (s *Service) ancestor(ctx context.Context, root []byte, slot uint64) ([]byt
 // the store's justified is not in chain with finalized check point.
 //
 // Spec definition:
-//   if (
-//            state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch
-//            or get_ancestor(store, store.justified_checkpoint.root, finalized_slot) != store.finalized_checkpoint.root
-//        ):
-//            store.justified_checkpoint = state.current_justified_checkpoint
+//   # Potentially update justified if different from store
+//        if store.justified_checkpoint != state.current_justified_checkpoint:
+//            # Update justified if new justified is later than store justified
+//            if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch:
+//                store.justified_checkpoint = state.current_justified_checkpoint
+//                return
+//            # Update justified if store justified is not in chain with finalized checkpoint
+//            finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)
+//            ancestor_at_finalized_slot = get_ancestor(store, store.justified_checkpoint.root, finalized_slot)
+//            if ancestor_at_finalized_slot != store.finalized_checkpoint.root:
+//                store.justified_checkpoint = state.current_justified_checkpoint
 func (s *Service) finalizedImpliesNewJustified(ctx context.Context, state *stateTrie.BeaconState) error {
-	finalizedBlkSigned, err := s.beaconDB.Block(ctx, bytesutil.ToBytes32(s.finalizedCheckpt.Root))
-	if err != nil || finalizedBlkSigned == nil || finalizedBlkSigned.Block == nil {
-		return errors.Wrap(err, "could not get finalized block")
-	}
-	finalizedBlk := finalizedBlkSigned.Block
+	// Update justified if it's different than the one cached in the store.
+	if !attestationutil.CheckPointIsEqual(s.justifiedCheckpt, state.CurrentJustifiedCheckpoint()) {
+		if state.CurrentJustifiedCheckpoint().Epoch > s.justifiedCheckpt.Epoch {
+			s.justifiedCheckpt = state.CurrentJustifiedCheckpoint()
+			if err := s.cacheJustifiedStateBalances(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root)); err != nil {
+				return err
+			}
+			return nil
+		}
 
-	anc, err := s.ancestor(ctx, s.justifiedCheckpt.Root, finalizedBlk.Slot)
-	if err != nil {
-		return err
+		// Update justified if store justified is not in chain with finalized check point.
+		finalizedSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
+		justifiedRoot := s.ensureRootNotZeros(bytesutil.ToBytes32(s.justifiedCheckpt.Root))
+		anc, err := s.ancestor(ctx, justifiedRoot[:], finalizedSlot)
+		if err != nil {
+			return err
+		}
+		if !bytes.Equal(anc, s.finalizedCheckpt.Root) {
+			s.justifiedCheckpt = state.CurrentJustifiedCheckpoint()
+			if err := s.cacheJustifiedStateBalances(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root)); err != nil {
+				return err
+			}
+		}
 	}
-
-	// Either the new justified is later than stored justified or not in chain with finalized check pint.
-	if cpt := state.CurrentJustifiedCheckpoint(); cpt != nil && cpt.Epoch > s.justifiedCheckpt.Epoch || !bytes.Equal(anc, s.finalizedCheckpt.Root) {
-		s.justifiedCheckpt = state.CurrentJustifiedCheckpoint()
-	}
-
 	return nil
 }
 
-// This retrieves missing blocks from DB (ie. the blocks that couldn't received over sync) and inserts them to fork choice store.
+// This retrieves missing blocks from DB (ie. the blocks that couldn't be received over sync) and inserts them to fork choice store.
 // This is useful for block tree visualizer and additional vote accounting.
 func (s *Service) fillInForkChoiceMissingBlocks(ctx context.Context, blk *ethpb.BeaconBlock, state *stateTrie.BeaconState) error {
 	pendingNodes := make([]*ethpb.BeaconBlock, 0)
@@ -452,13 +319,13 @@ func (s *Service) fillInForkChoiceMissingBlocks(ctx context.Context, blk *ethpb.
 	// Lower slots should be at the end of the list.
 	for i := len(pendingNodes) - 1; i >= 0; i-- {
 		b := pendingNodes[i]
-		r, err := ssz.HashTreeRoot(b)
+		r, err := stateutil.BlockRoot(b)
 		if err != nil {
 			return err
 		}
 
 		if err := s.forkChoiceStore.ProcessBlock(ctx,
-			b.Slot, r, bytesutil.ToBytes32(b.ParentRoot),
+			b.Slot, r, bytesutil.ToBytes32(b.ParentRoot), bytesutil.ToBytes32(b.Body.Graffiti),
 			state.CurrentJustifiedCheckpoint().Epoch,
 			state.FinalizedCheckpointEpoch()); err != nil {
 			return errors.Wrap(err, "could not process block for proto array fork choice")
@@ -484,3 +351,12 @@ func (s *Service) deletePoolAtts(atts []*ethpb.Attestation) error {
 
 	return nil
 }
+
+// This ensures that the input root defaults to using genesis root instead of zero hashes. This is needed for handling
+// fork choice justification routine.
+func (s *Service) ensureRootNotZeros(root [32]byte) [32]byte {
+	if root == params.BeaconConfig().ZeroHash {
+		return s.genesisRoot
+	}
+	return root
+}

beacon-chain/blockchain/process_block_test.go

@@ -9,28 +9,29 @@ import (
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/attestationutil"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/roughtime"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
 
 func TestStore_OnBlock(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	cfg := &Config{
 		BeaconDB: db,
-		StateGen: stategen.New(db, cache.NewStateSummaryCache()),
+		StateGen: stategen.New(db, sc),
 	}
 	service, err := NewService(ctx, cfg)
 	if err != nil {
@@ -42,7 +43,7 @@ func TestStore_OnBlock(t *testing.T) {
 	if err := db.SaveBlock(ctx, genesis); err != nil {
 		t.Error(err)
 	}
-	validGenesisRoot, err := ssz.HashTreeRoot(genesis.Block)
+	validGenesisRoot, err := stateutil.BlockRoot(genesis.Block)
 	if err != nil {
 		t.Error(err)
 	}
@@ -58,7 +59,7 @@ func TestStore_OnBlock(t *testing.T) {
 	if err := db.SaveBlock(ctx, random); err != nil {
 		t.Error(err)
 	}
-	randomParentRoot, err := ssz.HashTreeRoot(random.Block)
+	randomParentRoot, err := stateutil.BlockRoot(random.Block)
 	if err != nil {
 		t.Error(err)
 	}
@@ -87,13 +88,13 @@ func TestStore_OnBlock(t *testing.T) {
 			name:          "parent block root does not have a state",
 			blk:           &ethpb.BeaconBlock{},
 			s:             st.Copy(),
-			wantErrString: "provided block root does not have block saved in the db",
+			wantErrString: "could not reconstruct parent state",
 		},
 		{
-			name:          "block is from the feature",
+			name:          "block is from the future",
 			blk:           &ethpb.BeaconBlock{ParentRoot: randomParentRoot[:], Slot: params.BeaconConfig().FarFutureEpoch},
 			s:             st.Copy(),
-			wantErrString: "could not process slot from the future",
+			wantErrString: "far distant future",
 		},
 		{
 			name:          "could not get finalized block",
@@ -117,7 +118,11 @@ func TestStore_OnBlock(t *testing.T) {
 			service.prevFinalizedCheckpt = &ethpb.Checkpoint{Root: validGenesisRoot[:]}
 			service.finalizedCheckpt.Root = roots[0]
 
-			_, err := service.onBlock(ctx, &ethpb.SignedBeaconBlock{Block: tt.blk})
+			root, err := stateutil.BlockRoot(tt.blk)
+			if err != nil {
+				t.Error(err)
+			}
+			_, err = service.onBlock(ctx, &ethpb.SignedBeaconBlock{Block: tt.blk}, root)
 			if err == nil || !strings.Contains(err.Error(), tt.wantErrString) {
 				t.Errorf("Store.OnBlock() error = %v, wantErr = %v", err, tt.wantErrString)
 			}
@@ -125,90 +130,9 @@ func TestStore_OnBlock(t *testing.T) {
 	}
 }
 
-func TestRemoveStateSinceLastFinalized(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	params.UseMinimalConfig()
-	defer params.UseMainnetConfig()
-
-	cfg := &Config{BeaconDB: db}
-	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Save 100 blocks in DB, each has a state.
-	numBlocks := 100
-	totalBlocks := make([]*ethpb.SignedBeaconBlock, numBlocks)
-	blockRoots := make([][32]byte, 0)
-	for i := 0; i < len(totalBlocks); i++ {
-		totalBlocks[i] = &ethpb.SignedBeaconBlock{
-			Block: &ethpb.BeaconBlock{
-				Slot: uint64(i),
-			},
-		}
-		r, err := ssz.HashTreeRoot(totalBlocks[i].Block)
-		if err != nil {
-			t.Fatal(err)
-		}
-		s := testutil.NewBeaconState()
-		if err := s.SetSlot(uint64(i)); err != nil {
-			t.Fatal(err)
-		}
-		if err := service.beaconDB.SaveState(ctx, s, r); err != nil {
-			t.Fatal(err)
-		}
-		if err := service.beaconDB.SaveBlock(ctx, totalBlocks[i]); err != nil {
-			t.Fatal(err)
-		}
-		blockRoots = append(blockRoots, r)
-		if err := service.beaconDB.SaveHeadBlockRoot(ctx, r); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// New finalized epoch: 1
-	finalizedEpoch := uint64(1)
-	finalizedSlot := finalizedEpoch * params.BeaconConfig().SlotsPerEpoch
-	endSlot := helpers.StartSlot(finalizedEpoch+1) - 1 // Inclusive
-	if err := service.rmStatesOlderThanLastFinalized(ctx, 0, endSlot); err != nil {
-		t.Fatal(err)
-	}
-	for _, r := range blockRoots {
-		s, err := service.beaconDB.State(ctx, r)
-		if err != nil {
-			t.Fatal(err)
-		}
-		// Also verifies genesis state didnt get deleted
-		if s != nil && s.Slot() != finalizedSlot && s.Slot() != 0 && s.Slot() < endSlot {
-			t.Errorf("State with slot %d should not be in DB", s.Slot())
-		}
-	}
-
-	// New finalized epoch: 5
-	newFinalizedEpoch := uint64(5)
-	newFinalizedSlot := newFinalizedEpoch * params.BeaconConfig().SlotsPerEpoch
-	endSlot = helpers.StartSlot(newFinalizedEpoch+1) - 1 // Inclusive
-	if err := service.rmStatesOlderThanLastFinalized(ctx, helpers.StartSlot(finalizedEpoch+1)-1, endSlot); err != nil {
-		t.Fatal(err)
-	}
-	for _, r := range blockRoots {
-		s, err := service.beaconDB.State(ctx, r)
-		if err != nil {
-			t.Fatal(err)
-		}
-		// Also verifies genesis state didnt get deleted
-		if s != nil && s.Slot() != newFinalizedSlot && s.Slot() != finalizedSlot && s.Slot() != 0 && s.Slot() < endSlot {
-			t.Errorf("State with slot %d should not be in DB", s.Slot())
-		}
-	}
-}
-
 func TestRemoveStateSinceLastFinalized_EmptyStartSlot(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 	params.UseMinimalConfig()
 	defer params.UseMainnetConfig()
 
@@ -226,14 +150,16 @@ func TestRemoveStateSinceLastFinalized_EmptyStartSlot(t *testing.T) {
 	if !update {
 		t.Error("Should be able to update justified, received false")
 	}
-
-	lastJustifiedBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{ParentRoot: []byte{'G'}}}
-	lastJustifiedRoot, err := ssz.HashTreeRoot(lastJustifiedBlk.Block)
+	lastJustifiedBlk := testutil.NewBeaconBlock()
+	lastJustifiedBlk.Block.ParentRoot = bytesutil.PadTo([]byte{'G'}, 32)
+	lastJustifiedRoot, err := stateutil.BlockRoot(lastJustifiedBlk.Block)
 	if err != nil {
 		t.Fatal(err)
 	}
-	newJustifiedBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1, ParentRoot: lastJustifiedRoot[:]}}
-	newJustifiedRoot, err := ssz.HashTreeRoot(newJustifiedBlk.Block)
+	newJustifiedBlk := testutil.NewBeaconBlock()
+	newJustifiedBlk.Block.Slot = 1
+	newJustifiedBlk.Block.ParentRoot = bytesutil.PadTo(lastJustifiedRoot[:], 32)
+	newJustifiedRoot, err := stateutil.BlockRoot(newJustifiedBlk.Block)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -258,8 +184,7 @@ func TestRemoveStateSinceLastFinalized_EmptyStartSlot(t *testing.T) {
 
 func TestShouldUpdateJustified_ReturnFalse(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 	params.UseMinimalConfig()
 	defer params.UseMainnetConfig()
 
@@ -268,13 +193,15 @@ func TestShouldUpdateJustified_ReturnFalse(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	lastJustifiedBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{ParentRoot: []byte{'G'}}}
-	lastJustifiedRoot, err := ssz.HashTreeRoot(lastJustifiedBlk.Block)
+	lastJustifiedBlk := testutil.NewBeaconBlock()
+	lastJustifiedBlk.Block.ParentRoot = bytesutil.PadTo([]byte{'G'}, 32)
+	lastJustifiedRoot, err := stateutil.BlockRoot(lastJustifiedBlk.Block)
 	if err != nil {
 		t.Fatal(err)
 	}
-	newJustifiedBlk := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{ParentRoot: lastJustifiedRoot[:]}}
-	newJustifiedRoot, err := ssz.HashTreeRoot(newJustifiedBlk.Block)
+	newJustifiedBlk := testutil.NewBeaconBlock()
+	newJustifiedBlk.Block.ParentRoot = bytesutil.PadTo(lastJustifiedRoot[:], 32)
+	newJustifiedRoot, err := stateutil.BlockRoot(newJustifiedBlk.Block)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -299,13 +226,15 @@ func TestShouldUpdateJustified_ReturnFalse(t *testing.T) {
 }
 
 func TestCachedPreState_CanGetFromStateSummary(t *testing.T) {
+	resetCfg := featureconfig.InitWithReset(&featureconfig.Flags{NewStateMgmt: true})
+	defer resetCfg()
+
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	cfg := &Config{
 		BeaconDB: db,
-		StateGen: stategen.New(db, cache.NewStateSummaryCache()),
+		StateGen: stategen.New(db, sc),
 	}
 	service, err := NewService(ctx, cfg)
 	if err != nil {
@@ -335,13 +264,15 @@ func TestCachedPreState_CanGetFromStateSummary(t *testing.T) {
 }
 
 func TestCachedPreState_CanGetFromDB(t *testing.T) {
+	resetCfg := featureconfig.InitWithReset(&featureconfig.Flags{NewStateMgmt: true})
+	defer resetCfg()
+
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
 	cfg := &Config{
 		BeaconDB: db,
-		StateGen: stategen.New(db, cache.NewStateSummaryCache()),
+		StateGen: stategen.New(db, sc),
 	}
 	service, err := NewService(ctx, cfg)
 	if err != nil {
@@ -353,7 +284,7 @@ func TestCachedPreState_CanGetFromDB(t *testing.T) {
 
 	service.finalizedCheckpt = &ethpb.Checkpoint{Root: r[:]}
 	_, err = service.verifyBlkPreState(ctx, b)
-	wanted := "provided block root does not have block saved in the db"
+	wanted := "could not reconstruct parent state"
 	if err.Error() != wanted {
 		t.Error("Did not get wanted error")
 	}
@@ -378,58 +309,9 @@ func TestCachedPreState_CanGetFromDB(t *testing.T) {
 	}
 }
 
-func TestSaveInitState_CanSaveDelete(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	cfg := &Config{BeaconDB: db}
-	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for i := uint64(0); i < 64; i++ {
-		b := &ethpb.BeaconBlock{Slot: i}
-		s := testutil.NewBeaconState()
-		if err := s.SetSlot(i); err != nil {
-			t.Fatal(err)
-		}
-		r, err := ssz.HashTreeRoot(b)
-		if err != nil {
-			t.Fatal(err)
-		}
-		service.initSyncState[r] = s
-	}
-
-	// Set finalized root as slot 32
-	finalizedRoot, err := ssz.HashTreeRoot(&ethpb.BeaconBlock{Slot: 32})
-	if err != nil {
-		t.Fatal(err)
-	}
-	s := testutil.NewBeaconState()
-	if err := s.SetFinalizedCheckpoint(&ethpb.Checkpoint{
-		Epoch: 1, Root: finalizedRoot[:]}); err != nil {
-		t.Fatal(err)
-	}
-	if err := service.saveInitState(ctx, s); err != nil {
-		t.Fatal(err)
-	}
-
-	// Verify finalized state is saved in DB
-	finalizedState, err := service.beaconDB.State(ctx, finalizedRoot)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if finalizedState == nil {
-		t.Error("finalized state can't be nil")
-	}
-}
-
 func TestUpdateJustified_CouldUpdateBest(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 
 	cfg := &Config{BeaconDB: db}
 	service, err := NewService(ctx, cfg)
@@ -441,7 +323,7 @@ func TestUpdateJustified_CouldUpdateBest(t *testing.T) {
 	if err := db.SaveBlock(ctx, signedBlock); err != nil {
 		t.Fatal(err)
 	}
-	r, err := ssz.HashTreeRoot(signedBlock.Block)
+	r, err := stateutil.BlockRoot(signedBlock.Block)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -477,106 +359,9 @@ func TestUpdateJustified_CouldUpdateBest(t *testing.T) {
 	}
 }
 
-func TestFilterBlockRoots_CanFilter(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	cfg := &Config{BeaconDB: db}
-	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	fBlock := &ethpb.BeaconBlock{}
-	fRoot, err := ssz.HashTreeRoot(fBlock)
-	if err != nil {
-		t.Fatal(err)
-	}
-	hBlock := &ethpb.BeaconBlock{Slot: 1}
-	headRoot, err := ssz.HashTreeRoot(hBlock)
-	if err != nil {
-		t.Fatal(err)
-	}
-	st := testutil.NewBeaconState()
-	if err := service.beaconDB.SaveBlock(ctx, &ethpb.SignedBeaconBlock{Block: fBlock}); err != nil {
-		t.Fatal(err)
-	}
-	if err := service.beaconDB.SaveState(ctx, st.Copy(), fRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := service.beaconDB.SaveFinalizedCheckpoint(ctx, &ethpb.Checkpoint{Root: fRoot[:]}); err != nil {
-		t.Fatal(err)
-	}
-	if err := service.beaconDB.SaveBlock(ctx, &ethpb.SignedBeaconBlock{Block: hBlock}); err != nil {
-		t.Fatal(err)
-	}
-	if err := service.beaconDB.SaveState(ctx, st.Copy(), headRoot); err != nil {
-		t.Fatal(err)
-	}
-	if err := service.beaconDB.SaveHeadBlockRoot(ctx, headRoot); err != nil {
-		t.Fatal(err)
-	}
-
-	roots := [][32]byte{{'C'}, {'D'}, headRoot, {'E'}, fRoot, {'F'}}
-	wanted := [][32]byte{{'C'}, {'D'}, {'E'}, {'F'}}
-
-	received, err := service.filterBlockRoots(ctx, roots)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if !reflect.DeepEqual(wanted, received) {
-		t.Error("Did not filter correctly")
-	}
-}
-
-func TestPersistCache_CanSave(t *testing.T) {
-	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-
-	cfg := &Config{BeaconDB: db}
-	service, err := NewService(ctx, cfg)
-	if err != nil {
-		t.Fatal(err)
-	}
-	st := testutil.NewBeaconState()
-
-	for i := uint64(0); i < initialSyncCacheSize; i++ {
-		if err := st.SetSlot(i); err != nil {
-			t.Fatal(err)
-		}
-		root := [32]byte{}
-		copy(root[:], bytesutil.Bytes32(i))
-		service.initSyncState[root] = st.Copy()
-		service.boundaryRoots = append(service.boundaryRoots, root)
-	}
-
-	if err = service.persistCachedStates(ctx, initialSyncCacheSize); err != nil {
-		t.Fatal(err)
-	}
-
-	for i := uint64(0); i < initialSyncCacheSize-minimumCacheSize; i++ {
-		root := [32]byte{}
-		copy(root[:], bytesutil.Bytes32(i))
-		state, err := db.State(context.Background(), root)
-		if err != nil {
-			t.Errorf("State with root of %#x , could not be retrieved: %v", root, err)
-		}
-		if state == nil {
-			t.Errorf("State with root of %#x , does not exist", root)
-		}
-		if state.Slot() != i {
-			t.Errorf("Incorrect slot retrieved. Wanted %d but got %d", i, state.Slot())
-		}
-	}
-}
-
 func TestFillForkChoiceMissingBlocks_CanSave(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 
 	cfg := &Config{BeaconDB: db}
 	service, err := NewService(ctx, cfg)
@@ -591,7 +376,7 @@ func TestFillForkChoiceMissingBlocks_CanSave(t *testing.T) {
 	if err := db.SaveBlock(ctx, genesis); err != nil {
 		t.Error(err)
 	}
-	validGenesisRoot, err := ssz.HashTreeRoot(genesis.Block)
+	validGenesisRoot, err := stateutil.BlockRoot(genesis.Block)
 	if err != nil {
 		t.Error(err)
 	}
@@ -606,7 +391,7 @@ func TestFillForkChoiceMissingBlocks_CanSave(t *testing.T) {
 	}
 
 	beaconState, _ := testutil.DeterministicGenesisState(t, 32)
-	block := &ethpb.BeaconBlock{Slot: 9, ParentRoot: roots[8]}
+	block := &ethpb.BeaconBlock{Slot: 9, ParentRoot: roots[8], Body: &ethpb.BeaconBlockBody{Graffiti: []byte{}}}
 	if err := service.fillInForkChoiceMissingBlocks(context.Background(), block, beaconState); err != nil {
 		t.Fatal(err)
 	}
@@ -629,8 +414,7 @@ func TestFillForkChoiceMissingBlocks_CanSave(t *testing.T) {
 
 func TestFillForkChoiceMissingBlocks_FilterFinalized(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 
 	cfg := &Config{BeaconDB: db}
 	service, err := NewService(ctx, cfg)
@@ -646,7 +430,7 @@ func TestFillForkChoiceMissingBlocks_FilterFinalized(t *testing.T) {
 	if err := db.SaveBlock(ctx, genesis); err != nil {
 		t.Error(err)
 	}
-	validGenesisRoot, err := ssz.HashTreeRoot(genesis.Block)
+	validGenesisRoot, err := stateutil.BlockRoot(genesis.Block)
 	if err != nil {
 		t.Error(err)
 	}
@@ -657,23 +441,23 @@ func TestFillForkChoiceMissingBlocks_FilterFinalized(t *testing.T) {
 	}
 
 	// Define a tree branch, slot 63 <- 64 <- 65
-	b63 := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 63}}
+	b63 := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 63, Body: &ethpb.BeaconBlockBody{}}}
 	if err := service.beaconDB.SaveBlock(ctx, b63); err != nil {
 		t.Fatal(err)
 	}
-	r63, err := ssz.HashTreeRoot(b63.Block)
+	r63, err := stateutil.BlockRoot(b63.Block)
 	if err != nil {
 		t.Fatal(err)
 	}
-	b64 := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 64, ParentRoot: r63[:]}}
+	b64 := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 64, ParentRoot: r63[:], Body: &ethpb.BeaconBlockBody{}}}
 	if err := service.beaconDB.SaveBlock(ctx, b64); err != nil {
 		t.Fatal(err)
 	}
-	r64, err := ssz.HashTreeRoot(b64.Block)
+	r64, err := stateutil.BlockRoot(b64.Block)
 	if err != nil {
 		t.Fatal(err)
 	}
-	b65 := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 65, ParentRoot: r64[:]}}
+	b65 := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 65, ParentRoot: r64[:], Body: &ethpb.BeaconBlockBody{}}}
 	if err := service.beaconDB.SaveBlock(ctx, b65); err != nil {
 		t.Fatal(err)
 	}
@@ -743,10 +527,14 @@ func blockTree1(db db.Database, genesisRoot []byte) ([][]byte, error) {
 	st := testutil.NewBeaconState()
 
 	for _, b := range []*ethpb.BeaconBlock{b0, b1, b3, b4, b5, b6, b7, b8} {
-		if err := db.SaveBlock(context.Background(), &ethpb.SignedBeaconBlock{Block: b}); err != nil {
+		beaconBlock := testutil.NewBeaconBlock()
+		beaconBlock.Block.Slot = b.Slot
+		beaconBlock.Block.ParentRoot = bytesutil.PadTo(b.ParentRoot, 32)
+		beaconBlock.Block.Body = &ethpb.BeaconBlockBody{}
+		if err := db.SaveBlock(context.Background(), beaconBlock); err != nil {
 			return nil, err
 		}
-		if err := db.SaveState(context.Background(), st.Copy(), bytesutil.ToBytes32(b.ParentRoot)); err != nil {
+		if err := db.SaveState(context.Background(), st.Copy(), bytesutil.ToBytes32(beaconBlock.Block.ParentRoot)); err != nil {
 			return nil, err
 		}
 	}
@@ -761,3 +549,176 @@ func blockTree1(db db.Database, genesisRoot []byte) ([][]byte, error) {
 	}
 	return [][]byte{r0[:], r1[:], nil, r3[:], r4[:], r5[:], r6[:], r7[:], r8[:]}, nil
 }
+
+func TestCurrentSlot_HandlesOverflow(t *testing.T) {
+	svc := Service{genesisTime: roughtime.Now().Add(1 * time.Hour)}
+
+	slot := svc.CurrentSlot()
+
+	if slot != 0 {
+		t.Fatalf("Expected slot to be 0, got %d", slot)
+	}
+}
+
+func TestAncestor_HandleSkipSlot(t *testing.T) {
+	ctx := context.Background()
+	db, _ := testDB.SetupDB(t)
+
+	cfg := &Config{BeaconDB: db}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b1 := &ethpb.BeaconBlock{Slot: 1, ParentRoot: []byte{'a'}}
+	r1, err := ssz.HashTreeRoot(b1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	b100 := &ethpb.BeaconBlock{Slot: 100, ParentRoot: r1[:]}
+	r100, err := ssz.HashTreeRoot(b100)
+	if err != nil {
+		t.Fatal(err)
+	}
+	b200 := &ethpb.BeaconBlock{Slot: 200, ParentRoot: r100[:]}
+	r200, err := ssz.HashTreeRoot(b200)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, b := range []*ethpb.BeaconBlock{b1, b100, b200} {
+		beaconBlock := testutil.NewBeaconBlock()
+		beaconBlock.Block.Slot = b.Slot
+		beaconBlock.Block.ParentRoot = bytesutil.PadTo(b.ParentRoot, 32)
+		beaconBlock.Block.Body = &ethpb.BeaconBlockBody{}
+		if err := db.SaveBlock(context.Background(), beaconBlock); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// Slots 100 to 200 are skip slots. Requesting root at 150 will yield root at 100. The last physical block.
+	r, err := service.ancestor(context.Background(), r200[:], 150)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if bytesutil.ToBytes32(r) != r100 {
+		t.Error("Did not get correct root")
+	}
+
+	// Slots 1 to 100 are skip slots. Requesting root at 50 will yield root at 1. The last physical block.
+	r, err = service.ancestor(context.Background(), r200[:], 50)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if bytesutil.ToBytes32(r) != r1 {
+		t.Error("Did not get correct root")
+	}
+}
+
+func TestEnsureRootNotZeroHashes(t *testing.T) {
+	ctx := context.Background()
+	cfg := &Config{}
+	service, err := NewService(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	service.genesisRoot = [32]byte{'a'}
+
+	r := service.ensureRootNotZeros(params.BeaconConfig().ZeroHash)
+	if r != service.genesisRoot {
+		t.Error("Did not get wanted justified root")
+	}
+	root := [32]byte{'b'}
+	r = service.ensureRootNotZeros(root)
+	if r != root {
+		t.Error("Did not get wanted justified root")
+	}
+}
+
+func TestFinalizedImpliesNewJustified(t *testing.T) {
+	db, sc := testDB.SetupDB(t)
+	ctx := context.Background()
+	type args struct {
+		cachedCheckPoint        *ethpb.Checkpoint
+		stateCheckPoint         *ethpb.Checkpoint
+		diffFinalizedCheckPoint bool
+	}
+	tests := []struct {
+		name string
+		args args
+		want *ethpb.Checkpoint
+	}{
+		{
+			name: "Same justified, do nothing",
+			args: args{
+				cachedCheckPoint: &ethpb.Checkpoint{Epoch: 1, Root: []byte{'a'}},
+				stateCheckPoint:  &ethpb.Checkpoint{Epoch: 1, Root: []byte{'a'}},
+			},
+			want: &ethpb.Checkpoint{Epoch: 1, Root: []byte{'a'}},
+		},
+		{
+			name: "Different justified, higher epoch, cache new justified",
+			args: args{
+				cachedCheckPoint: &ethpb.Checkpoint{Epoch: 1, Root: []byte{'a'}},
+				stateCheckPoint:  &ethpb.Checkpoint{Epoch: 2, Root: []byte{'b'}},
+			},
+			want: &ethpb.Checkpoint{Epoch: 2, Root: []byte{'b'}},
+		},
+		{
+			name: "finalized has different justified, cache new justified",
+			args: args{
+				cachedCheckPoint:        &ethpb.Checkpoint{Epoch: 1, Root: []byte{'a'}},
+				stateCheckPoint:         &ethpb.Checkpoint{Epoch: 1, Root: []byte{'b'}},
+				diffFinalizedCheckPoint: true,
+			},
+			want: &ethpb.Checkpoint{Epoch: 1, Root: []byte{'b'}},
+		},
+	}
+	for _, test := range tests {
+		beaconState := testutil.NewBeaconState()
+		if err := beaconState.SetCurrentJustifiedCheckpoint(test.args.stateCheckPoint); err != nil {
+			t.Fatal(err)
+		}
+		service, err := NewService(ctx, &Config{BeaconDB: db, StateGen: stategen.New(db, sc)})
+		if err != nil {
+			t.Fatal(err)
+		}
+		service.justifiedCheckpt = test.args.cachedCheckPoint
+		if err := service.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{Root: bytesutil.PadTo(test.want.Root, 32)}); err != nil {
+			t.Fatal(err)
+		}
+		genesisState := testutil.NewBeaconState()
+		if err := service.beaconDB.SaveState(ctx, genesisState, bytesutil.ToBytes32(test.want.Root)); err != nil {
+			t.Fatal(err)
+		}
+
+		if test.args.diffFinalizedCheckPoint {
+			b1 := &ethpb.BeaconBlock{Slot: 1, ParentRoot: []byte{'a'}}
+			r1, err := ssz.HashTreeRoot(b1)
+			if err != nil {
+				t.Fatal(err)
+			}
+			b100 := &ethpb.BeaconBlock{Slot: 100, ParentRoot: r1[:]}
+			r100, err := ssz.HashTreeRoot(b100)
+			if err != nil {
+				t.Fatal(err)
+			}
+			for _, b := range []*ethpb.BeaconBlock{b1, b100} {
+				beaconBlock := testutil.NewBeaconBlock()
+				beaconBlock.Block.Slot = b.Slot
+				beaconBlock.Block.ParentRoot = bytesutil.PadTo(b.ParentRoot, 32)
+				if err := service.beaconDB.SaveBlock(context.Background(), beaconBlock); err != nil {
+					t.Fatal(err)
+				}
+			}
+			service.finalizedCheckpt = &ethpb.Checkpoint{Root: []byte{'c'}, Epoch: 1}
+			service.justifiedCheckpt.Root = r100[:]
+		}
+
+		if err := service.finalizedImpliesNewJustified(ctx, beaconState); err != nil {
+			t.Fatal(err)
+		}
+		if !attestationutil.CheckPointIsEqual(test.want, service.justifiedCheckpt) {
+			t.Error("Did not get wanted check point")
+		}
+	}
+}

beacon-chain/blockchain/receive_attestation.go

@@ -9,6 +9,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
@@ -22,9 +23,10 @@ import (
 type AttestationReceiver interface {
 	ReceiveAttestationNoPubsub(ctx context.Context, att *ethpb.Attestation) error
 	IsValidAttestation(ctx context.Context, att *ethpb.Attestation) bool
+	AttestationPreState(ctx context.Context, att *ethpb.Attestation) (*state.BeaconState, error)
 }
 
-// ReceiveAttestationNoPubsub is a function that defines the operations that are preformed on
+// ReceiveAttestationNoPubsub is a function that defines the operations that are performed on
 // attestation that is received from regular sync. The operations consist of:
 //  1. Validate attestation, update validator's latest vote
 //  2. Apply fork choice to the processed attestation
@@ -39,15 +41,10 @@ func (s *Service) ReceiveAttestationNoPubsub(ctx context.Context, att *ethpb.Att
 	}
 
 	if !featureconfig.Get().DisableUpdateHeadPerAttestation {
-		baseState, err := s.getAttPreState(ctx, att.Data.Target)
-		if err != nil {
-			return err
-		}
-
 		// This updates fork choice head, if a new head could not be updated due to
 		// long range or intermediate forking. It simply logs a warning and returns nil
 		// as that's more appropriate than returning errors.
-		if err := s.updateHead(ctx, baseState.Balances()); err != nil {
+		if err := s.updateHead(ctx, s.getJustifiedBalances()); err != nil {
 			log.Warnf("Resolving fork due to new attestation: %v", err)
 			return nil
 		}
@@ -58,9 +55,9 @@ func (s *Service) ReceiveAttestationNoPubsub(ctx context.Context, att *ethpb.Att
 
 // IsValidAttestation returns true if the attestation can be verified against its pre-state.
 func (s *Service) IsValidAttestation(ctx context.Context, att *ethpb.Attestation) bool {
-	baseState, err := s.getAttPreState(ctx, att.Data.Target)
+	baseState, err := s.AttestationPreState(ctx, att)
 	if err != nil {
-		log.WithError(err).Error("Failed to validate attestation")
+		log.WithError(err).Error("Failed to get attestation pre state")
 		return false
 	}
 
@@ -72,6 +69,11 @@ func (s *Service) IsValidAttestation(ctx context.Context, att *ethpb.Attestation
 	return true
 }
 
+// AttestationPreState returns the pre state of attestation.
+func (s *Service) AttestationPreState(ctx context.Context, att *ethpb.Attestation) (*state.BeaconState, error) {
+	return s.getAttPreState(ctx, att.Data.Target)
+}
+
 // This processes attestations from the attestation pool to account for validator votes and fork choice.
 func (s *Service) processAttestation(subscribedToStateEvents chan struct{}) {
 	// Wait for state to be initialized.
@@ -90,13 +92,15 @@ func (s *Service) processAttestation(subscribedToStateEvents chan struct{}) {
 			ctx := context.Background()
 			atts := s.attPool.ForkchoiceAttestations()
 			for _, a := range atts {
-				var hasState bool
-				if !featureconfig.Get().DisableNewStateMgmt {
-					hasState = s.stateGen.StateSummaryExists(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
-				} else {
-					hasState = s.beaconDB.HasState(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot)) && s.beaconDB.HasState(ctx, bytesutil.ToBytes32(a.Data.Target.Root))
+				// Based on the spec, don't process the attestation until the subsequent slot.
+				// This delays consideration in the fork choice until their slot is in the past.
+				// https://github.com/ethereum/eth2.0-specs/blob/dev/specs/phase0/fork-choice.md#validate_on_attestation
+				nextSlot := a.Data.Slot + 1
+				if err := helpers.VerifySlotTime(uint64(s.genesisTime.Unix()), nextSlot, params.BeaconNetworkConfig().MaximumGossipClockDisparity); err != nil {
+					continue
 				}
 
+				hasState := s.stateGen.StateSummaryExists(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
 				hasBlock := s.hasBlock(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
 				if !(hasState && hasBlock) {
 					continue

beacon-chain/blockchain/receive_attestation_test.go

@@ -5,14 +5,15 @@ import (
 	"time"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 )
 
 func TestVerifyCheckpointEpoch_Ok(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	helpers.ClearCache()
+	db, sc := testDB.SetupDB(t)
 
-	chainService := setupBeaconChain(t, db)
+	chainService := setupBeaconChain(t, db, sc)
 	chainService.genesisTime = time.Now()
 
 	if !chainService.verifyCheckpointEpoch(&ethpb.Checkpoint{}) {

beacon-chain/blockchain/receive_block.go

@@ -7,13 +7,11 @@ import (
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/traceutil"
 	"github.com/sirupsen/logrus"
@@ -22,37 +20,36 @@ import (
 
 // BlockReceiver interface defines the methods of chain service receive and processing new blocks.
 type BlockReceiver interface {
-	ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock) error
-	ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedBeaconBlock) error
-	ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *ethpb.SignedBeaconBlock) error
-	ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedBeaconBlock) error
+	ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error
+	ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error
+	ReceiveBlockInitialSync(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error
 	HasInitSyncBlock(root [32]byte) bool
 }
 
-// ReceiveBlock is a function that defines the operations that are preformed on
+// ReceiveBlock is a function that defines the operations that are performed on
 // blocks that is received from rpc service. The operations consists of:
 //   1. Gossip block to other peers
 //   2. Validate block, apply state transition and update check points
 //   3. Apply fork choice to the processed block
 //   4. Save latest head info
-func (s *Service) ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
+func (s *Service) ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
 	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlock")
 	defer span.End()
 
-	root, err := ssz.HashTreeRoot(block.Block)
-	if err != nil {
-		return errors.Wrap(err, "could not get signing root on received block")
-	}
-
 	// Broadcast the new block to the network.
 	if err := s.p2p.Broadcast(ctx, block); err != nil {
 		return errors.Wrap(err, "could not broadcast block")
 	}
 	log.WithFields(logrus.Fields{
-		"blockRoot": hex.EncodeToString(root[:]),
+		"blockRoot": hex.EncodeToString(blockRoot[:]),
 	}).Debug("Broadcasting block")
 
-	if err := s.ReceiveBlockNoPubsub(ctx, block); err != nil {
+	if err := captureSentTimeMetric(uint64(s.genesisTime.Unix()), block.Block.Slot); err != nil {
+		// If a node fails to capture metric, this shouldn't cause the block processing to fail.
+		log.Warnf("Could not capture block sent time metric: %v", err)
+	}
+
+	if err := s.ReceiveBlockNoPubsub(ctx, block, blockRoot); err != nil {
 		return err
 	}
 
@@ -60,17 +57,17 @@ func (s *Service) ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlo
 }
 
 // ReceiveBlockNoPubsub is a function that defines the the operations (minus pubsub)
-// that are preformed on blocks that is received from regular sync service. The operations consists of:
+// that are performed on blocks that is received from regular sync service. The operations consists of:
 //   1. Validate block, apply state transition and update check points
 //   2. Apply fork choice to the processed block
 //   3. Save latest head info
-func (s *Service) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
+func (s *Service) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
 	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlockNoPubsub")
 	defer span.End()
 	blockCopy := stateTrie.CopySignedBeaconBlock(block)
 
 	// Apply state transition on the new block.
-	postState, err := s.onBlock(ctx, blockCopy)
+	_, err := s.onBlock(ctx, blockCopy, blockRoot)
 	if err != nil {
 		err := errors.Wrap(err, "could not process block")
 		traceutil.AnnotateError(span, err)
@@ -90,17 +87,12 @@ func (s *Service) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedB
 	defer s.epochParticipationLock.Unlock()
 	s.epochParticipation[helpers.SlotToEpoch(blockCopy.Block.Slot)] = precompute.Balances
 
-	root, err := stateutil.BlockRoot(blockCopy.Block)
-	if err != nil {
-		return errors.Wrap(err, "could not get signing root on received block")
-	}
-
 	if featureconfig.Get().DisableForkChoice && block.Block.Slot > s.headSlot() {
-		if err := s.saveHead(ctx, root); err != nil {
+		if err := s.saveHead(ctx, blockRoot); err != nil {
 			return errors.Wrap(err, "could not save head")
 		}
 	} else {
-		if err := s.updateHead(ctx, postState.Balances()); err != nil {
+		if err := s.updateHead(ctx, s.getJustifiedBalances()); err != nil {
 			return errors.Wrap(err, "could not save head")
 		}
 	}
@@ -110,13 +102,16 @@ func (s *Service) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedB
 		Type: statefeed.BlockProcessed,
 		Data: &statefeed.BlockProcessedData{
 			Slot:      blockCopy.Block.Slot,
-			BlockRoot: root,
+			BlockRoot: blockRoot,
 			Verified:  true,
 		},
 	})
 
 	// Reports on block and fork choice metrics.
-	reportSlotMetrics(blockCopy.Block.Slot, s.headSlot(), s.finalizedCheckpt)
+	reportSlotMetrics(blockCopy.Block.Slot, s.headSlot(), s.CurrentSlot(), s.finalizedCheckpt)
+
+	// Log block sync status.
+	logBlockSyncStatus(blockCopy.Block, blockRoot, s.finalizedCheckpt)
 
 	// Log state transition data.
 	logStateTransitionData(blockCopy.Block)
@@ -124,87 +119,27 @@ func (s *Service) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedB
 	return nil
 }
 
-// ReceiveBlockNoPubsubForkchoice is a function that defines the all operations (minus pubsub and forkchoice)
-// that are preformed blocks that is received from initial sync service. The operations consists of:
-//   1. Validate block, apply state transition and update check points
-//   2. Save latest head info
-func (s *Service) ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
-	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlockNoForkchoice")
-	defer span.End()
-	blockCopy := stateTrie.CopySignedBeaconBlock(block)
-
-	// Apply state transition on the new block.
-	_, err := s.onBlock(ctx, blockCopy)
-	if err != nil {
-		err := errors.Wrap(err, "could not process block")
-		traceutil.AnnotateError(span, err)
-		return err
-	}
-
-	root, err := stateutil.BlockRoot(blockCopy.Block)
-	if err != nil {
-		return errors.Wrap(err, "could not get signing root on received block")
-	}
-	cachedHeadRoot, err := s.HeadRoot(ctx)
-	if err != nil {
-		return errors.Wrap(err, "could not get head root from cache")
-	}
-	if !bytes.Equal(root[:], cachedHeadRoot) {
-		if err := s.saveHead(ctx, root); err != nil {
-			return errors.Wrap(err, "could not save head")
-		}
-	}
-
-	// Send notification of the processed block to the state feed.
-	s.stateNotifier.StateFeed().Send(&feed.Event{
-		Type: statefeed.BlockProcessed,
-		Data: &statefeed.BlockProcessedData{
-			Slot:      blockCopy.Block.Slot,
-			BlockRoot: root,
-			Verified:  true,
-		},
-	})
-
-	// Reports on block and fork choice metrics.
-	reportSlotMetrics(blockCopy.Block.Slot, s.headSlot(), s.finalizedCheckpt)
-
-	// Log state transition data.
-	logStateTransitionData(blockCopy.Block)
-
-	s.epochParticipationLock.Lock()
-	defer s.epochParticipationLock.Unlock()
-	s.epochParticipation[helpers.SlotToEpoch(blockCopy.Block.Slot)] = precompute.Balances
-
-	return nil
-}
-
-// ReceiveBlockNoVerify runs state transition on a input block without verifying the block's BLS contents.
-// Depends on the security model, this is the "minimal" work a node can do to sync the chain.
-// It simulates light client behavior and assumes 100% trust with the syncing peer.
-func (s *Service) ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
+// ReceiveBlockInitialSync processes the input block for the purpose of initial syncing.
+// This method should only be used on blocks during initial syncing phase.
+func (s *Service) ReceiveBlockInitialSync(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
 	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlockNoVerify")
 	defer span.End()
 	blockCopy := stateTrie.CopySignedBeaconBlock(block)
 
 	// Apply state transition on the incoming newly received blockCopy without verifying its BLS contents.
-	if err := s.onBlockInitialSyncStateTransition(ctx, blockCopy); err != nil {
+	if err := s.onBlockInitialSyncStateTransition(ctx, blockCopy, blockRoot); err != nil {
 		err := errors.Wrap(err, "could not process block")
 		traceutil.AnnotateError(span, err)
 		return err
 	}
 
-	root, err := stateutil.BlockRoot(blockCopy.Block)
-	if err != nil {
-		return errors.Wrap(err, "could not get signing root on received blockCopy")
-	}
-
 	cachedHeadRoot, err := s.HeadRoot(ctx)
 	if err != nil {
 		return errors.Wrap(err, "could not get head root from cache")
 	}
 
-	if !bytes.Equal(root[:], cachedHeadRoot) {
-		if err := s.saveHeadNoDB(ctx, blockCopy, root); err != nil {
+	if !bytes.Equal(blockRoot[:], cachedHeadRoot) {
+		if err := s.saveHeadNoDB(ctx, blockCopy, blockRoot); err != nil {
 			err := errors.Wrap(err, "could not save head")
 			traceutil.AnnotateError(span, err)
 			return err
@@ -216,13 +151,13 @@ func (s *Service) ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedB
 		Type: statefeed.BlockProcessed,
 		Data: &statefeed.BlockProcessedData{
 			Slot:      blockCopy.Block.Slot,
-			BlockRoot: root,
+			BlockRoot: blockRoot,
 			Verified:  false,
 		},
 	})
 
 	// Reports on blockCopy and fork choice metrics.
-	reportSlotMetrics(blockCopy.Block.Slot, s.headSlot(), s.finalizedCheckpt)
+	reportSlotMetrics(blockCopy.Block.Slot, s.headSlot(), s.CurrentSlot(), s.finalizedCheckpt)
 
 	// Log state transition data.
 	log.WithFields(logrus.Fields{

beacon-chain/blockchain/service.go

@@ -1,6 +1,6 @@
-// Package blockchain defines the life-cycle and status of the beacon chain
-// as well as the Ethereum Serenity beacon chain fork-choice rule based on
-// Casper Proof of Stake finality.
+// Package blockchain defines the life-cycle of the blockchain at the core of
+// eth2, including processing of new blocks and attestations using casper
+// proof of stake.
 package blockchain
 
 import (
@@ -12,7 +12,6 @@ import (
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
@@ -33,50 +32,56 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/slotutil"
 	"go.opencensus.io/trace"
 )
 
 // Service represents a service that handles the internal
 // logic of managing the full PoS beacon chain.
 type Service struct {
-	ctx                    context.Context
-	cancel                 context.CancelFunc
-	beaconDB               db.HeadAccessDatabase
-	depositCache           *depositcache.DepositCache
-	chainStartFetcher      powchain.ChainStartFetcher
-	attPool                attestations.Pool
-	slashingPool           *slashings.Pool
-	exitPool               *voluntaryexits.Pool
-	genesisTime            time.Time
-	p2p                    p2p.Broadcaster
-	maxRoutines            int64
-	head                   *head
-	headLock               sync.RWMutex
-	stateNotifier          statefeed.Notifier
-	genesisRoot            [32]byte
-	epochParticipation     map[uint64]*precompute.Balance
-	epochParticipationLock sync.RWMutex
-	forkChoiceStore        f.ForkChoicer
-	justifiedCheckpt       *ethpb.Checkpoint
-	prevJustifiedCheckpt   *ethpb.Checkpoint
-	bestJustifiedCheckpt   *ethpb.Checkpoint
-	finalizedCheckpt       *ethpb.Checkpoint
-	prevFinalizedCheckpt   *ethpb.Checkpoint
-	nextEpochBoundarySlot  uint64
-	voteLock               sync.RWMutex
-	initSyncState          map[[32]byte]*stateTrie.BeaconState
-	boundaryRoots          [][32]byte
-	initSyncStateLock      sync.RWMutex
-	checkpointState        *cache.CheckpointStateCache
-	checkpointStateLock    sync.Mutex
-	stateGen               *stategen.State
-	opsService             *attestations.Service
-	initSyncBlocks         map[[32]byte]*ethpb.SignedBeaconBlock
-	initSyncBlocksLock     sync.RWMutex
+	ctx                       context.Context
+	cancel                    context.CancelFunc
+	beaconDB                  db.HeadAccessDatabase
+	depositCache              *depositcache.DepositCache
+	chainStartFetcher         powchain.ChainStartFetcher
+	attPool                   attestations.Pool
+	slashingPool              *slashings.Pool
+	exitPool                  *voluntaryexits.Pool
+	genesisTime               time.Time
+	p2p                       p2p.Broadcaster
+	maxRoutines               int64
+	head                      *head
+	headLock                  sync.RWMutex
+	stateNotifier             statefeed.Notifier
+	genesisRoot               [32]byte
+	epochParticipation        map[uint64]*precompute.Balance
+	epochParticipationLock    sync.RWMutex
+	forkChoiceStore           f.ForkChoicer
+	justifiedCheckpt          *ethpb.Checkpoint
+	prevJustifiedCheckpt      *ethpb.Checkpoint
+	bestJustifiedCheckpt      *ethpb.Checkpoint
+	finalizedCheckpt          *ethpb.Checkpoint
+	prevFinalizedCheckpt      *ethpb.Checkpoint
+	nextEpochBoundarySlot     uint64
+	voteLock                  sync.RWMutex
+	initSyncState             map[[32]byte]*stateTrie.BeaconState
+	boundaryRoots             [][32]byte
+	initSyncStateLock         sync.RWMutex
+	checkpointState           *cache.CheckpointStateCache
+	checkpointStateLock       sync.Mutex
+	stateGen                  *stategen.State
+	opsService                *attestations.Service
+	initSyncBlocks            map[[32]byte]*ethpb.SignedBeaconBlock
+	initSyncBlocksLock        sync.RWMutex
+	recentCanonicalBlocks     map[[32]byte]bool
+	recentCanonicalBlocksLock sync.RWMutex
+	justifiedBalances         []uint64
+	justifiedBalancesLock     sync.RWMutex
 }
 
 // Config options for the service.
@@ -101,25 +106,27 @@ type Config struct {
 func NewService(ctx context.Context, cfg *Config) (*Service, error) {
 	ctx, cancel := context.WithCancel(ctx)
 	return &Service{
-		ctx:                ctx,
-		cancel:             cancel,
-		beaconDB:           cfg.BeaconDB,
-		depositCache:       cfg.DepositCache,
-		chainStartFetcher:  cfg.ChainStartFetcher,
-		attPool:            cfg.AttPool,
-		exitPool:           cfg.ExitPool,
-		slashingPool:       cfg.SlashingPool,
-		p2p:                cfg.P2p,
-		maxRoutines:        cfg.MaxRoutines,
-		stateNotifier:      cfg.StateNotifier,
-		epochParticipation: make(map[uint64]*precompute.Balance),
-		forkChoiceStore:    cfg.ForkChoiceStore,
-		initSyncState:      make(map[[32]byte]*stateTrie.BeaconState),
-		boundaryRoots:      [][32]byte{},
-		checkpointState:    cache.NewCheckpointStateCache(),
-		opsService:         cfg.OpsService,
-		stateGen:           cfg.StateGen,
-		initSyncBlocks:     make(map[[32]byte]*ethpb.SignedBeaconBlock),
+		ctx:                   ctx,
+		cancel:                cancel,
+		beaconDB:              cfg.BeaconDB,
+		depositCache:          cfg.DepositCache,
+		chainStartFetcher:     cfg.ChainStartFetcher,
+		attPool:               cfg.AttPool,
+		exitPool:              cfg.ExitPool,
+		slashingPool:          cfg.SlashingPool,
+		p2p:                   cfg.P2p,
+		maxRoutines:           cfg.MaxRoutines,
+		stateNotifier:         cfg.StateNotifier,
+		epochParticipation:    make(map[uint64]*precompute.Balance),
+		forkChoiceStore:       cfg.ForkChoiceStore,
+		initSyncState:         make(map[[32]byte]*stateTrie.BeaconState),
+		boundaryRoots:         [][32]byte{},
+		checkpointState:       cache.NewCheckpointStateCache(),
+		opsService:            cfg.OpsService,
+		stateGen:              cfg.StateGen,
+		initSyncBlocks:        make(map[[32]byte]*ethpb.SignedBeaconBlock),
+		recentCanonicalBlocks: make(map[[32]byte]bool),
+		justifiedBalances:     make([]uint64, 0),
 	}, nil
 }
 
@@ -140,16 +147,9 @@ func (s *Service) Start() {
 	}
 
 	if beaconState == nil {
-		if !featureconfig.Get().DisableNewStateMgmt {
-			beaconState, err = s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(cp.Root))
-			if err != nil {
-				log.Fatalf("Could not fetch beacon state by root: %v", err)
-			}
-		} else {
-			beaconState, err = s.beaconDB.State(ctx, bytesutil.ToBytes32(cp.Root))
-			if err != nil {
-				log.Fatalf("Could not fetch beacon state: %v", err)
-			}
+		beaconState, err = s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(cp.Root))
+		if err != nil {
+			log.Fatalf("Could not fetch beacon state by root: %v", err)
 		}
 	}
 
@@ -164,6 +164,10 @@ func (s *Service) Start() {
 		if err := s.initializeChainInfo(ctx); err != nil {
 			log.Fatalf("Could not set up chain info: %v", err)
 		}
+
+		// We start a counter to genesis, if needed.
+		go slotutil.CountdownToGenesis(ctx, s.genesisTime, uint64(beaconState.NumValidators()))
+
 		justifiedCheckpoint, err := s.beaconDB.JustifiedCheckpoint(ctx)
 		if err != nil {
 			log.Fatalf("Could not get justified checkpoint: %v", err)
@@ -175,20 +179,15 @@ func (s *Service) Start() {
 
 		// Resume fork choice.
 		s.justifiedCheckpt = stateTrie.CopyCheckpoint(justifiedCheckpoint)
+		if err := s.cacheJustifiedStateBalances(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root)); err != nil {
+			log.Fatalf("Could not cache justified state balances: %v", err)
+		}
 		s.prevJustifiedCheckpt = stateTrie.CopyCheckpoint(justifiedCheckpoint)
 		s.bestJustifiedCheckpt = stateTrie.CopyCheckpoint(justifiedCheckpoint)
 		s.finalizedCheckpt = stateTrie.CopyCheckpoint(finalizedCheckpoint)
 		s.prevFinalizedCheckpt = stateTrie.CopyCheckpoint(finalizedCheckpoint)
 		s.resumeForkChoice(justifiedCheckpoint, finalizedCheckpoint)
 
-		if featureconfig.Get().DisableNewStateMgmt {
-			if finalizedCheckpoint.Epoch > 1 {
-				if err := s.pruneGarbageState(ctx, helpers.StartSlot(finalizedCheckpoint.Epoch)-params.BeaconConfig().SlotsPerEpoch); err != nil {
-					log.WithError(err).Warn("Could not prune old states")
-				}
-			}
-		}
-
 		s.stateNotifier.StateFeed().Send(&feed.Event{
 			Type: statefeed.Initialized,
 			Data: &statefeed.InitializedData{
@@ -242,6 +241,11 @@ func (s *Service) processChainStartTime(ctx context.Context, genesisTime time.Ti
 	if err != nil {
 		log.Fatalf("Could not initialize beacon chain: %v", err)
 	}
+	// We start a counter to genesis, if needed.
+	go slotutil.CountdownToGenesis(ctx, genesisTime, uint64(initializedState.NumValidators()))
+
+	// We send out a state initialized event to the rest of the services
+	// running in the beacon node.
 	s.stateNotifier.StateFeed().Send(&feed.Event{
 		Type: statefeed.Initialized,
 		Data: &statefeed.InitializedData{
@@ -300,7 +304,7 @@ func (s *Service) Stop() error {
 // Status always returns nil unless there is an error condition that causes
 // this service to be unhealthy.
 func (s *Service) Status() error {
-	if runtime.NumGoroutine() > int(s.maxRoutines) {
+	if int64(runtime.NumGoroutine()) > s.maxRoutines {
 		return fmt.Errorf("too many goroutines %d", runtime.NumGoroutine())
 	}
 	return nil
@@ -319,40 +323,39 @@ func (s *Service) saveGenesisData(ctx context.Context, genesisState *stateTrie.B
 		return err
 	}
 	genesisBlk := blocks.NewGenesisBlock(stateRoot[:])
-	genesisBlkRoot, err := ssz.HashTreeRoot(genesisBlk.Block)
+	genesisBlkRoot, err := stateutil.BlockRoot(genesisBlk.Block)
 	if err != nil {
 		return errors.Wrap(err, "could not get genesis block root")
 	}
+	s.genesisRoot = genesisBlkRoot
 
 	if err := s.beaconDB.SaveBlock(ctx, genesisBlk); err != nil {
 		return errors.Wrap(err, "could not save genesis block")
 	}
-	if !featureconfig.Get().DisableNewStateMgmt {
-		if err := s.stateGen.SaveState(ctx, genesisBlkRoot, genesisState); err != nil {
-			return errors.Wrap(err, "could not save genesis state")
-		}
-		if err := s.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{
-			Slot: 0,
-			Root: genesisBlkRoot[:],
-		}); err != nil {
-			return err
-		}
-	} else {
-		if err := s.beaconDB.SaveState(ctx, genesisState, genesisBlkRoot); err != nil {
-			return errors.Wrap(err, "could not save genesis state")
-		}
+	if err := s.stateGen.SaveState(ctx, genesisBlkRoot, genesisState); err != nil {
+		return errors.Wrap(err, "could not save genesis state")
 	}
+	if err := s.beaconDB.SaveStateSummary(ctx, &pb.StateSummary{
+		Slot: 0,
+		Root: genesisBlkRoot[:],
+	}); err != nil {
+		return err
+	}
+
 	if err := s.beaconDB.SaveHeadBlockRoot(ctx, genesisBlkRoot); err != nil {
 		return errors.Wrap(err, "could not save head block root")
 	}
 	if err := s.beaconDB.SaveGenesisBlockRoot(ctx, genesisBlkRoot); err != nil {
-		return errors.Wrap(err, "could save genesis block root")
+		return errors.Wrap(err, "could not save genesis block root")
 	}
 
-	genesisCheckpoint := &ethpb.Checkpoint{Root: genesisBlkRoot[:]}
+	// Finalized checkpoint at genesis is a zero hash.
+	genesisCheckpoint := genesisState.FinalizedCheckpoint()
 
-	// Add the genesis block to the fork choice store.
 	s.justifiedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
+	if err := s.cacheJustifiedStateBalances(ctx, genesisBlkRoot); err != nil {
+		return err
+	}
 	s.prevJustifiedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
 	s.bestJustifiedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
 	s.finalizedCheckpt = stateTrie.CopyCheckpoint(genesisCheckpoint)
@@ -362,6 +365,7 @@ func (s *Service) saveGenesisData(ctx context.Context, genesisState *stateTrie.B
 		genesisBlk.Block.Slot,
 		genesisBlkRoot,
 		params.BeaconConfig().ZeroHash,
+		[32]byte{},
 		genesisCheckpoint.Epoch,
 		genesisCheckpoint.Epoch); err != nil {
 		log.Fatalf("Could not process genesis block for fork choice: %v", err)
@@ -381,7 +385,7 @@ func (s *Service) initializeChainInfo(ctx context.Context) error {
 	if genesisBlock == nil {
 		return errors.New("no genesis block in db")
 	}
-	genesisBlkRoot, err := ssz.HashTreeRoot(genesisBlock.Block)
+	genesisBlkRoot, err := stateutil.BlockRoot(genesisBlock.Block)
 	if err != nil {
 		return errors.Wrap(err, "could not get signing root of genesis block")
 	}
@@ -392,7 +396,7 @@ func (s *Service) initializeChainInfo(ctx context.Context) error {
 		if err != nil {
 			return errors.Wrap(err, "could not retrieve head block")
 		}
-		headRoot, err := ssz.HashTreeRoot(headBlock.Block)
+		headRoot, err := stateutil.BlockRoot(headBlock.Block)
 		if err != nil {
 			return errors.Wrap(err, "could not hash head block")
 		}
@@ -415,20 +419,17 @@ func (s *Service) initializeChainInfo(ctx context.Context) error {
 	}
 	finalizedRoot := bytesutil.ToBytes32(finalized.Root)
 	var finalizedState *stateTrie.BeaconState
-	if !featureconfig.Get().DisableNewStateMgmt {
+
+	finalizedState, err = s.stateGen.Resume(ctx)
+	if err != nil {
+		return errors.Wrap(err, "could not get finalized state from db")
+	}
+	if !featureconfig.Get().SkipRegenHistoricalStates {
+		// Since historical states were skipped, the node should start from last finalized check point.
 		finalizedRoot = s.beaconDB.LastArchivedIndexRoot(ctx)
-		finalizedState, err = s.stateGen.Resume(ctx)
-		if err != nil {
-			return errors.Wrap(err, "could not get finalized state from db")
-		}
 		if finalizedRoot == params.BeaconConfig().ZeroHash {
 			finalizedRoot = bytesutil.ToBytes32(finalized.Root)
 		}
-	} else {
-		finalizedState, err = s.beaconDB.State(ctx, bytesutil.ToBytes32(finalized.Root))
-		if err != nil {
-			return errors.Wrap(err, "could not get finalized state from db")
-		}
 	}
 
 	finalizedBlock, err := s.beaconDB.Block(ctx, finalizedRoot)
@@ -436,6 +437,20 @@ func (s *Service) initializeChainInfo(ctx context.Context) error {
 		return errors.Wrap(err, "could not get finalized block from db")
 	}
 
+	// To skip the regeneration of historical state, the node has to generate the parent of the last finalized state.
+	// We don't need to do this for genesis.
+	atGenesis := s.CurrentSlot() == 0
+	if featureconfig.Get().SkipRegenHistoricalStates && !atGenesis {
+		parentRoot := bytesutil.ToBytes32(finalizedBlock.Block.ParentRoot)
+		parentState, err := s.generateState(ctx, finalizedRoot, parentRoot)
+		if err != nil {
+			return err
+		}
+		if s.beaconDB.SaveState(ctx, parentState, parentRoot) != nil {
+			return err
+		}
+	}
+
 	if finalizedState == nil || finalizedBlock == nil {
 		return errors.New("finalized state and block can't be nil")
 	}

beacon-chain/blockchain/service_norace_test.go

@@ -15,17 +15,16 @@ func init() {
 }
 
 func TestChainService_SaveHead_DataRace(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 	s := &Service{
 		beaconDB: db,
 	}
 	go func() {
-		if err := s.saveHead(context.Background(), [32]byte{}, ); err != nil {
+		if err := s.saveHead(context.Background(), [32]byte{}); err != nil {
 			t.Fatal(err)
 		}
 	}()
-	if err := s.saveHead(context.Background(), [32]byte{}, ); err != nil {
+	if err := s.saveHead(context.Background(), [32]byte{}); err != nil {
 		t.Fatal(err)
 	}
 }

beacon-chain/blockchain/service_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	ssz "github.com/prysmaticlabs/go-ssz"
+	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache"
 	b "github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
@@ -28,8 +28,10 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/powchain"
 	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	protodb "github.com/prysmaticlabs/prysm/proto/beacon/db"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/event"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
@@ -42,42 +44,6 @@ func init() {
 	logrus.SetOutput(ioutil.Discard)
 }
 
-type store struct {
-	headRoot []byte
-}
-
-func (s *store) OnBlock(ctx context.Context, b *ethpb.SignedBeaconBlock) (*beaconstate.BeaconState, error) {
-	return nil, nil
-}
-
-func (s *store) OnBlockCacheFilteredTree(ctx context.Context, b *ethpb.SignedBeaconBlock) (*beaconstate.BeaconState, error) {
-	return nil, nil
-}
-
-func (s *store) OnBlockInitialSyncStateTransition(ctx context.Context, b *ethpb.SignedBeaconBlock) (*beaconstate.BeaconState, error) {
-	return nil, nil
-}
-
-func (s *store) OnAttestation(ctx context.Context, a *ethpb.Attestation) ([]uint64, error) {
-	return nil, nil
-}
-
-func (s *store) GenesisStore(ctx context.Context, justifiedCheckpoint *ethpb.Checkpoint, finalizedCheckpoint *ethpb.Checkpoint) error {
-	return nil
-}
-
-func (s *store) FinalizedCheckpt() *ethpb.Checkpoint {
-	return nil
-}
-
-func (s *store) JustifiedCheckpt() *ethpb.Checkpoint {
-	return nil
-}
-
-func (s *store) Head(ctx context.Context) ([]byte, error) {
-	return s.headRoot, nil
-}
-
 type mockBeaconNode struct {
 	stateFeed *event.Feed
 }
@@ -99,10 +65,15 @@ func (mb *mockBroadcaster) Broadcast(_ context.Context, _ proto.Message) error {
 	return nil
 }
 
+func (mb *mockBroadcaster) BroadcastAttestation(_ context.Context, _ uint64, _ *ethpb.Attestation) error {
+	mb.broadcastCalled = true
+	return nil
+}
+
 var _ = p2p.Broadcaster(&mockBroadcaster{})
 
-func setupBeaconChain(t *testing.T, beaconDB db.Database) *Service {
-	endpoint := "ws://127.0.0.1"
+func setupBeaconChain(t *testing.T, beaconDB db.Database, sc *cache.StateSummaryCache) *Service {
+	endpoint := "http://127.0.0.1"
 	ctx := context.Background()
 	var web3Service *powchain.Service
 	var err error
@@ -127,7 +98,7 @@ func setupBeaconChain(t *testing.T, beaconDB db.Database) *Service {
 	}
 	web3Service, err = powchain.NewService(ctx, &powchain.Web3ServiceConfig{
 		BeaconDB:        beaconDB,
-		ETH1Endpoint:    endpoint,
+		HTTPEndPoint:    endpoint,
 		DepositContract: common.Address{},
 	})
 	if err != nil {
@@ -146,7 +117,7 @@ func setupBeaconChain(t *testing.T, beaconDB db.Database) *Service {
 		P2p:               &mockBroadcaster{},
 		StateNotifier:     &mockBeaconNode{},
 		AttPool:           attestations.NewPool(),
-		StateGen:          stategen.New(beaconDB, cache.NewStateSummaryCache()),
+		StateGen:          stategen.New(beaconDB, sc),
 		ForkChoiceStore:   protoarray.New(0, 0, params.BeaconConfig().ZeroHash),
 		OpsService:        opsService,
 	}
@@ -162,9 +133,8 @@ func setupBeaconChain(t *testing.T, beaconDB db.Database) *Service {
 
 func TestChainStartStop_Uninitialized(t *testing.T) {
 	hook := logTest.NewGlobal()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
-	chainService := setupBeaconChain(t, db)
+	db, sc := testDB.SetupDB(t)
+	chainService := setupBeaconChain(t, db, sc)
 
 	// Listen for state events.
 	stateSubChannel := make(chan *feed.Event, 1)
@@ -218,13 +188,12 @@ func TestChainStartStop_Uninitialized(t *testing.T) {
 func TestChainStartStop_Initialized(t *testing.T) {
 	hook := logTest.NewGlobal()
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 
-	chainService := setupBeaconChain(t, db)
+	chainService := setupBeaconChain(t, db, sc)
 
-	genesisBlk := b.NewGenesisBlock([]byte{})
-	blkRoot, err := ssz.HashTreeRoot(genesisBlk.Block)
+	genesisBlk := testutil.NewBeaconBlock()
+	blkRoot, err := stateutil.BlockRoot(genesisBlk.Block)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -263,11 +232,11 @@ func TestChainStartStop_Initialized(t *testing.T) {
 }
 
 func TestChainService_InitializeBeaconChain(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	helpers.ClearCache()
+	db, sc := testDB.SetupDB(t)
 	ctx := context.Background()
 
-	bc := setupBeaconChain(t, db)
+	bc := setupBeaconChain(t, db, sc)
 	var err error
 
 	// Set up 10 deposits pre chain start for validators to register
@@ -318,12 +287,11 @@ func TestChainService_InitializeBeaconChain(t *testing.T) {
 }
 
 func TestChainService_InitializeChainInfo(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 	ctx := context.Background()
 
-	genesis := b.NewGenesisBlock([]byte{})
-	genesisRoot, err := ssz.HashTreeRoot(genesis.Block)
+	genesis := testutil.NewBeaconBlock()
+	genesisRoot, err := stateutil.BlockRoot(genesis.Block)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -335,7 +303,9 @@ func TestChainService_InitializeChainInfo(t *testing.T) {
 	}
 
 	finalizedSlot := params.BeaconConfig().SlotsPerEpoch*2 + 1
-	headBlock := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: finalizedSlot, ParentRoot: genesisRoot[:]}}
+	headBlock := testutil.NewBeaconBlock()
+	headBlock.Block.Slot = finalizedSlot
+	headBlock.Block.ParentRoot = bytesutil.PadTo(genesisRoot[:], 32)
 	headState := testutil.NewBeaconState()
 	if err := headState.SetSlot(finalizedSlot); err != nil {
 		t.Fatal(err)
@@ -343,7 +313,7 @@ func TestChainService_InitializeChainInfo(t *testing.T) {
 	if err := headState.SetGenesisValidatorRoot(params.BeaconConfig().ZeroHash[:]); err != nil {
 		t.Fatal(err)
 	}
-	headRoot, err := ssz.HashTreeRoot(headBlock.Block)
+	headRoot, err := stateutil.BlockRoot(headBlock.Block)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -362,10 +332,7 @@ func TestChainService_InitializeChainInfo(t *testing.T) {
 	}); err != nil {
 		t.Fatal(err)
 	}
-	if err := db.SaveBlock(ctx, headBlock); err != nil {
-		t.Fatal(err)
-	}
-	c := &Service{beaconDB: db, stateGen: stategen.New(db, cache.NewStateSummaryCache())}
+	c := &Service{beaconDB: db, stateGen: stategen.New(db, sc)}
 	if err := c.initializeChainInfo(ctx); err != nil {
 		t.Fatal(err)
 	}
@@ -399,12 +366,11 @@ func TestChainService_InitializeChainInfo(t *testing.T) {
 }
 
 func TestChainService_SaveHeadNoDB(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, sc := testDB.SetupDB(t)
 	ctx := context.Background()
 	s := &Service{
 		beaconDB: db,
-		stateGen: stategen.New(db, cache.NewStateSummaryCache()),
+		stateGen: stategen.New(db, sc),
 	}
 	b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1}}
 	r, err := ssz.HashTreeRoot(b)
@@ -430,8 +396,7 @@ func TestChainService_SaveHeadNoDB(t *testing.T) {
 }
 
 func TestChainService_PruneOldStates(t *testing.T) {
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 	ctx := context.Background()
 	s := &Service{
 		beaconDB: db,
@@ -442,7 +407,7 @@ func TestChainService_PruneOldStates(t *testing.T) {
 		if err := s.beaconDB.SaveBlock(ctx, &ethpb.SignedBeaconBlock{Block: block}); err != nil {
 			t.Fatal(err)
 		}
-		r, err := ssz.HashTreeRoot(block)
+		r, err := stateutil.BlockRoot(block)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -479,15 +444,14 @@ func TestChainService_PruneOldStates(t *testing.T) {
 
 func TestHasBlock_ForkChoiceAndDB(t *testing.T) {
 	ctx := context.Background()
-	db := testDB.SetupDB(t)
-	defer testDB.TeardownDB(t, db)
+	db, _ := testDB.SetupDB(t)
 	s := &Service{
 		forkChoiceStore:  protoarray.New(0, 0, [32]byte{}),
 		finalizedCheckpt: &ethpb.Checkpoint{},
 		beaconDB:         db,
 	}
 	block := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Body: &ethpb.BeaconBlockBody{}}}
-	r, err := ssz.HashTreeRoot(block.Block)
+	r, err := stateutil.BlockRoot(block.Block)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -510,8 +474,7 @@ func TestHasBlock_ForkChoiceAndDB(t *testing.T) {
 }
 
 func BenchmarkHasBlockDB(b *testing.B) {
-	db := testDB.SetupDB(b)
-	defer testDB.TeardownDB(b, db)
+	db, _ := testDB.SetupDB(b)
 	ctx := context.Background()
 	s := &Service{
 		beaconDB: db,
@@ -520,7 +483,7 @@ func BenchmarkHasBlockDB(b *testing.B) {
 	if err := s.beaconDB.SaveBlock(ctx, block); err != nil {
 		b.Fatal(err)
 	}
-	r, err := ssz.HashTreeRoot(block.Block)
+	r, err := stateutil.BlockRoot(block.Block)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -535,15 +498,14 @@ func BenchmarkHasBlockDB(b *testing.B) {
 
 func BenchmarkHasBlockForkChoiceStore(b *testing.B) {
 	ctx := context.Background()
-	db := testDB.SetupDB(b)
-	defer testDB.TeardownDB(b, db)
+	db, _ := testDB.SetupDB(b)
 	s := &Service{
 		forkChoiceStore:  protoarray.New(0, 0, [32]byte{}),
 		finalizedCheckpt: &ethpb.Checkpoint{},
 		beaconDB:         db,
 	}
 	block := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Body: &ethpb.BeaconBlockBody{}}}
-	r, err := ssz.HashTreeRoot(block.Block)
+	r, err := stateutil.BlockRoot(block.Block)
 	if err != nil {
 		b.Fatal(err)
 	}

beacon-chain/blockchain/testing/BUILD.bazel

@@ -1,11 +1,14 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@prysm//tools/go:def.bzl", "go_library")
 
 go_library(
     name = "go_default_library",
     testonly = True,
     srcs = ["mock.go"],
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing",
-    visibility = ["//beacon-chain:__subpackages__"],
+    visibility = [
+        "//beacon-chain:__subpackages__",
+        "//fuzz:__pkg__",
+    ],
     deps = [
         "//beacon-chain/core/epoch/precompute:go_default_library",
         "//beacon-chain/core/feed/block:go_default_library",
@@ -13,13 +16,14 @@ go_library(
         "//beacon-chain/core/feed/state:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/db:go_default_library",
+        "//beacon-chain/forkchoice/protoarray:go_default_library",
         "//beacon-chain/state:go_default_library",
+        "//beacon-chain/state/stateutil:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/event:go_default_library",
         "//shared/params:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
-        "@com_github_prysmaticlabs_go_ssz//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
     ],
 )

beacon-chain/blockchain/testing/mock.go

@@ -1,3 +1,5 @@
+// Package testing includes useful mocks for writing unit
+// tests which depend on logic from the blockchain package.
 package testing
 
 import (
@@ -7,14 +9,15 @@ import (
 
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
 	blockfeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/block"
 	opfeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/operation"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/db"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/event"
 	"github.com/prysmaticlabs/prysm/shared/params"
@@ -32,12 +35,15 @@ type ChainService struct {
 	BlocksReceived              []*ethpb.SignedBeaconBlock
 	Balance                     *precompute.Balance
 	Genesis                     time.Time
+	ValidatorsRoot              [32]byte
 	Fork                        *pb.Fork
+	ETH1Data                    *ethpb.Eth1Data
 	DB                          db.Database
 	stateNotifier               statefeed.Notifier
 	blockNotifier               blockfeed.Notifier
 	opNotifier                  opfeed.Notifier
 	ValidAttestation            bool
+	ForkChoiceStore             *protoarray.Store
 }
 
 // StateNotifier mocks the same method in the chain service.
@@ -104,22 +110,12 @@ func (mon *MockOperationNotifier) OperationFeed() *event.Feed {
 }
 
 // ReceiveBlock mocks ReceiveBlock method in chain service.
-func (ms *ChainService) ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
+func (ms *ChainService) ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
 	return nil
 }
 
-// ReceiveBlockNoVerify mocks ReceiveBlockNoVerify method in chain service.
-func (ms *ChainService) ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
-	return nil
-}
-
-// ReceiveBlockNoPubsub mocks ReceiveBlockNoPubsub method in chain service.
-func (ms *ChainService) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
-	return nil
-}
-
-// ReceiveBlockNoPubsubForkchoice mocks ReceiveBlockNoPubsubForkchoice method in chain service.
-func (ms *ChainService) ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *ethpb.SignedBeaconBlock) error {
+// ReceiveBlockInitialSync mocks ReceiveBlockInitialSync method in chain service.
+func (ms *ChainService) ReceiveBlockInitialSync(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
 	if ms.State == nil {
 		ms.State = &stateTrie.BeaconState{}
 	}
@@ -130,7 +126,34 @@ func (ms *ChainService) ReceiveBlockNoPubsubForkchoice(ctx context.Context, bloc
 		return err
 	}
 	ms.BlocksReceived = append(ms.BlocksReceived, block)
-	signingRoot, err := ssz.HashTreeRoot(block.Block)
+	signingRoot, err := stateutil.BlockRoot(block.Block)
+	if err != nil {
+		return err
+	}
+	if ms.DB != nil {
+		if err := ms.DB.SaveBlock(ctx, block); err != nil {
+			return err
+		}
+		logrus.Infof("Saved block with root: %#x at slot %d", signingRoot, block.Block.Slot)
+	}
+	ms.Root = signingRoot[:]
+	ms.Block = block
+	return nil
+}
+
+// ReceiveBlockNoPubsub mocks ReceiveBlockNoPubsub method in chain service.
+func (ms *ChainService) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedBeaconBlock, blockRoot [32]byte) error {
+	if ms.State == nil {
+		ms.State = &stateTrie.BeaconState{}
+	}
+	if !bytes.Equal(ms.Root, block.Block.ParentRoot) {
+		return errors.Errorf("wanted %#x but got %#x", ms.Root, block.Block.ParentRoot)
+	}
+	if err := ms.State.SetSlot(block.Block.Slot); err != nil {
+		return err
+	}
+	ms.BlocksReceived = append(ms.BlocksReceived, block)
+	signingRoot, err := stateutil.BlockRoot(block.Block)
 	if err != nil {
 		return err
 	}
@@ -199,8 +222,13 @@ func (ms *ChainService) ReceiveAttestationNoPubsub(context.Context, *ethpb.Attes
 	return nil
 }
 
+// AttestationPreState mocks AttestationPreState method in chain service.
+func (ms *ChainService) AttestationPreState(ctx context.Context, att *ethpb.Attestation) (*stateTrie.BeaconState, error) {
+	return ms.State, nil
+}
+
 // HeadValidatorsIndices mocks the same method in the chain service.
-func (ms *ChainService) HeadValidatorsIndices(epoch uint64) ([]uint64, error) {
+func (ms *ChainService) HeadValidatorsIndices(ctx context.Context, epoch uint64) ([]uint64, error) {
 	if ms.State == nil {
 		return []uint64{}, nil
 	}
@@ -208,15 +236,30 @@ func (ms *ChainService) HeadValidatorsIndices(epoch uint64) ([]uint64, error) {
 }
 
 // HeadSeed mocks the same method in the chain service.
-func (ms *ChainService) HeadSeed(epoch uint64) ([32]byte, error) {
+func (ms *ChainService) HeadSeed(ctx context.Context, epoch uint64) ([32]byte, error) {
 	return helpers.Seed(ms.State, epoch, params.BeaconConfig().DomainBeaconAttester)
 }
 
+// HeadETH1Data provides the current ETH1Data of the head state.
+func (ms *ChainService) HeadETH1Data() *ethpb.Eth1Data {
+	return ms.ETH1Data
+}
+
+// ProtoArrayStore mocks the same method in the chain service.
+func (ms *ChainService) ProtoArrayStore() *protoarray.Store {
+	return ms.ForkChoiceStore
+}
+
 // GenesisTime mocks the same method in the chain service.
 func (ms *ChainService) GenesisTime() time.Time {
 	return ms.Genesis
 }
 
+// GenesisValidatorRoot mocks the same method in the chain service.
+func (ms *ChainService) GenesisValidatorRoot() [32]byte {
+	return ms.ValidatorsRoot
+}
+
 // CurrentSlot mocks the same method in the chain service.
 func (ms *ChainService) CurrentSlot() uint64 {
 	return uint64(time.Now().Unix()-ms.Genesis.Unix()) / params.BeaconConfig().SecondsPerSlot
@@ -232,6 +275,12 @@ func (ms *ChainService) IsValidAttestation(ctx context.Context, att *ethpb.Attes
 	return ms.ValidAttestation
 }
 
+// IsCanonical returns and determines whether a block with the provided root is part of
+// the canonical chain.
+func (ms *ChainService) IsCanonical(ctx context.Context, blockRoot [32]byte) (bool, error) {
+	return true, nil
+}
+
 // ClearCachedStates does nothing.
 func (ms *ChainService) ClearCachedStates() {}
 
@@ -239,3 +288,8 @@ func (ms *ChainService) ClearCachedStates() {}
 func (ms *ChainService) HasInitSyncBlock(root [32]byte) bool {
 	return false
 }
+
+// HeadGenesisValidatorRoot mocks HeadGenesisValidatorRoot method in chain service.
+func (ms *ChainService) HeadGenesisValidatorRoot() [32]byte {
+	return [32]byte{}
+}

beacon-chain/cache/BUILD.bazel

@@ -1,4 +1,5 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -6,16 +7,17 @@ go_library(
         "attestation_data.go",
         "checkpoint_state.go",
         "committee.go",
-        "committee_ids.go",
         "common.go",
-        "eth1_data.go",
+        "doc.go",
         "hot_state_cache.go",
         "skip_slot_cache.go",
         "state_summary.go",
+        "subnet_ids.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/cache",
     visibility = [
         "//beacon-chain:__subpackages__",
+        "//fuzz:__pkg__",
         "//tools:__subpackages__",
     ],
     deps = [
@@ -26,6 +28,7 @@ go_library(
         "//shared/params:go_default_library",
         "//shared/sliceutil:go_default_library",
         "@com_github_hashicorp_golang_lru//:go_default_library",
+        "@com_github_patrickmn_go_cache//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
@@ -41,12 +44,11 @@ go_test(
         "attestation_data_test.go",
         "checkpoint_state_test.go",
         "committee_fuzz_test.go",
-        "committee_ids_test.go",
         "committee_test.go",
-        "eth1_data_test.go",
         "feature_flag_test.go",
         "hot_state_cache_test.go",
         "skip_slot_cache_test.go",
+        "subnet_ids_test.go",
     ],
     embed = [":go_default_library"],
     deps = [

beacon-chain/cache/attestation_data.go

@@ -11,6 +11,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"k8s.io/client-go/tools/cache"
 )
 
@@ -97,6 +99,9 @@ func (c *AttestationCache) Get(ctx context.Context, req *ethpb.AttestationDataRe
 
 	if exists && item != nil && item.(*attestationReqResWrapper).res != nil {
 		attestationCacheHit.Inc()
+		if featureconfig.Get().ReduceAttesterStateCopy {
+			return state.CopyAttestationData(item.(*attestationReqResWrapper).res), nil
+		}
 		return item.(*attestationReqResWrapper).res, nil
 	}
 	attestationCacheMiss.Inc()
@@ -162,6 +167,9 @@ func wrapperToKey(i interface{}) (string, error) {
 }
 
 func reqToKey(req *ethpb.AttestationDataRequest) (string, error) {
+	if featureconfig.Get().ReduceAttesterStateCopy {
+		return fmt.Sprintf("%d", req.Slot), nil
+	}
 	return fmt.Sprintf("%d-%d", req.CommitteeIndex, req.Slot), nil
 }
 

beacon-chain/cache/checkpoint_state.go

@@ -20,7 +20,7 @@ var (
 	// maxCheckpointStateSize defines the max number of entries check point to state cache can contain.
 	// Choosing 10 to account for multiple forks, this allows 5 forks per epoch boundary with 2 epochs
 	// window to accept attestation based on latest spec.
-	maxCheckpointStateSize = 10
+	maxCheckpointStateSize = uint64(10)
 
 	// Metrics.
 	checkpointStateMiss = promauto.NewCounter(prometheus.CounterOpts{
@@ -93,7 +93,8 @@ func (c *CheckpointStateCache) StateByCheckpoint(cp *ethpb.Checkpoint) (*stateTr
 		return nil, ErrNotCheckpointState
 	}
 
-	return info.State.Copy(), nil
+	// Copy here is unnecessary since the return will only be used to verify attestation signature.
+	return info.State, nil
 }
 
 // AddCheckpointState adds CheckpointState object to the cache. This method also trims the least

beacon-chain/cache/checkpoint_state_test.go

@@ -5,12 +5,12 @@ import (
 	"testing"
 
 	"github.com/gogo/protobuf/proto"
-	"github.com/prysmaticlabs/prysm/shared/params"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/hashutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
 func TestCheckpointStateCacheKeyFn_OK(t *testing.T) {
@@ -118,12 +118,12 @@ func TestCheckpointStateCache_MaxSize(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	for i := 0; i < maxCheckpointStateSize+100; i++ {
-		if err := st.SetSlot(uint64(i)); err != nil {
+	for i := uint64(0); i < maxCheckpointStateSize+100; i++ {
+		if err := st.SetSlot(i); err != nil {
 			t.Fatal(err)
 		}
 		info := &CheckpointState{
-			Checkpoint: &ethpb.Checkpoint{Epoch: uint64(i)},
+			Checkpoint: &ethpb.Checkpoint{Epoch: i},
 			State:      st,
 		}
 		if err := c.AddCheckpointState(info); err != nil {
@@ -131,7 +131,7 @@ func TestCheckpointStateCache_MaxSize(t *testing.T) {
 		}
 	}
 
-	if len(c.cache.ListKeys()) != maxCheckpointStateSize {
+	if uint64(len(c.cache.ListKeys())) != maxCheckpointStateSize {
 		t.Errorf(
 			"Expected hash cache key size to be %d, got %d",
 			maxCheckpointStateSize,

beacon-chain/cache/committee.go

@@ -19,7 +19,7 @@ var (
 	// maxCommitteesCacheSize defines the max number of shuffled committees on per randao basis can cache.
 	// Due to reorgs, it's good to keep the old cache around for quickly switch over. 10 is a generous
 	// cache size as it considers 3 concurrent branches over 3 epochs.
-	maxCommitteesCacheSize = 10
+	maxCommitteesCacheSize = uint64(10)
 
 	// CommitteeCacheMiss tracks the number of committee requests that aren't present in the cache.
 	CommitteeCacheMiss = promauto.NewCounter(prometheus.CounterOpts{
@@ -96,7 +96,7 @@ func (c *CommitteeCache) Committee(slot uint64, seed [32]byte, index uint64) ([]
 	indexOffSet := index + (slot%params.BeaconConfig().SlotsPerEpoch)*committeeCountPerSlot
 	start, end := startEndIndices(item, indexOffSet)
 
-	if int(end) > len(item.ShuffledIndices) || end < start {
+	if end > uint64(len(item.ShuffledIndices)) || end < start {
 		return nil, errors.New("requested index out of bound")
 	}
 
@@ -169,6 +169,30 @@ func (c *CommitteeCache) ActiveIndices(seed [32]byte) ([]uint64, error) {
 	return item.SortedIndices, nil
 }
 
+// ActiveIndicesCount returns the active indices count of a given seed stored in cache.
+func (c *CommitteeCache) ActiveIndicesCount(seed [32]byte) (int, error) {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	obj, exists, err := c.CommitteeCache.GetByKey(key(seed))
+	if err != nil {
+		return 0, err
+	}
+
+	if exists {
+		CommitteeCacheHit.Inc()
+	} else {
+		CommitteeCacheMiss.Inc()
+		return 0, nil
+	}
+
+	item, ok := obj.(*Committees)
+	if !ok {
+		return 0, ErrNotCommittee
+	}
+
+	return len(item.SortedIndices), nil
+}
+
 // ProposerIndices returns the proposer indices of a given seed.
 func (c *CommitteeCache) ProposerIndices(seed [32]byte) ([]uint64, error) {
 	c.lock.RLock()

beacon-chain/cache/committee_fuzz_test.go

@@ -38,7 +38,7 @@ func TestCommitteeCache_FuzzCommitteesByEpoch(t *testing.T) {
 		}
 	}
 
-	if len(cache.CommitteeCache.ListKeys()) != maxCommitteesCacheSize {
+	if uint64(len(cache.CommitteeCache.ListKeys())) != maxCommitteesCacheSize {
 		t.Error("Incorrect key size")
 	}
 }
@@ -62,7 +62,7 @@ func TestCommitteeCache_FuzzActiveIndices(t *testing.T) {
 		}
 	}
 
-	if len(cache.CommitteeCache.ListKeys()) != maxCommitteesCacheSize {
+	if uint64(len(cache.CommitteeCache.ListKeys())) != maxCommitteesCacheSize {
 		t.Error("Incorrect key size")
 	}
 }

beacon-chain/cache/committee_ids.go

@@ -1,87 +0,0 @@
-package cache
-
-import (
-	"sync"
-
-	lru "github.com/hashicorp/golang-lru"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/prysmaticlabs/prysm/shared/sliceutil"
-)
-
-type committeeIDs struct {
-	attester       *lru.Cache
-	attesterLock   sync.RWMutex
-	aggregator     *lru.Cache
-	aggregatorLock sync.RWMutex
-}
-
-// CommitteeIDs for attester and aggregator.
-var CommitteeIDs = newCommitteeIDs()
-
-func newCommitteeIDs() *committeeIDs {
-	// Given a node can calculate committee assignments of current epoch and next epoch.
-	// Max size is set to 2 epoch length.
-	cacheSize := int(params.BeaconConfig().MaxCommitteesPerSlot * params.BeaconConfig().SlotsPerEpoch * 2)
-	attesterCache, err := lru.New(cacheSize)
-	if err != nil {
-		panic(err)
-	}
-	aggregatorCache, err := lru.New(cacheSize)
-	if err != nil {
-		panic(err)
-	}
-	return &committeeIDs{attester: attesterCache, aggregator: aggregatorCache}
-}
-
-// AddAttesterCommiteeID adds committee ID for subscribing subnet for the attester of a given slot.
-func (c *committeeIDs) AddAttesterCommiteeID(slot uint64, committeeID uint64) {
-	c.attesterLock.Lock()
-	defer c.attesterLock.Unlock()
-
-	ids := []uint64{committeeID}
-	val, exists := c.attester.Get(slot)
-	if exists {
-		ids = sliceutil.UnionUint64(append(val.([]uint64), ids...))
-	}
-	c.attester.Add(slot, ids)
-}
-
-// GetAttesterCommitteeIDs gets the committee ID for subscribing subnet for attester of the slot.
-func (c *committeeIDs) GetAttesterCommitteeIDs(slot uint64) []uint64 {
-	c.attesterLock.RLock()
-	defer c.attesterLock.RUnlock()
-
-	val, exists := c.attester.Get(slot)
-	if !exists {
-		return nil
-	}
-	if v, ok := val.([]uint64); ok {
-	   return v
-	}
-	return nil
-}
-
-// AddAggregatorCommiteeID adds committee ID for subscribing subnet for the aggregator of a given slot.
-func (c *committeeIDs) AddAggregatorCommiteeID(slot uint64, committeeID uint64) {
-	c.aggregatorLock.Lock()
-	defer c.aggregatorLock.Unlock()
-
-	ids := []uint64{committeeID}
-	val, exists := c.aggregator.Get(slot)
-	if exists {
-		ids = sliceutil.UnionUint64(append(val.([]uint64), ids...))
-	}
-	c.aggregator.Add(slot, ids)
-}
-
-// GetAggregatorCommitteeIDs gets the committee ID for subscribing subnet for aggregator of the slot.
-func (c *committeeIDs) GetAggregatorCommitteeIDs(slot uint64) []uint64 {
-	c.aggregatorLock.RLock()
-	defer c.aggregatorLock.RUnlock()
-
-	val, exists := c.aggregator.Get(slot)
-	if !exists {
-		return []uint64{}
-	}
-	return val.([]uint64)
-}

beacon-chain/cache/committee_ids_test.go

@@ -1,56 +0,0 @@
-package cache
-
-import (
-	"reflect"
-	"testing"
-)
-
-func TestCommitteeIDCache_RoundTrip(t *testing.T) {
-	c := newCommitteeIDs()
-	slot := uint64(100)
-	committeeIDs := c.GetAggregatorCommitteeIDs(slot)
-	if len(committeeIDs) != 0 {
-		t.Errorf("Empty cache returned an object: %v", committeeIDs)
-	}
-
-	c.AddAggregatorCommiteeID(slot, 1)
-	res := c.GetAggregatorCommitteeIDs(slot)
-	if !reflect.DeepEqual(res, []uint64{1}) {
-		t.Error("Expected equal value to return from cache")
-	}
-
-	c.AddAggregatorCommiteeID(slot, 2)
-	res = c.GetAggregatorCommitteeIDs(slot)
-	if !reflect.DeepEqual(res, []uint64{1, 2}) {
-		t.Error("Expected equal value to return from cache")
-	}
-
-	c.AddAggregatorCommiteeID(slot, 3)
-	res = c.GetAggregatorCommitteeIDs(slot)
-	if !reflect.DeepEqual(res, []uint64{1, 2, 3}) {
-		t.Error("Expected equal value to return from cache")
-	}
-
-	committeeIDs = c.GetAttesterCommitteeIDs(slot)
-	if len(committeeIDs) != 0 {
-		t.Errorf("Empty cache returned an object: %v", committeeIDs)
-	}
-
-	c.AddAttesterCommiteeID(slot, 11)
-	res = c.GetAttesterCommitteeIDs(slot)
-	if !reflect.DeepEqual(res, []uint64{11}) {
-		t.Error("Expected equal value to return from cache")
-	}
-
-	c.AddAttesterCommiteeID(slot, 22)
-	res = c.GetAttesterCommitteeIDs(slot)
-	if !reflect.DeepEqual(res, []uint64{11, 22}) {
-		t.Error("Expected equal value to return from cache")
-	}
-
-	c.AddAttesterCommiteeID(slot, 33)
-	res = c.GetAttesterCommitteeIDs(slot)
-	if !reflect.DeepEqual(res, []uint64{11, 22, 33}) {
-		t.Error("Expected equal value to return from cache")
-	}
-}

beacon-chain/cache/committee_test.go

@@ -97,6 +97,31 @@ func TestCommitteeCache_ActiveIndices(t *testing.T) {
 	}
 }
 
+func TestCommitteeCache_ActiveCount(t *testing.T) {
+	cache := NewCommitteesCache()
+
+	item := &Committees{Seed: [32]byte{'A'}, SortedIndices: []uint64{1, 2, 3, 4, 5, 6}}
+	count, err := cache.ActiveIndicesCount(item.Seed)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if count != 0 {
+		t.Error("Expected active count not to exist in empty cache")
+	}
+
+	if err := cache.AddCommitteeShuffledList(item); err != nil {
+		t.Fatal(err)
+	}
+
+	count, err = cache.ActiveIndicesCount(item.Seed)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if count != len(item.SortedIndices) {
+		t.Error("Did not receive correct active acount from cache")
+	}
+}
+
 func TestCommitteeCache_AddProposerIndicesList(t *testing.T) {
 	cache := NewCommitteesCache()
 
@@ -157,7 +182,7 @@ func TestCommitteeCache_CanRotate(t *testing.T) {
 	}
 
 	k := cache.CommitteeCache.ListKeys()
-	if len(k) != maxCommitteesCacheSize {
+	if uint64(len(k)) != maxCommitteesCacheSize {
 		t.Errorf("wanted: %d, got: %d", maxCommitteesCacheSize, len(k))
 	}
 

beacon-chain/cache/common.go

@@ -8,12 +8,12 @@ import (
 var (
 	// maxCacheSize is 4x of the epoch length for additional cache padding.
 	// Requests should be only accessing committees within defined epoch length.
-	maxCacheSize = int(4 * params.BeaconConfig().SlotsPerEpoch)
+	maxCacheSize = 4 * params.BeaconConfig().SlotsPerEpoch
 )
 
 // trim the FIFO queue to the maxSize.
-func trim(queue *cache.FIFO, maxSize int) {
-	for s := len(queue.ListKeys()); s > maxSize; s-- {
+func trim(queue *cache.FIFO, maxSize uint64) {
+	for s := uint64(len(queue.ListKeys())); s > maxSize; s-- {
 		_, err := queue.Pop(popProcessNoopFunc)
 		if err != nil {
 			// popProcessNoopFunc never returns an error, but we handle this anyway to make linter

beacon-chain/cache/depositcache/BUILD.bazel

@@ -1,4 +1,5 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_library(
     name = "go_default_library",

beacon-chain/cache/depositcache/deposits_cache.go

@@ -1,3 +1,7 @@
+// Package depositcache is the source of validator deposits maintained
+// in-memory by the beacon node – deposits processed from the
+// eth1 powchain are then stored in this cache to be accessed by
+// any other service during a beacon node's runtime.
 package depositcache
 
 import (
@@ -35,12 +39,11 @@ type DepositFetcher interface {
 // stores all the deposit related data that is required by the beacon-node.
 type DepositCache struct {
 	// Beacon chain deposits in memory.
-	pendingDeposits       []*dbpb.DepositContainer
-	deposits              []*dbpb.DepositContainer
-	depositsLock          sync.RWMutex
-	chainStartDeposits    []*ethpb.Deposit
-	chainstartPubkeys     map[string]bool
-	chainstartPubkeysLock sync.RWMutex
+	pendingDeposits    []*dbpb.DepositContainer
+	deposits           []*dbpb.DepositContainer
+	depositsLock       sync.RWMutex
+	chainStartDeposits []*ethpb.Deposit
+	chainStartPubkeys  map[string]bool
 }
 
 // NewDepositCache instantiates a new deposit cache
@@ -48,7 +51,7 @@ func NewDepositCache() *DepositCache {
 	return &DepositCache{
 		pendingDeposits:    []*dbpb.DepositContainer{},
 		deposits:           []*dbpb.DepositContainer{},
-		chainstartPubkeys:  make(map[string]bool),
+		chainStartPubkeys:  make(map[string]bool),
 		chainStartDeposits: make([]*ethpb.Deposit, 0),
 	}
 }
@@ -69,7 +72,7 @@ func (dc *DepositCache) InsertDeposit(ctx context.Context, d *ethpb.Deposit, blo
 	}
 	dc.depositsLock.Lock()
 	defer dc.depositsLock.Unlock()
-	// keep the slice sorted on insertion in order to avoid costly sorting on retrival.
+	// Keep the slice sorted on insertion in order to avoid costly sorting on retrieval.
 	heightIdx := sort.Search(len(dc.deposits), func(i int) bool { return dc.deposits[i].Index >= index })
 	newDeposits := append([]*dbpb.DepositContainer{{Deposit: d, Eth1BlockHeight: blockNum, DepositRoot: depositRoot[:], Index: index}}, dc.deposits[heightIdx:]...)
 	dc.deposits = append(dc.deposits[:heightIdx], newDeposits...)
@@ -102,21 +105,17 @@ func (dc *DepositCache) AllDepositContainers(ctx context.Context) []*dbpb.Deposi
 func (dc *DepositCache) MarkPubkeyForChainstart(ctx context.Context, pubkey string) {
 	ctx, span := trace.StartSpan(ctx, "DepositsCache.MarkPubkeyForChainstart")
 	defer span.End()
-	dc.chainstartPubkeysLock.Lock()
-	defer dc.chainstartPubkeysLock.Unlock()
-	dc.chainstartPubkeys[pubkey] = true
+	dc.chainStartPubkeys[pubkey] = true
 }
 
 // PubkeyInChainstart returns bool for whether the pubkey passed in has deposited.
 func (dc *DepositCache) PubkeyInChainstart(ctx context.Context, pubkey string) bool {
 	ctx, span := trace.StartSpan(ctx, "DepositsCache.PubkeyInChainstart")
 	defer span.End()
-	dc.chainstartPubkeysLock.Lock()
-	defer dc.chainstartPubkeysLock.Unlock()
-	if dc.chainstartPubkeys != nil {
-		return dc.chainstartPubkeys[pubkey]
+	if dc.chainStartPubkeys != nil {
+		return dc.chainStartPubkeys[pubkey]
 	}
-	dc.chainstartPubkeys = make(map[string]bool)
+	dc.chainStartPubkeys = make(map[string]bool)
 	return false
 }
 

beacon-chain/cache/depositcache/deposits_test.go

@@ -191,7 +191,7 @@ func TestBeaconDB_DepositsNumberAndRootAtHeight_ReturnsAppropriateCountAndRoot(t
 	}
 
 	n, root := dc.DepositsNumberAndRootAtHeight(context.Background(), big.NewInt(11))
-	if int(n) != 5 {
+	if n != 5 {
 		t.Errorf("Returned unexpected deposits number %d wanted %d", n, 5)
 	}
 
@@ -217,7 +217,7 @@ func TestBeaconDB_DepositsNumberAndRootAtHeight_ReturnsEmptyTrieIfBlockHeightLes
 	}
 
 	n, root := dc.DepositsNumberAndRootAtHeight(context.Background(), big.NewInt(2))
-	if int(n) != 0 {
+	if n != 0 {
 		t.Errorf("Returned unexpected deposits number %d wanted %d", n, 0)
 	}
 

beacon-chain/cache/doc.go

@@ -0,0 +1,5 @@
+// Package cache includes all important caches for the runtime
+// of an eth2 beacon node, ensuring the node does not spend
+// resources computing duplicate operations such as committee
+// calculations for validators during the same epoch, etc.
+package cache

beacon-chain/cache/eth1_data.go

@@ -1,139 +0,0 @@
-package cache
-
-import (
-	"errors"
-	"sync"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"k8s.io/client-go/tools/cache"
-)
-
-var (
-	// ErrNotEth1DataVote will be returned when a cache object is not a pointer to
-	// a Eth1DataVote struct.
-	ErrNotEth1DataVote = errors.New("object is not a eth1 data vote obj")
-
-	// maxEth1DataVoteSize defines the max number of eth1 data votes can cache.
-	maxEth1DataVoteSize = 1000
-
-	// Metrics.
-	eth1DataVoteCacheMiss = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "eth1_data_vote_cache_miss",
-		Help: "The number of eth1 data vote count requests that aren't present in the cache.",
-	})
-	eth1DataVoteCacheHit = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "eth1_data_vote_cache_hit",
-		Help: "The number of eth1 data vote count requests that are present in the cache.",
-	})
-)
-
-// Eth1DataVote defines the struct which keeps track of the vote count of individual deposit root.
-type Eth1DataVote struct {
-	Eth1DataHash [32]byte
-	VoteCount    uint64
-}
-
-// Eth1DataVoteCache is a struct with 1 queue for looking up eth1 data vote count by deposit root.
-type Eth1DataVoteCache struct {
-	eth1DataVoteCache *cache.FIFO
-	lock              sync.RWMutex
-}
-
-// eth1DataVoteKeyFn takes the eth1data hash as the key for the eth1 data vote count of a given eth1data object.
-func eth1DataVoteKeyFn(obj interface{}) (string, error) {
-	eInfo, ok := obj.(*Eth1DataVote)
-	if !ok {
-		return "", ErrNotEth1DataVote
-	}
-
-	return string(eInfo.Eth1DataHash[:]), nil
-}
-
-// NewEth1DataVoteCache creates a new eth1 data vote count cache for storing/accessing Eth1DataVote.
-func NewEth1DataVoteCache() *Eth1DataVoteCache {
-	return &Eth1DataVoteCache{
-		eth1DataVoteCache: cache.NewFIFO(eth1DataVoteKeyFn),
-	}
-}
-
-// Eth1DataVote fetches eth1 data vote count by the eth1data hash. Returns vote count,
-// if exists. Otherwise returns false, nil.
-func (c *Eth1DataVoteCache) Eth1DataVote(eth1DataHash [32]byte) (uint64, error) {
-	if !featureconfig.Get().EnableEth1DataVoteCache {
-		// Return a miss result if cache is not enabled.
-		eth1DataVoteCacheMiss.Inc()
-		return 0, nil
-	}
-
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-	obj, exists, err := c.eth1DataVoteCache.GetByKey(string(eth1DataHash[:]))
-	if err != nil {
-		return 0, err
-	}
-
-	if exists {
-		eth1DataVoteCacheHit.Inc()
-	} else {
-		eth1DataVoteCacheMiss.Inc()
-		return 0, nil
-	}
-
-	eInfo, ok := obj.(*Eth1DataVote)
-	if !ok {
-		return 0, ErrNotEth1DataVote
-	}
-
-	return eInfo.VoteCount, nil
-}
-
-// AddEth1DataVote adds eth1 data vote object to the cache. This method also trims the least
-// recently added Eth1DataVoteByEpoch object if the cache size has ready the max cache size limit.
-func (c *Eth1DataVoteCache) AddEth1DataVote(eth1DataVote *Eth1DataVote) error {
-	if !featureconfig.Get().EnableEth1DataVoteCache {
-		return nil
-	}
-
-	c.lock.Lock()
-	defer c.lock.Unlock()
-	if err := c.eth1DataVoteCache.Add(eth1DataVote); err != nil {
-		return err
-	}
-
-	trim(c.eth1DataVoteCache, maxEth1DataVoteSize)
-	return nil
-}
-
-// IncrementEth1DataVote increments the existing eth1 data object's vote count by 1,
-// and returns the vote count.
-func (c *Eth1DataVoteCache) IncrementEth1DataVote(eth1DataHash [32]byte) (uint64, error) {
-	if !featureconfig.Get().EnableEth1DataVoteCache {
-		return 0, nil
-	}
-
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-	obj, exists, err := c.eth1DataVoteCache.GetByKey(string(eth1DataHash[:]))
-	if err != nil {
-		return 0, err
-	}
-	if !exists {
-		return 0, errors.New("eth1 data vote object does not exist")
-	}
-
-	eth1DataVoteCacheHit.Inc()
-
-	eInfo, ok := obj.(*Eth1DataVote)
-	if !ok {
-		return 0, errors.New("cached value is not of type *Eth1DataVote")
-	}
-	eInfo.VoteCount++
-
-	if err := c.eth1DataVoteCache.Add(eInfo); err != nil {
-		return 0, err
-	}
-
-	return eInfo.VoteCount, nil
-}

beacon-chain/cache/eth1_data_test.go

@@ -1,113 +0,0 @@
-package cache
-
-import (
-	"strconv"
-	"testing"
-)
-
-func TestEth1DataVoteKeyFn_OK(t *testing.T) {
-	eInfo := &Eth1DataVote{
-		VoteCount:    44,
-		Eth1DataHash: [32]byte{'A'},
-	}
-
-	key, err := eth1DataVoteKeyFn(eInfo)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if key != string(eInfo.Eth1DataHash[:]) {
-		t.Errorf("Incorrect hash key: %s, expected %s", key, string(eInfo.Eth1DataHash[:]))
-	}
-}
-
-func TestEth1DataVoteKeyFn_InvalidObj(t *testing.T) {
-	_, err := eth1DataVoteKeyFn("bad")
-	if err != ErrNotEth1DataVote {
-		t.Errorf("Expected error %v, got %v", ErrNotEth1DataVote, err)
-	}
-}
-
-func TestEth1DataVoteCache_CanAdd(t *testing.T) {
-	cache := NewEth1DataVoteCache()
-
-	eInfo := &Eth1DataVote{
-		VoteCount:    55,
-		Eth1DataHash: [32]byte{'B'},
-	}
-	count, err := cache.Eth1DataVote(eInfo.Eth1DataHash)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if count != 0 {
-		t.Error("Expected seed not to exist in empty cache")
-	}
-
-	if err := cache.AddEth1DataVote(eInfo); err != nil {
-		t.Fatal(err)
-	}
-	count, err = cache.Eth1DataVote(eInfo.Eth1DataHash)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if count != eInfo.VoteCount {
-		t.Errorf(
-			"Expected vote count to be %d, got %d",
-			eInfo.VoteCount,
-			count,
-		)
-	}
-}
-
-func TestEth1DataVoteCache_CanIncrement(t *testing.T) {
-	cache := NewEth1DataVoteCache()
-
-	eInfo := &Eth1DataVote{
-		VoteCount:    55,
-		Eth1DataHash: [32]byte{'B'},
-	}
-
-	if err := cache.AddEth1DataVote(eInfo); err != nil {
-		t.Fatal(err)
-	}
-
-	_, err := cache.IncrementEth1DataVote(eInfo.Eth1DataHash)
-	if err != nil {
-		t.Fatal(err)
-	}
-	_, err = cache.IncrementEth1DataVote(eInfo.Eth1DataHash)
-	if err != nil {
-		t.Error(err)
-	}
-	count, err := cache.IncrementEth1DataVote(eInfo.Eth1DataHash)
-
-	if count != 58 {
-		t.Errorf(
-			"Expected vote count to be %d, got %d",
-			58,
-			count,
-		)
-	}
-}
-
-func TestEth1Data_MaxSize(t *testing.T) {
-	cache := NewEth1DataVoteCache()
-
-	for i := 0; i < maxEth1DataVoteSize+1; i++ {
-		var hash [32]byte
-		copy(hash[:], strconv.Itoa(i))
-		eInfo := &Eth1DataVote{
-			Eth1DataHash: hash,
-		}
-		if err := cache.AddEth1DataVote(eInfo); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	if len(cache.eth1DataVoteCache.ListKeys()) != maxEth1DataVoteSize {
-		t.Errorf(
-			"Expected hash cache key size to be %d, got %d",
-			maxEth1DataVoteSize,
-			len(cache.eth1DataVoteCache.ListKeys()),
-		)
-	}
-}

beacon-chain/cache/feature_flag_test.go

@@ -1,9 +1,14 @@
 package cache
 
-import "github.com/prysmaticlabs/prysm/shared/featureconfig"
+import (
+	"os"
+	"testing"
 
-func init() {
-	featureconfig.Init(&featureconfig.Flags{
-		EnableEth1DataVoteCache: true,
-	})
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
+)
+
+func TestMain(m *testing.M) {
+	resetCfg := featureconfig.InitWithReset(&featureconfig.Flags{EnableEth1DataVoteCache: true})
+	defer resetCfg()
+	os.Exit(m.Run())
 }

beacon-chain/cache/hot_state_cache.go

@@ -1,6 +1,8 @@
 package cache
 
 import (
+	"sync"
+
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
@@ -9,7 +11,7 @@ import (
 
 var (
 	// hotStateCacheSize defines the max number of hot state this can cache.
-	hotStateCacheSize = 16
+	hotStateCacheSize = 32
 	// Metrics
 	hotStateCacheHit = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "hot_state_cache_hit",
@@ -24,6 +26,7 @@ var (
 // HotStateCache is used to store the processed beacon state after finalized check point..
 type HotStateCache struct {
 	cache *lru.Cache
+	lock  sync.RWMutex
 }
 
 // NewHotStateCache initializes the map and underlying cache.
@@ -40,6 +43,8 @@ func NewHotStateCache() *HotStateCache {
 // Get returns a cached response via input block root, if any.
 // The response is copied by default.
 func (c *HotStateCache) Get(root [32]byte) *stateTrie.BeaconState {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
 	item, exists := c.cache.Get(root)
 
 	if exists && item != nil {
@@ -50,12 +55,36 @@ func (c *HotStateCache) Get(root [32]byte) *stateTrie.BeaconState {
 	return nil
 }
 
+// GetWithoutCopy returns a non-copied cached response via input block root.
+func (c *HotStateCache) GetWithoutCopy(root [32]byte) *stateTrie.BeaconState {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	item, exists := c.cache.Get(root)
+	if exists && item != nil {
+		hotStateCacheHit.Inc()
+		return item.(*stateTrie.BeaconState)
+	}
+	hotStateCacheMiss.Inc()
+	return nil
+}
+
 // Put the response in the cache.
 func (c *HotStateCache) Put(root [32]byte, state *stateTrie.BeaconState) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
 	c.cache.Add(root, state)
 }
 
 // Has returns true if the key exists in the cache.
 func (c *HotStateCache) Has(root [32]byte) bool {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
 	return c.cache.Contains(root)
 }
+
+// Delete deletes the key exists in the cache.
+func (c *HotStateCache) Delete(root [32]byte) bool {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return c.cache.Remove(root)
+}

beacon-chain/cache/hot_state_cache_test.go

@@ -38,4 +38,9 @@ func TestHotStateCache_RoundTrip(t *testing.T) {
 	if !reflect.DeepEqual(state.CloneInnerState(), res.CloneInnerState()) {
 		t.Error("Expected equal protos to return from cache")
 	}
+
+	c.Delete(root)
+	if c.Has(root) {
+		t.Error("Cache not suppose to have the object")
+	}
 }

beacon-chain/cache/subnet_ids.go

@@ -0,0 +1,134 @@
+package cache
+
+import (
+	"sync"
+	"time"
+
+	lru "github.com/hashicorp/golang-lru"
+	"github.com/patrickmn/go-cache"
+	"github.com/prysmaticlabs/prysm/shared/params"
+	"github.com/prysmaticlabs/prysm/shared/sliceutil"
+)
+
+type subnetIDs struct {
+	attester          *lru.Cache
+	attesterLock      sync.RWMutex
+	aggregator        *lru.Cache
+	aggregatorLock    sync.RWMutex
+	persistentSubnets *cache.Cache
+	subnetsLock       sync.RWMutex
+}
+
+// SubnetIDs for attester and aggregator.
+var SubnetIDs = newSubnetIDs()
+
+func newSubnetIDs() *subnetIDs {
+	// Given a node can calculate committee assignments of current epoch and next epoch.
+	// Max size is set to 2 epoch length.
+	cacheSize := params.BeaconConfig().MaxCommitteesPerSlot * params.BeaconConfig().SlotsPerEpoch * 2
+	attesterCache, err := lru.New(int(cacheSize))
+	if err != nil {
+		panic(err)
+	}
+	aggregatorCache, err := lru.New(int(cacheSize))
+	if err != nil {
+		panic(err)
+	}
+	epochDuration := time.Duration(params.BeaconConfig().SlotsPerEpoch * params.BeaconConfig().SecondsPerSlot)
+	subLength := epochDuration * time.Duration(params.BeaconNetworkConfig().EpochsPerRandomSubnetSubscription)
+	persistentCache := cache.New(subLength*time.Second, epochDuration*time.Second)
+	return &subnetIDs{attester: attesterCache, aggregator: aggregatorCache, persistentSubnets: persistentCache}
+}
+
+// AddAttesterSubnetID adds the subnet index for subscribing subnet for the attester of a given slot.
+func (c *subnetIDs) AddAttesterSubnetID(slot uint64, subnetID uint64) {
+	c.attesterLock.Lock()
+	defer c.attesterLock.Unlock()
+
+	ids := []uint64{subnetID}
+	val, exists := c.attester.Get(slot)
+	if exists {
+		ids = sliceutil.UnionUint64(append(val.([]uint64), ids...))
+	}
+	c.attester.Add(slot, ids)
+}
+
+// GetAttesterSubnetIDs gets the subnet IDs for subscribed subnets for attesters of the slot.
+func (c *subnetIDs) GetAttesterSubnetIDs(slot uint64) []uint64 {
+	c.attesterLock.RLock()
+	defer c.attesterLock.RUnlock()
+
+	val, exists := c.attester.Get(slot)
+	if !exists {
+		return nil
+	}
+	if v, ok := val.([]uint64); ok {
+		return v
+	}
+	return nil
+}
+
+// AddAggregatorSubnetID adds the subnet ID for subscribing subnet for the aggregator of a given slot.
+func (c *subnetIDs) AddAggregatorSubnetID(slot uint64, subnetID uint64) {
+	c.aggregatorLock.Lock()
+	defer c.aggregatorLock.Unlock()
+
+	ids := []uint64{subnetID}
+	val, exists := c.aggregator.Get(slot)
+	if exists {
+		ids = sliceutil.UnionUint64(append(val.([]uint64), ids...))
+	}
+	c.aggregator.Add(slot, ids)
+}
+
+// GetAggregatorSubnetIDs gets the subnet IDs for subscribing subnet for aggregator of the slot.
+func (c *subnetIDs) GetAggregatorSubnetIDs(slot uint64) []uint64 {
+	c.aggregatorLock.RLock()
+	defer c.aggregatorLock.RUnlock()
+
+	val, exists := c.aggregator.Get(slot)
+	if !exists {
+		return []uint64{}
+	}
+	return val.([]uint64)
+}
+
+// GetPersistentSubnets retrieves the persistent subnet and expiration time of that validator's
+// subscription.
+func (c *subnetIDs) GetPersistentSubnets(pubkey []byte) ([]uint64, bool, time.Time) {
+	c.subnetsLock.RLock()
+	defer c.subnetsLock.RUnlock()
+
+	id, duration, ok := c.persistentSubnets.GetWithExpiration(string(pubkey))
+	if !ok {
+		return []uint64{}, ok, time.Time{}
+	}
+	return id.([]uint64), ok, duration
+}
+
+// GetAllSubnets retrieves all the non-expired subscribed subnets of all the validators
+// in the cache.
+func (c *subnetIDs) GetAllSubnets() []uint64 {
+	c.subnetsLock.RLock()
+	defer c.subnetsLock.RUnlock()
+
+	itemsMap := c.persistentSubnets.Items()
+	committees := []uint64{}
+
+	for _, v := range itemsMap {
+		if v.Expired() {
+			continue
+		}
+		committees = append(committees, v.Object.([]uint64)...)
+	}
+	return sliceutil.SetUint64(committees)
+}
+
+// AddPersistentCommittee adds the relevant committee for that particular validator along with its
+// expiration period.
+func (c *subnetIDs) AddPersistentCommittee(pubkey []byte, comIndex []uint64, duration time.Duration) {
+	c.subnetsLock.Lock()
+	defer c.subnetsLock.Unlock()
+
+	c.persistentSubnets.Set(string(pubkey), comIndex, duration)
+}

beacon-chain/cache/subnet_ids_test.go

@@ -0,0 +1,84 @@
+package cache
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestSubnetIDsCache_RoundTrip(t *testing.T) {
+	c := newSubnetIDs()
+	slot := uint64(100)
+	committeeIDs := c.GetAggregatorSubnetIDs(slot)
+	if len(committeeIDs) != 0 {
+		t.Errorf("Empty cache returned an object: %v", committeeIDs)
+	}
+
+	c.AddAggregatorSubnetID(slot, 1)
+	res := c.GetAggregatorSubnetIDs(slot)
+	if !reflect.DeepEqual(res, []uint64{1}) {
+		t.Error("Expected equal value to return from cache")
+	}
+
+	c.AddAggregatorSubnetID(slot, 2)
+	res = c.GetAggregatorSubnetIDs(slot)
+	if !reflect.DeepEqual(res, []uint64{1, 2}) {
+		t.Error("Expected equal value to return from cache")
+	}
+
+	c.AddAggregatorSubnetID(slot, 3)
+	res = c.GetAggregatorSubnetIDs(slot)
+	if !reflect.DeepEqual(res, []uint64{1, 2, 3}) {
+		t.Error("Expected equal value to return from cache")
+	}
+
+	committeeIDs = c.GetAttesterSubnetIDs(slot)
+	if len(committeeIDs) != 0 {
+		t.Errorf("Empty cache returned an object: %v", committeeIDs)
+	}
+
+	c.AddAttesterSubnetID(slot, 11)
+	res = c.GetAttesterSubnetIDs(slot)
+	if !reflect.DeepEqual(res, []uint64{11}) {
+		t.Error("Expected equal value to return from cache")
+	}
+
+	c.AddAttesterSubnetID(slot, 22)
+	res = c.GetAttesterSubnetIDs(slot)
+	if !reflect.DeepEqual(res, []uint64{11, 22}) {
+		t.Error("Expected equal value to return from cache")
+	}
+
+	c.AddAttesterSubnetID(slot, 33)
+	res = c.GetAttesterSubnetIDs(slot)
+	if !reflect.DeepEqual(res, []uint64{11, 22, 33}) {
+		t.Error("Expected equal value to return from cache")
+	}
+}
+
+func TestSubnetIDsCache_PersistentCommitteeRoundtrip(t *testing.T) {
+	pubkeySet := [][48]byte{}
+	c := newSubnetIDs()
+
+	for i := 0; i < 20; i++ {
+		pubkey := [48]byte{byte(i)}
+		pubkeySet = append(pubkeySet, pubkey)
+		c.AddPersistentCommittee(pubkey[:], []uint64{uint64(i)}, 0)
+	}
+
+	for i := uint64(0); i < 20; i++ {
+		pubkey := [48]byte{byte(i)}
+
+		idxs, ok, _ := c.GetPersistentSubnets(pubkey[:])
+		if !ok {
+			t.Errorf("Couldn't find entry in cache for pubkey %#x", pubkey)
+			continue
+		}
+		if idxs[0] != i {
+			t.Fatalf("Wanted index of %d but got %d", i, idxs[0])
+		}
+	}
+	coms := c.GetAllSubnets()
+	if len(coms) != 20 {
+		t.Errorf("Number of committees is not %d but is %d", 20, len(coms))
+	}
+}

beacon-chain/core/blocks/BUILD.bazel

@@ -1,4 +1,5 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -9,10 +10,10 @@ go_library(
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/core/blocks",
     visibility = [
         "//beacon-chain:__subpackages__",
+        "//fuzz:__pkg__",
         "//shared/testutil:__pkg__",
     ],
     deps = [
-        "//beacon-chain/cache:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/validators:go_default_library",
         "//beacon-chain/state:go_default_library",
@@ -21,7 +22,6 @@ go_library(
         "//shared/attestationutil:go_default_library",
         "//shared/bls:go_default_library",
         "//shared/bytesutil:go_default_library",
-        "//shared/featureconfig:go_default_library",
         "//shared/hashutil:go_default_library",
         "//shared/mathutil:go_default_library",
         "//shared/params:go_default_library",
@@ -52,6 +52,8 @@ go_test(
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/stateutil:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
+        "//shared/aggregation:go_default_library",
+        "//shared/aggregation/attestations:go_default_library",
         "//shared/attestationutil:go_default_library",
         "//shared/bls:go_default_library",
         "//shared/bytesutil:go_default_library",

beacon-chain/core/blocks/block.go

@@ -1,6 +1,5 @@
-// Package blocks contains block processing libraries. These libraries
-// process and verify block specific messages such as PoW receipt root,
-// RANDAO, validator deposits, exits and slashing proofs.
+// Package blocks contains block processing libraries according to
+// the eth2spec.
 package blocks
 
 import (

beacon-chain/core/blocks/block_operations.go

@@ -5,14 +5,12 @@ import (
 	"context"
 	"encoding/binary"
 	"fmt"
-	"reflect"
 	"sort"
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/pkg/errors"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	v "github.com/prysmaticlabs/prysm/beacon-chain/core/validators"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
@@ -21,7 +19,6 @@ import (
 	"github.com/prysmaticlabs/prysm/shared/attestationutil"
 	"github.com/prysmaticlabs/prysm/shared/bls"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/hashutil"
 	"github.com/prysmaticlabs/prysm/shared/mathutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
@@ -33,8 +30,6 @@ import (
 
 var log = logrus.WithField("prefix", "blocks")
 
-var eth1DataCache = cache.NewEth1DataVoteCache()
-
 // Deprecated: This method uses deprecated ssz.SigningRoot.
 func verifyDepositDataSigningRoot(obj *ethpb.Deposit_Data, pub []byte, signature []byte, domain []byte) error {
 	publicKey, err := bls.PublicKeyFromBytes(pub)
@@ -49,15 +44,15 @@ func verifyDepositDataSigningRoot(obj *ethpb.Deposit_Data, pub []byte, signature
 	if err != nil {
 		return errors.Wrap(err, "could not get signing root")
 	}
-	sigRoot := &pb.SigningRoot{
+	signingData := &pb.SigningData{
 		ObjectRoot: root[:],
 		Domain:     domain,
 	}
-	ctrRoot, err := ssz.HashTreeRoot(sigRoot)
+	ctrRoot, err := ssz.HashTreeRoot(signingData)
 	if err != nil {
 		return errors.Wrap(err, "could not get container root")
 	}
-	if !sig.Verify(ctrRoot[:], publicKey) {
+	if !sig.Verify(publicKey, ctrRoot[:]) {
 		return helpers.ErrSigFailedToVerify
 	}
 	return nil
@@ -72,15 +67,15 @@ func verifySignature(signedData []byte, pub []byte, signature []byte, domain []b
 	if err != nil {
 		return errors.Wrap(err, "could not convert bytes to signature")
 	}
-	ctr := &pb.SigningRoot{
+	signingData := &pb.SigningData{
 		ObjectRoot: signedData,
 		Domain:     domain,
 	}
-	root, err := ssz.HashTreeRoot(ctr)
+	root, err := ssz.HashTreeRoot(signingData)
 	if err != nil {
 		return errors.Wrap(err, "could not hash container")
 	}
-	if !sig.Verify(root[:], publicKey) {
+	if !sig.Verify(publicKey, root[:]) {
 		return helpers.ErrSigFailedToVerify
 	}
 	return nil
@@ -117,7 +112,11 @@ func ProcessEth1DataInBlock(beaconState *stateTrie.BeaconState, block *ethpb.Bea
 	return beaconState, nil
 }
 
-func areEth1DataEqual(a, b *ethpb.Eth1Data) bool {
+// AreEth1DataEqual checks equality between two eth1 data objects.
+func AreEth1DataEqual(a, b *ethpb.Eth1Data) bool {
+	if a == nil && b == nil {
+		return true
+	}
 	if a == nil || b == nil {
 		return false
 	}
@@ -132,35 +131,11 @@ func areEth1DataEqual(a, b *ethpb.Eth1Data) bool {
 // votes to see if they match the eth1data.
 func Eth1DataHasEnoughSupport(beaconState *stateTrie.BeaconState, data *ethpb.Eth1Data) (bool, error) {
 	voteCount := uint64(0)
-	var eth1DataHash [32]byte
-	var err error
 	data = stateTrie.CopyETH1Data(data)
-	if featureconfig.Get().EnableEth1DataVoteCache {
-		eth1DataHash, err = hashutil.HashProto(data)
-		if err != nil {
-			return false, errors.Wrap(err, "could not hash eth1data")
-		}
-		voteCount, err = eth1DataCache.Eth1DataVote(eth1DataHash)
-		if err != nil {
-			return false, errors.Wrap(err, "could not retrieve eth1 data vote cache")
-		}
-	}
-	if voteCount == 0 {
-		for _, vote := range beaconState.Eth1DataVotes() {
-			if areEth1DataEqual(vote, data) {
-				voteCount++
-			}
-		}
-	} else {
-		voteCount++
-	}
 
-	if featureconfig.Get().EnableEth1DataVoteCache {
-		if err := eth1DataCache.AddEth1DataVote(&cache.Eth1DataVote{
-			Eth1DataHash: eth1DataHash,
-			VoteCount:    voteCount,
-		}); err != nil {
-			return false, errors.Wrap(err, "could not save eth1 data vote cache")
+	for _, vote := range beaconState.Eth1DataVotes() {
+		if AreEth1DataEqual(vote, data) {
+			voteCount++
 		}
 	}
 
@@ -180,7 +155,7 @@ func Eth1DataHasEnoughSupport(beaconState *stateTrie.BeaconState, data *ethpb.Et
 //     # Verify that proposer index is the correct index
 //    assert block.proposer_index == get_beacon_proposer_index(state)
 //    # Verify that the parent matches
-//    assert block.parent_root == signing_root(state.latest_block_header)
+//    assert block.parent_root == hash_tree_root(state.latest_block_header)
 //    # Save current block as the new latest block
 //    state.latest_block_header = BeaconBlockHeader(
 //        slot=block.slot,
@@ -204,15 +179,15 @@ func ProcessBlockHeader(
 	}
 
 	// Verify proposer signature.
-	if err := VerifyBlockHeaderSignature(beaconState, block); err != nil {
+	if err := VerifyBlockSignature(beaconState, block); err != nil {
 		return nil, err
 	}
 
 	return beaconState, nil
 }
 
-// VerifyBlockHeaderSignature verifies the proposer signature of a beacon block.
-func VerifyBlockHeaderSignature(beaconState *stateTrie.BeaconState, block *ethpb.SignedBeaconBlock) error {
+// VerifyBlockSignature verifies the proposer signature of a beacon block.
+func VerifyBlockSignature(beaconState *stateTrie.BeaconState, block *ethpb.SignedBeaconBlock) error {
 	proposer, err := beaconState.ValidatorAtIndex(block.Block.ProposerIndex)
 	if err != nil {
 		return err
@@ -223,7 +198,8 @@ func VerifyBlockHeaderSignature(beaconState *stateTrie.BeaconState, block *ethpb
 	if err != nil {
 		return err
 	}
-	return helpers.VerifySigningRoot(block.Block, proposer.PublicKey, block.Signature, domain)
+	proposerPubKey := proposer.PublicKey
+	return helpers.VerifyBlockSigningRoot(block.Block, proposerPubKey[:], block.Signature, domain)
 }
 
 // ProcessBlockHeaderNoVerify validates a block by its header but skips proposer
@@ -239,7 +215,7 @@ func VerifyBlockHeaderSignature(beaconState *stateTrie.BeaconState, block *ethpb
 //     # Verify that proposer index is the correct index
 //    assert block.proposer_index == get_beacon_proposer_index(state)
 //    # Verify that the parent matches
-//    assert block.parent_root == signing_root(state.latest_block_header)
+//    assert block.parent_root == hash_tree_root(state.latest_block_header)
 //    # Save current block as the new latest block
 //    state.latest_block_header = BeaconBlockHeader(
 //        slot=block.slot,
@@ -268,7 +244,11 @@ func ProcessBlockHeaderNoVerify(
 	if block.ProposerIndex != idx {
 		return nil, fmt.Errorf("proposer index: %d is different than calculated: %d", block.ProposerIndex, idx)
 	}
-	parentRoot, err := stateutil.BlockHeaderRoot(beaconState.LatestBlockHeader())
+	parentHeader := beaconState.LatestBlockHeader()
+	if parentHeader.Slot >= block.Slot {
+		return nil, fmt.Errorf("block.Slot %d must be greater than state.LatestBlockHeader.Slot %d", block.Slot, parentHeader.Slot)
+	}
+	parentRoot, err := stateutil.BlockHeaderRoot(parentHeader)
 	if err != nil {
 		return nil, err
 	}
@@ -279,11 +259,11 @@ func ProcessBlockHeaderNoVerify(
 			block.ParentRoot, parentRoot)
 	}
 
-	proposer, err := beaconState.ValidatorAtIndex(idx)
+	proposer, err := beaconState.ValidatorAtIndexReadOnly(idx)
 	if err != nil {
 		return nil, err
 	}
-	if proposer.Slashed {
+	if proposer.Slashed() {
 		return nil, fmt.Errorf("proposer at index %d was previously slashed", idx)
 	}
 
@@ -309,19 +289,14 @@ func ProcessBlockHeaderNoVerify(
 //
 // Spec pseudocode definition:
 //   def process_randao(state: BeaconState, body: BeaconBlockBody) -> None:
-//     proposer = state.validator_registry[get_beacon_proposer_index(state)]
-//     # Verify that the provided randao value is valid
-//     assert bls_verify(
-//         proposer.pubkey,
-//         hash_tree_root(get_current_epoch(state)),
-//         body.randao_reveal,
-//         get_domain(state, DOMAIN_RANDAO),
-//     )
-//     # Mix it in
-//     state.latest_randao_mixes[get_current_epoch(state) % LATEST_RANDAO_MIXES_LENGTH] = (
-//         xor(get_randao_mix(state, get_current_epoch(state)),
-//             hash(body.randao_reveal))
-//     )
+//    epoch = get_current_epoch(state)
+//    # Verify RANDAO reveal
+//    proposer = state.validators[get_beacon_proposer_index(state)]
+//    signing_root = compute_signing_root(epoch, get_domain(state, DOMAIN_RANDAO))
+//    assert bls.Verify(proposer.pubkey, signing_root, body.randao_reveal)
+//    # Mix in RANDAO reveal
+//    mix = xor(get_randao_mix(state, epoch), hash(body.randao_reveal))
+//    state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] = mix
 func ProcessRandao(
 	beaconState *stateTrie.BeaconState,
 	body *ethpb.BeaconBlockBody,
@@ -379,7 +354,7 @@ func ProcessRandaoNoVerify(
 	for i, x := range blockRandaoReveal {
 		latestMixSlice[i] ^= x
 	}
-	if err := beaconState.UpdateRandaoMixesAtIndex(latestMixSlice, currentEpoch%latestMixesLength); err != nil {
+	if err := beaconState.UpdateRandaoMixesAtIndex(currentEpoch%latestMixesLength, latestMixSlice); err != nil {
 		return nil, err
 	}
 	return beaconState, nil
@@ -421,7 +396,7 @@ func ProcessProposerSlashings(
 			return nil, errors.Wrapf(err, "could not verify proposer slashing %d", idx)
 		}
 		beaconState, err = v.SlashValidator(
-			beaconState, slashing.Header_1.Header.ProposerIndex, 0, /* proposer is whistleblower */
+			beaconState, slashing.Header_1.Header.ProposerIndex,
 		)
 		if err != nil {
 			return nil, errors.Wrapf(err, "could not slash proposer index %d", slashing.Header_1.Header.ProposerIndex)
@@ -447,12 +422,12 @@ func VerifyProposerSlashing(
 	if proto.Equal(slashing.Header_1, slashing.Header_2) {
 		return errors.New("expected slashing headers to differ")
 	}
-	proposer, err := beaconState.ValidatorAtIndex(slashing.Header_1.Header.ProposerIndex)
+	proposer, err := beaconState.ValidatorAtIndexReadOnly(slashing.Header_1.Header.ProposerIndex)
 	if err != nil {
 		return err
 	}
-	if !helpers.IsSlashableValidator(proposer, helpers.SlotToEpoch(beaconState.Slot())) {
-		return fmt.Errorf("validator with key %#x is not slashable", proposer.PublicKey)
+	if !helpers.IsSlashableValidatorUsingTrie(proposer, helpers.SlotToEpoch(beaconState.Slot())) {
+		return fmt.Errorf("validator with key %#x is not slashable", proposer.PublicKey())
 	}
 	// Using headerEpoch1 here because both of the headers should have the same epoch.
 	domain, err := helpers.Domain(beaconState.Fork(), helpers.SlotToEpoch(slashing.Header_1.Header.Slot), params.BeaconConfig().DomainBeaconProposer, beaconState.GenesisValidatorRoot())
@@ -461,7 +436,8 @@ func VerifyProposerSlashing(
 	}
 	headers := []*ethpb.SignedBeaconBlockHeader{slashing.Header_1, slashing.Header_2}
 	for _, header := range headers {
-		if err := helpers.VerifySigningRoot(header.Header, proposer.PublicKey, header.Signature, domain); err != nil {
+		proposerPubKey := proposer.PublicKey()
+		if err := helpers.VerifySigningRoot(header.Header, proposerPubKey[:], header.Signature, domain); err != nil {
 			return errors.Wrap(err, "could not verify beacon block header")
 		}
 	}
@@ -510,7 +486,7 @@ func ProcessAttesterSlashings(
 				return nil, err
 			}
 			if helpers.IsSlashableValidator(val, currentEpoch) {
-				beaconState, err = v.SlashValidator(beaconState, validatorIndex, 0)
+				beaconState, err = v.SlashValidator(beaconState, validatorIndex)
 				if err != nil {
 					return nil, errors.Wrapf(err, "could not slash validator index %d",
 						validatorIndex)
@@ -569,7 +545,7 @@ func IsSlashableAttestationData(data1 *ethpb.AttestationData, data2 *ethpb.Attes
 	if data1 == nil || data2 == nil || data1.Target == nil || data2.Target == nil || data1.Source == nil || data2.Source == nil {
 		return false
 	}
-	isDoubleVote := !proto.Equal(data1, data2) && data1.Target.Epoch == data2.Target.Epoch
+	isDoubleVote := !attestationutil.AttDataIsEqual(data1, data2) && data1.Target.Epoch == data2.Target.Epoch
 	isSurroundVote := data1.Source.Epoch < data2.Source.Epoch && data2.Target.Epoch < data1.Target.Epoch
 	return isDoubleVote || isSurroundVote
 }
@@ -584,8 +560,7 @@ func slashableAttesterIndices(slashing *ethpb.AttesterSlashing) []uint64 {
 }
 
 // ProcessAttestations applies processing operations to a block's inner attestation
-// records. This function returns a list of pending attestations which can then be
-// appended to the BeaconState's latest attestations.
+// records.
 func ProcessAttestations(
 	ctx context.Context,
 	beaconState *stateTrie.BeaconState,
@@ -764,83 +739,39 @@ func ProcessAttestationNoVerify(
 // Spec pseudocode definition:
 //  def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool:
 //    """
-//    Check if ``indexed_attestation`` has valid indices and signature.
+//    Check if ``indexed_attestation`` is not empty, has sorted and unique indices and has a valid aggregate signature.
 //    """
-//    indices = indexed_attestation.attesting_indices
-//
-//    # Verify max number of indices
-//    if not len(indices) <= MAX_VALIDATORS_PER_COMMITTEE:
-//        return False
 //    # Verify indices are sorted and unique
-//        if not indices == sorted(set(indices)):
-//    # Verify aggregate signature
-//    if not bls_verify(
-//        pubkey=bls_aggregate_pubkeys([state.validators[i].pubkey for i in indices]),
-//        message_hash=hash_tree_root(indexed_attestation.data),
-//        signature=indexed_attestation.signature,
-//        domain=get_domain(state, DOMAIN_BEACON_ATTESTER, indexed_attestation.data.target.epoch),
-//    ):
+//    indices = indexed_attestation.attesting_indices
+//    if len(indices) == 0 or not indices == sorted(set(indices)):
 //        return False
-//    return True
+//    # Verify aggregate signature
+//    pubkeys = [state.validators[i].pubkey for i in indices]
+//    domain = get_domain(state, DOMAIN_BEACON_ATTESTER, indexed_attestation.data.target.epoch)
+//    signing_root = compute_signing_root(indexed_attestation.data, domain)
+//    return bls.FastAggregateVerify(pubkeys, signing_root, indexed_attestation.signature)
 func VerifyIndexedAttestation(ctx context.Context, beaconState *stateTrie.BeaconState, indexedAtt *ethpb.IndexedAttestation) error {
 	ctx, span := trace.StartSpan(ctx, "core.VerifyIndexedAttestation")
 	defer span.End()
-	if indexedAtt == nil || indexedAtt.Data == nil || indexedAtt.Data.Target == nil {
-		return errors.New("nil or missing indexed attestation data")
-	}
-	indices := indexedAtt.AttestingIndices
 
-	if uint64(len(indices)) > params.BeaconConfig().MaxValidatorsPerCommittee {
-		return fmt.Errorf("validator indices count exceeds MAX_VALIDATORS_PER_COMMITTEE, %d > %d", len(indices), params.BeaconConfig().MaxValidatorsPerCommittee)
+	if err := attestationutil.IsValidAttestationIndices(ctx, indexedAtt); err != nil {
+		return err
 	}
-
-	set := make(map[uint64]bool)
-	setIndices := make([]uint64, 0, len(indices))
-	for _, i := range indices {
-		if ok := set[i]; ok {
-			continue
-		}
-		setIndices = append(setIndices, i)
-		set[i] = true
-	}
-	sort.SliceStable(setIndices, func(i, j int) bool {
-		return setIndices[i] < setIndices[j]
-	})
-	if !reflect.DeepEqual(setIndices, indices) {
-		return errors.New("attesting indices is not uniquely sorted")
-	}
-
 	domain, err := helpers.Domain(beaconState.Fork(), indexedAtt.Data.Target.Epoch, params.BeaconConfig().DomainBeaconAttester, beaconState.GenesisValidatorRoot())
 	if err != nil {
 		return err
 	}
-	pubkeys := []*bls.PublicKey{}
-	if len(indices) > 0 {
-		for i := 0; i < len(indices); i++ {
-			pubkeyAtIdx := beaconState.PubkeyAtIndex(indices[i])
-			pk, err := bls.PublicKeyFromBytes(pubkeyAtIdx[:])
-			if err != nil {
-				return errors.Wrap(err, "could not deserialize validator public key")
-			}
-			pubkeys = append(pubkeys, pk)
+	indices := indexedAtt.AttestingIndices
+	pubkeys := []bls.PublicKey{}
+	for i := 0; i < len(indices); i++ {
+		pubkeyAtIdx := beaconState.PubkeyAtIndex(indices[i])
+		pk, err := bls.PublicKeyFromBytes(pubkeyAtIdx[:])
+		if err != nil {
+			return errors.Wrap(err, "could not deserialize validator public key")
 		}
+		pubkeys = append(pubkeys, pk)
 	}
-
-	messageHash, err := helpers.ComputeSigningRoot(indexedAtt.Data, domain)
-	if err != nil {
-		return errors.Wrap(err, "could not get signing root of object")
-	}
-
-	sig, err := bls.SignatureFromBytes(indexedAtt.Signature)
-	if err != nil {
-		return errors.Wrap(err, "could not convert bytes to signature")
-	}
-
-	voted := len(indices) > 0
-	if voted && !sig.FastAggregateVerify(pubkeys, messageHash) {
-		return helpers.ErrSigFailedToVerify
-	}
-	return nil
+	return attestationutil.VerifyIndexedAttestationSig(ctx, indexedAtt, pubkeys, domain)
 }
 
 // VerifyAttestation converts and attestation into an indexed attestation and verifies
@@ -857,6 +788,109 @@ func VerifyAttestation(ctx context.Context, beaconState *stateTrie.BeaconState,
 	return VerifyIndexedAttestation(ctx, beaconState, indexedAtt)
 }
 
+// VerifyAttestations will verify the signatures of the provided attestations. This method performs
+// a single BLS verification call to verify the signatures of all of the provided attestations. All
+// of the provided attestations must have valid signatures or this method will return an error.
+// This method does not determine which attestation signature is invalid, only that one or more
+// attestation signatures were not valid.
+func VerifyAttestations(ctx context.Context, beaconState *stateTrie.BeaconState, atts []*ethpb.Attestation) error {
+	ctx, span := trace.StartSpan(ctx, "core.VerifyAttestations")
+	defer span.End()
+	span.AddAttributes(trace.Int64Attribute("attestations", int64(len(atts))))
+
+	if len(atts) == 0 {
+		return nil
+	}
+
+	fork := beaconState.Fork()
+	gvr := beaconState.GenesisValidatorRoot()
+	dt := params.BeaconConfig().DomainBeaconAttester
+
+	// Split attestations by fork. Note: the signature domain will differ based on the fork.
+	var preForkAtts []*ethpb.Attestation
+	var postForkAtts []*ethpb.Attestation
+	for _, a := range atts {
+		if helpers.SlotToEpoch(a.Data.Slot) < fork.Epoch {
+			preForkAtts = append(preForkAtts, a)
+		} else {
+			postForkAtts = append(postForkAtts, a)
+		}
+	}
+
+	// Check attestations from before the fork.
+	if fork.Epoch > 0 { // Check to prevent underflow.
+		prevDomain, err := helpers.Domain(fork, fork.Epoch-1, dt, gvr)
+		if err != nil {
+			return err
+		}
+		if err := verifyAttestationsWithDomain(ctx, beaconState, preForkAtts, prevDomain); err != nil {
+			return err
+		}
+	} else if len(preForkAtts) > 0 {
+		// This is a sanity check that preForkAtts were not ignored when fork.Epoch == 0. This
+		// condition is not possible, but it doesn't hurt to check anyway.
+		return errors.New("some attestations were not verified from previous fork before genesis")
+	}
+
+	// Then check attestations from after the fork.
+	currDomain, err := helpers.Domain(fork, fork.Epoch, dt, gvr)
+	if err != nil {
+		return err
+	}
+
+	return verifyAttestationsWithDomain(ctx, beaconState, postForkAtts, currDomain)
+}
+
+// Inner method to verify attestations. This abstraction allows for the domain to be provided as an
+// argument.
+func verifyAttestationsWithDomain(ctx context.Context, beaconState *stateTrie.BeaconState, atts []*ethpb.Attestation, domain []byte) error {
+	if len(atts) == 0 {
+		return nil
+	}
+
+	sigs := make([]bls.Signature, len(atts))
+	pks := make([]bls.PublicKey, len(atts))
+	msgs := make([][32]byte, len(atts))
+	for i, a := range atts {
+		sig, err := bls.SignatureFromBytes(a.Signature)
+		if err != nil {
+			return err
+		}
+		sigs[i] = sig
+		c, err := helpers.BeaconCommitteeFromState(beaconState, a.Data.Slot, a.Data.CommitteeIndex)
+		if err != nil {
+			return err
+		}
+		ia := attestationutil.ConvertToIndexed(ctx, a, c)
+		indices := ia.AttestingIndices
+		var pk bls.PublicKey
+		for i := 0; i < len(indices); i++ {
+			pubkeyAtIdx := beaconState.PubkeyAtIndex(indices[i])
+			p, err := bls.PublicKeyFromBytes(pubkeyAtIdx[:])
+			if err != nil {
+				return errors.Wrap(err, "could not deserialize validator public key")
+			}
+			if pk == nil {
+				pk = p
+			} else {
+				pk.Aggregate(p)
+			}
+		}
+		pks[i] = pk
+
+		root, err := helpers.ComputeSigningRoot(ia.Data, domain)
+		if err != nil {
+			return errors.Wrap(err, "could not get signing root of object")
+		}
+		msgs[i] = root
+	}
+	as := bls.AggregateSignatures(sigs)
+	if !as.AggregateVerify(pks, msgs) {
+		return errors.New("one or more attestation signatures did not verify")
+	}
+	return nil
+}
+
 // ProcessDeposits is one of the operations performed on each processed
 // beacon block to verify queued validators from the Ethereum 1.0 Deposit Contract
 // into the beacon chain.
@@ -913,7 +947,7 @@ func ProcessPreGenesisDeposit(
 		validator.ActivationEligibilityEpoch = 0
 		validator.ActivationEpoch = 0
 	}
-	if err := beaconState.UpdateValidatorAtIndex(uint64(index), validator); err != nil {
+	if err := beaconState.UpdateValidatorAtIndex(index, validator); err != nil {
 		return nil, err
 	}
 	return beaconState, nil
@@ -928,7 +962,7 @@ func ProcessPreGenesisDeposit(
 //    assert is_valid_merkle_branch(
 //        leaf=hash_tree_root(deposit.data),
 //        branch=deposit.proof,
-//        depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1,  # Add 1 for the `List` length mix-in
+//        depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1,  # Add 1 for the List length mix-in
 //        index=state.eth1_deposit_index,
 //        root=state.eth1_data.deposit_root,
 //    )
@@ -940,23 +974,19 @@ func ProcessPreGenesisDeposit(
 //    amount = deposit.data.amount
 //    validator_pubkeys = [v.pubkey for v in state.validators]
 //    if pubkey not in validator_pubkeys:
-//        # Verify the deposit signature (proof of possession) for new validators.
-//        # Note: The deposit contract does not check signatures.
-//        # Note: Deposits are valid across forks, thus the deposit domain is retrieved directly from `compute_domain`.
-//        domain = compute_domain(DOMAIN_DEPOSIT)
-//        if not bls_verify(pubkey, signing_root(deposit.data), deposit.data.signature, domain):
+//        # Verify the deposit signature (proof of possession) which is not checked by the deposit contract
+//        deposit_message = DepositMessage(
+//            pubkey=deposit.data.pubkey,
+//            withdrawal_credentials=deposit.data.withdrawal_credentials,
+//            amount=deposit.data.amount,
+//        )
+//        domain = compute_domain(DOMAIN_DEPOSIT)  # Fork-agnostic domain since deposits are valid across forks
+//        signing_root = compute_signing_root(deposit_message, domain)
+//        if not bls.Verify(pubkey, signing_root, deposit.data.signature):
 //            return
 //
 //        # Add validator and balance entries
-//        state.validators.append(Validator(
-//            pubkey=pubkey,
-//            withdrawal_credentials=deposit.data.withdrawal_credentials,
-//            activation_eligibility_epoch=FAR_FUTURE_EPOCH,
-//            activation_epoch=FAR_FUTURE_EPOCH,
-//            exit_epoch=FAR_FUTURE_EPOCH,
-//            withdrawable_epoch=FAR_FUTURE_EPOCH,
-//            effective_balance=min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE),
-//        ))
+//        state.validators.append(get_validator_from_deposit(state, deposit))
 //        state.balances.append(amount)
 //    else:
 //        # Increase balance by deposit amount
@@ -978,7 +1008,6 @@ func ProcessDeposit(
 	pubKey := deposit.Data.PublicKey
 	amount := deposit.Data.Amount
 	index, ok := beaconState.ValidatorIndexByPubkey(bytesutil.ToBytes48(pubKey))
-	numVals := beaconState.NumValidators()
 	if !ok {
 		domain, err := helpers.ComputeDomain(params.BeaconConfig().DomainDeposit, nil, nil)
 		if err != nil {
@@ -987,7 +1016,7 @@ func ProcessDeposit(
 		depositSig := deposit.Data.Signature
 		if err := verifyDepositDataSigningRoot(deposit.Data, pubKey, depositSig, domain); err != nil {
 			// Ignore this error as in the spec pseudo code.
-			log.Errorf("Skipping deposit: could not verify deposit data signature: %v", err)
+			log.Debugf("Skipping deposit: could not verify deposit data signature: %v", err)
 			return beaconState, nil
 		}
 
@@ -1009,10 +1038,8 @@ func ProcessDeposit(
 		if err := beaconState.AppendBalance(amount); err != nil {
 			return nil, err
 		}
-		numVals++
-		beaconState.SetValidatorIndexByPubkey(bytesutil.ToBytes48(pubKey), uint64(numVals-1))
 	} else {
-		if err := helpers.IncreaseBalance(beaconState, uint64(index), amount); err != nil {
+		if err := helpers.IncreaseBalance(beaconState, index, amount); err != nil {
 			return nil, err
 		}
 	}
@@ -1082,14 +1109,14 @@ func ProcessVoluntaryExits(
 		if exit == nil || exit.Exit == nil {
 			return nil, errors.New("nil voluntary exit in block body")
 		}
-		if int(exit.Exit.ValidatorIndex) >= beaconState.NumValidators() {
+		if exit.Exit.ValidatorIndex >= uint64(beaconState.NumValidators()) {
 			return nil, fmt.Errorf(
 				"validator index out of bound %d > %d",
 				exit.Exit.ValidatorIndex,
 				beaconState.NumValidators(),
 			)
 		}
-		val, err := beaconState.ValidatorAtIndex(exit.Exit.ValidatorIndex)
+		val, err := beaconState.ValidatorAtIndexReadOnly(exit.Exit.ValidatorIndex)
 		if err != nil {
 			return nil, err
 		}
@@ -1144,7 +1171,7 @@ func ProcessVoluntaryExitsNoVerify(
 //    # Verify signature
 //    domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, exit.epoch)
 //    assert bls_verify(validator.pubkey, signing_root(exit), exit.signature, domain)
-func VerifyExit(validator *ethpb.Validator, currentSlot uint64, fork *pb.Fork, signed *ethpb.SignedVoluntaryExit, genesisRoot []byte) error {
+func VerifyExit(validator *stateTrie.ReadOnlyValidator, currentSlot uint64, fork *pb.Fork, signed *ethpb.SignedVoluntaryExit, genesisRoot []byte) error {
 	if signed == nil || signed.Exit == nil {
 		return errors.New("nil exit")
 	}
@@ -1152,36 +1179,32 @@ func VerifyExit(validator *ethpb.Validator, currentSlot uint64, fork *pb.Fork, s
 	exit := signed.Exit
 	currentEpoch := helpers.SlotToEpoch(currentSlot)
 	// Verify the validator is active.
-	if !helpers.IsActiveValidator(validator, currentEpoch) {
+	if !helpers.IsActiveValidatorUsingTrie(validator, currentEpoch) {
 		return errors.New("non-active validator cannot exit")
 	}
 	// Verify the validator has not yet exited.
-	if validator.ExitEpoch != params.BeaconConfig().FarFutureEpoch {
-		return fmt.Errorf("validator has already exited at epoch: %v", validator.ExitEpoch)
+	if validator.ExitEpoch() != params.BeaconConfig().FarFutureEpoch {
+		return fmt.Errorf("validator has already exited at epoch: %v", validator.ExitEpoch())
 	}
 	// Exits must specify an epoch when they become valid; they are not valid before then.
 	if currentEpoch < exit.Epoch {
 		return fmt.Errorf("expected current epoch >= exit epoch, received %d < %d", currentEpoch, exit.Epoch)
 	}
 	// Verify the validator has been active long enough.
-	if currentEpoch < validator.ActivationEpoch+params.BeaconConfig().PersistentCommitteePeriod {
+	if currentEpoch < validator.ActivationEpoch()+params.BeaconConfig().ShardCommitteePeriod {
 		return fmt.Errorf(
 			"validator has not been active long enough to exit, wanted epoch %d >= %d",
 			currentEpoch,
-			validator.ActivationEpoch+params.BeaconConfig().PersistentCommitteePeriod,
+			validator.ActivationEpoch()+params.BeaconConfig().ShardCommitteePeriod,
 		)
 	}
 	domain, err := helpers.Domain(fork, exit.Epoch, params.BeaconConfig().DomainVoluntaryExit, genesisRoot)
 	if err != nil {
 		return err
 	}
-	if err := helpers.VerifySigningRoot(exit, validator.PublicKey, signed.Signature, domain); err != nil {
+	valPubKey := validator.PublicKey()
+	if err := helpers.VerifySigningRoot(exit, valPubKey[:], signed.Signature, domain); err != nil {
 		return helpers.ErrSigFailedToVerify
 	}
 	return nil
 }
-
-// ClearEth1DataVoteCache clears the eth1 data vote count cache.
-func ClearEth1DataVoteCache() {
-	eth1DataCache = cache.NewEth1DataVoteCache()
-}

beacon-chain/core/blocks/block_operations_fuzz_test.go

@@ -25,12 +25,8 @@ func TestFuzzProcessAttestationNoVerify_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(att)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
-		if _, err = ProcessAttestationNoVerify(ctx, s, att); err != nil {
-			t.Log(err)
-		}
+		_, err = ProcessAttestationNoVerify(ctx, s, att)
+		_ = err
 	}
 }
 
@@ -44,12 +40,8 @@ func TestFuzzProcessBlockHeader_10000(t *testing.T) {
 		fuzzer.Fuzz(block)
 
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
-		if _, err = ProcessBlockHeader(s, block); err != nil {
-			t.Log(err)
-		}
+		_, err = ProcessBlockHeader(s, block)
+		_ = err
 	}
 }
 
@@ -70,13 +62,9 @@ func TestFuzzverifyDepositDataSigningRoot_10000(t *testing.T) {
 		fuzzer.Fuzz(&p)
 		fuzzer.Fuzz(&s)
 		fuzzer.Fuzz(&d)
-		if err := verifySignature(ba, pubkey[:], sig[:], domain[:]); err != nil {
-			t.Log(err)
-		}
-		if err := verifySignature(ba, p, s, d); err != nil {
-			t.Log(err)
-		}
-
+		err := verifySignature(ba, pubkey[:], sig[:], domain[:])
+		err = verifySignature(ba, p, s, d)
+		_ = err
 	}
 }
 
@@ -102,8 +90,8 @@ func TestFuzzareEth1DataEqual_10000(t *testing.T) {
 	for i := 0; i < 10000; i++ {
 		fuzzer.Fuzz(eth1data)
 		fuzzer.Fuzz(eth1data2)
-		areEth1DataEqual(eth1data, eth1data2)
-		areEth1DataEqual(eth1data, eth1data)
+		AreEth1DataEqual(eth1data, eth1data2)
+		AreEth1DataEqual(eth1data, eth1data)
 	}
 }
 
@@ -117,12 +105,8 @@ func TestFuzzEth1DataHasEnoughSupport_10000(t *testing.T) {
 		s, err := beaconstate.InitializeFromProto(&ethereum_beacon_p2p_v1.BeaconState{
 			Eth1DataVotes: stateVotes,
 		})
-		if err != nil {
-			t.Log(err)
-		}
-		if _, err := Eth1DataHasEnoughSupport(s, eth1data); err != nil {
-			t.Log(err)
-		}
+		_, err = Eth1DataHasEnoughSupport(s, eth1data)
+		_ = err
 	}
 
 }
@@ -136,12 +120,8 @@ func TestFuzzProcessBlockHeaderNoVerify_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(block)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
-		if _, err = ProcessBlockHeaderNoVerify(s, block); err != nil {
-			t.Log(err)
-		}
+		_, err = ProcessBlockHeaderNoVerify(s, block)
+		_ = err
 	}
 }
 
@@ -154,9 +134,6 @@ func TestFuzzProcessRandao_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(blockBody)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessRandao(s, blockBody)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
@@ -173,9 +150,6 @@ func TestFuzzProcessRandaoNoVerify_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(blockBody)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessRandaoNoVerify(s, blockBody)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
@@ -192,9 +166,6 @@ func TestFuzzProcessProposerSlashings_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(blockBody)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessProposerSlashings(ctx, s, blockBody)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
@@ -210,12 +181,8 @@ func TestFuzzVerifyProposerSlashing_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(proposerSlashing)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
-		if err := VerifyProposerSlashing(s, proposerSlashing); err != nil {
-			t.Log(err)
-		}
+		err = VerifyProposerSlashing(s, proposerSlashing)
+		_ = err
 	}
 }
 
@@ -228,9 +195,6 @@ func TestFuzzProcessAttesterSlashings_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(blockBody)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessAttesterSlashings(ctx, s, blockBody)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
@@ -247,12 +211,8 @@ func TestFuzzVerifyAttesterSlashing_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(attesterSlashing)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
-		if err := VerifyAttesterSlashing(ctx, s, attesterSlashing); err != nil {
-			t.Log(err)
-		}
+		err = VerifyAttesterSlashing(ctx, s, attesterSlashing)
+		_ = err
 	}
 }
 
@@ -287,9 +247,6 @@ func TestFuzzProcessAttestations_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(blockBody)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessAttestations(ctx, s, blockBody)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
@@ -306,9 +263,6 @@ func TestFuzzProcessAttestationsNoVerify_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(blockBody)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessAttestationsNoVerify(ctx, s, blockBody)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
@@ -325,9 +279,6 @@ func TestFuzzProcessAttestation_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(attestation)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessAttestation(ctx, s, attestation)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, attestation)
@@ -344,12 +295,8 @@ func TestFuzzVerifyIndexedAttestationn_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(idxAttestation)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
-		if err := VerifyIndexedAttestation(ctx, s, idxAttestation); err != nil {
-			t.Log(err)
-		}
+		err = VerifyIndexedAttestation(ctx, s, idxAttestation)
+		_ = err
 	}
 }
 
@@ -362,12 +309,8 @@ func TestFuzzVerifyAttestation_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(attestation)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
-		if err := VerifyAttestation(ctx, s, attestation); err != nil {
-			t.Log(err)
-		}
+		err = VerifyAttestation(ctx, s, attestation)
+		_ = err
 	}
 }
 
@@ -380,9 +323,6 @@ func TestFuzzProcessDeposits_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(blockBody)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessDeposits(ctx, s, blockBody)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
@@ -400,9 +340,6 @@ func TestFuzzProcessPreGenesisDeposit_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(deposit)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessPreGenesisDeposit(ctx, s, deposit)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, deposit)
@@ -419,9 +356,6 @@ func TestFuzzProcessDeposit_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(deposit)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessDeposit(s, deposit)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, deposit)
@@ -437,12 +371,8 @@ func TestFuzzverifyDeposit_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(deposit)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
-		if err := verifyDeposit(s, deposit); err != nil {
-			t.Log(err)
-		}
+		err = verifyDeposit(s, deposit)
+		_ = err
 	}
 }
 
@@ -455,9 +385,6 @@ func TestFuzzProcessVoluntaryExits_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(blockBody)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessVoluntaryExits(ctx, s, blockBody)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
@@ -473,9 +400,6 @@ func TestFuzzProcessVoluntaryExitsNoVerify_10000(t *testing.T) {
 		fuzzer.Fuzz(state)
 		fuzzer.Fuzz(blockBody)
 		s, err := beaconstate.InitializeFromProtoUnsafe(state)
-		if err != nil {
-			t.Log(err)
-		}
 		r, err := ProcessVoluntaryExitsNoVerify(s, blockBody)
 		if err != nil && r != nil {
 			t.Fatalf("return value should be nil on err. found: %v on error: %v for state: %v and block: %v", r, err, state, blockBody)
@@ -486,7 +410,7 @@ func TestFuzzProcessVoluntaryExitsNoVerify_10000(t *testing.T) {
 func TestFuzzVerifyExit_10000(t *testing.T) {
 	fuzzer := fuzz.NewWithSeed(0)
 	ve := &eth.SignedVoluntaryExit{}
-	val := &eth.Validator{}
+	val := &beaconstate.ReadOnlyValidator{}
 	fork := &pb.Fork{}
 	var slot uint64
 
@@ -495,8 +419,7 @@ func TestFuzzVerifyExit_10000(t *testing.T) {
 		fuzzer.Fuzz(val)
 		fuzzer.Fuzz(fork)
 		fuzzer.Fuzz(&slot)
-		if err := VerifyExit(val, slot, fork, ve, params.BeaconConfig().ZeroHash[:]); err != nil {
-			t.Log(err)
-		}
+		err := VerifyExit(val, slot, fork, ve, params.BeaconConfig().ZeroHash[:])
+		_ = err
 	}
 }

beacon-chain/core/blocks/block_operations_test.go

@@ -19,6 +19,8 @@ import (
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateutil"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/aggregation"
+	attaggregation "github.com/prysmaticlabs/prysm/shared/aggregation/attestations"
 	"github.com/prysmaticlabs/prysm/shared/attestationutil"
 	"github.com/prysmaticlabs/prysm/shared/bls"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
@@ -38,6 +40,9 @@ func TestProcessBlockHeader_WrongProposerSig(t *testing.T) {
 	if err := beaconState.SetLatestBlockHeader(&ethpb.BeaconBlockHeader{Slot: 9}); err != nil {
 		t.Fatal(err)
 	}
+	if err := beaconState.SetSlot(10); err != nil {
+		t.Error(err)
+	}
 
 	lbhdr, err := stateutil.BlockHeaderRoot(beaconState.LatestBlockHeader())
 	if err != nil {
@@ -52,7 +57,7 @@ func TestProcessBlockHeader_WrongProposerSig(t *testing.T) {
 	block := &ethpb.SignedBeaconBlock{
 		Block: &ethpb.BeaconBlock{
 			ProposerIndex: proposerIdx,
-			Slot:          0,
+			Slot:          10,
 			Body: &ethpb.BeaconBlockBody{
 				RandaoReveal: []byte{'A', 'B', 'C'},
 			},
@@ -89,7 +94,7 @@ func TestProcessBlockHeader_DifferentSlots(t *testing.T) {
 
 	state, err := stateTrie.InitializeFromProto(&pb.BeaconState{
 		Validators:        validators,
-		Slot:              0,
+		Slot:              10,
 		LatestBlockHeader: &ethpb.BeaconBlockHeader{Slot: 9},
 		Fork: &pb.Fork{
 			PreviousVersion: []byte{0, 0, 0, 0},
@@ -146,7 +151,7 @@ func TestProcessBlockHeader_PreviousBlockRootNotSignedRoot(t *testing.T) {
 
 	state, err := stateTrie.InitializeFromProto(&pb.BeaconState{
 		Validators:        validators,
-		Slot:              0,
+		Slot:              10,
 		LatestBlockHeader: &ethpb.BeaconBlockHeader{Slot: 9},
 		Fork: &pb.Fork{
 			PreviousVersion: []byte{0, 0, 0, 0},
@@ -170,10 +175,14 @@ func TestProcessBlockHeader_PreviousBlockRootNotSignedRoot(t *testing.T) {
 	}
 	blockSig := priv.Sign(root[:])
 	validators[5896].PublicKey = priv.PublicKey().Marshal()
+	pID, err := helpers.BeaconProposerIndex(state)
+	if err != nil {
+		t.Error(err)
+	}
 	block := &ethpb.SignedBeaconBlock{
 		Block: &ethpb.BeaconBlock{
-			ProposerIndex: 5669,
-			Slot:          0,
+			ProposerIndex: pID,
+			Slot:          10,
 			Body: &ethpb.BeaconBlockBody{
 				RandaoReveal: []byte{'A', 'B', 'C'},
 			},
@@ -200,7 +209,7 @@ func TestProcessBlockHeader_SlashedProposer(t *testing.T) {
 
 	state, err := stateTrie.InitializeFromProto(&pb.BeaconState{
 		Validators:        validators,
-		Slot:              0,
+		Slot:              10,
 		LatestBlockHeader: &ethpb.BeaconBlockHeader{Slot: 9},
 		Fork: &pb.Fork{
 			PreviousVersion: []byte{0, 0, 0, 0},
@@ -227,11 +236,16 @@ func TestProcessBlockHeader_SlashedProposer(t *testing.T) {
 		t.Error(err)
 	}
 	blockSig := priv.Sign(root[:])
+
 	validators[12683].PublicKey = priv.PublicKey().Marshal()
+	pID, err := helpers.BeaconProposerIndex(state)
+	if err != nil {
+		t.Error(err)
+	}
 	block := &ethpb.SignedBeaconBlock{
 		Block: &ethpb.BeaconBlock{
-			ProposerIndex: 5669,
-			Slot:          0,
+			ProposerIndex: pID,
+			Slot:          10,
 			Body: &ethpb.BeaconBlockBody{
 				RandaoReveal: []byte{'A', 'B', 'C'},
 			},
@@ -258,7 +272,7 @@ func TestProcessBlockHeader_OK(t *testing.T) {
 
 	state, err := stateTrie.InitializeFromProto(&pb.BeaconState{
 		Validators:        validators,
-		Slot:              0,
+		Slot:              10,
 		LatestBlockHeader: &ethpb.BeaconBlockHeader{Slot: 9},
 		Fork: &pb.Fork{
 			PreviousVersion: []byte{0, 0, 0, 0},
@@ -281,10 +295,14 @@ func TestProcessBlockHeader_OK(t *testing.T) {
 		t.Fatalf("Failed to get domain form state: %v", err)
 	}
 	priv := bls.RandKey()
+	pID, err := helpers.BeaconProposerIndex(state)
+	if err != nil {
+		t.Error(err)
+	}
 	block := &ethpb.SignedBeaconBlock{
 		Block: &ethpb.BeaconBlock{
-			ProposerIndex: 5669,
-			Slot:          0,
+			ProposerIndex: pID,
+			Slot:          10,
 			Body: &ethpb.BeaconBlockBody{
 				RandaoReveal: []byte{'A', 'B', 'C'},
 			},
@@ -297,7 +315,7 @@ func TestProcessBlockHeader_OK(t *testing.T) {
 	}
 	blockSig := priv.Sign(signingRoot[:])
 	block.Signature = blockSig.Marshal()[:]
-	bodyRoot, err := ssz.HashTreeRoot(block.Block.Body)
+	bodyRoot, err := stateutil.BlockBodyRoot(block.Block.Body)
 	if err != nil {
 		t.Fatalf("Failed to hash block bytes got: %v", err)
 	}
@@ -320,7 +338,7 @@ func TestProcessBlockHeader_OK(t *testing.T) {
 	var zeroHash [32]byte
 	nsh := newState.LatestBlockHeader()
 	expected := &ethpb.BeaconBlockHeader{
-		ProposerIndex: 5669,
+		ProposerIndex: pID,
 		Slot:          block.Block.Slot,
 		ParentRoot:    latestBlockSignedRoot[:],
 		BodyRoot:      bodyRoot[:],
@@ -331,6 +349,78 @@ func TestProcessBlockHeader_OK(t *testing.T) {
 	}
 }
 
+func TestProcessBlockHeader_ImproperBlockSlot(t *testing.T) {
+	validators := make([]*ethpb.Validator, params.BeaconConfig().MinGenesisActiveValidatorCount)
+	for i := 0; i < len(validators); i++ {
+		validators[i] = &ethpb.Validator{
+			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
+			Slashed:   true,
+		}
+	}
+
+	state, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Validators:        validators,
+		Slot:              10,
+		LatestBlockHeader: &ethpb.BeaconBlockHeader{Slot: 10}, // Must be less than block.Slot
+		Fork: &pb.Fork{
+			PreviousVersion: []byte{0, 0, 0, 0},
+			CurrentVersion:  []byte{0, 0, 0, 0},
+		},
+		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	latestBlockSignedRoot, err := stateutil.BlockHeaderRoot(state.LatestBlockHeader())
+	if err != nil {
+		t.Error(err)
+	}
+
+	currentEpoch := helpers.CurrentEpoch(state)
+	dt, err := helpers.Domain(state.Fork(), currentEpoch, params.BeaconConfig().DomainBeaconProposer, state.GenesisValidatorRoot())
+	if err != nil {
+		t.Fatalf("Failed to get domain form state: %v", err)
+	}
+	priv := bls.RandKey()
+	pID, err := helpers.BeaconProposerIndex(state)
+	if err != nil {
+		t.Error(err)
+	}
+	block := &ethpb.SignedBeaconBlock{
+		Block: &ethpb.BeaconBlock{
+			ProposerIndex: pID,
+			Slot:          10,
+			Body: &ethpb.BeaconBlockBody{
+				RandaoReveal: []byte{'A', 'B', 'C'},
+			},
+			ParentRoot: latestBlockSignedRoot[:],
+		},
+	}
+	signingRoot, err := helpers.ComputeSigningRoot(block.Block, dt)
+	if err != nil {
+		t.Fatalf("Failed to get signing root of block: %v", err)
+	}
+	blockSig := priv.Sign(signingRoot[:])
+	block.Signature = blockSig.Marshal()[:]
+
+	proposerIdx, err := helpers.BeaconProposerIndex(state)
+	if err != nil {
+		t.Fatal(err)
+	}
+	validators[proposerIdx].Slashed = false
+	validators[proposerIdx].PublicKey = priv.PublicKey().Marshal()
+	err = state.UpdateValidatorAtIndex(proposerIdx, validators[proposerIdx])
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = blocks.ProcessBlockHeader(state, block)
+	if err == nil || err.Error() != "block.Slot 10 must be greater than state.LatestBlockHeader.Slot 10" {
+		t.Fatalf("did not get expected error, got %v", err)
+	}
+}
+
 func TestProcessRandao_IncorrectProposerFailsVerification(t *testing.T) {
 	beaconState, privKeys := testutil.DeterministicGenesisState(t, 100)
 	// We fetch the proposer's index as that is whom the RANDAO will be verified against.
@@ -345,7 +435,7 @@ func TestProcessRandao_IncorrectProposerFailsVerification(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	root, err := ssz.HashTreeRoot(&pb.SigningRoot{ObjectRoot: buf, Domain: domain})
+	root, err := ssz.HashTreeRoot(&pb.SigningData{ObjectRoot: buf, Domain: domain})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -549,7 +639,7 @@ func TestProcessProposerSlashings_ValidatorNotSlashable(t *testing.T) {
 	}
 	want := fmt.Sprintf(
 		"validator with key %#x is not slashable",
-		beaconState.Validators()[0].PublicKey,
+		bytesutil.ToBytes48(beaconState.Validators()[0].PublicKey),
 	)
 
 	_, err = blocks.ProcessProposerSlashings(context.Background(), beaconState, block.Body)
@@ -746,7 +836,7 @@ func TestProcessAttesterSlashings_AppliesCorrectStatus(t *testing.T) {
 	}
 	sig0 := privKeys[0].Sign(signingRoot[:])
 	sig1 := privKeys[1].Sign(signingRoot[:])
-	aggregateSig := bls.AggregateSignatures([]*bls.Signature{sig0, sig1})
+	aggregateSig := bls.AggregateSignatures([]bls.Signature{sig0, sig1})
 	att1.Signature = aggregateSig.Marshal()[:]
 
 	att2 := &ethpb.IndexedAttestation{
@@ -762,7 +852,7 @@ func TestProcessAttesterSlashings_AppliesCorrectStatus(t *testing.T) {
 	}
 	sig0 = privKeys[0].Sign(signingRoot[:])
 	sig1 = privKeys[1].Sign(signingRoot[:])
-	aggregateSig = bls.AggregateSignatures([]*bls.Signature{sig0, sig1})
+	aggregateSig = bls.AggregateSignatures([]bls.Signature{sig0, sig1})
 	att2.Signature = aggregateSig.Marshal()[:]
 
 	slashings := []*ethpb.AttesterSlashing{
@@ -1058,7 +1148,7 @@ func TestProcessAttestations_OK(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	sigs := make([]*bls.Signature, len(attestingIndices))
+	sigs := make([]bls.Signature, len(attestingIndices))
 	for i, indice := range attestingIndices {
 		sig := privKeys[indice].Sign(hashTreeRoot[:])
 		sigs[i] = sig
@@ -1122,7 +1212,7 @@ func TestProcessAggregatedAttestation_OverlappingBits(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	sigs := make([]*bls.Signature, len(attestingIndices1))
+	sigs := make([]bls.Signature, len(attestingIndices1))
 	for i, indice := range attestingIndices1 {
 		sig := privKeys[indice].Sign(hashTreeRoot[:])
 		sigs[i] = sig
@@ -1150,14 +1240,14 @@ func TestProcessAggregatedAttestation_OverlappingBits(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	sigs = make([]*bls.Signature, len(attestingIndices2))
+	sigs = make([]bls.Signature, len(attestingIndices2))
 	for i, indice := range attestingIndices2 {
 		sig := privKeys[indice].Sign(hashTreeRoot[:])
 		sigs[i] = sig
 	}
 	att2.Signature = bls.AggregateSignatures(sigs).Marshal()[:]
 
-	if _, err = helpers.AggregateAttestation(att1, att2); err != helpers.ErrAttestationAggregationBitsOverlap {
+	if _, err = attaggregation.AggregatePair(att1, att2); err != aggregation.ErrBitsOverlap {
 		t.Error("Did not receive wanted error")
 	}
 }
@@ -1204,7 +1294,7 @@ func TestProcessAggregatedAttestation_NoOverlappingBits(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	sigs := make([]*bls.Signature, len(attestingIndices1))
+	sigs := make([]bls.Signature, len(attestingIndices1))
 	for i, indice := range attestingIndices1 {
 		sig := privKeys[indice].Sign(hashTreeRoot[:])
 		sigs[i] = sig
@@ -1231,14 +1321,14 @@ func TestProcessAggregatedAttestation_NoOverlappingBits(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	sigs = make([]*bls.Signature, len(attestingIndices2))
+	sigs = make([]bls.Signature, len(attestingIndices2))
 	for i, indice := range attestingIndices2 {
 		sig := privKeys[indice].Sign(hashTreeRoot[:])
 		sigs[i] = sig
 	}
 	att2.Signature = bls.AggregateSignatures(sigs).Marshal()[:]
 
-	aggregatedAtt, err := helpers.AggregateAttestation(att1, att2)
+	aggregatedAtt, err := attaggregation.AggregatePair(att1, att2)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1450,7 +1540,7 @@ func TestVerifyIndexedAttestation_OK(t *testing.T) {
 		if err != nil {
 			t.Error(err)
 		}
-		var sig []*bls.Signature
+		var sig []bls.Signature
 		for _, idx := range tt.attestation.AttestingIndices {
 			validatorSig := keys[idx].Sign(root[:])
 			sig = append(sig, validatorSig)
@@ -1930,7 +2020,7 @@ func TestProcessVoluntaryExits_AppliesCorrectStatus(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = state.SetSlot(state.Slot() + (params.BeaconConfig().PersistentCommitteePeriod * params.BeaconConfig().SlotsPerEpoch))
+	err = state.SetSlot(state.Slot() + (params.BeaconConfig().ShardCommitteePeriod * params.BeaconConfig().SlotsPerEpoch))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1970,3 +2060,239 @@ func TestProcessVoluntaryExits_AppliesCorrectStatus(t *testing.T) {
 			helpers.ActivationExitEpoch(state.Slot()/params.BeaconConfig().SlotsPerEpoch), newRegistry[0].ExitEpoch)
 	}
 }
+
+func TestVerifyAttestations_VerifiesMultipleAttestations(t *testing.T) {
+	ctx := context.Background()
+	numOfValidators := 4 * params.BeaconConfig().SlotsPerEpoch
+	validators := make([]*ethpb.Validator, numOfValidators)
+	_, keys, err := testutil.DeterministicDepositsAndKeys(numOfValidators)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < len(validators); i++ {
+		validators[i] = &ethpb.Validator{
+			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
+			PublicKey: keys[i].PublicKey().Marshal(),
+		}
+	}
+
+	st, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot:       5,
+		Validators: validators,
+		Fork: &pb.Fork{
+			Epoch:           0,
+			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
+			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
+		},
+		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+	})
+
+	comm1, err := helpers.BeaconCommitteeFromState(st, 1 /*slot*/, 0 /*committeeIndex*/)
+	if err != nil {
+		t.Fatal(err)
+	}
+	att1 := &ethpb.Attestation{
+		AggregationBits: bitfield.NewBitlist(uint64(len(comm1))),
+		Data: &ethpb.AttestationData{
+			Slot:           1,
+			CommitteeIndex: 0,
+		},
+		Signature: nil,
+	}
+	domain, err := helpers.Domain(st.Fork(), st.Fork().Epoch, params.BeaconConfig().DomainBeaconAttester, st.GenesisValidatorRoot())
+	if err != nil {
+		t.Fatal(err)
+	}
+	root, err := helpers.ComputeSigningRoot(att1.Data, domain)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var sigs []bls.Signature
+	for i, u := range comm1 {
+		att1.AggregationBits.SetBitAt(uint64(i), true)
+		sigs = append(sigs, keys[u].Sign(root[:]))
+	}
+	att1.Signature = bls.AggregateSignatures(sigs).Marshal()
+
+	comm2, err := helpers.BeaconCommitteeFromState(st, 1 /*slot*/, 1 /*committeeIndex*/)
+	if err != nil {
+		t.Fatal(err)
+	}
+	att2 := &ethpb.Attestation{
+		AggregationBits: bitfield.NewBitlist(uint64(len(comm2))),
+		Data: &ethpb.AttestationData{
+			Slot:           1,
+			CommitteeIndex: 1,
+		},
+		Signature: nil,
+	}
+	root, err = helpers.ComputeSigningRoot(att2.Data, domain)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sigs = nil
+	for i, u := range comm2 {
+		att2.AggregationBits.SetBitAt(uint64(i), true)
+		sigs = append(sigs, keys[u].Sign(root[:]))
+	}
+	att2.Signature = bls.AggregateSignatures(sigs).Marshal()
+
+	if err := blocks.VerifyAttestations(ctx, st, []*ethpb.Attestation{att1, att2}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestVerifyAttestations_HandlesPlannedFork(t *testing.T) {
+	// In this test, att1 is from the prior fork and att2 is from the new fork.
+	ctx := context.Background()
+	numOfValidators := 4 * params.BeaconConfig().SlotsPerEpoch
+	validators := make([]*ethpb.Validator, numOfValidators)
+	_, keys, err := testutil.DeterministicDepositsAndKeys(numOfValidators)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < len(validators); i++ {
+		validators[i] = &ethpb.Validator{
+			ExitEpoch: params.BeaconConfig().FarFutureEpoch,
+			PublicKey: keys[i].PublicKey().Marshal(),
+		}
+	}
+
+	st, err := stateTrie.InitializeFromProto(&pb.BeaconState{
+		Slot:       35,
+		Validators: validators,
+		Fork: &pb.Fork{
+			Epoch:           1,
+			CurrentVersion:  []byte{0, 1, 2, 3},
+			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
+		},
+		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+	})
+
+	comm1, err := helpers.BeaconCommitteeFromState(st, 1 /*slot*/, 0 /*committeeIndex*/)
+	if err != nil {
+		t.Fatal(err)
+	}
+	att1 := &ethpb.Attestation{
+		AggregationBits: bitfield.NewBitlist(uint64(len(comm1))),
+		Data: &ethpb.AttestationData{
+			Slot:           1,
+			CommitteeIndex: 0,
+		},
+		Signature: nil,
+	}
+	prevDomain, err := helpers.Domain(st.Fork(), st.Fork().Epoch-1, params.BeaconConfig().DomainBeaconAttester, st.GenesisValidatorRoot())
+	if err != nil {
+		t.Fatal(err)
+	}
+	root, err := helpers.ComputeSigningRoot(att1.Data, prevDomain)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var sigs []bls.Signature
+	for i, u := range comm1 {
+		att1.AggregationBits.SetBitAt(uint64(i), true)
+		sigs = append(sigs, keys[u].Sign(root[:]))
+	}
+	att1.Signature = bls.AggregateSignatures(sigs).Marshal()
+
+	comm2, err := helpers.BeaconCommitteeFromState(st, 1*params.BeaconConfig().SlotsPerEpoch+1 /*slot*/, 1 /*committeeIndex*/)
+	if err != nil {
+		t.Fatal(err)
+	}
+	att2 := &ethpb.Attestation{
+		AggregationBits: bitfield.NewBitlist(uint64(len(comm2))),
+		Data: &ethpb.AttestationData{
+			Slot:           1*params.BeaconConfig().SlotsPerEpoch + 1,
+			CommitteeIndex: 1,
+		},
+		Signature: nil,
+	}
+	currDomain, err := helpers.Domain(st.Fork(), st.Fork().Epoch, params.BeaconConfig().DomainBeaconAttester, st.GenesisValidatorRoot())
+	root, err = helpers.ComputeSigningRoot(att2.Data, currDomain)
+	if err != nil {
+		t.Fatal(err)
+	}
+	sigs = nil
+	for i, u := range comm2 {
+		att2.AggregationBits.SetBitAt(uint64(i), true)
+		sigs = append(sigs, keys[u].Sign(root[:]))
+	}
+	att2.Signature = bls.AggregateSignatures(sigs).Marshal()
+
+	if err := blocks.VerifyAttestations(ctx, st, []*ethpb.Attestation{att1, att2}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestAreEth1DataEqual(t *testing.T) {
+	type args struct {
+		a *ethpb.Eth1Data
+		b *ethpb.Eth1Data
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "true when both are nil",
+			args: args{
+				a: nil,
+				b: nil,
+			},
+			want: true,
+		},
+		{
+			name: "false when only one is nil",
+			args: args{
+				a: nil,
+				b: &ethpb.Eth1Data{
+					DepositRoot:  make([]byte, 32),
+					DepositCount: 0,
+					BlockHash:    make([]byte, 32),
+				},
+			},
+			want: false,
+		},
+		{
+			name: "true when real equality",
+			args: args{
+				a: &ethpb.Eth1Data{
+					DepositRoot:  make([]byte, 32),
+					DepositCount: 0,
+					BlockHash:    make([]byte, 32),
+				},
+				b: &ethpb.Eth1Data{
+					DepositRoot:  make([]byte, 32),
+					DepositCount: 0,
+					BlockHash:    make([]byte, 32),
+				},
+			},
+			want: true,
+		},
+		{
+			name: "false is field value differs",
+			args: args{
+				a: &ethpb.Eth1Data{
+					DepositRoot:  make([]byte, 32),
+					DepositCount: 0,
+					BlockHash:    make([]byte, 32),
+				},
+				b: &ethpb.Eth1Data{
+					DepositRoot:  make([]byte, 32),
+					DepositCount: 64,
+					BlockHash:    make([]byte, 32),
+				},
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := blocks.AreEth1DataEqual(tt.args.a, tt.args.b); got != tt.want {
+				t.Errorf("AreEth1DataEqual() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

beacon-chain/core/blocks/block_regression_test.go

@@ -50,7 +50,7 @@ func TestProcessAttesterSlashings_RegressionSlashableIndices(t *testing.T) {
 	if err != nil {
 		t.Errorf("Could not get signing root of beacon block header: %v", err)
 	}
-	aggSigs := []*bls.Signature{}
+	aggSigs := []bls.Signature{}
 	for _, index := range setA {
 		sig := privKeys[index].Sign(signingRoot[:])
 		aggSigs = append(aggSigs, sig)
@@ -70,7 +70,7 @@ func TestProcessAttesterSlashings_RegressionSlashableIndices(t *testing.T) {
 	if err != nil {
 		t.Errorf("Could not get signing root of beacon block header: %v", err)
 	}
-	aggSigs = []*bls.Signature{}
+	aggSigs = []bls.Signature{}
 	for _, index := range setB {
 		sig := privKeys[index].Sign(signingRoot[:])
 		aggSigs = append(aggSigs, sig)

beacon-chain/core/blocks/eth1_data_test.go

@@ -11,9 +11,9 @@ import (
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
-func FakeDeposits(n int) []*ethpb.Eth1Data {
+func FakeDeposits(n uint64) []*ethpb.Eth1Data {
 	deposits := make([]*ethpb.Eth1Data, n)
-	for i := 0; i < n; i++ {
+	for i := uint64(0); i < n; i++ {
 		deposits[i] = &ethpb.Eth1Data{
 			DepositCount: 1,
 			DepositRoot:  []byte("root"),
@@ -30,7 +30,7 @@ func TestEth1DataHasEnoughSupport(t *testing.T) {
 		votingPeriodLength uint64
 	}{
 		{
-			stateVotes: FakeDeposits(4 * int(params.BeaconConfig().SlotsPerEpoch)),
+			stateVotes: FakeDeposits(4 * params.BeaconConfig().SlotsPerEpoch),
 			data: &ethpb.Eth1Data{
 				DepositCount: 1,
 				DepositRoot:  []byte("root"),
@@ -38,7 +38,7 @@ func TestEth1DataHasEnoughSupport(t *testing.T) {
 			hasSupport:         true,
 			votingPeriodLength: 7,
 		}, {
-			stateVotes: FakeDeposits(4 * int(params.BeaconConfig().SlotsPerEpoch)),
+			stateVotes: FakeDeposits(4 * params.BeaconConfig().SlotsPerEpoch),
 			data: &ethpb.Eth1Data{
 				DepositCount: 1,
 				DepositRoot:  []byte("root"),
@@ -46,7 +46,7 @@ func TestEth1DataHasEnoughSupport(t *testing.T) {
 			hasSupport:         false,
 			votingPeriodLength: 8,
 		}, {
-			stateVotes: FakeDeposits(4 * int(params.BeaconConfig().SlotsPerEpoch)),
+			stateVotes: FakeDeposits(4 * params.BeaconConfig().SlotsPerEpoch),
 			data: &ethpb.Eth1Data{
 				DepositCount: 1,
 				DepositRoot:  []byte("root"),
@@ -56,6 +56,7 @@ func TestEth1DataHasEnoughSupport(t *testing.T) {
 		},
 	}
 
+	params.SetupTestConfigCleanup(t)
 	for i, tt := range tests {
 		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
 			c := params.BeaconConfig()

beacon-chain/core/blocks/spectest/BUILD.bazel

@@ -1,4 +1,5 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_library(
     name = "go_default_library",

beacon-chain/core/blocks/spectest/attestation_test.go

@@ -5,14 +5,13 @@ import (
 	"testing"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/shared/params/spectest"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
 
 func runAttestationTest(t *testing.T, config string) {
-	if err := spectest.SetConfig(config); err != nil {
+	if err := spectest.SetConfig(t, config); err != nil {
 		t.Fatal(err)
 	}
 
@@ -25,7 +24,7 @@ func runAttestationTest(t *testing.T, config string) {
 				t.Fatal(err)
 			}
 			att := &ethpb.Attestation{}
-			if err := ssz.Unmarshal(attestationFile, att); err != nil {
+			if err := att.UnmarshalSSZ(attestationFile); err != nil {
 				t.Fatalf("Failed to unmarshal: %v", err)
 			}
 

beacon-chain/core/blocks/spectest/attester_slashing_test.go

@@ -5,14 +5,13 @@ import (
 	"testing"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/shared/params/spectest"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
 
 func runAttesterSlashingTest(t *testing.T, config string) {
-	if err := spectest.SetConfig(config); err != nil {
+	if err := spectest.SetConfig(t, config); err != nil {
 		t.Fatal(err)
 	}
 
@@ -25,7 +24,7 @@ func runAttesterSlashingTest(t *testing.T, config string) {
 				t.Fatal(err)
 			}
 			attSlashing := &ethpb.AttesterSlashing{}
-			if err := ssz.Unmarshal(attSlashingFile, attSlashing); err != nil {
+			if err := attSlashing.UnmarshalSSZ(attSlashingFile); err != nil {
 				t.Fatalf("Failed to unmarshal: %v", err)
 			}
 

beacon-chain/core/blocks/spectest/block_header_test.go

@@ -9,9 +9,8 @@ import (
 	"github.com/bazelbuild/rules_go/go/tools/bazel"
 	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params/spectest"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
@@ -19,7 +18,7 @@ import (
 )
 
 func runBlockHeaderTest(t *testing.T, config string) {
-	if err := spectest.SetConfig(config); err != nil {
+	if err := spectest.SetConfig(t, config); err != nil {
 		t.Fatal(err)
 	}
 
@@ -31,7 +30,7 @@ func runBlockHeaderTest(t *testing.T, config string) {
 				t.Fatal(err)
 			}
 			block := &ethpb.BeaconBlock{}
-			if err := ssz.Unmarshal(blockFile, block); err != nil {
+			if err := block.UnmarshalSSZ(blockFile); err != nil {
 				t.Fatalf("Failed to unmarshal: %v", err)
 			}
 
@@ -40,10 +39,10 @@ func runBlockHeaderTest(t *testing.T, config string) {
 				t.Fatal(err)
 			}
 			preBeaconStateBase := &pb.BeaconState{}
-			if err := ssz.Unmarshal(preBeaconStateFile, preBeaconStateBase); err != nil {
+			if err := preBeaconStateBase.UnmarshalSSZ(preBeaconStateFile); err != nil {
 				t.Fatalf("Failed to unmarshal: %v", err)
 			}
-			preBeaconState, err := beaconstate.InitializeFromProto(preBeaconStateBase)
+			preBeaconState, err := stateTrie.InitializeFromProto(preBeaconStateBase)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -70,7 +69,7 @@ func runBlockHeaderTest(t *testing.T, config string) {
 				}
 
 				postBeaconState := &pb.BeaconState{}
-				if err := ssz.Unmarshal(postBeaconStateFile, postBeaconState); err != nil {
+				if err := postBeaconState.UnmarshalSSZ(postBeaconStateFile); err != nil {
 					t.Fatalf("Failed to unmarshal: %v", err)
 				}
 				if !proto.Equal(beaconState.CloneInnerState(), postBeaconState) {

beacon-chain/core/blocks/spectest/block_processing_test.go

@@ -1,3 +1,5 @@
+// Package spectest contains all comformity specification tests
+// for block processing according to the eth2 spec.
 package spectest
 
 import (
@@ -11,10 +13,9 @@ import (
 	"github.com/bazelbuild/rules_go/go/tools/bazel"
 	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
-	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
+	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/params/spectest"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
@@ -26,7 +27,7 @@ func init() {
 }
 
 func runBlockProcessingTest(t *testing.T, config string) {
-	if err := spectest.SetConfig(config); err != nil {
+	if err := spectest.SetConfig(t, config); err != nil {
 		t.Fatal(err)
 	}
 
@@ -39,10 +40,10 @@ func runBlockProcessingTest(t *testing.T, config string) {
 				t.Fatal(err)
 			}
 			beaconStateBase := &pb.BeaconState{}
-			if err := ssz.Unmarshal(preBeaconStateFile, beaconStateBase); err != nil {
+			if err := beaconStateBase.UnmarshalSSZ(preBeaconStateFile); err != nil {
 				t.Fatalf("Failed to unmarshal: %v", err)
 			}
-			beaconState, err := beaconstate.InitializeFromProto(beaconStateBase)
+			beaconState, err := stateTrie.InitializeFromProto(beaconStateBase)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -65,7 +66,7 @@ func runBlockProcessingTest(t *testing.T, config string) {
 					t.Fatal(err)
 				}
 				block := &ethpb.SignedBeaconBlock{}
-				if err := ssz.Unmarshal(blockFile, block); err != nil {
+				if err := block.UnmarshalSSZ(blockFile); err != nil {
 					t.Fatalf("Failed to unmarshal: %v", err)
 				}
 				beaconState, transitionError = state.ExecuteStateTransition(context.Background(), beaconState, block)
@@ -94,7 +95,7 @@ func runBlockProcessingTest(t *testing.T, config string) {
 				}
 
 				postBeaconState := &pb.BeaconState{}
-				if err := ssz.Unmarshal(postBeaconStateFile, postBeaconState); err != nil {
+				if err := postBeaconState.UnmarshalSSZ(postBeaconStateFile); err != nil {
 					t.Fatalf("Failed to unmarshal: %v", err)
 				}
 

beacon-chain/core/blocks/spectest/deposit_test.go

@@ -5,14 +5,13 @@ import (
 	"testing"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/shared/params/spectest"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
 
 func runDepositTest(t *testing.T, config string) {
-	if err := spectest.SetConfig(config); err != nil {
+	if err := spectest.SetConfig(t, config); err != nil {
 		t.Fatal(err)
 	}
 
@@ -25,7 +24,7 @@ func runDepositTest(t *testing.T, config string) {
 				t.Fatal(err)
 			}
 			deposit := &ethpb.Deposit{}
-			if err := ssz.Unmarshal(depositFile, deposit); err != nil {
+			if err := deposit.UnmarshalSSZ(depositFile); err != nil {
 				t.Fatalf("Failed to unmarshal: %v", err)
 			}
 

beacon-chain/core/blocks/spectest/proposer_slashing_test.go

@@ -5,14 +5,13 @@ import (
 	"testing"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/shared/params/spectest"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
 
 func runProposerSlashingTest(t *testing.T, config string) {
-	if err := spectest.SetConfig(config); err != nil {
+	if err := spectest.SetConfig(t, config); err != nil {
 		t.Fatal(err)
 	}
 
@@ -25,7 +24,7 @@ func runProposerSlashingTest(t *testing.T, config string) {
 				t.Fatal(err)
 			}
 			proposerSlashing := &ethpb.ProposerSlashing{}
-			if err := ssz.Unmarshal(proposerSlashingFile, proposerSlashing); err != nil {
+			if err := proposerSlashing.UnmarshalSSZ(proposerSlashingFile); err != nil {
 				t.Fatalf("Failed to unmarshal: %v", err)
 			}
 

beacon-chain/core/blocks/spectest/voluntary_exit_test.go

@@ -5,14 +5,13 @@ import (
 	"testing"
 
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-ssz"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/shared/params/spectest"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
 
 func runVoluntaryExitTest(t *testing.T, config string) {
-	if err := spectest.SetConfig(config); err != nil {
+	if err := spectest.SetConfig(t, config); err != nil {
 		t.Fatal(err)
 	}
 
@@ -25,7 +24,7 @@ func runVoluntaryExitTest(t *testing.T, config string) {
 				t.Fatal(err)
 			}
 			voluntaryExit := &ethpb.SignedVoluntaryExit{}
-			if err := ssz.Unmarshal(exitFile, voluntaryExit); err != nil {
+			if err := voluntaryExit.UnmarshalSSZ(exitFile); err != nil {
 				t.Fatalf("Failed to unmarshal: %v", err)
 			}
 

beacon-chain/core/epoch/BUILD.bazel

@@ -1,4 +1,5 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_library(
     name = "go_default_library",

beacon-chain/core/epoch/epoch_processing.go

@@ -1,6 +1,6 @@
-// Package epoch contains epoch processing libraries. These libraries
-// process new balance for the validators, justify and finalize new
-// check points, shuffle and reassign validators to different slots and
+// Package epoch contains epoch processing libraries according to spec, able to
+// process new balance for validators, justify and finalize new
+// check points, and shuffle validators to different slots and
 // shards.
 package epoch
 
@@ -80,10 +80,12 @@ func ProcessRegistryUpdates(state *stateTrie.BeaconState) (*stateTrie.BeaconStat
 	currentEpoch := helpers.CurrentEpoch(state)
 	vals := state.Validators()
 	var err error
+	ejectionBal := params.BeaconConfig().EjectionBalance
+	activationEligibilityEpoch := helpers.CurrentEpoch(state) + 1
 	for idx, validator := range vals {
 		// Process the validators for activation eligibility.
 		if helpers.IsEligibleForActivationQueue(validator) {
-			validator.ActivationEligibilityEpoch = helpers.CurrentEpoch(state) + 1
+			validator.ActivationEligibilityEpoch = activationEligibilityEpoch
 			if err := state.UpdateValidatorAtIndex(uint64(idx), validator); err != nil {
 				return nil, err
 			}
@@ -91,7 +93,7 @@ func ProcessRegistryUpdates(state *stateTrie.BeaconState) (*stateTrie.BeaconStat
 
 		// Process the validators for ejection.
 		isActive := helpers.IsActiveValidator(validator, currentEpoch)
-		belowEjectionBalance := validator.EffectiveBalance <= params.BeaconConfig().EjectionBalance
+		belowEjectionBalance := validator.EffectiveBalance <= ejectionBal
 		if isActive && belowEjectionBalance {
 			state, err = validators.InitiateValidatorExit(state, uint64(idx))
 			if err != nil {
@@ -111,7 +113,7 @@ func ProcessRegistryUpdates(state *stateTrie.BeaconState) (*stateTrie.BeaconStat
 	sort.Sort(sortableIndices{indices: activationQ, validators: vals})
 
 	// Only activate just enough validators according to the activation churn limit.
-	limit := len(activationQ)
+	limit := uint64(len(activationQ))
 	activeValidatorCount, err := helpers.ActiveValidatorCount(state, currentEpoch)
 	if err != nil {
 		return nil, errors.Wrap(err, "could not get active validator count")
@@ -123,16 +125,17 @@ func ProcessRegistryUpdates(state *stateTrie.BeaconState) (*stateTrie.BeaconStat
 	}
 
 	// Prevent churn limit cause index out of bound.
-	if int(churnLimit) < limit {
-		limit = int(churnLimit)
+	if churnLimit < limit {
+		limit = churnLimit
 	}
 
+	activationExitEpoch := helpers.ActivationExitEpoch(currentEpoch)
 	for _, index := range activationQ[:limit] {
 		validator, err := state.ValidatorAtIndex(index)
 		if err != nil {
 			return nil, err
 		}
-		validator.ActivationEpoch = helpers.ActivationExitEpoch(currentEpoch)
+		validator.ActivationEpoch = activationExitEpoch
 		if err := state.UpdateValidatorAtIndex(index, validator); err != nil {
 			return nil, err
 		}
@@ -170,11 +173,11 @@ func ProcessSlashings(state *stateTrie.BeaconState) (*stateTrie.BeaconState, err
 
 	// a callback is used here to apply the following actions  to all validators
 	// below equally.
+	increment := params.BeaconConfig().EffectiveBalanceIncrement
 	err = state.ApplyToEveryValidator(func(idx int, val *ethpb.Validator) (bool, error) {
 		correctEpoch := (currentEpoch + exitLength/2) == val.WithdrawableEpoch
 		if val.Slashed && correctEpoch {
 			minSlashing := mathutil.Min(totalSlashing*3, totalBalance)
-			increment := params.BeaconConfig().EffectiveBalanceIncrement
 			penaltyNumerator := val.EffectiveBalance / increment * minSlashing
 			penalty := penaltyNumerator / totalBalance * increment
 			if err := helpers.DecreaseBalance(state, uint64(idx), penalty); err != nil {
@@ -206,6 +209,7 @@ func ProcessSlashings(state *stateTrie.BeaconState) (*stateTrie.BeaconState, err
 //            balance + DOWNWARD_THRESHOLD < validator.effective_balance
 //            or validator.effective_balance + UPWARD_THRESHOLD < balance
 //        ):
+//        validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE)
 //    index_epoch = Epoch(next_epoch + ACTIVATION_EXIT_DELAY)
 //    index_root_position = index_epoch % EPOCHS_PER_HISTORICAL_VECTOR
 //    indices_list = List[ValidatorIndex, VALIDATOR_REGISTRY_LIMIT](get_active_validator_indices(state, index_epoch))
@@ -237,6 +241,12 @@ func ProcessFinalUpdates(state *stateTrie.BeaconState) (*stateTrie.BeaconState,
 		}
 	}
 
+	effBalanceInc := params.BeaconConfig().EffectiveBalanceIncrement
+	maxEffBalance := params.BeaconConfig().MaxEffectiveBalance
+	hysteresisInc := effBalanceInc / params.BeaconConfig().HysteresisQuotient
+	downwardThreshold := hysteresisInc * params.BeaconConfig().HysteresisDownwardMultiplier
+	upwardThreshold := hysteresisInc * params.BeaconConfig().HysteresisUpwardMultiplier
+
 	bals := state.Balances()
 	// Update effective balances with hysteresis.
 	validatorFunc := func(idx int, val *ethpb.Validator) (bool, error) {
@@ -247,14 +257,11 @@ func ProcessFinalUpdates(state *stateTrie.BeaconState) (*stateTrie.BeaconState,
 			return false, fmt.Errorf("validator index exceeds validator length in state %d >= %d", idx, len(state.Balances()))
 		}
 		balance := bals[idx]
-		hysteresisInc := params.BeaconConfig().EffectiveBalanceIncrement / params.BeaconConfig().HysteresisQuotient
-		downwardThreshold := hysteresisInc * params.BeaconConfig().HysteresisDownwardMultiplier
-		upwardThreshold := hysteresisInc * params.BeaconConfig().HysteresisUpwardMultiplier
 
 		if balance+downwardThreshold < val.EffectiveBalance || val.EffectiveBalance+upwardThreshold < balance {
-			val.EffectiveBalance = params.BeaconConfig().MaxEffectiveBalance
-			if val.EffectiveBalance > balance-balance%params.BeaconConfig().EffectiveBalanceIncrement {
-				val.EffectiveBalance = balance - balance%params.BeaconConfig().EffectiveBalanceIncrement
+			val.EffectiveBalance = maxEffBalance
+			if val.EffectiveBalance > balance-balance%effBalanceInc {
+				val.EffectiveBalance = balance - balance%effBalanceInc
 			}
 			return true, nil
 		}
@@ -267,22 +274,22 @@ func ProcessFinalUpdates(state *stateTrie.BeaconState) (*stateTrie.BeaconState,
 
 	// Set total slashed balances.
 	slashedExitLength := params.BeaconConfig().EpochsPerSlashingsVector
-	slashedEpoch := int(nextEpoch % slashedExitLength)
+	slashedEpoch := nextEpoch % slashedExitLength
 	slashings := state.Slashings()
-	if len(slashings) != int(slashedExitLength) {
+	if uint64(len(slashings)) != slashedExitLength {
 		return nil, fmt.Errorf(
 			"state slashing length %d different than EpochsPerHistoricalVector %d",
 			len(slashings),
 			slashedExitLength,
 		)
 	}
-	if err := state.UpdateSlashingsAtIndex(uint64(slashedEpoch) /* index */, 0 /* value */); err != nil {
+	if err := state.UpdateSlashingsAtIndex(slashedEpoch /* index */, 0 /* value */); err != nil {
 		return nil, err
 	}
 
 	// Set RANDAO mix.
 	randaoMixLength := params.BeaconConfig().EpochsPerHistoricalVector
-	if state.RandaoMixesLength() != int(randaoMixLength) {
+	if uint64(state.RandaoMixesLength()) != randaoMixLength {
 		return nil, fmt.Errorf(
 			"state randao length %d different than EpochsPerHistoricalVector %d",
 			state.RandaoMixesLength(),
@@ -293,7 +300,7 @@ func ProcessFinalUpdates(state *stateTrie.BeaconState) (*stateTrie.BeaconState,
 	if err != nil {
 		return nil, err
 	}
-	if err := state.UpdateRandaoMixesAtIndex(mix, nextEpoch%randaoMixLength); err != nil {
+	if err := state.UpdateRandaoMixesAtIndex(nextEpoch%randaoMixLength, mix); err != nil {
 		return nil, err
 	}
 
@@ -357,11 +364,11 @@ func unslashedAttestingIndices(state *stateTrie.BeaconState, atts []*pb.PendingA
 	sort.Slice(setIndices, func(i, j int) bool { return setIndices[i] < setIndices[j] })
 	// Remove the slashed validator indices.
 	for i := 0; i < len(setIndices); i++ {
-		v, err := state.ValidatorAtIndex(setIndices[i])
+		v, err := state.ValidatorAtIndexReadOnly(setIndices[i])
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to look up validator")
 		}
-		if v != nil && v.Slashed {
+		if v != nil && v.Slashed() {
 			setIndices = append(setIndices[:i], setIndices[i+1:]...)
 		}
 	}
@@ -385,11 +392,11 @@ func BaseReward(state *stateTrie.BeaconState, index uint64) (uint64, error) {
 	if err != nil {
 		return 0, errors.Wrap(err, "could not calculate active balance")
 	}
-	val, err := state.ValidatorAtIndex(index)
+	val, err := state.ValidatorAtIndexReadOnly(index)
 	if err != nil {
 		return 0, err
 	}
-	effectiveBalance := val.EffectiveBalance
+	effectiveBalance := val.EffectiveBalance()
 	baseReward := effectiveBalance * params.BeaconConfig().BaseRewardFactor /
 		mathutil.IntegerSquareRoot(totalBalance) / params.BeaconConfig().BaseRewardsPerEpoch
 	return baseReward, nil

beacon-chain/core/epoch/epoch_processing_fuzz_test.go

@@ -19,8 +19,5 @@ func TestFuzzFinalUpdates_10000(t *testing.T) {
 			t.Fatal(err)
 		}
 		_, err = ProcessFinalUpdates(s)
-		if err != nil {
-			t.Log(err)
-		}
 	}
 }

beacon-chain/core/epoch/epoch_processing_test.go

@@ -13,13 +13,6 @@ import (
 	"github.com/prysmaticlabs/prysm/shared/params"
 )
 
-func init() {
-	// TODO(2312): remove this and use the mainnet count.
-	c := params.BeaconConfig()
-	c.MinGenesisActiveValidatorCount = 16384
-	params.OverrideBeaconConfig(c)
-}
-
 func TestUnslashedAttestingIndices_CanSortAndFilter(t *testing.T) {
 	// Generate 2 attestations.
 	atts := make([]*pb.PendingAttestation, 2)
@@ -406,7 +399,7 @@ func TestProcessRegistryUpdates_EligibleToActivate(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	for i := 0; i < int(limit)+10; i++ {
+	for i := uint64(0); i < limit+10; i++ {
 		base.Validators = append(base.Validators, &ethpb.Validator{
 			ActivationEligibilityEpoch: params.BeaconConfig().FarFutureEpoch,
 			EffectiveBalance:           params.BeaconConfig().MaxEffectiveBalance,
@@ -424,11 +417,11 @@ func TestProcessRegistryUpdates_EligibleToActivate(t *testing.T) {
 			t.Errorf("Could not update registry %d, wanted activation eligibility epoch %d got %d",
 				i, currentEpoch, validator.ActivationEligibilityEpoch)
 		}
-		if i < int(limit) && validator.ActivationEpoch != helpers.ActivationExitEpoch(currentEpoch) {
+		if uint64(i) < limit && validator.ActivationEpoch != helpers.ActivationExitEpoch(currentEpoch) {
 			t.Errorf("Could not update registry %d, validators failed to activate: wanted activation epoch %d, got %d",
 				i, helpers.ActivationExitEpoch(currentEpoch), validator.ActivationEpoch)
 		}
-		if i >= int(limit) && validator.ActivationEpoch != params.BeaconConfig().FarFutureEpoch {
+		if uint64(i) >= limit && validator.ActivationEpoch != params.BeaconConfig().FarFutureEpoch {
 			t.Errorf("Could not update registry %d, validators should not have been activated, wanted activation epoch: %d, got %d",
 				i, params.BeaconConfig().FarFutureEpoch, validator.ActivationEpoch)
 		}

beacon-chain/core/epoch/precompute/BUILD.bazel

@@ -1,4 +1,5 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -42,6 +43,7 @@ go_test(
         "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//shared/attestationutil:go_default_library",
+        "//shared/mathutil:go_default_library",
         "//shared/params:go_default_library",
         "//shared/testutil:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",

beacon-chain/core/epoch/precompute/attestation.go

@@ -9,6 +9,7 @@ import (
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/attestationutil"
+	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/traceutil"
 	"go.opencensus.io/trace"
 )
@@ -23,7 +24,7 @@ func ProcessAttestations(
 	ctx context.Context,
 	state *stateTrie.BeaconState,
 	vp []*Validator,
-	bp *Balance,
+	pBal *Balance,
 ) ([]*Validator, *Balance, error) {
 	ctx, span := trace.StartSpan(ctx, "precomputeEpoch.ProcessAttestations")
 	defer span.End()
@@ -51,10 +52,10 @@ func ProcessAttestations(
 		vp = UpdateValidator(vp, v, indices, a, a.Data.Slot)
 	}
 
-	bp = UpdateBalance(vp, bp)
-	Balances = bp
+	pBal = UpdateBalance(vp, pBal)
+	Balances = pBal
 
-	return vp, bp, nil
+	return vp, pBal, nil
 }
 
 // AttestedCurrentEpoch returns true if attestation `a` attested once in current epoch and/or epoch boundary block.
@@ -90,12 +91,14 @@ func AttestedPrevEpoch(s *stateTrie.BeaconState, a *pb.PendingAttestation) (bool
 			votedTarget = true
 		}
 
-		same, err = SameHead(s, a)
-		if err != nil {
-			return false, false, false, errors.Wrap(err, "could not check same head")
-		}
-		if same {
-			votedHead = true
+		if votedTarget {
+			same, err = SameHead(s, a)
+			if err != nil {
+				return false, false, false, errors.Wrap(err, "could not check same head")
+			}
+			if same {
+				votedHead = true
+			}
 		}
 	}
 	return votedPrevEpoch, votedTarget, votedHead, nil
@@ -156,25 +159,54 @@ func UpdateValidator(vp []*Validator, record *Validator, indices []uint64, a *pb
 }
 
 // UpdateBalance updates pre computed balance store.
-func UpdateBalance(vp []*Validator, bp *Balance) *Balance {
+func UpdateBalance(vp []*Validator, bBal *Balance) *Balance {
 	for _, v := range vp {
 		if !v.IsSlashed {
 			if v.IsCurrentEpochAttester {
-				bp.CurrentEpochAttesters += v.CurrentEpochEffectiveBalance
+				bBal.CurrentEpochAttested += v.CurrentEpochEffectiveBalance
 			}
 			if v.IsCurrentEpochTargetAttester {
-				bp.CurrentEpochTargetAttesters += v.CurrentEpochEffectiveBalance
+				bBal.CurrentEpochTargetAttested += v.CurrentEpochEffectiveBalance
 			}
 			if v.IsPrevEpochAttester {
-				bp.PrevEpochAttesters += v.CurrentEpochEffectiveBalance
+				bBal.PrevEpochAttested += v.CurrentEpochEffectiveBalance
 			}
 			if v.IsPrevEpochTargetAttester {
-				bp.PrevEpochTargetAttesters += v.CurrentEpochEffectiveBalance
+				bBal.PrevEpochTargetAttested += v.CurrentEpochEffectiveBalance
 			}
 			if v.IsPrevEpochHeadAttester {
-				bp.PrevEpochHeadAttesters += v.CurrentEpochEffectiveBalance
+				bBal.PrevEpochHeadAttested += v.CurrentEpochEffectiveBalance
 			}
 		}
 	}
-	return bp
+
+	return EnsureBalancesLowerBound(bBal)
+}
+
+// EnsureBalancesLowerBound ensures all the balances such as active current epoch, active previous epoch and more
+// have EffectiveBalanceIncrement(1 eth) as a lower bound.
+func EnsureBalancesLowerBound(bBal *Balance) *Balance {
+	ebi := params.BeaconConfig().EffectiveBalanceIncrement
+	if ebi > bBal.ActiveCurrentEpoch {
+		bBal.ActiveCurrentEpoch = ebi
+	}
+	if ebi > bBal.ActivePrevEpoch {
+		bBal.ActivePrevEpoch = ebi
+	}
+	if ebi > bBal.CurrentEpochAttested {
+		bBal.CurrentEpochAttested = ebi
+	}
+	if ebi > bBal.CurrentEpochTargetAttested {
+		bBal.CurrentEpochTargetAttested = ebi
+	}
+	if ebi > bBal.PrevEpochAttested {
+		bBal.PrevEpochAttested = ebi
+	}
+	if ebi > bBal.PrevEpochTargetAttested {
+		bBal.PrevEpochTargetAttested = ebi
+	}
+	if ebi > bBal.PrevEpochHeadAttested {
+		bBal.PrevEpochHeadAttested = ebi
+	}
+	return bBal
 }

beacon-chain/core/epoch/precompute/attestation_test.go

@@ -50,24 +50,26 @@ func TestUpdateValidator_InclusionOnlyCountsPrevEpoch(t *testing.T) {
 
 func TestUpdateBalance(t *testing.T) {
 	vp := []*precompute.Validator{
-		{IsCurrentEpochAttester: true, CurrentEpochEffectiveBalance: 100},
-		{IsCurrentEpochTargetAttester: true, IsCurrentEpochAttester: true, CurrentEpochEffectiveBalance: 100},
-		{IsCurrentEpochTargetAttester: true, CurrentEpochEffectiveBalance: 100},
-		{IsPrevEpochAttester: true, CurrentEpochEffectiveBalance: 100},
-		{IsPrevEpochAttester: true, IsPrevEpochTargetAttester: true, CurrentEpochEffectiveBalance: 100},
-		{IsPrevEpochHeadAttester: true, CurrentEpochEffectiveBalance: 100},
-		{IsPrevEpochAttester: true, IsPrevEpochHeadAttester: true, CurrentEpochEffectiveBalance: 100},
-		{IsSlashed: true, IsCurrentEpochAttester: true, CurrentEpochEffectiveBalance: 100},
+		{IsCurrentEpochAttester: true, CurrentEpochEffectiveBalance: 100 * params.BeaconConfig().EffectiveBalanceIncrement},
+		{IsCurrentEpochTargetAttester: true, IsCurrentEpochAttester: true, CurrentEpochEffectiveBalance: 100 * params.BeaconConfig().EffectiveBalanceIncrement},
+		{IsCurrentEpochTargetAttester: true, CurrentEpochEffectiveBalance: 100 * params.BeaconConfig().EffectiveBalanceIncrement},
+		{IsPrevEpochAttester: true, CurrentEpochEffectiveBalance: 100 * params.BeaconConfig().EffectiveBalanceIncrement},
+		{IsPrevEpochAttester: true, IsPrevEpochTargetAttester: true, CurrentEpochEffectiveBalance: 100 * params.BeaconConfig().EffectiveBalanceIncrement},
+		{IsPrevEpochHeadAttester: true, CurrentEpochEffectiveBalance: 100 * params.BeaconConfig().EffectiveBalanceIncrement},
+		{IsPrevEpochAttester: true, IsPrevEpochHeadAttester: true, CurrentEpochEffectiveBalance: 100 * params.BeaconConfig().EffectiveBalanceIncrement},
+		{IsSlashed: true, IsCurrentEpochAttester: true, CurrentEpochEffectiveBalance: 100 * params.BeaconConfig().EffectiveBalanceIncrement},
 	}
-	wantedBp := &precompute.Balance{
-		CurrentEpochAttesters:       200,
-		CurrentEpochTargetAttesters: 200,
-		PrevEpochAttesters:          300,
-		PrevEpochTargetAttesters:    100,
-		PrevEpochHeadAttesters:      200,
+	wantedPBal := &precompute.Balance{
+		ActiveCurrentEpoch:         params.BeaconConfig().EffectiveBalanceIncrement,
+		ActivePrevEpoch:            params.BeaconConfig().EffectiveBalanceIncrement,
+		CurrentEpochAttested:       200 * params.BeaconConfig().EffectiveBalanceIncrement,
+		CurrentEpochTargetAttested: 200 * params.BeaconConfig().EffectiveBalanceIncrement,
+		PrevEpochAttested:          300 * params.BeaconConfig().EffectiveBalanceIncrement,
+		PrevEpochTargetAttested:    100 * params.BeaconConfig().EffectiveBalanceIncrement,
+		PrevEpochHeadAttested:      200 * params.BeaconConfig().EffectiveBalanceIncrement,
 	}
-	bp := precompute.UpdateBalance(vp, &precompute.Balance{})
-	if !reflect.DeepEqual(bp, wantedBp) {
+	pBal := precompute.UpdateBalance(vp, &precompute.Balance{})
+	if !reflect.DeepEqual(pBal, wantedPBal) {
 		t.Error("Incorrect balance calculations")
 	}
 }
@@ -220,7 +222,7 @@ func TestProcessAttestations(t *testing.T) {
 	if err := beaconState.SetBlockRoots(br); err != nil {
 		t.Fatal(err)
 	}
-	att2.Data.Target.Root = rt[:]
+	att2.Data.Target.Root = newRt[:]
 	att2.Data.BeaconBlockRoot = newRt[:]
 	err := beaconState.SetPreviousEpochAttestations([]*pb.PendingAttestation{{Data: att1.Data, AggregationBits: bf}})
 	if err != nil {
@@ -231,12 +233,12 @@ func TestProcessAttestations(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	vp := make([]*precompute.Validator, validators)
-	for i := 0; i < len(vp); i++ {
-		vp[i] = &precompute.Validator{CurrentEpochEffectiveBalance: 100}
+	pVals := make([]*precompute.Validator, validators)
+	for i := 0; i < len(pVals); i++ {
+		pVals[i] = &precompute.Validator{CurrentEpochEffectiveBalance: 100}
 	}
-	bp := &precompute.Balance{}
-	vp, bp, err = precompute.ProcessAttestations(context.Background(), beaconState, vp, bp)
+	pBal := &precompute.Balance{}
+	pVals, pBal, err = precompute.ProcessAttestations(context.Background(), beaconState, pVals, pBal)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -247,7 +249,7 @@ func TestProcessAttestations(t *testing.T) {
 	}
 	indices := attestationutil.AttestingIndices(att1.AggregationBits, committee)
 	for _, i := range indices {
-		if !vp[i].IsPrevEpochAttester {
+		if !pVals[i].IsPrevEpochAttester {
 			t.Error("Not a prev epoch attester")
 		}
 	}
@@ -257,11 +259,40 @@ func TestProcessAttestations(t *testing.T) {
 	}
 	indices = attestationutil.AttestingIndices(att2.AggregationBits, committee)
 	for _, i := range indices {
-		if !vp[i].IsPrevEpochAttester {
+		if !pVals[i].IsPrevEpochAttester {
 			t.Error("Not a prev epoch attester")
 		}
-		if !vp[i].IsPrevEpochHeadAttester {
+		if !pVals[i].IsPrevEpochTargetAttester {
+			t.Error("Not a prev epoch target attester")
+		}
+		if !pVals[i].IsPrevEpochHeadAttester {
 			t.Error("Not a prev epoch head attester")
 		}
 	}
 }
+
+func TestEnsureBalancesLowerBound(t *testing.T) {
+	b := &precompute.Balance{}
+	b = precompute.EnsureBalancesLowerBound(b)
+	if b.ActiveCurrentEpoch != params.BeaconConfig().EffectiveBalanceIncrement {
+		t.Error("Did not get wanted active current balance")
+	}
+	if b.ActivePrevEpoch != params.BeaconConfig().EffectiveBalanceIncrement {
+		t.Error("Did not get wanted active previous balance")
+	}
+	if b.CurrentEpochAttested != params.BeaconConfig().EffectiveBalanceIncrement {
+		t.Error("Did not get wanted current attested balance")
+	}
+	if b.CurrentEpochTargetAttested != params.BeaconConfig().EffectiveBalanceIncrement {
+		t.Error("Did not get wanted target attested balance")
+	}
+	if b.PrevEpochAttested != params.BeaconConfig().EffectiveBalanceIncrement {
+		t.Error("Did not get wanted prev attested balance")
+	}
+	if b.PrevEpochTargetAttested != params.BeaconConfig().EffectiveBalanceIncrement {
+		t.Error("Did not get wanted prev target attested balance")
+	}
+	if b.PrevEpochHeadAttested != params.BeaconConfig().EffectiveBalanceIncrement {
+		t.Error("Did not get wanted prev head attested balance")
+	}
+}

beacon-chain/core/epoch/precompute/justification_finalization.go

@@ -10,7 +10,45 @@ import (
 // ProcessJustificationAndFinalizationPreCompute processes justification and finalization during
 // epoch processing. This is where a beacon node can justify and finalize a new epoch.
 // Note: this is an optimized version by passing in precomputed total and attesting balances.
-func ProcessJustificationAndFinalizationPreCompute(state *stateTrie.BeaconState, p *Balance) (*stateTrie.BeaconState, error) {
+// def process_justification_and_finalization(state: BeaconState) -> None:
+//    if get_current_epoch(state) <= GENESIS_EPOCH + 1:
+//        return
+//
+//    previous_epoch = get_previous_epoch(state)
+//    current_epoch = get_current_epoch(state)
+//    old_previous_justified_checkpoint = state.previous_justified_checkpoint
+//    old_current_justified_checkpoint = state.current_justified_checkpoint
+//
+//    # Process justifications
+//    state.previous_justified_checkpoint = state.current_justified_checkpoint
+//    state.justification_bits[1:] = state.justification_bits[:-1]
+//    state.justification_bits[0] = 0b0
+//    matching_target_attestations = get_matching_target_attestations(state, previous_epoch)  # Previous epoch
+//    if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2:
+//        state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch,
+//                                                        root=get_block_root(state, previous_epoch))
+//        state.justification_bits[1] = 0b1
+//    matching_target_attestations = get_matching_target_attestations(state, current_epoch)  # Current epoch
+//    if get_attesting_balance(state, matching_target_attestations) * 3 >= get_total_active_balance(state) * 2:
+//        state.current_justified_checkpoint = Checkpoint(epoch=current_epoch,
+//                                                        root=get_block_root(state, current_epoch))
+//        state.justification_bits[0] = 0b1
+//
+//    # Process finalizations
+//    bits = state.justification_bits
+//    # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source
+//    if all(bits[1:4]) and old_previous_justified_checkpoint.epoch + 3 == current_epoch:
+//        state.finalized_checkpoint = old_previous_justified_checkpoint
+//    # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source
+//    if all(bits[1:3]) and old_previous_justified_checkpoint.epoch + 2 == current_epoch:
+//        state.finalized_checkpoint = old_previous_justified_checkpoint
+//    # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source
+//    if all(bits[0:3]) and old_current_justified_checkpoint.epoch + 2 == current_epoch:
+//        state.finalized_checkpoint = old_current_justified_checkpoint
+//    # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source
+//    if all(bits[0:2]) and old_current_justified_checkpoint.epoch + 1 == current_epoch:
+//        state.finalized_checkpoint = old_current_justified_checkpoint
+func ProcessJustificationAndFinalizationPreCompute(state *stateTrie.BeaconState, pBal *Balance) (*stateTrie.BeaconState, error) {
 	if state.Slot() <= helpers.StartSlot(2) {
 		return state, nil
 	}
@@ -34,7 +72,7 @@ func ProcessJustificationAndFinalizationPreCompute(state *stateTrie.BeaconState,
 	// We will use that paradigm here for consistency with the godoc spec definition.
 
 	// If 2/3 or more of total balance attested in the previous epoch.
-	if 3*p.PrevEpochTargetAttesters >= 2*p.CurrentEpoch {
+	if 3*pBal.PrevEpochTargetAttested >= 2*pBal.ActiveCurrentEpoch {
 		blockRoot, err := helpers.BlockRoot(state, prevEpoch)
 		if err != nil {
 			return nil, errors.Wrapf(err, "could not get block root for previous epoch %d", prevEpoch)
@@ -50,7 +88,7 @@ func ProcessJustificationAndFinalizationPreCompute(state *stateTrie.BeaconState,
 	}
 
 	// If 2/3 or more of the total balance attested in the current epoch.
-	if 3*p.CurrentEpochTargetAttesters >= 2*p.CurrentEpoch {
+	if 3*pBal.CurrentEpochTargetAttested >= 2*pBal.ActiveCurrentEpoch {
 		blockRoot, err := helpers.BlockRoot(state, currentEpoch)
 		if err != nil {
 			return nil, errors.Wrapf(err, "could not get block root for current epoch %d", prevEpoch)

beacon-chain/core/epoch/precompute/justification_finalization_test.go

@@ -40,7 +40,7 @@ func TestProcessJustificationAndFinalizationPreCompute_ConsecutiveEpochs(t *test
 		t.Fatal(err)
 	}
 	attestedBalance := 4 * e * 3 / 2
-	b := &precompute.Balance{PrevEpochTargetAttesters: attestedBalance}
+	b := &precompute.Balance{PrevEpochTargetAttested: attestedBalance}
 	newState, err := precompute.ProcessJustificationAndFinalizationPreCompute(state, b)
 	if err != nil {
 		t.Fatal(err)
@@ -95,7 +95,7 @@ func TestProcessJustificationAndFinalizationPreCompute_JustifyCurrentEpoch(t *te
 		t.Fatal(err)
 	}
 	attestedBalance := 4 * e * 3 / 2
-	b := &precompute.Balance{PrevEpochTargetAttesters: attestedBalance}
+	b := &precompute.Balance{PrevEpochTargetAttested: attestedBalance}
 	newState, err := precompute.ProcessJustificationAndFinalizationPreCompute(state, b)
 	if err != nil {
 		t.Fatal(err)
@@ -149,7 +149,7 @@ func TestProcessJustificationAndFinalizationPreCompute_JustifyPrevEpoch(t *testi
 		t.Fatal(err)
 	}
 	attestedBalance := 4 * e * 3 / 2
-	b := &precompute.Balance{PrevEpochTargetAttesters: attestedBalance}
+	b := &precompute.Balance{PrevEpochTargetAttested: attestedBalance}
 	newState, err := precompute.ProcessJustificationAndFinalizationPreCompute(state, b)
 	if err != nil {
 		t.Fatal(err)

beacon-chain/core/epoch/precompute/new.go

@@ -1,3 +1,6 @@
+// Package precompute provides gathering of nicely-structured
+// data important to feed into epoch processing, such as attesting
+// records and balances, for faster computation.
 package precompute
 
 import (
@@ -16,39 +19,39 @@ import (
 func New(ctx context.Context, state *stateTrie.BeaconState) ([]*Validator, *Balance, error) {
 	ctx, span := trace.StartSpan(ctx, "precomputeEpoch.New")
 	defer span.End()
-	vp := make([]*Validator, state.NumValidators())
-	bp := &Balance{}
+	pValidators := make([]*Validator, state.NumValidators())
+	pBal := &Balance{}
 
 	currentEpoch := helpers.CurrentEpoch(state)
 	prevEpoch := helpers.PrevEpoch(state)
 
 	if err := state.ReadFromEveryValidator(func(idx int, val *stateTrie.ReadOnlyValidator) error {
 		// Was validator withdrawable or slashed
-		withdrawable := currentEpoch >= val.WithdrawableEpoch()
-		p := &Validator{
+		withdrawable := prevEpoch+1 >= val.WithdrawableEpoch()
+		pVal := &Validator{
 			IsSlashed:                    val.Slashed(),
 			IsWithdrawableCurrentEpoch:   withdrawable,
 			CurrentEpochEffectiveBalance: val.EffectiveBalance(),
 		}
 		// Was validator active current epoch
 		if helpers.IsActiveValidatorUsingTrie(val, currentEpoch) {
-			p.IsActiveCurrentEpoch = true
-			bp.CurrentEpoch += val.EffectiveBalance()
+			pVal.IsActiveCurrentEpoch = true
+			pBal.ActiveCurrentEpoch += val.EffectiveBalance()
 		}
 		// Was validator active previous epoch
 		if helpers.IsActiveValidatorUsingTrie(val, prevEpoch) {
-			p.IsActivePrevEpoch = true
-			bp.PrevEpoch += val.EffectiveBalance()
+			pVal.IsActivePrevEpoch = true
+			pBal.ActivePrevEpoch += val.EffectiveBalance()
 		}
 		// Set inclusion slot and inclusion distance to be max, they will be compared and replaced
 		// with the lower values
-		p.InclusionSlot = params.BeaconConfig().FarFutureEpoch
-		p.InclusionDistance = params.BeaconConfig().FarFutureEpoch
+		pVal.InclusionSlot = params.BeaconConfig().FarFutureEpoch
+		pVal.InclusionDistance = params.BeaconConfig().FarFutureEpoch
 
-		vp[idx] = p
+		pValidators[idx] = pVal
 		return nil
 	}); err != nil {
 		return nil, nil, errors.Wrap(err, "failed to initialize precompute")
 	}
-	return vp, bp, nil
+	return pValidators, pBal, nil
 }

beacon-chain/core/epoch/precompute/new_test.go

@@ -53,8 +53,8 @@ func TestNew(t *testing.T) {
 	}
 
 	wantedBalances := &precompute.Balance{
-		CurrentEpoch: 100,
-		PrevEpoch:    200,
+		ActiveCurrentEpoch: 100,
+		ActivePrevEpoch:    200,
 	}
 	if !reflect.DeepEqual(b, wantedBalances) {
 		t.Error("Incorrect wanted balance")