rest of slasher beacon changes

Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com>

.bazelrc

@@ -0,0 +1,229 @@
+# Print warnings for tests with inappropriate test size or timeout.
+test --test_verbose_timeout_warnings
+
+# Only build test targets when running bazel test //...
+test --build_tests_only
+test --test_output=errors
+
+# E2E run with debug gotag
+test:e2e --define gotags=debug
+
+# Clearly indicate that coverage is enabled to disable certain nogo checks.
+coverage --define=coverage_enabled=1
+
+# Fix for rules_docker. See: https://github.com/bazelbuild/rules_docker/issues/842
+build --host_force_python=PY2
+test --host_force_python=PY2
+run --host_force_python=PY2
+
+# Networking is blocked for tests by default, add "requires-network" tag to your test if networking
+# is required within the sandbox. Network sandboxing only works on linux.
+build --sandbox_default_allow_network=false
+
+# Stamp binaries with git information
+build --workspace_status_command=./hack/workspace_status.sh
+build --stamp
+
+# Prevent PATH changes from rebuilding when switching from IDE to command line.
+build --incompatible_strict_action_env
+test --incompatible_strict_action_env
+run --incompatible_strict_action_env
+
+build --define blst_disabled=false
+test --define blst_disabled=false
+run --define blst_disabled=false
+
+build:blst_disabled --define blst_disabled=true
+build:blst_disabled --define gotags=blst_disabled
+
+# Release flags
+build:release --compilation_mode=opt
+build:release --config=llvm
+
+# LLVM compiler for building C/C++ dependencies.
+build:llvm --crosstool_top=@llvm_toolchain//:toolchain
+build:llvm --define compiler=llvm
+build:llvm --copt -fno-sanitize=vptr,function
+build:llvm --linkopt -fno-sanitize=vptr,function
+
+build:asan --copt -fsanitize=address,undefined
+build:asan --copt -fno-omit-frame-pointer
+build:asan --linkopt -fsanitize=address,undefined
+build:asan --copt -fno-sanitize=vptr,function
+build:asan --linkopt -fno-sanitize=vptr,function
+build:asan --copt -DADDRESS_SANITIZER=1
+build:asan --copt -D__SANITIZE_ADDRESS__
+build:asan --linkopt -ldl
+
+build:llvm-asan --config=llvm
+build:llvm-asan --config=asan
+build:llvm-asan --linkopt -fuse-ld=ld.lld
+
+build:fuzz --define=gotags=libfuzzer
+build:fuzz --config=llvm-asan
+build:fuzz --copt=-fsanitize=fuzzer-no-link
+build:fuzz --linkopt=-fsanitize=fuzzer
+build:fuzz --copt=-fno-omit-frame-pointer
+build:fuzz --define=FUZZING_ENGINE=libfuzzer
+build:fuzz --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+build:fuzz --linkopt -Wl,--no-as-needed
+build:fuzz --define=gc_goopts=-d=libfuzzer,checkptr
+build:fuzz --run_under=//tools:fuzz_wrapper
+build:fuzz --compilation_mode=opt
+build:fuzz --define=blst_disabled=true
+
+test:fuzz --local_test_jobs="HOST_CPUS*.5"
+
+# Build binary with cgo symbolizer for debugging / profiling.
+build:cgo_symbolizer --config=llvm
+build:cgo_symbolizer --copt=-g
+build:cgo_symbolizer --define=USE_CGO_SYMBOLIZER=true
+build:cgo_symbolizer -c dbg
+build:cgo_symbolizer --define=gotags=cgosymbolizer_enabled
+
+# multi-arch cross-compiling toolchain configs:
+-----------------------------------------------
+build:cross --crosstool_top=@prysm_toolchains//:multiarch_toolchain
+build:cross --host_platform=@io_bazel_rules_go//go/toolchain:linux_amd64
+build:cross --host_crosstool_top=@prysm_toolchains//:hostonly_toolchain
+
+# linux_amd64 config for cross compiler toolchain, not strictly necessary since host/exec env is amd64
+build:linux_amd64 --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64_cgo
+
+# osx_amd64 config for cross compiler toolchain
+build:osx_amd64 --config=cross
+build:osx_amd64 --platforms=@io_bazel_rules_go//go/toolchain:darwin_amd64_cgo
+build:osx_amd64 --compiler=osxcross
+
+# windows
+build:windows_amd64 --config=cross
+build:windows_amd64  --platforms=@io_bazel_rules_go//go/toolchain:windows_amd64_cgo
+build:windows_amd64 --compiler=mingw-w64
+
+# linux_arm64 conifg for cross compiler toolchain
+build:linux_arm64 --config=cross
+build:linux_arm64 --platforms=@io_bazel_rules_go//go/toolchain:linux_arm64_cgo
+build:linux_arm64 --copt=-funsafe-math-optimizations
+build:linux_arm64 --copt=-ftree-vectorize
+build:linux_arm64 --copt=-fomit-frame-pointer
+build:linux_arm64 --cpu=aarch64
+build:linux_arm64 --compiler=clang
+build:linux_arm64 --copt=-march=armv8-a
+
+
+# toolchain build debug configs
+#------------------------------
+build:debug --sandbox_debug
+build:debug --toolchain_resolution_debug
+build:debug --verbose_failures
+build:debug -s
+
+# windows debug
+build:windows_amd64_debug --config=windows_amd64
+build:windows_amd64_debug --config=debug
+
+# osx_amd64 debug config
+build:osx_amd64_debug --config=debug
+build:osx_amd64_debug --config=osx_amd64
+
+# linux_arm64_debug
+build:linux_arm64_debug --config=linux_arm64
+build:linux_arm64_debug --config=debug
+
+# linux_amd64_debug
+build:linux_amd64_debug --config=linux_amd64
+build:linux_amd64_debug --config=debug
+
+
+# Docker Sandbox Configs
+#-----------------------
+# Note all docker sandbox configs must run from a linux x86_64 host
+# build:docker-sandbox --experimental_docker_image=gcr.io/prysmaticlabs/rbe-worker:latest
+build:docker-sandbox --spawn_strategy=docker --strategy=Javac=docker --genrule_strategy=docker
+build:docker-sandbox --define=EXECUTOR=remote
+build:docker-sandbox --experimental_docker_verbose
+build:docker-sandbox --experimental_enable_docker_sandbox
+build:docker-sandbox --crosstool_top=@rbe_ubuntu_clang//cc:toolchain 
+build:docker-sandbox --host_javabase=@rbe_ubuntu_clang//java:jdk 
+build:docker-sandbox --javabase=@rbe_ubuntu_clang//java:jdk 
+build:docker-sandbox --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 
+build:docker-sandbox --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 
+build:docker-sandbox --extra_execution_platforms=@rbe_ubuntu_clang//config:platform 
+build:docker-sandbox --host_platform=@rbe_ubuntu_clang//config:platform 
+build:docker-sandbox --platforms=@rbe_ubuntu_clang//config:platform
+build:docker-sandbox --extra_toolchains=@prysm_toolchains//:cc-toolchain-multiarch
+
+# windows_amd64 docker sandbox build config
+build:windows_amd64_docker --config=docker-sandbox --config=windows_amd64
+build:windows_amd64_docker_debug --config=windows_amd64_docker --config=debug
+
+# osx_amd64 docker sandbox build config
+build:osx_amd64_docker --config=docker-sandbox --config=osx_amd64
+build:osx_amd64_docker_debug --config=osx_amd64_docker --config=debug
+
+# linux_arm64 docker sandbox build config
+build:linux_arm64_docker --config=docker-sandbox --config=linux_arm64
+build:linux_arm64_docker_debug --config=linux_arm64_docker --config=debug
+
+# linux_amd64 docker sandbox build config
+build:linux_amd64_docker --config=docker-sandbox --config=linux_amd64
+build:linux_amd64_docker_debug --config=linux_amd64_docker --config=debug
+
+
+# Remote Build Execution
+#-----------------------
+# Originally from https://github.com/bazelbuild/bazel-toolchains/blob/master/bazelrc/bazel-2.0.0.bazelrc
+#
+# Depending on how many machines are in the remote execution instance, setting
+# this higher can make builds faster by allowing more jobs to run in parallel.
+# Setting it too high can result in jobs that timeout, however, while waiting
+# for a remote machine to execute them.
+build:remote --jobs=50
+
+# Set several flags related to specifying the platform, toolchain and java
+# properties.
+# These flags should only be used as is for the rbe-ubuntu16-04 container
+# and need to be adapted to work with other toolchain containers.
+build:remote --host_javabase=@rbe_ubuntu_clang//java:jdk
+build:remote --javabase=@rbe_ubuntu_clang//java:jdk
+build:remote --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
+build:remote --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
+build:remote --crosstool_top=@rbe_ubuntu_clang//cc:toolchain
+build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
+# Platform flags:
+# The toolchain container used for execution is defined in the target indicated
+# by "extra_execution_platforms", "host_platform" and "platforms".
+# More about platforms: https://docs.bazel.build/versions/master/platforms.html
+build:remote --extra_toolchains=@rbe_ubuntu_clang//config:cc-toolchain
+build:remote --extra_execution_platforms=@rbe_ubuntu_clang//config:platform
+build:remote --host_platform=@rbe_ubuntu_clang//config:platform
+build:remote --platforms=@rbe_ubuntu_clang//config:platform
+
+# Starting with Bazel 0.27.0 strategies do not need to be explicitly
+# defined. See https://github.com/bazelbuild/bazel/issues/7480
+build:remote --define=EXECUTOR=remote
+
+# Enable remote execution so actions are performed on the remote systems.
+# build:remote --remote_executor=grpcs://remotebuildexecution.googleapis.com
+
+# Enforce stricter environment rules, which eliminates some non-hermetic
+# behavior and therefore improves both the remote cache hit rate and the
+# correctness and repeatability of the build.
+build:remote --incompatible_strict_action_env=true
+
+# Set a higher timeout value, just in case.
+build:remote --remote_timeout=3600
+
+# Enable authentication. This will pick up application default credentials by
+# default. You can use --google_credentials=some_file.json to use a service
+# account credential instead.
+# build:remote --google_default_credentials=true
+
+# Enable build without the bytes
+# See: https://github.com/bazelbuild/bazel/issues/6862
+build:remote --experimental_remote_download_outputs=toplevel --experimental_inmemory_jdeps_files --experimental_inmemory_dotd_files
+
+build:remote --remote_local_fallback
+
+# Ignore GoStdLib with remote caching
+build --modify_execution_info='GoStdlib.*=+no-remote-cache'

.bazelversion

@@ -0,0 +1 @@
+3.7.0

.buildkite-bazelrc

@@ -2,21 +2,31 @@
 # across machines, developers, and workspaces.
 # 
 # This config is loaded from https://github.com/bazelbuild/bazel-toolchains/blob/master/bazelrc/latest.bazelrc
-build:remote-cache --remote_cache=remotebuildexecution.googleapis.com
-build:remote-cache --tls_enabled=true
-build:remote-cache --remote_timeout=3600
-build:remote-cache --auth_enabled=true
-build:remote-cache --spawn_strategy=standalone
-build:remote-cache --strategy=Javac=standalone
-build:remote-cache --strategy=Closure=standalone
-build:remote-cache --strategy=Genrule=standalone
+#build:remote-cache --remote_timeout=3600
+#build:remote-cache --spawn_strategy=standalone
+#build:remote-cache --strategy=Javac=standalone
+#build:remote-cache --strategy=Closure=standalone
+#build:remote-cache --strategy=Genrule=standalone
 
 # Prysm specific remote-cache properties.
-build:remote-cache --disk_cache=
-build:remote-cache --jobs=50
-build:remote-cache --host_platform_remote_properties_override='properties:{name:\"cache-silo-key\" value:\"prysm\"}' 
-build:remote-cache --remote_instance_name=projects/prysmaticlabs/instances/default_instance
+#build:remote-cache --disk_cache=
+build:remote-cache --remote_download_minimal
+build:remote-cache --remote_cache=grpc://bazel-remote-cache:9092
+build:remote-cache --experimental_remote_downloader=grpc://bazel-remote-cache:9092
+build:remote-cache --remote_local_fallback
+# Enforce stricter environment rules, which eliminates some non-hermetic
+# behavior and therefore improves both the remote cache hit rate and the
+# correctness and repeatability of the build.
+build:remote-cache --incompatible_strict_action_env=true
 
+# Import workspace options.
+import %workspace%/.bazelrc
+
+startup --host_jvm_args=-Xmx2g --host_jvm_args=-Xms2g
+query --repository_cache=/tmp/repositorycache
+query --experimental_repository_cache_hardlinks
+build --repository_cache=/tmp/repositorycache
+build --experimental_repository_cache_hardlinks
 build --experimental_strict_action_env
 build --disk_cache=/tmp/bazelbuilds
 build --experimental_multi_threaded_digest
@@ -24,10 +34,16 @@ build --sandbox_tmpfs_path=/tmp
 build --verbose_failures
 build --announce_rc
 build --show_progress_rate_limit=5
-build --curses=yes --color=yes
+build --curses=yes --color=no
 build --keep_going
 build --test_output=errors
 build --flaky_test_attempts=5
-build --test_timeout=5,60,-1,-1
 # Disabled race detection due to unstable test results under constrained environment build kite
 # build --features=race
+
+# Disable flaky test detection for fuzzing.
+test:fuzz --flaky_test_attempts=1
+
+# Better caching
+build:nostamp --nostamp
+build:nostamp --workspace_status_command=./hack/workspace_status_ci.sh

.codecov.yml

@@ -27,3 +27,7 @@ comment:
   layout: "header, diff"
   behavior: default
   require_changes: no
+
+ignore:
+  - "**/*.pb.go"
+  - "**/testing/**/*"

.deepsource.toml

@@ -0,0 +1,26 @@
+version = 1
+
+exclude_patterns = [
+  "tools/analyzers/**",
+  "validator/keymanager/remote/keymanager_test.go",
+  "validator/web/site_data.go"
+]
+
+[[analyzers]]
+name = "go"
+enabled = true
+
+[analyzers.meta]
+import_paths = ["github.com/prysmaticlabs/prysm"]
+
+[[analyzers]]
+name = "test-coverage"
+enabled = true
+
+[[analyzers]]
+name = "shell"
+enabled = true
+
+[[analyzers]]
+name = "secrets"
+enabled = true

.dockerignore

@@ -0,0 +1,2 @@
+bazel-*
+.git

.github/CODEOWNERS

@@ -0,0 +1,8 @@
+# Automatically require code review from core-team.
+* @prysmaticlabs/core-team
+
+# Starlark code owners
+*.bzl @prestonvanloon
+
+# Anyone on prylabs team can approve dependency updates.
+deps.bzl @prysmaticlabs/core-team

.github/FUNDING.yml

@@ -0,0 +1 @@
+custom: https://gitcoin.co/grants/24/prysm-by-prysmatic-labs

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,59 @@
+---
+name: "\U0001F41EBug report"
+about: Report a bug or problem with running Prysm
+---
+<!--💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎
+
+Hellooo! 😄 
+
+To help us tend to your issue faster, please search our currently open issues before submitting a new one.
+Existing issues often contain information about workarounds, resolution, or progress updates.
+
+💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎-->
+
+# 🐞 Bug Report
+
+### Description
+
+<!-- ✍️--> A clear and concise description of the problem...
+
+### Has this worked before in a previous version?
+
+<!-- Did this behavior use to work in the previous version? -->
+<!-- ✍️--> Yes, the previous version in which this bug was not present was: ....
+
+## 🔬 Minimal Reproduction
+
+<!--
+Please let us know how we can reproduce this issue. Include the exact method you used to run Prysm along with any flags used in your beacon chain and/or validator. Make sure you don't upload any confidential files or private keys.
+-->
+
+## 🔥 Error
+
+<pre><code>
+<!-- If the issue is accompanied by an error, please share the error logs with us below. If you have a lot of logs, place make a paste bin with your logs and share the link with us here: -->
+<!-- ✍️-->
+
+</code></pre>
+
+
+## 🌍  Your Environment
+
+**Operating System:**
+
+<pre>
+  <code>
+
+  </code>
+</pre>
+
+**What version of Prysm are you running? (Which release)**
+
+<pre>
+  <code>
+
+  </code>
+</pre>
+
+**Anything else relevant (validator index / public key)?**
+

.github/ISSUE_TEMPLATE/discussion.md

@@ -0,0 +1,21 @@
+---
+name: "\U0001F48EGeneral issue / discussion"
+about: Any other type of general issue or discussion
+
+---
+<!--💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎
+
+Hellooo! 😄 
+
+To help us tend to your issue faster, please search our currently open issues before submitting a new one.
+Existing issues often contain information about workarounds, resolution, or progress updates.
+
+💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎-->
+
+# 💎 Issue
+
+### Background
+
+<!-- ✍️--> Context and background information on the discussion...
+
+### Description

.github/ISSUE_TEMPLATE/feature_flag.md

@@ -0,0 +1,38 @@
+---
+name: "\U0001F984Feature Flag Tracking"
+about: Track a new feature, in development, in Prysm. This issue template should only be used by developers or contributors!
+labels: Tracking
+---
+<!--💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎
+
+Hellooo! 😄 
+
+Thanks for taking the time to file a tracking issue for your new feature. These issues really help
+us track progress of features as they work their way through development. Be sure to review 
+shared/featureconfig/README.md for the latest documentation around feature flags.
+
+💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎-->
+
+# 🦄 Feature Tracking
+
+**Current status** (opt-in or opt-out): 
+
+**Opt-in feature flag name**:
+
+**Opt-out feature flag name**: 
+
+### Feature Description
+
+<!-- ✍️--> A clear and concise description of the new feature. Provide links to your technical
+design document or other supporting information.
+
+### Tasks
+
+- [ ] Feature flag created as opt-in
+- [ ] Feature implemented / code complete
+- [ ] Feature tested in production for adequate amount of time
+- [ ] Feature flag is inverted to be opt-out and the opt-in flag is deprecated
+- [ ] Feature has made it to a tagged production release as an opt-out flag
+- [ ] Opt-out feature flag is deprecated and old code paths are cleaned up
+
+This issue should be closed after all of the above tasks are complete.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,27 @@
+---
+name: "\U0001F680Feature request"
+about: Suggest a feature for Prysm
+
+---
+<!--💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎
+
+Hellooo! 😄 
+
+To help us tend to your issue faster, please search our currently open issues before submitting a new one.
+Existing issues often contain information about workarounds, resolution, or progress updates.
+
+💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎💎-->
+
+# 🚀 Feature Request
+
+### Description
+
+<!-- ✍️--> A clear and concise description of the problem or missing capability...
+
+### Describe the solution you'd like
+
+<!-- ✍️--> If you have a solution in mind, please describe it.
+
+### Describe alternatives you've considered
+
+<!-- ✍️--> Have you considered any alternative solutions or workarounds?

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,30 @@
+<!-- Thanks for sending a PR! Before submitting:
+
+1. If this is your first PR, check out our contribution guide here https://docs.prylabs.network/docs/contribute/contribution-guidelines
+   You will then need to sign our Contributor License Agreement (CLA), which will show up as a comment from a bot in this pull request after you open it. We cannot review code without a signed CLA.
+2. Please file an associated tracking issue if this pull request is non-trivial and requires context for our team to understand. All features and most bug fixes should have
+   an associated issue with a design discussed and decided upon. Small bug
+   fixes and documentation improvements don't need issues.
+3. New features and bug fixes must have tests. Documentation may need to
+   be updated. If you're unsure what to update, send the PR, and we'll discuss
+   in review.
+4. Note that PRs updating dependencies and new Go versions are not accepted.
+   Please file an issue instead.
+-->
+
+**What type of PR is this?**
+
+> Uncomment one line below and remove others.
+>
+> Bug fix
+> Feature
+> Documentation
+> Other
+
+**What does this PR do? Why is it needed?**
+
+**Which issues(s) does this PR fix?**
+
+Fixes #
+
+**Other notes for review**

.github/actions/gofmt/Dockerfile

@@ -0,0 +1,5 @@
+FROM cytopia/gofmt
+
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

.github/actions/gofmt/action.yml

@@ -0,0 +1,12 @@
+name: 'Gofmt checker'
+description: 'Checks that all project files have been properly formatted.'
+inputs:
+  path:
+    description: 'Path to check'
+    required: true
+    default: './'
+runs:
+  using: 'docker'
+  image: 'Dockerfile'
+  args:
+    - ${{ inputs.path }}

.github/actions/gofmt/entrypoint.sh

@@ -0,0 +1,15 @@
+#!/bin/sh -l
+set -e
+
+cd $GITHUB_WORKSPACE
+
+# Check if any files are not formatted.
+nonformatted="$(gofmt -l $1 2>&1)"
+
+# Return if `go fmt` passes.
+[ -z "$nonformatted" ] && exit 0
+
+# Notify of issues with formatting.
+echo "Following files need to be properly formatted:"
+echo "$nonformatted"
+exit 1

.github/actions/gomodtidy/Dockerfile

@@ -0,0 +1,5 @@
+FROM golang:alpine
+
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

.github/actions/gomodtidy/action.yml

@@ -0,0 +1,5 @@
+name: 'Go mod tidy checker'
+description: 'Checks that `go mod tidy` has been applied.'
+runs:
+  using: 'docker'
+  image: 'Dockerfile'

.github/actions/gomodtidy/entrypoint.sh

@@ -0,0 +1,34 @@
+#!/bin/sh -l
+set -e
+export PATH=$PATH:/usr/local/go/bin
+
+cd $GITHUB_WORKSPACE
+
+cp go.mod go.mod.orig
+cp go.sum go.sum.orig
+
+go mod tidy
+
+echo "Checking go.mod and go.sum:"
+checks=0
+if [ "$(diff -s go.mod.orig go.mod | grep -c 'Files go.mod.orig and go.mod are identical')" = 1 ]; then
+  echo "- go.mod is up to date."
+  checks=$((checks + 1))
+else
+  echo "- go.mod is NOT up to date."
+fi
+
+if [ "$(diff -s go.sum.orig go.sum | grep -c 'Files go.sum.orig and go.sum are identical')" = 1 ]; then
+  echo "- go.sum is up to date."
+  checks=$((checks + 1))
+else
+  echo "- go.sum is NOT up to date."
+fi
+
+if [ $checks -eq 2 ]; then
+  exit 0
+fi
+
+# Notify of any issues.
+echo "Run 'go mod tidy' to update."
+exit 1

.github/no-response.yml

@@ -0,0 +1,13 @@
+# Configuration for probot-no-response - https://github.com/probot/no-response
+
+# Number of days of inactivity before an Issue is closed for lack of response
+daysUntilClose: 14
+# Label requiring a response
+responseRequiredLabel: Need-Info
+# Comment to post when closing an Issue for lack of response. Set to `false` to disable
+closeComment: >
+  This issue has been automatically closed because there has been no response
+  to our request for more information from the original author. With only the
+  information that is currently in the issue, we don't have enough information
+  to take action. Please reach out if you have or find the answers we need so
+  that we can investigate further.

.github/workflows/dappnode-release-trigger.yml

@@ -0,0 +1,41 @@
+name: Update DAppNodePackages
+
+on:
+  push:
+    tags:
+    - '*'
+
+jobs:
+  dappnode-update-beacon-chain:
+    name: Trigger a beacon-chain release
+    runs-on: ubuntu-latest
+    steps:
+    - name: Get latest tag
+      id: get_tag
+      run: echo ::set-output name=TAG::${GITHUB_REF/refs\/tags\//}
+    - name: Send dispatch event to DAppNodePackage-prysm-beacon-chain
+      env: 
+        DISPATCH_REPO: dappnode/DAppNodePackage-prysm-beacon-chain
+      run: |
+        curl -v -X POST -u "${{ secrets.PAT_GITHUB }}" \
+        -H "Accept: application/vnd.github.everest-preview+json" \
+        -H "Content-Type: application/json" \
+        --data '{"event_type":"new_release", "client_payload": { "tag":"${{ steps.get_tag.outputs.TAG }}"}}' \
+        https://api.github.com/repos/$DISPATCH_REPO/dispatches
+        
+  dappnode-update-validator:
+    name: Trigger a validator release
+    runs-on: ubuntu-latest
+    steps:
+    - name: Get latest tag
+      id: get_tag
+      run: echo ::set-output name=TAG::${GITHUB_REF/refs\/tags\//}
+    - name: Send dispatch event to DAppNodePackage validator repository
+      env: 
+        DISPATCH_REPO: dappnode/DAppNodePackage-prysm-validator
+      run: |
+        curl -v -X POST -u "${{ secrets.PAT_GITHUB }}" \
+        -H "Accept: application/vnd.github.everest-preview+json" \
+        -H "Content-Type: application/json" \
+        --data '{"event_type":"new_release", "client_payload": { "tag":"${{ steps.get_tag.outputs.TAG }}"}}' \
+        https://api.github.com/repos/$DISPATCH_REPO/dispatches

.github/workflows/go.yml

@@ -0,0 +1,61 @@
+name: Go
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ '*' ]
+
+jobs:
+
+  check:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+
+      - name: Go mod tidy checker
+        id: gomodtidy
+        uses: ./.github/actions/gomodtidy
+
+      - name: Gofmt checker
+        id: gofmt
+        uses: ./.github/actions/gofmt
+        with:
+          path: ./
+      
+      - name: GoImports checker
+        id: goimports
+        uses: Jerome1337/goimports-action@v1.0.2
+        with:
+          goimports-path: ./
+
+      - name: Gosec security scanner
+        uses: securego/gosec@master
+        with:
+          args: '-exclude-dir=crypto/bls/herumi ./...'
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.14
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Get dependencies
+        run: |
+          go get -v -t -d ./...
+
+      - name: Build
+        # Use blst tag to allow go and bazel builds for blst.
+        run: go build -v ./...
+
+# Tests run via Bazel for now...
+#      - name: Test
+#        run: go test -v ./...

.gitignore

@@ -4,6 +4,9 @@ bazel-*
 .DS_Store
 .swp
 
+# Ignore VI/Vim swapfiles
+.*.sw?
+
 # IntelliJ
 .idea
 .ijwb
@@ -14,6 +17,7 @@ bazel-*
 # Coverage outputs
 coverage.txt
 profile.out
+profile.grind
 
 # Nodejs
 node_modules
@@ -21,4 +25,14 @@ yarn-error.log
 .vscode/
 
 # Ignore password file
-password.txt 
+password.txt
+
+# Dist files
+dist
+
+# libfuzzer
+oom-*
+crash-*
+
+# deepsource cli
+bin

.policy.yml

@@ -0,0 +1,91 @@
+policy:
+  approval:
+    - or:
+        - only test files are changed
+        - only proto files are changed
+        - touches consensus critical code
+        - touches sync/blockchain/p2p code
+        - large line count
+        - is critical priority
+approval_rules:
+  - name: only test files are changed
+    if:
+      only_changed_files:
+        paths:
+          - "*_test.go"
+          - "*.bazel"
+    options:
+      ignore_commits_by:
+        users: ["bulldozer[bot]"]
+    requires:
+      count: 1
+      teams:
+        - "prysmaticlabs/core-team"
+  - name: only proto files are changed
+    if:
+      only_changed_files:
+        paths:
+          - "*pb.go"
+          - "*pb.gw.go"
+          - "*.bazel"
+    options:
+      ignore_commits_by:
+        users: ["bulldozer[bot]"]
+    requires:
+      count: 1
+      teams:
+        - "prysmaticlabs/core-team"
+  - name: touches consensus critical code
+    if:
+      only_changed_files:
+        paths:
+          - "beacon-chain/core/*"
+          - "beacon-chain/state/*"
+    options:
+      ignore_commits_by:
+        users: ["bulldozer[bot]"]
+    requires:
+      count: 2
+      teams:
+        - "prysmaticlabs/core-team"
+  - name: touches sync/blockchain/p2p code
+    if:
+      only_changed_files:
+        paths:
+          - "beacon-chain/blockchain/*"
+          - "beacon-chain/sync/*"
+          - "beacon-chain/p2p/*"
+    options:
+      ignore_commits_by:
+        users: ["bulldozer[bot]"]
+    requires:
+      count: 2
+      teams:
+        - "prysmaticlabs/core-team"
+  - name: large line count
+    if:
+      modified_lines:
+        total: "> 1000"
+      changed_files:
+        ignore:
+          - "*pb.go"
+          - "*pb.gw.go"
+          - "*.bazel"
+    options:
+      ignore_commits_by:
+        users: ["bulldozer[bot]"]
+    requires:
+      count: 2
+      teams:
+        - "prysmaticlabs/core-team"
+  - name: is critical priority
+    if:
+      has_labels:
+        - "Priority: Critical"
+    options:
+      ignore_commits_by:
+        users: ["bulldozer[bot]"]
+    requires:
+      count: 3
+      teams:
+        - "prysmaticlabs/core-team"

.soliumrc.json

@@ -1,18 +0,0 @@
-{
-  "extends": "solium:recommended",
-  "plugins": [
-    "security"
-  ],
-  "rules": {
-    "quotes": [
-      "error",
-      "double"
-    ],
-    "security/no-inline-assembly": ["warning"],
-
-    "indentation": [
-      "error",
-      4
-    ]
-  }
-}

.travis.yml

@@ -12,7 +12,7 @@ matrix:
         - go get ${gobuild_args} -t ./...
         - go get ${gobuild_args} github.com/golangci/golangci-lint/cmd/golangci-lint
       script:
-        - golangci-lint run
+        - golangci-lint run --skip-dirs ./proto
       email: false
       after_success:
         - wget https://raw.githubusercontent.com/k3rn31p4nic/travis-ci-discord-webhook/master/send.sh

.well-known/security.pub

@@ -0,0 +1,218 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBFpZUO0BEAC/tqN6QctJKSOF+A7jrBzvNvs6rVEi9Hn55u/scKNIPkSRJRTy
+8kVsDMha+EKXka8oQiVM+sHoHMIMCa3tdicm1/k5f67X8dqadQmFP4DhYRYIKCKT
+TkntNG568ynf/yUs/YY9Ce8H17JvgytkLK50mxMxycUlYREMaRPMR8Wt1Arrd9QT
+U0I2cZemeqUORKuiVuZjj/7BVDHvSXHvzpi5zKI86sJQTnzcuGqyNsrUU4n4sYrb
+I+0TfEHzmSdxoAXSaMYjomPLmbaSdBiK/CeNKF8xuFCKJRd8wOUe5FcIN3DCdk0e
+yoFye5RMC+W09Ro+tK/jTQ/sTUuNLJm0VUlddQQeoGlhQdLLwiU4PJqcyeL4KaN1
+l8cVml7xr1CdemhGV4wmEqAgxnNFN5mKnS2KcDaHxRz7MoGNdgVVQuxxaE0+OsdJ
+zCKtA12Q/OcR24qYVFg5+O+bUNhy23mqIxx0HiQ0DsK+IDvcLLWqta0aP9wd9wxG
+eObh9WkCxELTLg8xlbe0d7R3juaRjBLdD5d3UyjqGh+7zUflMsFhpUPraNXdKzvm
+AqT25cveadM7q/CNzFXeCwmKjZab8Jic8VB80KcikmX6y9eGOHwjFixBogxowlBq
+3KpeNTqJMNHYBzEb0V18P8huKVO0SMfg11Z1/nU4NA4lcjiVImb5xnJS0wARAQAB
+tCxQcmVzdG9uIFZhbiBMb29uIDxwcmVzdG9uQHByeXNtYXRpY2xhYnMuY29tPokC
+NwQTAQgAIQUCWuZ7uwIbAwULCQgHAgYVCAkKCwIEFgIDAQIeAQIXgAAKCRBy4z5N
+8aUDbtchD/4hQDTj7qsccQKpaL8furw+OWuoZtX38smO9LrcSoLaZlk5NRxFNlhQ
+FaL6iyDLoDkWUXEPg94x1dMFBUfMdIuh6iEF2pcvCN6WG3Pd2j2kG6OxaIx+rI7a
+eaD2Wds312etYATu+7AI6pxlQVqPeZi6kZVvXo8r9qzEnl1nifo7d3L4ut6FerDz
+/ptvIuhW++BEgksILkwMA44U7owTllkZ5wSbXRJer0bI/jLPLTaRETVMgWmF5L2n
+PWKhBnhBXf/P00zTHVoba0peJ/RGdzlb1JZH+uCQk0wGBl0rBMUmiihIB8DZBZDX
+5prwMdoHG9qAT9SuJ8ZOjPKaBHVVVw4JOU6rX2+p9E49CVATsdoPfWVbNyVRGi5f
+Oo0bJPZU3uO10Q09CIeyqT6JPDCZYS7po2bpfFjQTDkoIv0uPWOsV5l3cvFxVlcD
+Pvir4669xhujmoVBOrp18mn/W/rMft2TJ84inp0seKSKdwBUeEyIZwwu1YTorFe4
+bgJghDu/Y+K4y0wq7rpPimoiXSoJOaaIPrOzsKAVu20zJB4eoulXlPwHexix8wwf
+xeVH4bGl3wtTFWKja0+EQVdxpt+uUlABvrheQbNlNGz5ROOAtjvwASI3YW/jVVaG
+wSbOUuMEHfC1rSnj5Y9fN6FbilxJAZ2UbvZE6iXqqemfRdFuB5yedbQmUHJlc3Rv
+biBWYW4gTG9vbiA8cHJlc3RvbjkwQGdtYWlsLmNvbT6JAjcEEwEIACEFAlqrvbMC
+GwMFCwkIBwIGFQgJCgsCBBYCAwECHgECF4AACgkQcuM+TfGlA269zg/9FynobfLE
+Vh+Aid7l5C6pprHflHpdNVhdHzjvosLKfPMCcBJLyCLAIueKC0vvKbt5kyNV6Adl
+6fibdO/vRrNsEdmKXTdfDnN03yVX+VO54rqyuweR09CbLHakECh5awWpk0x9E/Gc
+3MlU8YDCVNPcWAw8JPZZ0y6s50/MFFXya+/opw45neH6WXrbqbIapzZ4V1bhkloF
+TTz7LYp1dk6qjus/f1jHgujclJ6FazG+OqCpzJ22lnuwNYs8sv1DclW+lEUfIin5
+PvzcSRCCd6NilFRSGzfBhM7wxZrAp0JzWXpM1jmd2WHtgErusTaTXRTATjPf9Chg
+SE9UT3EJvJ5fPxuxm+qOAowpJwe8Irv+YuMzL8C6C2BhZIV8XID3/cErNeQbPocj
+QFksmBEwpe9EG3Yhd5SmSXRhCtBFnyFKyOqPpAE2s+U0b7qsnWc50UFbIytPEm4C
+YmdNL6IVilp/+LYFcRWWK/ppOtvtgbj8r7+Foidi/tCauJGt2BzhWH7DkLEdXGxk
+PfR/rTgyuAQZowl03DaNwAPKegY2SIuROO9kpQACwTWZNg4l2yrZv09qrKBhohyn
+b2i4pwHPZ5/bWLikdzENcJuvbH5qhf6tV0FIaNGbTkWX00uI++bvWAxxE25jZv56
+e7VTaWuMt8Ly4DBcwl5JyWuvZ74+yv7QGp20WlByZXN0b24gVmFuIExvb24gKDB4
+ZjcxRTlDNzY2Q2RmMTY5ZURGYkUyNzQ5NDkwOTQzQzFEQzZiOEE1NSkgPHByZXN0
+b25AbWFjaGluZXBvd2VyZWQuY29tPokCNwQTAQgAIQUCWllQ7QIbAwULCQgHAgYV
+CAkKCwIEFgIDAQIeAQIXgAAKCRBy4z5N8aUDbmvdD/4jkT1IXJyj65sgHus0HHYC
+MBp6mzvtpDcPljg/5Nl1WXydv4zYGnEnIwWIqDYwwvokOKllxAjgevcplHqZa0cb
+7XKCFk5h+56FLPHv9e0WK1fOFeo2ad3imNRstSHgaumGAWHJMg9vWbDisGv1zjQ0
+usFkrMKoX34YzMt8KBD7IuQyeBkYNupT8EfByeA9Ta+wkvS+X6BojsFB1UWDAFCY
+z8RgTcWIFjWtGZwIkWUKCzebqXbElpJF8ckZU9q4qVhyZ2DT+8BS/Lm0Sf4z6VTC
+4EN10ZuN+F+J3DMR2Zuudp4X5OUGkDG4KuU/kvj6EJRWpbTS1D9ReK7hoApIbV72
+Um6Mf7kC9EKvxgL1gOfl4aj3Io9M0R8FT/0WSMduQzf/jI3XqdNH0nYo2kTDdZm8
+S972cyvt6frKYy6RsOlPz1+S61iCmupsRY+HyKDTa0vnpXg2c4dE1neF5eMI6SVw
+viJvCG2cflctJ2PLiINZJYAg87gV5Rm1i/Lt2YG5zdxAXXJt2R0uuU9fv4DcHZLU
+yx69Yuh+5UiEaXYU7xoRCdZJYyHbvaC2hPcDGnEa3K1QbgnI5hGAtG3cwdbYC5e3
+bcaJb/LdwzkdRnHLgpxa/UTAIfejEI1U2kvVvNoe/HvEXq/OPrhFDvE4rW8AzbX+
+ISTWWRY0lLSr8/SD0TDJMbkBDQRam2geAQgA0kESOibOO3CcXrHtqP9lPGmj6rVe
+G18fRmPDJiWtx863QqJeByuuUKwrGkPW/sqtIa5Ano+iXVHpk7m955nRjBmz4gd8
+xqSGZd9XpNObYPA5iirAO8ztpgQvuvsHH9y9Ge50NnR7hQSMUbGVeCUU/vljwT60
+/U+UPnsTAmkhvkEI72x50l5Ih9ihcBcr5jJpi08XktE3sFOoannac0kZQJ6WXFrY
+4ILbv8fVqcRf44xMKOlFB9qHhleGW0H9ZUjTKs9quRt7r5D1MOiwrZDjNN3LqMco
+oWj37hb+3KkmIIsAYB2DjnWYkMPu2B0O4uSOHYAWfEJtRvA8qW7sWj+q1wARAQAB
+iQIlBBgBCAAPBQJam2geAhsgBQkB4TOAAAoJEHLjPk3xpQNulz0P/2m9veAAGGCO
+yPsqiNVVa8lrhmLGh/W+BaoszQ/r+pfin4xTOV5K5h3MC5CVJM0JxP/cxNol+Nmr
+cYGbZgq4QhLlH6PdQ7cReL5ED6Wi+eHb4ocvXJqUuZ2Gl8Z7gzQzp+WFbLrn8vXj
+LcyGGEETV84jy+X5cKu24eAjTFK+unfqwfxXFZP5vhIEVe7+uG6A/pMB5eLDqEUh
+FQUcWqCF2Imt08Kcn7HL31GoIY0ABHdD+ICXZE5lTm6iJGzpFBKdTMm/e5igCJ3v
+/tgtWZbnvyseLR/T/gU1JyheS9kNXP5sICwicBbY/vnGdEP6OKpDOSfQam5x4BBj
+cvsBnsNuonY7OJn4iLY6LviQ0MM91PbaJUUJrp9Uyi4hj9iO/MZBaG0Giu0FKjG6
+Vk+RlCYPjYIvHflQLZHe9BWLPN1AvaLKszt3IYaxS5seXCu0ZqHDGCBVqVCNDDJk
+MJbHlrOVZ9T6mZhA+aQ1EJvTWk3MNj1AOyCJdiXtOOdg+4Fm5dN7nLpumLIg2yP2
+afI7YfrPGA7sm+T0IMnOWxkK+KODC7OV9h/QjyBJDcUYGXLapuK9eP80cr8hKvH7
+S3G4Top/rXDpnBNQ2azCqmUwaJWNRhLucC80Vd00d4pWIOwAPWpiV70Fq8OhQFhT
+PNXwFXVLwtNfPvPxN1s+Vk+BBXp+M19AuQENBFqbaC8BCADSq89Z9xWRS2fvWI/N
++yEWliIU8XiqC9Ch+/6mS2LEyzB1pPLIIQcRvM6rq2dxXIRveVGpb63ck9vUtuJG
+//es+DnDkO7re+ZmWHX+LAqMYNdaobSYxHkkR4CcY2HbPSEUbb//Zwk4BDyp3g3W
+bKK9noVbslZuSwWNrxjX/Hieh/dIGYkNFeWOlmNfUYsevzqNDjsziOagyXKxLc++
+hUM3GKgzXRQJBvBpgzQc4bRY+YGHXtZurk9AiZ4ZBhWv2Qrb5OYYislE9sdv3KWV
+Iv1EBpbkAJ9MM1HgS8bkIOIpNs4XxHY6fTWWdrXi+NgZXQwQRYaWTQsXL3mktarS
+fFfTABEBAAGJAiUEGAEIAA8FAlqbaC8CGwwFCQHhM4AACgkQcuM+TfGlA24vqg/8
+CsVBHO4mh8SSaxdWNEU+mG4pU230BRCwrfn42wuSKb7WNLTTcWMDNI0KY/JNcWSq
+G2qa6lngiAyOS72Jd635ptZ6Wvjd0WtBt90NN2jtn+aRqwQ8uItlYMQscofFzskj
+QnBF+NWER+44nxboghuQ041m6aI2XmYUErSOBZi6onbC3svH6coMldkkiueWFbdB
+qac3UXue4LUcaGR5T9pCQsLgTl3D8V5icEM+HpTVVGQZkVrOuKMKe6T9N5FS/WFu
+T6CuFf/OyU15a6RE4WW9QqKYsaHl8B6+0P7uqPoVIxs8hfJcwaUu9XwIiZYBZg7N
+yYCQ7JBapC5KZlIcTCfFSxby8lCJoZbIM3Pk/pgCuKxGaS9rHHUGuIvz8TfnM9FO
+2zlxl4E6k3NFgGKF3p6oiKayC74o6EOw50p6DMjrisG7kkWVsTRCMINF7CcfeVme
+MI9ljGAMB1hPtHPhl27hMRfq+/iIbR9gb7+Xw2yKQL2QRjMDIGGxP3q4NjD4tV8P
+VyTbAAwNARG8oMpNM628v3tsW+WYNot1SPUQuZbIx1pCwMeTqljWsdQxWRVW5UxM
+dnXTCqZhQwH0ICG/jbepbP6ciVB/CSa7TVohEK6jiTMorhqynRMMJ6p48Z6HIlyY
+0Ss8q9K29eXaqmLB8Oy3HmupqxH95TqMntqivzf6i6e5AQ0EWptoPQEIAL1OdDIl
+7E3VKAEWH5GnMzdnl9bju/ovoMjLqGevHS9Gyz4OPyZvCQ2pS8k0sgqwsn4F8vWM
+7L3xKTyQTSYOPby3do58+kxUrdTNlqGKEEcZDG+MSzxKyft7m+37fzbg6tcU+O3L
+/7m8nYWQSRKJeist7Q8HrQJzehuzcgAnNmpeBqXHnAwRBvtqORvz5cQAIQ4EsEvP
+f/unTjw95JtL1LtBOaOaynF9ap/TZ34OvDdARmZSdqPpRtEvjfgIboIYYt1dNmDH
+kiSaaKaqBLCJTD2z5KT8ccDeF8lzYHAYzNy8v2lTc9vagCZH+lf3d2d6umNcr4y1
+NGEN4ZEhrmt/lP0AEQEAAYkDRAQYAQgADwUCWptoPQIbAgUJAeEzgAEpCRBy4z5N
+8aUDbsBdIAQZAQgABgUCWptoPQAKCRD619WmVzqkJDTvB/49MQNQk8YJTwAnbdSH
+7stU2uQFBbkljE8Juz5WJK53lL6xUVUp/3gKrQio1F+z9uRVqRh0cQnqX6mPMe5a
+2dlHEIDHTJjSlR5GCCBRDbssV6SN72xSVgbxVGZ9L32qUYtiwGnxwXQC9D9KsonD
+YfGfUhD1CLAldr6HwhJkOq4QKz5GF4ty8sMKEcpM5UaR2sa2y6Ebn9Vzma10YaVp
+X7RlZM/iTiDhTmWuxLh7e21DI95k/eFqHpKA912N0n1BdzZPbwk83nVRxXg793NU
+RFpegzlLawtaCW9oVJOYqErMGOYN5nbXAHXsr5X7Or70v1Yo1khgzNOfnirO57T9
+VjT0J1QP/j1xobgMuNda7Fpwc0xo2oIKEf+SWY9GQrkUK7wCLDbpgbGVxTmREkyE
+6oyDzW1QpQXRjLS9Wvtun0J3Wn6r6Aar0VaGKa7uiiq8UORWtFkWQAzzyBj87Rjx
+eWZWV1dzLK7eMJdyN0gsOzcejrsOqf1sydzvhm4K66byjDZ78piv0DdyPIb4OsiQ
+QU2GH+QwGRYIkYlU9f9g+hSasAfzvrATHlJZNrOjOCgXbut/yP9ug3DHKj76wmoU
+n/Y4rpmskAzIQrZIpimkpNBmZVTGr4bkWcokVzrRFor3NCpl1qA1K9Cd43wARH8t
+Zwk4evI4abbqUId0vVNCKSSDyCCjgNwRsmU8RXdn7r0vs4ObKuWfY9Yl2y8tq3qO
+YPHr0r50YXWtKqUNy5JUc3Ow9DFR1p4O4yfmiSyTLUyOYbglfvtnO32OJkrrZ8Kq
+iSDnyiq9u2nYJAEHk7AchF5TJrTCnd8yWNIjohild+wc+rMKktspoEcxmT6zaK4T
+AymmEe3jzQhlxptpBm68t2E5kaYFQh+NnizXlRcmFjBgEhH2CxCoerUpK82Y+MuE
+S/ODmF9kULre14aXAAspj4vldQ/LJm0SYEfTkE1sDipJQUzv6oS5AFloQDXZPWTB
+15P+Xsj8sJV9hWCfJZVmppWuPg/pCYXwdjUHUYouTz3ZL6qnUbm2uQINBFpZUO0B
+EADZkfsbXPpDcMALUgVmB3cJU90tFqc8Q5guK9oSs3ibx4SmyhBVmeF2TF6PQNoK
+YvpUR50hAcx2AbwtY51u0XxrAr8kFiL6R5ce/0pVMfXGchcC58CJ3uf+O+OPt080
+ftyVSTsY9xEnBohMoJescn0L/IPaM6KkkIFIMAI4Ct3QCHox7WHgPLrqB7Efx2Dj
+Qkgjbioqj8zVKDwxTs0S3sknch675gOhsCkaI7KMcRGACDinKRF3wQha4brNa8En
+vPTV01Cv0ttCo6NCcbQzQi8QQYpMQcWVkquEJWweZQvL/OvYWdT13JgnIp66pkmo
++JvGTOqQpSnIx6OQnV9yqwqsg4E0dkCE+9rDYAHpwvmRkaI2sItjN4KAEQdTWES8
+RgGVgfCtvNH87PqpaPIgarMDY/j5KqTDp/7Nc5oGLCmhwZiYzQa7Bm5uVNnYIyOO
+d1IjfclgVdVAzMOmrFytvXFCBcga1khL15taC7s6Vuld89TgMZdSot2RVz8W8Xc+
+39ZrBvzrCeYsPq5/U0Z85cnOSw4skwh06wsxTvL1D70SilI2c0YdR1sVgbhq04HN
+7FyE7SDQ1GqxyTJAU+OPH3Pk97Bl25vWD43RCCIjSUHhKzQRPno2ItObFepE+QJH
+lSO1YMXmZDAfsRts3dca3VSDOdAQely6G7HQu5kXWXGtRQARAQABiQIfBBgBCAAJ
+BQJaWVDtAhsMAAoJEHLjPk3xpQNuRlsQAKnkyjjXRvfMB3eKZ1PrWf7DBx5WL8Hk
+r2QAnv0diDeRTzotQyzKivf3/W3gQc9kQi/ilaPuuXeW+yKiNQaIbai7EC0DdyHa
+qG9s8F7UDoojFOGCc3OVpQvuucVbv4a6EpqXBI6ayrQW0jMXakiuIF/dL5uCGRLn
+sPMB87xJfMrkqEziymJMooQTRckgj2S9J2QYDZFxmKChPl1kXMlmx6Y4FhzrsYwo
+I47hW9hHG1+8129FOgwYTwELEuX6FKShcKpyy77b4Tar3UvdzNkfaysFPvX3O7Oh
+Bes2VgfslEZSgK2CScigvLIz9Ehe9CUw6WEC6LZW3bbC+Cbz624gOsm/GYI9Llsc
+5/NMMwQTCoTRm+b0UAYQvzHDS7km9yceNSfFlkpE/lWnIt9E0H+8bOwEbYF8y2qy
+yLXIm7MYEm4UnUZ0j8kihP+wSHsZP2xPhjEUcQw1ZWhWLACvzlEC0a3zsovoJ6U8
+zrqqfEBtuQgfwwJRWArMLQn/rlEJSrTrFfehwhvH3dPVSU36N5nBA8ls5YlSHBQv
+38dChpBXu3wzFhetkSuHxdlfopeZMtDmljnIkBNTEFg01oqJNPbyiX2JQcfKizCt
+maCIiJY+hOYIKkJdkt1FyOhADBciebxCQrpIIzWupeyQNuVj3I4g6YaQX00+kgiB
+H1XKrAg/dpMNmQENBFrHiEoBCADASg9zHYzaSU0/1q1hcmTHU6H4syCQXHHd3zF7
+n/RcsGnt4RBuUi/BUvNp3zYR6uFOyyk6LPV1hq2Ca8e/oSFrDYxqLladESQd9GNN
+stDeK3HinsWJCVwSbkzNJbUtyr6SclmWt66vNqBZngMallJRQe8QDqpg0ZSZj/0d
+VGxPBR16zc/2ddGnXJFe/V5XAWAap9SEo44pyGK4xf87Bgq8jT33LuQtd8exOk3E
+atkK5jLEn9xmiheoSePEhOoQSrJMHfMjFka0PYZlCeaaHw7r7yXb/VoHFOAPxb6k
+a1cunbp39b4z7Jy9kLBy0tbDnAs/sLp4LUN3Vx1JLoXBSIsHABEBAAG0JFJhdWwg
+Sm9yZGFuIDxyYXVsQHByeXNtYXRpY2xhYnMuY29tPokBNwQTAQoAIQUCWseISgIb
+AwULCQgHAwUVCgkICwUWAgMBAAIeAQIXgAAKCRCVRSpwGBD+26lmB/wJxChWwva0
+StjMRSk3V7JTBPi5RNQm3VY8UydUNjsXzHVvtjUczOj+zHfo8tYUYa/ypWujXEX9
+bVYVMr9JGNjm+rysI1gmW1gcx9pZr9gde4CR4Owfpwjh8oFnSrBBrcaelb0Krrv4
+Okms43fEw1bWpllayal5SqknOIxpw1ZiNpG3jVe/C9n1/3Nw4XF5R0RHDxXTu6Hu
+H3t7zRQwAJcOod6ccRdzNR9CsGdnOoNPG3pqs9w6zWUp0QETMAMcEpSLCJumaVfQ
+24PK+MF92F5JRwNVt80xSLA9KMyNnv/ZvxZXXLjkIhsAmwYkHUs3WSm5HJ4qqOCX
+V8VwFbwnea3eiQIzBBABCgAdFiEECuAFHWR7o8GpF69AcuM+TfGlA24FAl9zs2AA
+CgkQcuM+TfGlA24dMA/+N31SPAXVgwiNasBknb+YSq2/OQPogxQUc8tupvDcQ62H
+u0kvA7pUPUFjITJ1xsm8CRXfb7Hge9rNUw9Y2MNKf4sIDQs3bFeKOiAGVbO8Z055
+5VCTWWPhXzjWoR84YWrx0Go8WT/V3Lahy4frGA3Vsza6wzi2P6c7LF4jqX2iBD1l
+OpAYNGyt0sX/RLp3s2jOTWJwVyRR4UKSZOgpi9OTGLXrq5oU2dpwEIzBmhaWIs+u
+oD5/4TaAt4lEFu3Sxk5w8fJlUXbM4IG8A1l+dnRPF5rtjtfvuX0GeQcDtJ5e1qHj
+7PvVj8HjGPwxqsAjYHpg6QjyQCdtHYHdboEIU+OXMYRPRh2Iv0GUoeuMqoSsvFgn
+c9PGN8Ai5u+oNzKeLJhpxpLV7s9zAbsripdnvDBn7RwuNx2ziqZayxoYvFRCAQl5
+wzr0/eo/wq36D9EI7uJ2I3yt1g/VkwWQsAeE2skuGGbwed263cWTkl6g1w+vlDY3
+/jc3a8HcS0zIUX194ChrbbNezGb74mQFoLk6PksLfhXseAKCs8bvPaTwIm0JGTMT
+YzkRufrv1+I9KPFy3fTpvnMZTbud4nPCLsK179whk+Jrdv866i+E2WZ1JyzIZ9bB
+P1x+ABi82PMdzhTw+pTkR1CVHmrmOiSi2MHRYGMedM9ECGnPIdab5d75r1qkuvO5
+AQ0EWseISgEIAKHrgTVeZ0CWZMETk7NFjDTvBpoTMQDT7DDe2vFQHc8D2AvR3Aod
++CUaOeY0Yp0GOemd7BEcUbpYbCQk1B9G8vHNcQqTMLjBgujXwF6w2c+lqEu2/tyI
+2/uGCOBDs4wuQZo+akCCMekqYjdZBfZV6nQzf/l7eQHIq+sNtnSiZmHdH46PLi7/
+17wR9GxzKvuv1R73DCyekCcz1Puo0b7lfGD28kESJK1Mg9SAOqVjtaS58Oxo+Y1M
+ZWRqh0tkAkgOBpdyddGy6TX/9c3a0U3eBQweRpNDh0eCEh5UsYDluL4NtXj7rVYd
+4mHONJzI3h1LnKWvpVYmk703MPmtgeJwNzEAEQEAAYkBHwQYAQoACQUCWseISgIb
+DAAKCRCVRSpwGBD+25PxCACNBj/2HBSpdYAxEGhNHSaw62y7Jwuf7NbsKIAqygzi
+m2+dxae7PbAm64jEXYJA5GaCOs4xO52S/2+1N87e5J86VRNg0vlA9/ivFzERAxEg
+rgIUyGXYfS8oA/4r5+PfKt/NvXO2wH3MPakrqZqXhOv+1IPvOt03wWoZKmYyVT6g
+YnzutOsvH6cbhUh/D0WpuU8ZgkrFu5Xe7ynZoLm1qQOA23pkxxQbeNs0uoKUja9v
+bx4eBtaliLc6rsXt+1WzdXA5ZRNqkdn87Tz5IU0FuQYVblYvPz9QoUlvx5siWVaP
+Mmc7dIctWaWyaJmgjkLKdy36ydS5axhqn158yIjOZV3emQINBF8h6lMBEACovC4z
+oieJ9iMZpWfylsLQKkeEmfSXnjcjW4RLUjs2CRWj+W6H7eCRy/MBOdx+6zAb8TE/
+n/TSlP5l5Xx40BGDAMUZVFrJVEkMPK5kUmNzybg7PiuMd3qZE+pNyHEXXlLU77Dn
+VO26TD9RvpKXdjm+ATsnQ6rvDNszkYI2Bj1tPxwZ7bRi96U/upL3WfYOsTLHirM3
+pEkI3zfMZj8ufDX9XlmGrQ384E/hTgjpLXmDm/jMRlX3mRzDkV1HO+gEicfePm5+
+K4eWlMCNXN5bcGwoEFY2LwAojRcbRaH/UH2S3btkG2eSK2fOFEwQ0G4vIyoLF60R
+cEk1s6cYIgk3kVsmNSzA5iJ81nD5bbfTceUKsjTZiw5RmN0Vh5g75pw+3uoXB+55
+egabEIt/GTZZlP6zhd/CKjTQR0R4+ZaEbZFOCtABukC5xfWGCRdNmXAWMBe0MYpW
+ub2TRLISLfNNc7GWM4Y17d9aRhaql9gY6QLIRNGYuvGPiRMaJADZx46LtbWo8YiG
+xLId7H8D+/0MSzMOg7RhELqYScYDigiVEXkTHA3QSprf/ohjm8woRhQY5CjCEZp5
+8MvhD40VAo4Gxgx1V8lwB3CD3kDgFEaUZh2H+N/fmRegACN2lzEm1krOiS7sAB48
+/Av99/1VXalLoBbzFTnuAYwnmLk8vdparT5LlwARAQABtChUZXJlbmNlIFRzYW8g
+PHRlcmVuY2VAcHJ5c21hdGljbGFicy5jb20+iQJOBBMBCAA4FiEEMX1ukQWPjzwj
+A7p3VjE+RFgSl6YFAl8h6lMCGwMFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQ
+VjE+RFgSl6YJgBAAqIAMzMFf/+qE896kuXYSlmaYtzhrMXgQVD09UgSq4JfVYudE
+e8EfamWz8RFV7znxHO/5Fp4yoWeDGc2b40DhivvRb1p4XSJi+F64moeHt+qn4Hay
+CDs5wGv40KJk0+2r7iILc3Gw4NP6r4c2HU2EqSW8fn/bYM3yxLsYEXt5zhgipKpR
+CB4WmqojW+CxXkj8dv4CyY+HlKJ6nZWstK42h736njM44rjMDSDt3lv0ZYykopTw
+09THnHkWOXY4PSrKOzE+OqsPMPOD3Gq3bYPqQ0ZgdS4FeuDesqqHjJabRX+7DgJK
+lLEnylfDlvDKf6I/tqrAfqCCdQvEjpdQXG84GSdK//wT1biunHvGeyHM/PZx9V5O
+X8YoecswcYjozU1F7x8buk8GxV7ZuJ2pbdXr7jTou1dA7e4Nas9Qu1RcK39MoTYc
+Ub467ONFCX+Y3MzELTnEyx4hSwTxlWg7VjCQ63zrFOd8VG0WUydlqiHgIE8WCJVE
+pQlrr2v/A6ieM1HY1Ftjt+mk/qncqibCdyIofyV4o5FrVo8arx74jR+Lhvkp1T9N
+uRj8V6M3g9vxJdAsy1O5/ZrdwKJ6Yy1n62+9b/uxhklYONK7DpDlWU0ZLWHlKEIV
+zzfILCiLf+nxMQdVwuthlcysg8JwYEE8xDXc851ciuj0ygv3Wm0BY44V6/m5Ag0E
+XyHqUwEQALuKzOAlo75p2vSYD7TecE/E0hBOS+cjs8kRH+oIzm5fx7sf00SC1jU2
+q5QLYLixNeT+l0bD70R7b8r2uFu1aZL7pQqbGIGisLHlxu8611+PCpE5AsQi3Wui
+IZ6Y8K7grJ28vviBiZUBY3iCCRH0LuvyZN3R0zgyMGbzouQk5wuGJUkRHJKtV5by
+FVEl3CYudRtAp5LPFw6j7UzT5yQqBmY6tXp5WMmmAvOtnu8ohpRhzY21dJMlSykX
+Ne9rcARy8mVWNdcXJUIc85z0BmyrdiA4YY0XiZTHD9mslj+af4QHsaS+p3aorTLD
+5BKkp5Ek79a1BUxBjrao4W2fljYf129/SHbwds/Dup26zB2vi/fhbfVPvevXLPpi
+Vm4uz8fE4D5lPYZAdu5GtO2V9kWbhDtU1R1SJSdFDI9Sev3B+NLrstclGfdbFQKF
+shbUxydjSX56OJvh5hee50PcCz+Ab+usoyUPkcOfET/L55AdqJo3cVYtnAKpOJG/
+mKP5Ih3LZVm9wCdWTCzboEtDfLlcjCc5kLiE9Pq7sKMnXm/NcXNFWBkRuaHg+Mt5
+Yk659/Q6oUG3yUrS2d7cSeuNRWNlaRlnk3hZtB13xpmoHGYmrMOe7wiEE5xEnOju
+1ctRRRX6lNUPlSvID83Y9JSYL9hYMbidRmFY28AIddT/PjVTgfi1ABEBAAGJAjYE
+GAEIACAWIQQxfW6RBY+PPCMDundWMT5EWBKXpgUCXyHqUwIbDAAKCRBWMT5EWBKX
+poMOD/4/sRLnK94pY2qtSSfNZEumOdQVvges08u34saG4mT1iLD9TcRgsFvgwcTV
+Pec9Git4iJmoJpFylvOzoBWmtDzJ7TB3JxvtwUfFH2uZ22k12eChyAFHI8TxASqI
+TV2YwkIgB2PTmva1GrdNKM6y5bfS1BdS0u+Etgp4zN9nyQECY2sM2accUi2S7JBV
+7o6lEJWCRgFUwIfY6fdfxhZo/w0g2u5wftUQYlbXWSQLImFFZBc/13k8EMxsfbMi
+LQ/wxKEfe1ZwFoXqIS6cq8CHTdj70QO7K9ah6krGPmL23LVuxlkQIR7mT5yFJtv2
+VRT4IYgOUl6rAd08B6MOy6MOMa14FpNrAybPK8Yd/eSdUNciVf17q33Yc15x+trs
+En1hHbDS82xaLJ/Ld4mANeJVt3FodgVmzZGpKheFEOFXXeuX1ZZ4c9TgX+0wBKoh
+aBb8EGBVo4aV4GbZ/gEa8wls5NNTwd56H4G94hiq+qFM39x4YjhAvkM0hLRYzR05
+tSCdlEbkh2K6RbOhBPsNHCbypWRt2fN8/q4uLPJJt7NRQ2zi6H/x04HGblhXdX6H
+mmYjJv1LTbem9VptcpvPauNsibeIvIWA2nYM2ncDWt6gJpOH9Zh4fHuaG4aCdNmJ
+hPNeJNnmLcpDQvR9wU5w7e5tC/ZSeTZ5ul1zOKa1qZ4lJ50BDQ==
+=a30p
+-----END PGP PUBLIC KEY BLOCK-----

.well-known/security.txt

@@ -0,0 +1,25 @@
+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA512
+
+Contact: mailto:security@prysmaticlabs.com
+Encryption: openpgp4fpr:0AE0051D647BA3C1A917AF4072E33E4DF1A5036E
+Encryption: openpgp4fpr:CD08DE68C60B82D3EE2A3F7D95452A701810FEDB
+Encryption: openpgp4fpr:317D6E91058F8F3C2303BA7756313E44581297A6
+Preferred-Languages: en
+Canonical: https://github.com/prysmaticlabs/prysm/tree/master/.well-known/security.txt
+-----BEGIN PGP SIGNATURE-----
+
+iQIzBAEBCgAdFiEECuAFHWR7o8GpF69AcuM+TfGlA24FAl++klgACgkQcuM+TfGl
+A27rQw/6A29p1W20J0v+h218p8XWLSUpTIGLnZTxw6KqdyVXMzlsQK0YG4G2s2AB
+0LKh7Ae/Di5E0U+Z4AjUW5nc5eaCxK36GMscH9Ah0rgJwNYxEJw7/2o8ZqVT/Ip2
++56rFihRqxFZfaCNKFVuZFaL9jKewV9FKYP38ID6/SnTcrOHiu2AoAlyZGmB03p+
+iT57SPRHatygeY4xb/gwcfREFWEv+VHGyBTv8A+6ABZDxyurboCFMERHzFICrbmk
+8UdHxxlWZDnHAbAUyAwpERC5znx6IHXQJwF8TMtu6XY6a6axT2XBOyJDF9/mZOz+
+kdkz6loX5uxaQBGLtTv6Kqf1yUGANOZ16VhHvWwL209LmHmigIVQ+qSM6c79PsW/
+vrsqdz3GBsiMC5Fq2vYgnbgzpfE8Atjn0y7E+j4R7IvwOAE/Ro/b++nqnc4YqhME
+P/yTcfGftaCrdSNnQCXeoV9JxpFM5Xy8KV3eexvNKbcgA/9DtgxL5i+s5ZJkUT9A
++qJvoRrRyIym32ghkHgtFJKB3PLCdobeoOVRk6EnMo9zKSiSK2rZEJW8Ccbo515D
+W9qUOn3GF7lNVuUFAU/YKEdmDp/AVaViZ7vH+8aq0LC0HBkZ8XlzWnWoArS8sMhw
+fX0R9g/HMgrwNte/d0mwim5lJ2Plgv60Bh4grJqwZJeWbU0zi1U=
+=uW+X
+-----END PGP SIGNATURE-----

BUILD.bazel

@@ -3,12 +3,19 @@ load("@com_github_atlassian_bazel_tools//gometalinter:def.bzl", "gometalinter")
 load("@com_github_atlassian_bazel_tools//goimports:def.bzl", "goimports")
 load("@io_kubernetes_build//defs:run_in_workspace.bzl", "workspace_binary")
 load("@io_bazel_rules_go//go:def.bzl", "nogo")
+load("@graknlabs_bazel_distribution//common:rules.bzl", "assemble_targz", "assemble_versioned")
+load("@bazel_skylib//rules:common_settings.bzl", "string_setting")
 
 prefix = "github.com/prysmaticlabs/prysm"
 
-exports_files(["genesis.json"])
+exports_files([
+    "LICENSE.md",
+])
 
 # gazelle:prefix github.com/prysmaticlabs/prysm
+# gazelle:map_kind go_library go_library @prysm//tools/go:def.bzl
+# gazelle:map_kind go_test go_test @prysm//tools/go:def.bzl
+# gazelle:map_kind go_repository go_repository @prysm//tools/go:def.bzl
 gazelle(
     name = "gazelle",
     prefix = prefix,
@@ -17,7 +24,16 @@ gazelle(
 # Protobuf compiler (non-gRPC)
 alias(
     name = "proto_compiler",
-    actual = "@io_bazel_rules_go//proto:gogofast_proto",
+    actual = "@io_bazel_rules_go//proto:go_proto",
+    visibility = [
+        "//proto:__subpackages__",
+    ],
+)
+
+# Cast protobuf compiler (non-gRPC)
+alias(
+    name = "cast_proto_compiler",
+    actual = "@com_github_prysmaticlabs_protoc_gen_go_cast//:go_cast",
     visibility = [
         "//proto:__subpackages__",
     ],
@@ -26,10 +42,22 @@ alias(
 # Protobuf compiler (gRPC)
 alias(
     name = "grpc_proto_compiler",
-    actual = "@io_bazel_rules_go//proto:gogofast_grpc",
-    visibility = [
-        "//proto:__subpackages__",
-    ],
+    actual = "@io_bazel_rules_go//proto:go_grpc",
+    visibility = ["//visibility:public"],
+)
+
+# Cast protobuf compiler (gRPC)
+alias(
+    name = "cast_grpc_proto_compiler",
+    actual = "@com_github_prysmaticlabs_protoc_gen_go_cast//:go_cast_grpc",
+    visibility = ["//visibility:public"],
+)
+
+# Protobuf gRPC gateway compiler
+alias(
+    name = "grpc_gateway_proto_compiler",
+    actual = "@com_github_grpc_ecosystem_grpc_gateway_v2//protoc-gen-grpc-gateway:go_gen_grpc_gateway",
+    visibility = ["//visibility:public"],
 )
 
 gometalinter(
@@ -44,8 +72,8 @@ gometalinter(
 goimports(
     name = "goimports",
     display_diffs = True,
-    write = False,
     prefix = prefix,
+    write = False,
 )
 
 workspace_binary(
@@ -55,6 +83,8 @@ workspace_binary(
 
 nogo(
     name = "nogo",
+    config = "nogo_config.json",
+    visibility = ["//visibility:public"],
     deps = [
         "@org_golang_x_tools//go/analysis/passes/unsafeptr:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/unreachable:go_tool_library",
@@ -68,14 +98,12 @@ nogo(
         "@org_golang_x_tools//go/analysis/passes/pkgfact:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/nilness:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/nilfunc:go_tool_library",
-        # "@org_golang_x_tools//go/analysis/passes/lostcancel:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/loopclosure:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/httpresponse:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/findcall:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/deepequalerrors:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/ctrlflow:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/copylock:go_tool_library",
-        "@org_golang_x_tools//go/analysis/passes/composite:go_tool_library",
         #  "@org_golang_x_tools//go/analysis/passes/cgocall:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/buildtag:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/buildssa:go_tool_library",
@@ -85,7 +113,54 @@ nogo(
         "@org_golang_x_tools//go/analysis/passes/assign:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/inspect:go_tool_library",
         "@org_golang_x_tools//go/analysis/passes/asmdecl:go_tool_library",
-    ],
-    visibility = ["//visibility:public"],
-    config = "nogo_config.json",
+        "//tools/analyzers/maligned:go_tool_library",
+        "//tools/analyzers/cryptorand:go_tool_library",
+        "//tools/analyzers/errcheck:go_tool_library",
+        "//tools/analyzers/featureconfig:go_tool_library",
+        "//tools/analyzers/comparesame:go_tool_library",
+        "//tools/analyzers/shadowpredecl:go_tool_library",
+        "//tools/analyzers/nop:go_tool_library",
+        "//tools/analyzers/slicedirect:go_tool_library",
+        "//tools/analyzers/interfacechecker:go_tool_library",
+        "//tools/analyzers/ineffassign:go_tool_library",
+        "//tools/analyzers/properpermissions:go_tool_library",
+    ] + select({
+        # nogo checks that fail with coverage enabled.
+        ":coverage_enabled": [],
+        "//conditions:default": [
+            "@org_golang_x_tools//go/analysis/passes/lostcancel:go_tool_library",
+            "@org_golang_x_tools//go/analysis/passes/composite:go_tool_library",
+        ],
+    }),
+)
+
+config_setting(
+    name = "coverage_enabled",
+    values = {"define": "coverage_enabled=1"},
+)
+
+common_files = {
+    "//:LICENSE.md": "LICENSE.md",
+    "//:README.md": "README.md",
+}
+
+toolchain(
+    name = "built_cmake_toolchain",
+    toolchain = "@rules_foreign_cc//tools/build_defs/native_tools:built_cmake",
+    toolchain_type = "@rules_foreign_cc//tools/build_defs:cmake_toolchain",
+)
+
+string_setting(
+    name = "gotags",
+    build_setting_default = "",
+    values = [
+        "",
+        "libfuzzer",
+    ],
+)
+
+sh_binary(
+    name = "prysm_sh",
+    srcs = ["prysm.sh"],
+    visibility = ["//visibility:public"],
 )

CONTRIBUTING.md

@@ -1,8 +1,10 @@
 # Contribution Guidelines
 
-Excited by our work and want to get involved in building out our sharding releases? Or maybe you haven't learned as much about the Ethereum protocol but are a savvy developer? Our [READINGS.md](https://github.com/prysmaticlabs/prysm/blob/master/docs/READINGS.md) doc includes comprehensive information on Ethereum and sharding for both part-time and core contributors to the project.
+Note: The latest and most up to date documenation can be found on our [docs portal](https://docs.prylabs.network/docs/contribute/contribution-guidelines).
 
-You can explore our [Open Issues](https://github.com/prysmaticlabs/prysm/issues) in-the works for our different releases. Feel free to fork our repo and start creating PR’s after assigning yourself to an issue of interest. We are always chatting on [Discord](https://discord.gg/che9auJ) or [Gitter](https://gitter.im/prysmaticlabs/geth-sharding) drop us a line there if you want to get more involved or have any questions on our implementation!
+Excited by our work and want to get involved in building out our sharding releases? Or maybe you haven't learned as much about the Ethereum protocol but are a savvy developer? 
+
+You can explore our [Open Issues](https://github.com/prysmaticlabs/prysm/issues) in-the works for our different releases. Feel free to fork our repo and start creating PR’s after assigning yourself to an issue of interest. We are always chatting on [Discord](https://discord.gg/CTYGPUJ) drop us a line there if you want to get more involved or have any questions on our implementation!
 
 ## Contribution Steps
 
@@ -60,12 +62,6 @@ Changes that only affect a single file can be tested with
 $ go test <file_you_are_working_on>
 ```
 
-Changes that affect multiple files can be tested with ...
-
-```
-$ gometalinter && bazel test
-```
-
 **10. Stage the file or files that you want to commit.**
 
 ```
@@ -88,10 +84,10 @@ You can use the –amend flag to include previous commits that have not yet been
 $ git fetch prysm
 ```
 
-**13. Rebase your branch atop of the latest version of Prysm.**
+**13. Pull latest version of Prysm.**
 
 ```
-$ git rebase prysm/master
+$ git pull origin master
 ```
 
 If there are conflicts between your edits and those made by others since you started work Git will ask you to resolve them. To find out which files have conflicts run ...
@@ -115,10 +111,10 @@ The code from the Prysm repo is inserted between <<< and === while the change yo
 
 **14. Push your changes to your fork of the Prysm repo.**
 
-Rebasing a pull request changes the history on your branch, so Git will reject a normal git push after a rebase. Use a force push to move your changes to your fork of the repo.
+Use git push to move your changes to your fork of the repo.
 
 ```
-$ git push myrepo feature-in-progress-branch -f
+$ git push myrepo feature-in-progress-branch
 ```
 
 **15. Check to be sure your fork of the Prysm repo contains your feature branch with the latest edits.**
@@ -179,7 +175,7 @@ We consider two types of contributions to our repo and categorize them as follow
 
 ### Part-Time Contributors
 
-Anyone can become a part-time contributor and help out on implementing sharding. The responsibilities of a part-time contributor include:
+Anyone can become a part-time contributor and help out on implementing Ethereum consensus. The responsibilities of a part-time contributor include:
 
 -   Engaging in Gitter conversations, asking the questions on how to begin contributing to the project
 -   Opening up github issues to express interest in code to implement
@@ -190,8 +186,6 @@ Anyone can become a part-time contributor and help out on implementing sharding.
 -   Follow up on open PRs
     -   Have an estimated timeframe to completion and let the core contributors know if a PR will take longer than expected
 
-We do not expect all part-time contributors to be experts on all the latest sharding documentation, but all contributors should at least be familiarized with our sharding [README.md](https://github.com/prysmaticlabs/prysm/blob/master/validator/README.md) and have gone through the required Ethereum readings as posted on our [READINGS.md](https://github.com/prysmaticlabs/prysm/blob/master/docs/READINGS.md) document.
-
 ### Core Contributors
 
 Core contributors are remote contractors of Prysmatic Labs, LLC. and are considered critical team members of our organization. Core devs have all of the responsibilities of part-time contributors plus the majority of the following:

DEPENDENCIES.md

@@ -0,0 +1,80 @@
+# Dependency Managagement in Prysm
+
+Prysm is go project with many complicated dependencies, including some c++ based libraries. There
+are two parts to Prysm's dependency management. Go modules and bazel managed dependencies. Be sure 
+to read [Why Bazel?](https://github.com/prysmaticlabs/documentation/issues/138) to fully
+understand the reasoning behind an additional layer of build tooling via Bazel rather than a pure
+"go build" project.
+
+## Go Module support
+
+The Prysm project officially supports go modules with some caveats. 
+
+### Caveat 1: Some c++ libraries are precompiled archives
+
+Given some of Prysm's c++ dependencies have very complicated project structures which make building
+difficult or impossible with "go build" alone. Additionally, building c++ dependencies with certain
+compilers, like clang / LLVM, offer a significant performance improvement. To get around this 
+issue, c++ dependencies have been precompiled as linkable archives. While there isn't necessarily
+anything bad about precompiled archives, these files are a "blackbox" which a 3rd party author
+could have compiled anything for this archive and detecting undesired behavior would be nearly
+impossible. If your risk tolerance is low, always compile everything from source yourself, 
+including complicated c++ dependencies.
+
+*Recommendation: Use go build only for local development and use bazel build for production.*
+
+### Caveat 2: Generated gRPC protobuf libraries
+
+One key advantage of Bazel over vanilla `go build` is that Bazel automatically (re)builds generated
+pb.go files at build time when file changes are present in any protobuf definition file or after
+any updates to the protobuf compiler or other relevant dependencies. Vanilla go users should run
+the following scripts often to ensure their generated files are up to date. Further more, Prysm
+generates SSZ marshal related code based on defined data structures. These generated files must
+also be updated and checked in as frequently.
+
+```bash
+./hack/update-go-pbs.sh
+./hack/update-go-ssz.sh
+```
+
+*Recommendation: Use go build only for local development and use bazel build for production.*
+
+### Caveat 3: Compile-time optimizations 
+
+When Prysmatic Labs builds production binaries, they use the "release" configuration of bazel to
+compile with several compiler optimizations and recommended production build configurations.
+Additionally, the release build properly stamps the built binaries to include helpful metadata
+about how and when the binary was built. 
+
+*Recommendation: Use go build only for local development and use bazel build for production.*
+
+```bash
+bazel build //beacon-chain --config=release
+```
+ 
+## Adding / updating dependencies
+
+1. Add your dependency as you would with go modules. I.e. `go get ...`
+1. Run `gazelle update-repos -from_file=go.mod` to update the bazel managed dependencies.
+
+Example:
+
+```bash
+go get github.com/prysmaticlabs/example@v1.2.3
+bazel run //:gazelle -- update-repos -from_file=go.mod -to_macro=deps.bzl%prysm_deps -prune=true
+```
+
+The deps.bzl file should have been updated with the dependency and any transitive dependencies. 
+
+Do NOT add new `go_repository` to the WORKSPACE file. All dependencies should live in deps.bzl.
+
+## Running tests
+
+To enable conditional compilation and custom configuration for tests (where compiled code has more 
+debug info, while not being completely optimized), we rely on Go's build tags/constraints mechanism 
+(see official docs on [build constraints](https://golang.org/pkg/go/build/#hdr-Build_Constraints)). 
+Therefore, whenever using `go test`, do not forget to pass in extra build tag, eg:
+
+```bash
+go test ./beacon-chain/sync/initial-sync -tags develop 
+```

INTEROP.md

@@ -0,0 +1,84 @@
+# Prysm Client Interoperability Guide
+
+This README details how to setup Prysm for interop testing for usage with other Ethereum consensus clients.
+
+## Installation & Setup
+
+1. Install [Bazel](https://docs.bazel.build/versions/master/install.html) **(Recommended)**
+2. `git clone https://github.com/prysmaticlabs/prysm && cd prysm`
+3. `bazel build //...`
+
+## Starting from Genesis
+
+Prysm supports a few ways to quickly launch a beacon node from basic configurations:
+
+- `NumValidators + GenesisTime`: Launches a beacon node by deterministically generating a state from a num-validators flag along with a genesis time **(Recommended)**
+- `SSZ Genesis`: Launches a beacon node  from a .ssz file containing a SSZ-encoded, genesis beacon state
+
+## Generating a Genesis State
+
+To setup the necessary files for these quick starts, Prysm provides a tool to generate a `genesis.ssz` from
+a deterministically generated set of validator private keys following the official interop YAML format 
+[here](https://github.com/ethereum/eth2.0-pm/blob/master/interop/mocked_start).
+
+You can use `bazel run //tools/genesis-state-gen` to create a deterministic genesis state for interop.
+
+### Usage
+
+- **--genesis-time** uint: Unix timestamp used as the genesis time in the generated genesis state (defaults to now)
+- **--mainnet-config** bool: Select whether genesis state should be generated with mainnet or minimal (default) params
+- **--num-validators** int: Number of validators to deterministically include in the generated genesis state
+- **--output-ssz** string: Output filename of the SSZ marshaling of the generated genesis state
+
+The example below creates 64 validator keys, instantiates a genesis state with those 64 validators and with genesis unix timestamp 1567542540,
+and finally writes a ssz encoded output to ~/Desktop/genesis.ssz. This file can be used to kickstart the beacon chain in the next section.
+
+```
+bazel run //tools/genesis-state-gen -- --output-ssz ~/Desktop/genesis.ssz --num-validators 64 --genesis-time 1567542540
+```
+
+## Launching a Beacon Node + Validator Client
+
+### Launching from Pure CLI Flags
+
+Open up two terminal windows, run:
+
+```
+bazel run //beacon-chain -- \
+--bootstrap-node= \
+--deposit-contract $(curl -s https://prylabs.net/contract) \
+--force-clear-db \
+--interop-num-validators 64 \
+--interop-eth1data-votes
+```
+
+This will deterministically generate a beacon genesis state and start
+the system with 64 validators and the genesis time set to the current unix timestamp.
+Wait a bit until your beacon chain starts, and in the other window:
+
+```
+bazel run //validator -- --keymanager=interop --keymanageropts='{"keys":64}'
+```
+
+This will launch and kickstart the system with your 64 validators performing their duties accordingly.
+
+### Launching from `genesis.ssz`
+
+Assuming you generated a `genesis.ssz` file with 64 validators, open up two terminal windows, run:
+
+```
+ bazel run //beacon-chain -- \
+--bootstrap-node= \
+--deposit-contract $(curl -s https://prylabs.net/contract) \
+--force-clear-db \
+--interop-genesis-state /path/to/genesis.ssz \
+--interop-eth1data-votes
+```
+
+Wait a bit until your beacon chain starts, and in the other window:
+
+```
+bazel run //validator -- --keymanager=interop --keymanageropts='{"keys":64}'
+```
+
+This will launch and kickstart the system with your 64 validators performing their duties accordingly.

PULL_REQUEST_TEMPLATE.md

@@ -1,13 +0,0 @@
-TODO(you): choose "part of" or "resolves" and the associated github issue #.
-
-[Part of|Resolves] #531 
-
----
-
-# Description
-
-**Write why you are making the changes in this pull request**
-
-**Write a summary of the changes you are making**
-
-**Link anything that would be helpful or relevant to the reviewers**

README.md

@@ -1,122 +1,35 @@
-# Prysmatic Labs Ethereum Serenity Implementation
+# Prysm: An Ethereum Consensus Implementation Written in Go
 
-[![Build status](https://badge.buildkite.com/b555891daf3614bae4284dcf365b2340cefc0089839526f096.svg)](https://buildkite.com/prysmatic-labs/prysm)
+[![Build status](https://badge.buildkite.com/b555891daf3614bae4284dcf365b2340cefc0089839526f096.svg?branch=master)](https://buildkite.com/prysmatic-labs/prysm)
+[![Go Report Card](https://goreportcard.com/badge/github.com/prysmaticlabs/prysm)](https://goreportcard.com/report/github.com/prysmaticlabs/prysm)
+[![ETH2.0_Spec_Version 1.0.0](https://img.shields.io/badge/ETH2.0%20Spec%20Version-v1.0.0-blue.svg)](https://github.com/ethereum/consensus-specs/tree/v1.0.0)
+[![Discord](https://user-images.githubusercontent.com/7288322/34471967-1df7808a-efbb-11e7-9088-ed0b04151291.png)](https://discord.gg/CTYGPUJ)
 
-This is the main repository for the Go implementation of the Ethereum 2.0 Serenity [Prysmatic Labs](https://prysmaticlabs.com).
+This is the core repository for Prysm, a [Golang](https://golang.org/) implementation of the [Ethereum Consensus](https://ethereum.org/en/eth2/) specification, developed by [Prysmatic Labs](https://prysmaticlabs.com). See the [Changelog](https://github.com/prysmaticlabs/prysm/releases) for details of the latest releases and upcoming breaking changes.
 
-Before you begin, check out our [official documentation portal](https://prysmaticlabs.gitbook.io/prysm/) and join our active chat room on Discord or Gitter below:
+### Getting Started
 
-[![Discord](https://user-images.githubusercontent.com/7288322/34471967-1df7808a-efbb-11e7-9088-ed0b04151291.png)](https://discord.gg/KSA7rPr)
-[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/prysmaticlabs/geth-sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
+A detailed set of installation and usage instructions as well as breakdowns of each individual component are available in the [official documentation portal](https://docs.prylabs.network). If you still have questions, feel free to stop by our [Discord](https://discord.gg/CTYGPUJ).
 
-Also, read our [Roadmap Reference Implementation Doc](https://github.com/prysmaticlabs/prysm/blob/master/docs/ROADMAP.md). This doc provides a background on the milestones we aim for the project to achieve.
+### Staking on Mainnet
+
+To participate in staking, you can join the [official eth2 launchpad](https://launchpad.ethereum.org). The launchpad is the only recommended way to become a validator on mainnet. You can explore validator rewards/penalties via Bitfly's block explorer: [beaconcha.in](https://beaconcha.in), and follow the latest blocks added to the chain on [beaconscan](https://beaconscan.com).
 
 
-# Table of Contents
+## Contributing
+### Branches
+Prysm maintains two permanent branches:
 
-- [Join Our Testnet](#join-our-testnet)
-- [Installation](#installation)
-    - [Run Via Docker](#run-via-docker-recommended)
-    - [Run Via Bazel](#run-via-bazel)
-  - [Prysm Main Components](#prysm-main-components)
-    - [Running an Ethereum 2.0 Beacon Node](#running-an-ethereum-20-beacon-node)
-    - [Staking ETH: Running a Validator Client](#staking-eth-running-a-validator-client)
--   [Testing](#testing)
--   [Contributing](#contributing)
--   [License](#license)
+* [master](https://github.com/prysmaticlabs/prysm/tree/master): This points to the latest stable release. It is ideal for most users.
+* [develop](https://github.com/prysmaticlabs/prysm/tree/develop): This is used for development, it contains the latest PRs. Developers should base their PRs on this branch.
 
-# Join Our Testnet
+### Guide
+Want to get involved? Check out our [Contribution Guide](https://docs.prylabs.network/docs/contribute/contribution-guidelines/) to learn more!
 
-You can now participate in our public testnet release for Ethereum 2.0 phase 0. Visit [prylabs.net](https://prylabs.net) 💎 to participate!
-
-# Installing Prysm
-
-### Installation Options
-You can either choose to run our system via:
-- Our latest [release](https://github.com/prysmaticlabs/prysm/releases) **(Easiest)**
-- Using Docker **(Recommended)**
-- Using Our Build Tool, Bazel
-
-### Fetching via Docker (Recommended)
-Docker is a convenient way to run Prysm, as all you need to do is fetch the latest images:
-
-```
-docker pull gcr.io/prysmaticlabs/prysm/validator:latest
-docker pull gcr.io/prysmaticlabs/prysm/beacon-chain:latest
-```
-
-### Build Via Bazel
-First, clone our repository:
-
-```
-git clone https://github.com/prysmaticlabs/prysm
-```
-
-Download the Bazel build tool by Google here and ensure it works by typing:
-
-```
-bazel version
-```
-
-Bazel manages all of the dependencies for you (including go and necessary compilers) so you are all set to build prysm. Then, build both parts of our system: a beacon chain node implementation, and a validator client:
-
-```
-bazel build //beacon-chain:beacon-chain
-bazel build //validator:validator
-```
-
-# Prysm Main Components
-Prysm ships with two important components: a beacon node and a validator client. The beacon node is the server that performs the heavy lifting of Ethereum 2.0., A validator client is another piece of software that securely connects to the beacon node and allows you to stake 3.2 Goerli ETH in order to secure the network. You'll be mostly interacting with the validator client to manage your stake.
-Another critical component of Ethereum 2.0 is the Validator Deposit Contract, which is a smart contract deployed on the Ethereum 1.0 chain which can be used for current holders of ETH to do a one-way transfer into Ethereum 2.0.
-
-### Running an Ethereum 2.0 Beacon Node
-With docker:
-
-```
-docker run -v /tmp/prysm-data:/data -p 4000:4000 \
-  gcr.io/prysmaticlabs/prysm/beacon-chain:latest \
-  --datadir=/data
-  --clear-db
-```
-
-To start your beacon node with bazel:
-
-```
-bazel run //beacon-chain -- --clear-db --datadir=/tmp/prysm-data
-```
-
-This will sync you up with the latest head block in the network, and then you'll have a ready beacon node.
-The chain will then be waiting for you to deposit 3.2 Goerli ETH into the Validator Deposit Contract before your validator can become active! Now, you'll need to create a validator client to connect to this node and stake 3.2 Goerli ETH to participate as a validator in Ethereum 2.0's Proof of Stake system.
-
-### Staking ETH: Running a Validator Client
-Once your beacon node is up, you'll need to attach a validator client as a separate process. Each validator represents 3.2 Goerli ETH being staked in the system, so you can spin up as many as you want to have more at stake in the network
-
-**Activating Your Validator: Depositing 3.2 Goerli ETH**
-
-Using your validator deposit data from the previous step, use the instructions in https://alpha.prylabs.net/participate to deposit.
-
-It'll take a while for the nodes in the network to process your deposit, but once you're active, your validator will begin doing its responsibility! In your validator client, you'll be able to frequently see your validator balance as it goes up. If you ever go offline for a while, you'll start gradually losing your deposit until you get kicked out of the system. Congratulations, you are now running Ethereum 2.0 Phase 0 :).
-
-# Testing
-
-To run the unit tests of our system do:
-
-```
-bazel test //...
-```
-
-To run our linter, make sure you have [golangci-lint](https://https://github.com/golangci/golangci-lint) installed and then run:
-
-```
-golangci-lint run
-```
-
-# Contributing
-
-We have put all of our contribution guidelines into [CONTRIBUTING.md](https://github.com/prysmaticlabs/prysm/blob/master/CONTRIBUTING.md)! Check it out to get started.
-
-![nyancat](https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcRBSus2ozk_HuGdHMHKWjb1W5CmwwoxmYIjIBmERE1u-WeONpJJXg)
-
-# License
+## License
 
 [GNU General Public License v3.0](https://www.gnu.org/licenses/gpl-3.0.en.html)
+
+## Legal Disclaimer
+
+[Terms of Use](/TERMS_OF_SERVICE.md)

SECURITY.md

@@ -0,0 +1,11 @@
+# Security Policy
+
+## Supported Versions
+
+[Releases](https://github.com/prysmaticlabs/prysm/releases/) contains all available releases. We recommend using the [most recently released version](https://github.com/prysmaticlabs/prysm/releases/latest).
+
+## Reporting a Vulnerability
+
+Please see our signed [security.txt](https://github.com/prysmaticlabs/prysm/blob/develop/.well-known/security.txt) for preferred encryption and reporting destination.
+
+**Please do not file a public ticket** mentioning the vulnerability, as doing so could increase the likelihood of the vulnerability being used before a fix has been created, released and installed on the network.

TERMS_OF_SERVICE.md

@@ -0,0 +1,45 @@
+## Terms of Use
+
+Effective as of Oct 14, 2020
+
+By downloading, accessing or using the Prysm implementation (“Prysm”), you (referenced herein as “you” or the “user”) certify that you have read and agreed to the terms and conditions below (the “Terms”) which form a binding contract between you and Prysmatic Labs (referenced herein as “we” or “us”). If you do not agree to the Terms, do not download or use Prysm.
+
+### About Prysm
+Prysm is a client implementation for Ethereum consensus protocol for a proof-of-stake blockchain. To participate in the network, a user must send ETH from the Eth1.0 chain into a validator deposit contract, which will queue in the user as a validator in the system. Validators participate in proposing and voting on blocks in the protocol, and the network applies rewards/penalties based on their behavior. A detailed set of installation and usage instructions as well as breakdowns of each individual component are available in the official documentation portal, however, we do not warrant the accuracy, completeness or usefulness of this documentation. Any reliance you place on such information is strictly at your own risk.
+
+### Licensing Terms
+Prysm is a fully open-source software program licensed pursuant to the GNU General Public License v3.0.
+
+The Prysmatic Labs name, the term “Prysm” and all related names, logos, product and service names, designs and slogans are trademarks of Prysmatic Labs or its affiliates and/or licensors. You must not use such marks without our prior written permission.
+
+### Risks of Operating Prysm
+The use of Prysm and acting as a validator on the Ethereum network can lead to loss of money. Ethereum is still an experimental system and ETH remains a risky investment. You alone are responsible for your actions on Prysm including the security of your ETH and meeting any applicable minimum system requirements.
+
+Use of Prysm and the ability to receive rewards or penalties may be affected at any time by mistakes made by the user or other users, software problems such as bugs, errors, incorrectly constructed transactions, unsafe cryptographic libraries or malware affecting the network, technical failures in the hardware of a user, security problems experienced by a user and/or actions or inactions of third parties and/or events experienced by third parties, among other risks. We cannot and do not guarantee that any user of Prysm will make money, that the Prysm network will operate in accordance with the documentation or that transactions will be effective or secure.
+
+We make no claims that Prysm is appropriate or permitted for use in any specific jurisdiction. Access to Prysm may not be legal by certain persons or in certain jurisdictions or countries. If you access Prysm, you do so on your own initiative and are responsible for compliance with local laws.
+
+Some Internet plans will charge an additional amount for any excess upload bandwidth used that isn’t included in the plan and may terminate your connection without warning because of overuse. We advise that you check whether your Internet connection is subjected to such limitations and monitor your bandwidth use so that you can stop Prysm before you reach your upload limit.
+
+### Warranty Disclaimer
+PRYSM IS PROVIDED ON AN “AS-IS” BASIS AND MAY INCLUDE ERRORS, OMISSIONS, OR OTHER INACCURACIES. PRYSMATIC LABS AND ITS CONTRIBUTORS MAKE NO REPRESENTATIONS OR WARRANTIES ABOUT PRYSM FOR ANY PURPOSE, AND HEREBY EXPRESSLY DISCLAIM ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY WARRANTY OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT OR ANY OTHER IMPLIED WARRANTY UNDER THE UNIFORM COMPUTER INFORMATION TRANSACTIONS ACT AS ENACTED BY ANY STATE. WE ALSO MAKE NO REPRESENTATIONS OR WARRANTIES THAT PRYSM WILL OPERATE ERROR-FREE, UNINTERRUPTED, OR IN A MANNER THAT WILL MEET YOUR REQUIREMENTS AND/OR NEEDS. THEREFORE, YOU ASSUME THE ENTIRE RISK REGARDING THE QUALITY AND/OR PERFORMANCE OF PRYSM AND ANY TRANSACTIONS ENTERED INTO THEREON.
+
+### Limitation of Liability
+In no event will Prysmatic Labs or any of its contributors be liable, whether in contract, warranty, tort (including negligence, whether active, passive or imputed), product liability, strict liability or other theory, breach of statutory duty or otherwise arising out of, or in connection with, your use of Prysm, for any direct, indirect, incidental, special or consequential damages (including any loss of profits or data, business interruption or other pecuniary loss, or damage, loss or other compromise of data, in each case whether direct, indirect, incidental, special or consequential) arising out of use Prysm, even if we or other users have been advised of the possibility of such damages. The foregoing limitations and disclaimers shall apply to the maximum extent permitted by applicable law, even if any remedy fails of its essential purpose. You acknowledge and agree that the limitations of liability afforded us hereunder constitute a material and actual inducement and condition to entering into these Terms, and are reasonable, fair and equitable in scope to protect our legitimate interests in light of the fact that we are not receiving consideration from you for providing Prysm.
+
+### Indemnification
+To the maximum extent permitted by law, you will defend, indemnify and hold Prysmatic Labs and its contributors harmless from and against any and all claims, actions, suits, investigations, or proceedings by any third party (including any party or purported party to or beneficiary or purported beneficiary of any transaction on Prysm), as well as any and all losses, liabilities,
+damages, costs, and expenses (including reasonable attorneys’ fees) arising out of, accruing from, or in any way related to (i) your breach of the terms of this Agreement, (ii) any transaction, or the failure to occur of any transaction on Prysm, and (iii) your negligence, fraud, or willful misconduct.
+
+### Compliance with Laws and Tax Obligations
+Your use of Prysm is subject to all applicable laws of any governmental authority, including, without limitation, federal, state and foreign securities laws, tax laws, tariff and trade laws, ordinances, judgments, decrees, injunctions, writs and orders or like actions of any governmental authority and rules, regulations, orders, interpretations, licenses, and permits of any federal,
+regional, state, county, municipal or other governmental authority and you agree to comply with all such laws in your use of Prysm. The users of Prysm are solely responsible to determinate what, if any, taxes apply to their ETH transactions. The owners of, or contributors to, Prysm are not responsible for determining the taxes that apply to ETH transactions.
+
+### Miscellaneous
+These Terms will be construed and enforced in accordance with the laws of the state of Illinois as applied to agreements entered into and completely performed in Illinois. You agree to the personal jurisdiction by and venue in Illinois and waive any objection to such jurisdiction or venue.
+
+We reserve the right to revise these Terms, and your rights and obligations are at all times subject to the then-current Terms provided on Prysm. Your continued use of Prysm constitutes acceptance of such revised Terms.
+
+These Terms constitute the entire agreement between you and Prysmatic Labs regarding use of Prysm and will supersede all prior agreements whether, written or oral. No usage of trade or other regular practice or method of dealing between the parties will be used to modify, interpret, supplement, or alter the terms of these Terms.
+
+If any portion of these Terms is held invalid or unenforceable, such invalidity or enforceability will not affect the other provisions of these Terms, which will remain in full force and effect, and the invalid or unenforceable portion will be given effect to the greatest extent possible. The failure of a party to require performance of any provision will not affect that party’s right to require performance at any time thereafter, nor will a waiver of any breach or default of these Terms or any provision of these Terms constitute a waiver of any subsequent breach or default or a waiver of the provision itself.

api/gateway/BUILD.bazel

@@ -0,0 +1,41 @@
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "gateway.go",
+        "log.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/api/gateway",
+    visibility = [
+        "//beacon-chain:__subpackages__",
+        "//validator:__subpackages__",
+    ],
+    deps = [
+        "//api/gateway/apimiddleware:go_default_library",
+        "//runtime:go_default_library",
+        "@com_github_gorilla_mux//:go_default_library",
+        "@com_github_grpc_ecosystem_grpc_gateway_v2//runtime:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
+        "@com_github_rs_cors//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+        "@org_golang_google_grpc//:go_default_library",
+        "@org_golang_google_grpc//connectivity:go_default_library",
+        "@org_golang_google_grpc//credentials:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["gateway_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//api/gateway/apimiddleware:go_default_library",
+        "//cmd/beacon-chain/flags:go_default_library",
+        "//testing/assert:go_default_library",
+        "//testing/require:go_default_library",
+        "@com_github_gorilla_mux//:go_default_library",
+        "@com_github_sirupsen_logrus//hooks/test:go_default_library",
+        "@com_github_urfave_cli_v2//:go_default_library",
+    ],
+)

api/gateway/apimiddleware/BUILD.bazel

@@ -0,0 +1,40 @@
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "api_middleware.go",
+        "log.go",
+        "param_handling.go",
+        "process_field.go",
+        "process_request.go",
+        "structs.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/api/gateway/apimiddleware",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//api/grpc:go_default_library",
+        "//encoding/bytesutil:go_default_library",
+        "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
+        "@com_github_gorilla_mux//:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+        "@com_github_wealdtech_go_bytesutil//:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "param_handling_test.go",
+        "process_request_test.go",
+    ],
+    embed = [":go_default_library"],
+    deps = [
+        "//api/grpc:go_default_library",
+        "//testing/assert:go_default_library",
+        "//testing/require:go_default_library",
+        "@com_github_gorilla_mux//:go_default_library",
+        "@com_github_sirupsen_logrus//hooks/test:go_default_library",
+    ],
+)

api/gateway/apimiddleware/api_middleware.go

@@ -0,0 +1,233 @@
+package apimiddleware
+
+import (
+	"net/http"
+	"reflect"
+
+	"github.com/gorilla/mux"
+)
+
+// ApiProxyMiddleware is a proxy between an Ethereum consensus API HTTP client and grpc-gateway.
+// The purpose of the proxy is to handle HTTP requests and gRPC responses in such a way that:
+//   - Ethereum consensus API requests can be handled by grpc-gateway correctly
+//   - gRPC responses can be returned as spec-compliant Ethereum consensus API responses
+type ApiProxyMiddleware struct {
+	GatewayAddress  string
+	EndpointCreator EndpointFactory
+}
+
+// EndpointFactory is responsible for creating new instances of Endpoint values.
+type EndpointFactory interface {
+	Create(path string) (*Endpoint, error)
+	Paths() []string
+	IsNil() bool
+}
+
+// Endpoint is a representation of an API HTTP endpoint that should be proxied by the middleware.
+type Endpoint struct {
+	Path               string          // The path of the HTTP endpoint.
+	GetResponse        interface{}     // The struct corresponding to the JSON structure used in a GET response.
+	PostRequest        interface{}     // The struct corresponding to the JSON structure used in a POST request.
+	PostResponse       interface{}     // The struct corresponding to the JSON structure used in a POST response.
+	RequestURLLiterals []string        // Names of URL parameters that should not be base64-encoded.
+	RequestQueryParams []QueryParam    // Query parameters of the request.
+	Err                ErrorJson       // The struct corresponding to the error that should be returned in case of a request failure.
+	Hooks              HookCollection  // A collection of functions that can be invoked at various stages of the request/response cycle.
+	CustomHandlers     []CustomHandler // Functions that will be executed instead of the default request/response behaviour.
+}
+
+// RunDefault expresses whether the default processing logic should be carried out after running a pre hook.
+type RunDefault bool
+
+// DefaultEndpoint returns an Endpoint with default configuration, e.g. DefaultErrorJson for error handling.
+func DefaultEndpoint() Endpoint {
+	return Endpoint{
+		Err: &DefaultErrorJson{},
+	}
+}
+
+// QueryParam represents a single query parameter's metadata.
+type QueryParam struct {
+	Name string
+	Hex  bool
+	Enum bool
+}
+
+// CustomHandler is a function that can be invoked at the very beginning of the request,
+// essentially replacing the whole default request/response logic with custom logic for a specific endpoint.
+type CustomHandler = func(m *ApiProxyMiddleware, endpoint Endpoint, w http.ResponseWriter, req *http.Request) (handled bool)
+
+// HookCollection contains hooks that can be used to amend the default request/response cycle with custom logic for a specific endpoint.
+type HookCollection struct {
+	OnPreDeserializeRequestBodyIntoContainer      func(endpoint *Endpoint, w http.ResponseWriter, req *http.Request) (RunDefault, ErrorJson)
+	OnPostDeserializeRequestBodyIntoContainer     func(endpoint *Endpoint, w http.ResponseWriter, req *http.Request) ErrorJson
+	OnPreDeserializeGrpcResponseBodyIntoContainer func([]byte, interface{}) (RunDefault, ErrorJson)
+	OnPreSerializeMiddlewareResponseIntoJson      func(interface{}) (RunDefault, []byte, ErrorJson)
+}
+
+// fieldProcessor applies the processing function f to a value when the tag is present on the field.
+type fieldProcessor struct {
+	tag string
+	f   func(value reflect.Value) error
+}
+
+// Run starts the proxy, registering all proxy endpoints.
+func (m *ApiProxyMiddleware) Run(gatewayRouter *mux.Router) {
+	for _, path := range m.EndpointCreator.Paths() {
+		m.handleApiPath(gatewayRouter, path, m.EndpointCreator)
+	}
+}
+
+func (m *ApiProxyMiddleware) handleApiPath(gatewayRouter *mux.Router, path string, endpointFactory EndpointFactory) {
+	gatewayRouter.HandleFunc(path, func(w http.ResponseWriter, req *http.Request) {
+		endpoint, err := endpointFactory.Create(path)
+		if err != nil {
+			errJson := InternalServerErrorWithMessage(err, "could not create endpoint")
+			WriteError(w, errJson, nil)
+		}
+
+		for _, handler := range endpoint.CustomHandlers {
+			if handler(m, *endpoint, w, req) {
+				return
+			}
+		}
+
+		if req.Method == "POST" {
+			if errJson := deserializeRequestBodyIntoContainerWrapped(endpoint, req, w); errJson != nil {
+				WriteError(w, errJson, nil)
+				return
+			}
+
+			if errJson := ProcessRequestContainerFields(endpoint.PostRequest); errJson != nil {
+				WriteError(w, errJson, nil)
+				return
+			}
+			if errJson := SetRequestBodyToRequestContainer(endpoint.PostRequest, req); errJson != nil {
+				WriteError(w, errJson, nil)
+				return
+			}
+		}
+
+		if errJson := m.PrepareRequestForProxying(*endpoint, req); errJson != nil {
+			WriteError(w, errJson, nil)
+			return
+		}
+		grpcResp, errJson := ProxyRequest(req)
+		if errJson != nil {
+			WriteError(w, errJson, nil)
+			return
+		}
+		grpcRespBody, errJson := ReadGrpcResponseBody(grpcResp.Body)
+		if errJson != nil {
+			WriteError(w, errJson, nil)
+			return
+		}
+
+		var respJson []byte
+		if !GrpcResponseIsEmpty(grpcRespBody) {
+			if errJson := DeserializeGrpcResponseBodyIntoErrorJson(endpoint.Err, grpcRespBody); errJson != nil {
+				WriteError(w, errJson, nil)
+				return
+			}
+			if endpoint.Err.Msg() != "" {
+				HandleGrpcResponseError(endpoint.Err, grpcResp, w)
+				return
+			}
+
+			var resp interface{}
+			if req.Method == "GET" {
+				resp = endpoint.GetResponse
+			} else {
+				resp = endpoint.PostResponse
+			}
+			if errJson := deserializeGrpcResponseBodyIntoContainerWrapped(endpoint, grpcRespBody, resp); errJson != nil {
+				WriteError(w, errJson, nil)
+				return
+			}
+			if errJson := ProcessMiddlewareResponseFields(resp); errJson != nil {
+				WriteError(w, errJson, nil)
+				return
+			}
+
+			respJson, errJson = serializeMiddlewareResponseIntoJsonWrapped(endpoint, respJson, resp)
+			if errJson != nil {
+				WriteError(w, errJson, nil)
+				return
+			}
+		}
+
+		if errJson := WriteMiddlewareResponseHeadersAndBody(grpcResp, respJson, w); errJson != nil {
+			WriteError(w, errJson, nil)
+			return
+		}
+		if errJson := Cleanup(grpcResp.Body); errJson != nil {
+			WriteError(w, errJson, nil)
+			return
+		}
+	})
+}
+
+func deserializeRequestBodyIntoContainerWrapped(endpoint *Endpoint, req *http.Request, w http.ResponseWriter) ErrorJson {
+	runDefault := true
+	if endpoint.Hooks.OnPreDeserializeRequestBodyIntoContainer != nil {
+		run, errJson := endpoint.Hooks.OnPreDeserializeRequestBodyIntoContainer(endpoint, w, req)
+		if errJson != nil {
+			return errJson
+		}
+		if !run {
+			runDefault = false
+		}
+	}
+	if runDefault {
+		if errJson := DeserializeRequestBodyIntoContainer(req.Body, endpoint.PostRequest); errJson != nil {
+			return errJson
+		}
+	}
+	if endpoint.Hooks.OnPostDeserializeRequestBodyIntoContainer != nil {
+		if errJson := endpoint.Hooks.OnPostDeserializeRequestBodyIntoContainer(endpoint, w, req); errJson != nil {
+			return errJson
+		}
+	}
+	return nil
+}
+
+func deserializeGrpcResponseBodyIntoContainerWrapped(endpoint *Endpoint, grpcResponseBody []byte, resp interface{}) ErrorJson {
+	runDefault := true
+	if endpoint.Hooks.OnPreDeserializeGrpcResponseBodyIntoContainer != nil {
+		run, errJson := endpoint.Hooks.OnPreDeserializeGrpcResponseBodyIntoContainer(grpcResponseBody, resp)
+		if errJson != nil {
+			return errJson
+		}
+		if !run {
+			runDefault = false
+		}
+	}
+	if runDefault {
+		if errJson := DeserializeGrpcResponseBodyIntoContainer(grpcResponseBody, resp); errJson != nil {
+			return errJson
+		}
+	}
+	return nil
+}
+
+func serializeMiddlewareResponseIntoJsonWrapped(endpoint *Endpoint, respJson []byte, resp interface{}) ([]byte, ErrorJson) {
+	runDefault := true
+	var errJson ErrorJson
+	if endpoint.Hooks.OnPreSerializeMiddlewareResponseIntoJson != nil {
+		var run RunDefault
+		run, respJson, errJson = endpoint.Hooks.OnPreSerializeMiddlewareResponseIntoJson(resp)
+		if errJson != nil {
+			return nil, errJson
+		}
+		if !run {
+			runDefault = false
+		}
+	}
+	if runDefault {
+		respJson, errJson = SerializeMiddlewareResponseIntoJson(resp)
+		if errJson != nil {
+			return nil, errJson
+		}
+	}
+	return respJson, nil
+}

api/gateway/apimiddleware/log.go

@@ -0,0 +1,5 @@
+package apimiddleware
+
+import "github.com/sirupsen/logrus"
+
+var log = logrus.WithField("prefix", "apimiddleware")

api/gateway/apimiddleware/param_handling.go

@@ -0,0 +1,103 @@
+package apimiddleware
+
+import (
+	"encoding/base64"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/gorilla/mux"
+	butil "github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	"github.com/wealdtech/go-bytesutil"
+)
+
+// HandleURLParameters processes URL parameters, allowing parameterized URLs to be safely and correctly proxied to grpc-gateway.
+func HandleURLParameters(url string, req *http.Request, literals []string) ErrorJson {
+	segments := strings.Split(url, "/")
+
+segmentsLoop:
+	for i, s := range segments {
+		// We only care about segments which are parameterized.
+		if isRequestParam(s) {
+			// Don't do anything with parameters which should be forwarded literally to gRPC.
+			for _, l := range literals {
+				if s == "{"+l+"}" {
+					continue segmentsLoop
+				}
+			}
+
+			routeVar := mux.Vars(req)[s[1:len(s)-1]]
+			bRouteVar := []byte(routeVar)
+			if butil.IsHex(bRouteVar) {
+				var err error
+				bRouteVar, err = bytesutil.FromHexString(string(bRouteVar))
+				if err != nil {
+					return InternalServerErrorWithMessage(err, "could not process URL parameter")
+				}
+			}
+			// Converting hex to base64 may result in a value which malforms the URL.
+			// We use URLEncoding to safely escape such values.
+			base64RouteVar := base64.URLEncoding.EncodeToString(bRouteVar)
+
+			// Merge segments back into the full URL.
+			splitPath := strings.Split(req.URL.Path, "/")
+			splitPath[i] = base64RouteVar
+			req.URL.Path = strings.Join(splitPath, "/")
+		}
+	}
+	return nil
+}
+
+// HandleQueryParameters processes query parameters, allowing them to be safely and correctly proxied to grpc-gateway.
+func HandleQueryParameters(req *http.Request, params []QueryParam) ErrorJson {
+	queryParams := req.URL.Query()
+
+	normalizeQueryValues(queryParams)
+
+	for key, vals := range queryParams {
+		for _, p := range params {
+			if key == p.Name {
+				if p.Hex {
+					queryParams.Del(key)
+					for _, v := range vals {
+						b := []byte(v)
+						if butil.IsHex(b) {
+							var err error
+							b, err = bytesutil.FromHexString(v)
+							if err != nil {
+								return InternalServerErrorWithMessage(err, "could not process query parameter")
+							}
+						}
+						queryParams.Add(key, base64.URLEncoding.EncodeToString(b))
+					}
+				}
+				if p.Enum {
+					queryParams.Del(key)
+					for _, v := range vals {
+						// gRPC expects uppercase enum values.
+						queryParams.Add(key, strings.ToUpper(v))
+					}
+				}
+			}
+		}
+	}
+	req.URL.RawQuery = queryParams.Encode()
+	return nil
+}
+
+// isRequestParam verifies whether the passed string is a request parameter.
+// Request parameters are enclosed in { and }.
+func isRequestParam(s string) bool {
+	return len(s) > 2 && s[0] == '{' && s[len(s)-1] == '}'
+}
+
+func normalizeQueryValues(queryParams url.Values) {
+	// Replace comma-separated values with individual values.
+	for key, vals := range queryParams {
+		splitVals := make([]string, 0)
+		for _, v := range vals {
+			splitVals = append(splitVals, strings.Split(v, ",")...)
+		}
+		queryParams[key] = splitVals
+	}
+}

api/gateway/apimiddleware/param_handling_test.go

@@ -0,0 +1,124 @@
+package apimiddleware
+
+import (
+	"bytes"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gorilla/mux"
+	"github.com/prysmaticlabs/prysm/testing/assert"
+	"github.com/prysmaticlabs/prysm/testing/require"
+)
+
+func TestHandleURLParameters(t *testing.T) {
+	var body bytes.Buffer
+
+	t.Run("no_params", func(t *testing.T) {
+		request := httptest.NewRequest("GET", "http://foo.example/bar", &body)
+
+		errJson := HandleURLParameters("/not_param", request, []string{})
+		require.Equal(t, true, errJson == nil)
+		assert.Equal(t, "/bar", request.URL.Path)
+	})
+
+	t.Run("with_params", func(t *testing.T) {
+		muxVars := make(map[string]string)
+		muxVars["bar_param"] = "bar"
+		muxVars["quux_param"] = "quux"
+		request := httptest.NewRequest("GET", "http://foo.example/bar/baz/quux", &body)
+		request = mux.SetURLVars(request, muxVars)
+
+		errJson := HandleURLParameters("/{bar_param}/not_param/{quux_param}", request, []string{})
+		require.Equal(t, true, errJson == nil)
+		assert.Equal(t, "/YmFy/baz/cXV1eA==", request.URL.Path)
+	})
+
+	t.Run("with_literal", func(t *testing.T) {
+		muxVars := make(map[string]string)
+		muxVars["bar_param"] = "bar"
+		request := httptest.NewRequest("GET", "http://foo.example/bar/baz", &body)
+		request = mux.SetURLVars(request, muxVars)
+
+		errJson := HandleURLParameters("/{bar_param}/not_param/", request, []string{"bar_param"})
+		require.Equal(t, true, errJson == nil)
+		assert.Equal(t, "/bar/baz", request.URL.Path)
+	})
+
+	t.Run("with_hex", func(t *testing.T) {
+		muxVars := make(map[string]string)
+		muxVars["hex_param"] = "0x626172"
+		request := httptest.NewRequest("GET", "http://foo.example/0x626172/baz", &body)
+		request = mux.SetURLVars(request, muxVars)
+
+		errJson := HandleURLParameters("/{hex_param}/not_param/", request, []string{})
+		require.Equal(t, true, errJson == nil)
+		assert.Equal(t, "/YmFy/baz", request.URL.Path)
+	})
+}
+
+func TestHandleQueryParameters(t *testing.T) {
+	var body bytes.Buffer
+
+	t.Run("regular_params", func(t *testing.T) {
+		request := httptest.NewRequest("GET", "http://foo.example?bar=bar&baz=baz", &body)
+
+		errJson := HandleQueryParameters(request, []QueryParam{{Name: "bar"}, {Name: "baz"}})
+		require.Equal(t, true, errJson == nil)
+		query := request.URL.Query()
+		v, ok := query["bar"]
+		require.Equal(t, true, ok, "query param not found")
+		require.Equal(t, 1, len(v), "wrong number of query param values")
+		assert.Equal(t, "bar", v[0])
+		v, ok = query["baz"]
+		require.Equal(t, true, ok, "query param not found")
+		require.Equal(t, 1, len(v), "wrong number of query param values")
+		assert.Equal(t, "baz", v[0])
+	})
+
+	t.Run("hex_and_enum_params", func(t *testing.T) {
+		request := httptest.NewRequest("GET", "http://foo.example?hex=0x626172&baz=baz", &body)
+
+		errJson := HandleQueryParameters(request, []QueryParam{{Name: "hex", Hex: true}, {Name: "baz", Enum: true}})
+		require.Equal(t, true, errJson == nil)
+		query := request.URL.Query()
+		v, ok := query["hex"]
+		require.Equal(t, true, ok, "query param not found")
+		require.Equal(t, 1, len(v), "wrong number of query param values")
+		assert.Equal(t, "YmFy", v[0])
+		v, ok = query["baz"]
+		require.Equal(t, true, ok, "query param not found")
+		require.Equal(t, 1, len(v), "wrong number of query param values")
+		assert.Equal(t, "BAZ", v[0])
+	})
+}
+
+func TestIsRequestParam(t *testing.T) {
+	tests := []struct {
+		s string
+		b bool
+	}{
+		{"", false},
+		{"{", false},
+		{"}", false},
+		{"{}", false},
+		{"{x}", true},
+		{"{very_long_parameter_name_with_underscores}", true},
+	}
+	for _, tt := range tests {
+		b := isRequestParam(tt.s)
+		assert.Equal(t, tt.b, b)
+	}
+}
+
+func TestNormalizeQueryValues(t *testing.T) {
+	input := make(map[string][]string)
+	input["key"] = []string{"value1", "value2,value3,value4", "value5"}
+
+	normalizeQueryValues(input)
+	require.Equal(t, 5, len(input["key"]))
+	assert.Equal(t, "value1", input["key"][0])
+	assert.Equal(t, "value2", input["key"][1])
+	assert.Equal(t, "value3", input["key"][2])
+	assert.Equal(t, "value4", input["key"][3])
+	assert.Equal(t, "value5", input["key"][4])
+}

api/gateway/apimiddleware/process_field.go

@@ -0,0 +1,108 @@
+package apimiddleware
+
+import (
+	"encoding/base64"
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/pkg/errors"
+	"github.com/wealdtech/go-bytesutil"
+)
+
+// processField calls each processor function on any field that has the matching tag set.
+// It is a recursive function.
+func processField(s interface{}, processors []fieldProcessor) error {
+	kind := reflect.TypeOf(s).Kind()
+	if kind != reflect.Ptr && kind != reflect.Slice && kind != reflect.Array {
+		return fmt.Errorf("processing fields of kind '%v' is unsupported", kind)
+	}
+
+	t := reflect.TypeOf(s).Elem()
+	v := reflect.Indirect(reflect.ValueOf(s))
+
+	for i := 0; i < t.NumField(); i++ {
+		switch v.Field(i).Kind() {
+		case reflect.Slice:
+			sliceElem := t.Field(i).Type.Elem()
+			kind := sliceElem.Kind()
+			// Recursively process slices to struct pointers.
+			if kind == reflect.Ptr && sliceElem.Elem().Kind() == reflect.Struct {
+				for j := 0; j < v.Field(i).Len(); j++ {
+					if err := processField(v.Field(i).Index(j).Interface(), processors); err != nil {
+						return errors.Wrapf(err, "could not process field '%s'", t.Field(i).Name)
+					}
+				}
+			}
+			// Process each string in string slices.
+			if kind == reflect.String {
+				for _, proc := range processors {
+					_, hasTag := t.Field(i).Tag.Lookup(proc.tag)
+					if hasTag {
+						for j := 0; j < v.Field(i).Len(); j++ {
+							if err := proc.f(v.Field(i).Index(j)); err != nil {
+								return errors.Wrapf(err, "could not process field '%s'", t.Field(i).Name)
+							}
+						}
+					}
+				}
+
+			}
+		// Recursively process struct pointers.
+		case reflect.Ptr:
+			if v.Field(i).Elem().Kind() == reflect.Struct {
+				if err := processField(v.Field(i).Interface(), processors); err != nil {
+					return errors.Wrapf(err, "could not process field '%s'", t.Field(i).Name)
+				}
+			}
+		default:
+			field := t.Field(i)
+			for _, proc := range processors {
+				if _, hasTag := field.Tag.Lookup(proc.tag); hasTag {
+					if err := proc.f(v.Field(i)); err != nil {
+						return errors.Wrapf(err, "could not process field '%s'", t.Field(i).Name)
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func hexToBase64Processor(v reflect.Value) error {
+	b, err := bytesutil.FromHexString(v.String())
+	if err != nil {
+		return err
+	}
+	v.SetString(base64.StdEncoding.EncodeToString(b))
+	return nil
+}
+
+func base64ToHexProcessor(v reflect.Value) error {
+	if v.String() == "" {
+		return nil
+	}
+	b, err := base64.StdEncoding.DecodeString(v.String())
+	if err != nil {
+		return err
+	}
+	v.SetString(hexutil.Encode(b))
+	return nil
+}
+
+func enumToLowercaseProcessor(v reflect.Value) error {
+	v.SetString(strings.ToLower(v.String()))
+	return nil
+}
+
+func timeToUnixProcessor(v reflect.Value) error {
+	t, err := time.Parse(time.RFC3339, v.String())
+	if err != nil {
+		return err
+	}
+	v.SetString(strconv.FormatUint(uint64(t.Unix()), 10))
+	return nil
+}

api/gateway/apimiddleware/process_request.go

@@ -0,0 +1,225 @@
+package apimiddleware
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/prysmaticlabs/prysm/api/grpc"
+)
+
+// DeserializeRequestBodyIntoContainer deserializes the request's body into an endpoint-specific struct.
+func DeserializeRequestBodyIntoContainer(body io.Reader, requestContainer interface{}) ErrorJson {
+	if err := json.NewDecoder(body).Decode(&requestContainer); err != nil {
+		return InternalServerErrorWithMessage(err, "could not decode request body")
+	}
+	return nil
+}
+
+// ProcessRequestContainerFields processes fields of an endpoint-specific container according to field tags.
+func ProcessRequestContainerFields(requestContainer interface{}) ErrorJson {
+	if err := processField(requestContainer, []fieldProcessor{
+		{
+			tag: "hex",
+			f:   hexToBase64Processor,
+		},
+	}); err != nil {
+		return InternalServerErrorWithMessage(err, "could not process request data")
+	}
+	return nil
+}
+
+// SetRequestBodyToRequestContainer makes the endpoint-specific container the new body of the request.
+func SetRequestBodyToRequestContainer(requestContainer interface{}, req *http.Request) ErrorJson {
+	// Serialize the struct, which now includes a base64-encoded value, into JSON.
+	j, err := json.Marshal(requestContainer)
+	if err != nil {
+		return InternalServerErrorWithMessage(err, "could not marshal request")
+	}
+	// Set the body to the new JSON.
+	req.Body = ioutil.NopCloser(bytes.NewReader(j))
+	req.Header.Set("Content-Length", strconv.Itoa(len(j)))
+	req.ContentLength = int64(len(j))
+	return nil
+}
+
+// PrepareRequestForProxying applies additional logic to the request so that it can be correctly proxied to grpc-gateway.
+func (m *ApiProxyMiddleware) PrepareRequestForProxying(endpoint Endpoint, req *http.Request) ErrorJson {
+	req.URL.Scheme = "http"
+	req.URL.Host = m.GatewayAddress
+	req.RequestURI = ""
+	if errJson := HandleURLParameters(endpoint.Path, req, endpoint.RequestURLLiterals); errJson != nil {
+		return errJson
+	}
+	if errJson := HandleQueryParameters(req, endpoint.RequestQueryParams); errJson != nil {
+		return errJson
+	}
+	// We have to add the prefix after handling parameters because adding the prefix changes URL segment indexing.
+	req.URL.Path = "/internal" + req.URL.Path
+	return nil
+}
+
+// ProxyRequest proxies the request to grpc-gateway.
+func ProxyRequest(req *http.Request) (*http.Response, ErrorJson) {
+	// We do not use http.DefaultClient because it does not have any timeout.
+	netClient := &http.Client{Timeout: time.Minute * 2}
+	grpcResp, err := netClient.Do(req)
+	if err != nil {
+		return nil, InternalServerErrorWithMessage(err, "could not proxy request")
+	}
+	if grpcResp == nil {
+		return nil, &DefaultErrorJson{Message: "nil response from gRPC-gateway", Code: http.StatusInternalServerError}
+	}
+	return grpcResp, nil
+}
+
+// ReadGrpcResponseBody reads the body from the grpc-gateway's response.
+func ReadGrpcResponseBody(r io.Reader) ([]byte, ErrorJson) {
+	body, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, InternalServerErrorWithMessage(err, "could not read response body")
+	}
+	return body, nil
+}
+
+// DeserializeGrpcResponseBodyIntoErrorJson deserializes the body from the grpc-gateway's response into an error struct.
+// The struct can be later examined to check if the request resulted in an error.
+func DeserializeGrpcResponseBodyIntoErrorJson(errJson ErrorJson, body []byte) ErrorJson {
+	if err := json.Unmarshal(body, errJson); err != nil {
+		return InternalServerErrorWithMessage(err, "could not unmarshal error")
+	}
+	return nil
+}
+
+// HandleGrpcResponseError acts on an error that resulted from a grpc-gateway's response.
+func HandleGrpcResponseError(errJson ErrorJson, resp *http.Response, w http.ResponseWriter) {
+	// Something went wrong, but the request completed, meaning we can write headers and the error message.
+	for h, vs := range resp.Header {
+		for _, v := range vs {
+			w.Header().Set(h, v)
+		}
+	}
+	// Set code to HTTP code because unmarshalled body contained gRPC code.
+	errJson.SetCode(resp.StatusCode)
+	WriteError(w, errJson, resp.Header)
+}
+
+// GrpcResponseIsEmpty determines whether the grpc-gateway's response body contains no data.
+func GrpcResponseIsEmpty(grpcResponseBody []byte) bool {
+	return len(grpcResponseBody) == 0 || string(grpcResponseBody) == "{}"
+}
+
+// DeserializeGrpcResponseBodyIntoContainer deserializes the grpc-gateway's response body into an endpoint-specific struct.
+func DeserializeGrpcResponseBodyIntoContainer(body []byte, responseContainer interface{}) ErrorJson {
+	if err := json.Unmarshal(body, &responseContainer); err != nil {
+		return InternalServerErrorWithMessage(err, "could not unmarshal response")
+	}
+	return nil
+}
+
+// ProcessMiddlewareResponseFields processes fields of an endpoint-specific container according to field tags.
+func ProcessMiddlewareResponseFields(responseContainer interface{}) ErrorJson {
+	if err := processField(responseContainer, []fieldProcessor{
+		{
+			tag: "hex",
+			f:   base64ToHexProcessor,
+		},
+		{
+			tag: "enum",
+			f:   enumToLowercaseProcessor,
+		},
+		{
+			tag: "time",
+			f:   timeToUnixProcessor,
+		},
+	}); err != nil {
+		return InternalServerErrorWithMessage(err, "could not process response data")
+	}
+	return nil
+}
+
+// SerializeMiddlewareResponseIntoJson serializes the endpoint-specific response struct into a JSON representation.
+func SerializeMiddlewareResponseIntoJson(responseContainer interface{}) (jsonResponse []byte, errJson ErrorJson) {
+	j, err := json.Marshal(responseContainer)
+	if err != nil {
+		return nil, InternalServerErrorWithMessage(err, "could not marshal response")
+	}
+	return j, nil
+}
+
+// WriteMiddlewareResponseHeadersAndBody populates headers and the body of the final response.
+func WriteMiddlewareResponseHeadersAndBody(grpcResp *http.Response, responseJson []byte, w http.ResponseWriter) ErrorJson {
+	var statusCodeHeader string
+	for h, vs := range grpcResp.Header {
+		// We don't want to expose any gRPC metadata in the HTTP response, so we skip forwarding metadata headers.
+		if strings.HasPrefix(h, "Grpc-Metadata") {
+			if h == "Grpc-Metadata-"+grpc.HttpCodeMetadataKey {
+				statusCodeHeader = vs[0]
+			}
+		} else {
+			for _, v := range vs {
+				w.Header().Set(h, v)
+			}
+		}
+	}
+	if !GrpcResponseIsEmpty(responseJson) {
+		w.Header().Set("Content-Length", strconv.Itoa(len(responseJson)))
+		if statusCodeHeader != "" {
+			code, err := strconv.Atoi(statusCodeHeader)
+			if err != nil {
+				return InternalServerErrorWithMessage(err, "could not parse status code")
+			}
+			w.WriteHeader(code)
+		} else {
+			w.WriteHeader(grpcResp.StatusCode)
+		}
+		if _, err := io.Copy(w, ioutil.NopCloser(bytes.NewReader(responseJson))); err != nil {
+			return InternalServerErrorWithMessage(err, "could not write response message")
+		}
+	} else {
+		w.Header().Set("Content-Length", "0")
+		w.WriteHeader(grpcResp.StatusCode)
+	}
+	return nil
+}
+
+// WriteError writes the error by manipulating headers and the body of the final response.
+func WriteError(w http.ResponseWriter, errJson ErrorJson, responseHeader http.Header) {
+	// Include custom error in the error JSON.
+	if responseHeader != nil {
+		customError, ok := responseHeader["Grpc-Metadata-"+grpc.CustomErrorMetadataKey]
+		if ok {
+			// Assume header has only one value and read the 0 index.
+			if err := json.Unmarshal([]byte(customError[0]), errJson); err != nil {
+				log.WithError(err).Error("Could not unmarshal custom error message")
+				return
+			}
+		}
+	}
+
+	j, err := json.Marshal(errJson)
+	if err != nil {
+		log.WithError(err).Error("Could not marshal error message")
+		return
+	}
+
+	w.Header().Set("Content-Length", strconv.Itoa(len(j)))
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(errJson.StatusCode())
+	if _, err := io.Copy(w, ioutil.NopCloser(bytes.NewReader(j))); err != nil {
+		log.WithError(err).Error("Could not write error message")
+	}
+}
+
+// Cleanup performs final cleanup on the initial response from grpc-gateway.
+func Cleanup(grpcResponseBody io.ReadCloser) ErrorJson {
+	if err := grpcResponseBody.Close(); err != nil {
+		return InternalServerErrorWithMessage(err, "could not close response body")
+	}
+	return nil
+}

api/gateway/apimiddleware/process_request_test.go

@@ -0,0 +1,414 @@
+package apimiddleware
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/api/grpc"
+	"github.com/prysmaticlabs/prysm/testing/assert"
+	"github.com/prysmaticlabs/prysm/testing/require"
+	"github.com/sirupsen/logrus/hooks/test"
+)
+
+type testRequestContainer struct {
+	TestString    string
+	TestHexString string `hex:"true"`
+}
+
+func defaultRequestContainer() *testRequestContainer {
+	return &testRequestContainer{
+		TestString:    "test string",
+		TestHexString: "0x666F6F", // hex encoding of "foo"
+	}
+}
+
+type testResponseContainer struct {
+	TestString string
+	TestHex    string `hex:"true"`
+	TestEnum   string `enum:"true"`
+	TestTime   string `time:"true"`
+}
+
+func defaultResponseContainer() *testResponseContainer {
+	return &testResponseContainer{
+		TestString: "test string",
+		TestHex:    "Zm9v", // base64 encoding of "foo"
+		TestEnum:   "Test Enum",
+		TestTime:   "2006-01-02T15:04:05Z",
+	}
+}
+
+type testErrorJson struct {
+	Message     string
+	Code        int
+	CustomField string
+}
+
+// StatusCode returns the error's underlying error code.
+func (e *testErrorJson) StatusCode() int {
+	return e.Code
+}
+
+// Msg returns the error's underlying message.
+func (e *testErrorJson) Msg() string {
+	return e.Message
+}
+
+// SetCode sets the error's underlying error code.
+func (e *testErrorJson) SetCode(code int) {
+	e.Code = code
+}
+
+func TestDeserializeRequestBodyIntoContainer(t *testing.T) {
+	t.Run("ok", func(t *testing.T) {
+		var bodyJson bytes.Buffer
+		err := json.NewEncoder(&bodyJson).Encode(defaultRequestContainer())
+		require.NoError(t, err)
+
+		container := &testRequestContainer{}
+		errJson := DeserializeRequestBodyIntoContainer(&bodyJson, container)
+		require.Equal(t, true, errJson == nil)
+		assert.Equal(t, "test string", container.TestString)
+	})
+
+	t.Run("error", func(t *testing.T) {
+		var bodyJson bytes.Buffer
+		bodyJson.Write([]byte("foo"))
+		errJson := DeserializeRequestBodyIntoContainer(&bodyJson, &testRequestContainer{})
+		require.NotNil(t, errJson)
+		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not decode request body"))
+		assert.Equal(t, http.StatusInternalServerError, errJson.StatusCode())
+	})
+}
+
+func TestProcessRequestContainerFields(t *testing.T) {
+	t.Run("ok", func(t *testing.T) {
+		container := defaultRequestContainer()
+
+		errJson := ProcessRequestContainerFields(container)
+		require.Equal(t, true, errJson == nil)
+		assert.Equal(t, "Zm9v", container.TestHexString)
+	})
+
+	t.Run("error", func(t *testing.T) {
+		errJson := ProcessRequestContainerFields("foo")
+		require.NotNil(t, errJson)
+		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not process request data"))
+		assert.Equal(t, http.StatusInternalServerError, errJson.StatusCode())
+	})
+}
+
+func TestSetRequestBodyToRequestContainer(t *testing.T) {
+	var body bytes.Buffer
+	request := httptest.NewRequest("GET", "http://foo.example", &body)
+
+	errJson := SetRequestBodyToRequestContainer(defaultRequestContainer(), request)
+	require.Equal(t, true, errJson == nil)
+	container := &testRequestContainer{}
+	require.NoError(t, json.NewDecoder(request.Body).Decode(container))
+	assert.Equal(t, "test string", container.TestString)
+	contentLengthHeader, ok := request.Header["Content-Length"]
+	require.Equal(t, true, ok)
+	require.Equal(t, 1, len(contentLengthHeader), "wrong number of header values")
+	assert.Equal(t, "55", contentLengthHeader[0])
+	assert.Equal(t, int64(55), request.ContentLength)
+}
+
+func TestPrepareRequestForProxying(t *testing.T) {
+	middleware := &ApiProxyMiddleware{
+		GatewayAddress: "http://gateway.example",
+	}
+	// We will set some params to make the request more interesting.
+	endpoint := Endpoint{
+		Path:               "/{url_param}",
+		RequestURLLiterals: []string{"url_param"},
+		RequestQueryParams: []QueryParam{{Name: "query_param"}},
+	}
+	var body bytes.Buffer
+	request := httptest.NewRequest("GET", "http://foo.example?query_param=bar", &body)
+
+	errJson := middleware.PrepareRequestForProxying(endpoint, request)
+	require.Equal(t, true, errJson == nil)
+	assert.Equal(t, "http", request.URL.Scheme)
+	assert.Equal(t, middleware.GatewayAddress, request.URL.Host)
+	assert.Equal(t, "", request.RequestURI)
+}
+
+func TestReadGrpcResponseBody(t *testing.T) {
+	var b bytes.Buffer
+	b.Write([]byte("foo"))
+
+	body, jsonErr := ReadGrpcResponseBody(&b)
+	require.Equal(t, true, jsonErr == nil)
+	assert.Equal(t, "foo", string(body))
+}
+
+func TestDeserializeGrpcResponseBodyIntoErrorJson(t *testing.T) {
+	t.Run("ok", func(t *testing.T) {
+		e := &testErrorJson{
+			Message: "foo",
+			Code:    500,
+		}
+		body, err := json.Marshal(e)
+		require.NoError(t, err)
+
+		eToDeserialize := &testErrorJson{}
+		errJson := DeserializeGrpcResponseBodyIntoErrorJson(eToDeserialize, body)
+		require.Equal(t, true, errJson == nil)
+		assert.Equal(t, "foo", eToDeserialize.Msg())
+		assert.Equal(t, 500, eToDeserialize.StatusCode())
+	})
+
+	t.Run("error", func(t *testing.T) {
+		errJson := DeserializeGrpcResponseBodyIntoErrorJson(nil, nil)
+		require.NotNil(t, errJson)
+		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not unmarshal error"))
+	})
+}
+
+func TestHandleGrpcResponseError(t *testing.T) {
+	response := &http.Response{
+		StatusCode: 400,
+		Header: http.Header{
+			"Foo": []string{"foo"},
+			"Bar": []string{"bar"},
+		},
+	}
+	writer := httptest.NewRecorder()
+	errJson := &testErrorJson{
+		Message: "foo",
+		Code:    500,
+	}
+
+	HandleGrpcResponseError(errJson, response, writer)
+	v, ok := writer.Header()["Foo"]
+	require.Equal(t, true, ok, "header not found")
+	require.Equal(t, 1, len(v), "wrong number of header values")
+	assert.Equal(t, "foo", v[0])
+	v, ok = writer.Header()["Bar"]
+	require.Equal(t, true, ok, "header not found")
+	require.Equal(t, 1, len(v), "wrong number of header values")
+	assert.Equal(t, "bar", v[0])
+	assert.Equal(t, 400, errJson.StatusCode())
+}
+
+func TestGrpcResponseIsEmpty(t *testing.T) {
+	t.Run("nil", func(t *testing.T) {
+		assert.Equal(t, true, GrpcResponseIsEmpty(nil))
+	})
+	t.Run("empty_slice", func(t *testing.T) {
+		assert.Equal(t, true, GrpcResponseIsEmpty(make([]byte, 0)))
+	})
+	t.Run("empty_brackets", func(t *testing.T) {
+		assert.Equal(t, true, GrpcResponseIsEmpty([]byte("{}")))
+	})
+	t.Run("non_empty", func(t *testing.T) {
+		assert.Equal(t, false, GrpcResponseIsEmpty([]byte("{\"foo\":\"bar\"})")))
+	})
+}
+
+func TestDeserializeGrpcResponseBodyIntoContainer(t *testing.T) {
+	t.Run("ok", func(t *testing.T) {
+		body, err := json.Marshal(defaultRequestContainer())
+		require.NoError(t, err)
+
+		container := &testRequestContainer{}
+		errJson := DeserializeGrpcResponseBodyIntoContainer(body, container)
+		require.Equal(t, true, errJson == nil)
+		assert.Equal(t, "test string", container.TestString)
+	})
+
+	t.Run("error", func(t *testing.T) {
+		var bodyJson bytes.Buffer
+		bodyJson.Write([]byte("foo"))
+		errJson := DeserializeGrpcResponseBodyIntoContainer(bodyJson.Bytes(), &testRequestContainer{})
+		require.NotNil(t, errJson)
+		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not unmarshal response"))
+		assert.Equal(t, http.StatusInternalServerError, errJson.StatusCode())
+	})
+}
+
+func TestProcessMiddlewareResponseFields(t *testing.T) {
+	t.Run("Ok", func(t *testing.T) {
+		container := defaultResponseContainer()
+
+		errJson := ProcessMiddlewareResponseFields(container)
+		require.Equal(t, true, errJson == nil)
+		assert.Equal(t, "0x666f6f", container.TestHex)
+		assert.Equal(t, "test enum", container.TestEnum)
+		assert.Equal(t, "1136214245", container.TestTime)
+	})
+
+	t.Run("error", func(t *testing.T) {
+		errJson := ProcessMiddlewareResponseFields("foo")
+		require.NotNil(t, errJson)
+		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not process response data"))
+		assert.Equal(t, http.StatusInternalServerError, errJson.StatusCode())
+	})
+}
+
+func TestSerializeMiddlewareResponseIntoJson(t *testing.T) {
+	container := defaultResponseContainer()
+	j, errJson := SerializeMiddlewareResponseIntoJson(container)
+	assert.Equal(t, true, errJson == nil)
+	cToDeserialize := &testResponseContainer{}
+	require.NoError(t, json.Unmarshal(j, cToDeserialize))
+	assert.Equal(t, "test string", cToDeserialize.TestString)
+}
+
+func TestWriteMiddlewareResponseHeadersAndBody(t *testing.T) {
+	t.Run("GET", func(t *testing.T) {
+		response := &http.Response{
+			Header: http.Header{
+				"Foo": []string{"foo"},
+				"Grpc-Metadata-" + grpc.HttpCodeMetadataKey: []string{"204"},
+			},
+		}
+		container := defaultResponseContainer()
+		responseJson, err := json.Marshal(container)
+		require.NoError(t, err)
+		writer := httptest.NewRecorder()
+		writer.Body = &bytes.Buffer{}
+
+		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
+		require.Equal(t, true, errJson == nil)
+		v, ok := writer.Header()["Foo"]
+		require.Equal(t, true, ok, "header not found")
+		require.Equal(t, 1, len(v), "wrong number of header values")
+		assert.Equal(t, "foo", v[0])
+		v, ok = writer.Header()["Content-Length"]
+		require.Equal(t, true, ok, "header not found")
+		require.Equal(t, 1, len(v), "wrong number of header values")
+		assert.Equal(t, "102", v[0])
+		assert.Equal(t, 204, writer.Code)
+		assert.DeepEqual(t, responseJson, writer.Body.Bytes())
+	})
+
+	t.Run("GET_no_grpc_status_code_header", func(t *testing.T) {
+		response := &http.Response{
+			Header:     http.Header{},
+			StatusCode: 204,
+		}
+		container := defaultResponseContainer()
+		responseJson, err := json.Marshal(container)
+		require.NoError(t, err)
+		writer := httptest.NewRecorder()
+
+		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
+		require.Equal(t, true, errJson == nil)
+		assert.Equal(t, 204, writer.Code)
+	})
+
+	t.Run("GET_invalid_status_code", func(t *testing.T) {
+		response := &http.Response{
+			Header: http.Header{},
+		}
+
+		// Set invalid status code.
+		response.Header["Grpc-Metadata-"+grpc.HttpCodeMetadataKey] = []string{"invalid"}
+
+		container := defaultResponseContainer()
+		responseJson, err := json.Marshal(container)
+		require.NoError(t, err)
+		writer := httptest.NewRecorder()
+
+		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
+		require.Equal(t, false, errJson == nil)
+		assert.Equal(t, true, strings.Contains(errJson.Msg(), "could not parse status code"))
+		assert.Equal(t, http.StatusInternalServerError, errJson.StatusCode())
+	})
+
+	t.Run("POST", func(t *testing.T) {
+		response := &http.Response{
+			Header:     http.Header{},
+			StatusCode: 204,
+		}
+		container := defaultResponseContainer()
+		responseJson, err := json.Marshal(container)
+		require.NoError(t, err)
+		writer := httptest.NewRecorder()
+
+		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
+		require.Equal(t, true, errJson == nil)
+		assert.Equal(t, 204, writer.Code)
+	})
+
+	t.Run("POST_with_response_body", func(t *testing.T) {
+		response := &http.Response{
+			Header:     http.Header{},
+			StatusCode: 204,
+		}
+		container := defaultResponseContainer()
+		responseJson, err := json.Marshal(container)
+		require.NoError(t, err)
+		writer := httptest.NewRecorder()
+		writer.Body = &bytes.Buffer{}
+
+		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
+		require.Equal(t, true, errJson == nil)
+		assert.Equal(t, 204, writer.Code)
+		assert.DeepEqual(t, responseJson, writer.Body.Bytes())
+	})
+
+	t.Run("POST_with_empty_json_body", func(t *testing.T) {
+		response := &http.Response{
+			Header:     http.Header{},
+			StatusCode: 204,
+		}
+		responseJson, err := json.Marshal(struct{}{})
+		require.NoError(t, err)
+		writer := httptest.NewRecorder()
+		writer.Body = &bytes.Buffer{}
+
+		errJson := WriteMiddlewareResponseHeadersAndBody(response, responseJson, writer)
+		require.Equal(t, true, errJson == nil)
+		assert.Equal(t, 204, writer.Code)
+		assert.DeepEqual(t, []byte(nil), writer.Body.Bytes())
+		assert.Equal(t, "0", writer.Header()["Content-Length"][0])
+	})
+}
+
+func TestWriteError(t *testing.T) {
+	t.Run("ok", func(t *testing.T) {
+		responseHeader := http.Header{
+			"Grpc-Metadata-" + grpc.CustomErrorMetadataKey: []string{"{\"CustomField\":\"bar\"}"},
+		}
+		errJson := &testErrorJson{
+			Message: "foo",
+			Code:    500,
+		}
+		writer := httptest.NewRecorder()
+		writer.Body = &bytes.Buffer{}
+
+		WriteError(writer, errJson, responseHeader)
+		v, ok := writer.Header()["Content-Length"]
+		require.Equal(t, true, ok, "header not found")
+		require.Equal(t, 1, len(v), "wrong number of header values")
+		assert.Equal(t, "48", v[0])
+		v, ok = writer.Header()["Content-Type"]
+		require.Equal(t, true, ok, "header not found")
+		require.Equal(t, 1, len(v), "wrong number of header values")
+		assert.Equal(t, "application/json", v[0])
+		assert.Equal(t, 500, writer.Code)
+		eDeserialize := &testErrorJson{}
+		require.NoError(t, json.Unmarshal(writer.Body.Bytes(), eDeserialize))
+		assert.Equal(t, "foo", eDeserialize.Message)
+		assert.Equal(t, 500, eDeserialize.Code)
+		assert.Equal(t, "bar", eDeserialize.CustomField)
+	})
+
+	t.Run("invalid_custom_error_header", func(t *testing.T) {
+		logHook := test.NewGlobal()
+
+		responseHeader := http.Header{
+			"Grpc-Metadata-" + grpc.CustomErrorMetadataKey: []string{"invalid"},
+		}
+
+		WriteError(httptest.NewRecorder(), &testErrorJson{}, responseHeader)
+		assert.LogsContain(t, logHook, "Could not unmarshal custom error message")
+	})
+}

api/gateway/apimiddleware/structs.go

@@ -0,0 +1,56 @@
+package apimiddleware
+
+import (
+	"net/http"
+
+	"github.com/pkg/errors"
+)
+
+// ---------------
+// Error handling.
+// ---------------
+
+// ErrorJson describes common functionality of all JSON error representations.
+type ErrorJson interface {
+	StatusCode() int
+	SetCode(code int)
+	Msg() string
+}
+
+// DefaultErrorJson is a JSON representation of a simple error value, containing only a message and an error code.
+type DefaultErrorJson struct {
+	Message string `json:"message"`
+	Code    int    `json:"code"`
+}
+
+// InternalServerErrorWithMessage returns a DefaultErrorJson with 500 code and a custom message.
+func InternalServerErrorWithMessage(err error, message string) *DefaultErrorJson {
+	e := errors.Wrapf(err, message)
+	return &DefaultErrorJson{
+		Message: e.Error(),
+		Code:    http.StatusInternalServerError,
+	}
+}
+
+// InternalServerError returns a DefaultErrorJson with 500 code.
+func InternalServerError(err error) *DefaultErrorJson {
+	return &DefaultErrorJson{
+		Message: err.Error(),
+		Code:    http.StatusInternalServerError,
+	}
+}
+
+// StatusCode returns the error's underlying error code.
+func (e *DefaultErrorJson) StatusCode() int {
+	return e.Code
+}
+
+// Msg returns the error's underlying message.
+func (e *DefaultErrorJson) Msg() string {
+	return e.Message
+}
+
+// SetCode sets the error's underlying error code.
+func (e *DefaultErrorJson) SetCode(code int) {
+	e.Code = code
+}

api/gateway/gateway.go

@@ -0,0 +1,281 @@
+// Package gateway defines a grpc-gateway server that serves HTTP-JSON traffic and acts a proxy between HTTP and gRPC.
+package gateway
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/gorilla/mux"
+	gwruntime "github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/api/gateway/apimiddleware"
+	"github.com/prysmaticlabs/prysm/runtime"
+	"github.com/rs/cors"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials"
+)
+
+var _ runtime.Service = (*Gateway)(nil)
+
+// PbMux serves grpc-gateway requests for selected patterns using registered protobuf handlers.
+type PbMux struct {
+	Registrations []PbHandlerRegistration // Protobuf registrations to be registered in Mux.
+	Patterns      []string                // URL patterns that will be handled by Mux.
+	Mux           *gwruntime.ServeMux     // The router that will be used for grpc-gateway requests.
+}
+
+// PbHandlerRegistration is a function that registers a protobuf handler.
+type PbHandlerRegistration func(context.Context, *gwruntime.ServeMux, *grpc.ClientConn) error
+
+// MuxHandler is a function that implements the mux handler functionality.
+type MuxHandler func(http.Handler, http.ResponseWriter, *http.Request)
+
+// Gateway is the gRPC gateway to serve HTTP JSON traffic as a proxy and forward it to the gRPC server.
+type Gateway struct {
+	conn                         *grpc.ClientConn
+	pbHandlers                   []*PbMux
+	muxHandler                   MuxHandler
+	maxCallRecvMsgSize           uint64
+	router                       *mux.Router
+	server                       *http.Server
+	cancel                       context.CancelFunc
+	remoteCert                   string
+	gatewayAddr                  string
+	apiMiddlewareEndpointFactory apimiddleware.EndpointFactory
+	ctx                          context.Context
+	startFailure                 error
+	remoteAddr                   string
+	allowedOrigins               []string
+}
+
+// New returns a new instance of the Gateway.
+func New(
+	ctx context.Context,
+	pbHandlers []*PbMux,
+	muxHandler MuxHandler,
+	remoteAddr,
+	gatewayAddress string,
+) *Gateway {
+	g := &Gateway{
+		pbHandlers:     pbHandlers,
+		muxHandler:     muxHandler,
+		router:         mux.NewRouter(),
+		gatewayAddr:    gatewayAddress,
+		ctx:            ctx,
+		remoteAddr:     remoteAddr,
+		allowedOrigins: []string{},
+	}
+	return g
+}
+
+// WithRouter allows adding a custom mux router to the gateway.
+func (g *Gateway) WithRouter(r *mux.Router) *Gateway {
+	g.router = r
+	return g
+}
+
+// WithAllowedOrigins allows adding a set of allowed origins to the gateway.
+func (g *Gateway) WithAllowedOrigins(origins []string) *Gateway {
+	g.allowedOrigins = origins
+	return g
+}
+
+// WithRemoteCert allows adding a custom certificate to the gateway,
+func (g *Gateway) WithRemoteCert(cert string) *Gateway {
+	g.remoteCert = cert
+	return g
+}
+
+// WithMaxCallRecvMsgSize allows specifying the maximum allowed gRPC message size.
+func (g *Gateway) WithMaxCallRecvMsgSize(size uint64) *Gateway {
+	g.maxCallRecvMsgSize = size
+	return g
+}
+
+// WithApiMiddleware allows adding API Middleware proxy to the gateway.
+func (g *Gateway) WithApiMiddleware(endpointFactory apimiddleware.EndpointFactory) *Gateway {
+	g.apiMiddlewareEndpointFactory = endpointFactory
+	return g
+}
+
+// Start the gateway service.
+func (g *Gateway) Start() {
+	ctx, cancel := context.WithCancel(g.ctx)
+	g.cancel = cancel
+
+	conn, err := g.dial(ctx, "tcp", g.remoteAddr)
+	if err != nil {
+		log.WithError(err).Error("Failed to connect to gRPC server")
+		g.startFailure = err
+		return
+	}
+	g.conn = conn
+
+	for _, h := range g.pbHandlers {
+		for _, r := range h.Registrations {
+			if err := r(ctx, h.Mux, g.conn); err != nil {
+				log.WithError(err).Error("Failed to register handler")
+				g.startFailure = err
+				return
+			}
+		}
+		for _, p := range h.Patterns {
+			g.router.PathPrefix(p).Handler(h.Mux)
+		}
+	}
+
+	corsMux := g.corsMiddleware(g.router)
+
+	if g.muxHandler != nil {
+		g.router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+			g.muxHandler(corsMux, w, r)
+		})
+	}
+
+	if g.apiMiddlewareEndpointFactory != nil && !g.apiMiddlewareEndpointFactory.IsNil() {
+		go g.registerApiMiddleware()
+	}
+
+	g.server = &http.Server{
+		Addr:    g.gatewayAddr,
+		Handler: g.router,
+	}
+
+	go func() {
+		log.WithField("address", g.gatewayAddr).Info("Starting gRPC gateway")
+		if err := g.server.ListenAndServe(); err != http.ErrServerClosed {
+			log.WithError(err).Error("Failed to start gRPC gateway")
+			g.startFailure = err
+			return
+		}
+	}()
+}
+
+// Status of grpc gateway. Returns an error if this service is unhealthy.
+func (g *Gateway) Status() error {
+	if g.startFailure != nil {
+		return g.startFailure
+	}
+
+	if s := g.conn.GetState(); s != connectivity.Ready {
+		return fmt.Errorf("grpc server is %s", s)
+	}
+
+	return nil
+}
+
+// Stop the gateway with a graceful shutdown.
+func (g *Gateway) Stop() error {
+	if g.server != nil {
+		shutdownCtx, shutdownCancel := context.WithTimeout(g.ctx, 2*time.Second)
+		defer shutdownCancel()
+		if err := g.server.Shutdown(shutdownCtx); err != nil {
+			if errors.Is(err, context.DeadlineExceeded) {
+				log.Warn("Existing connections terminated")
+			} else {
+				log.WithError(err).Error("Failed to gracefully shut down server")
+			}
+		}
+	}
+
+	if g.cancel != nil {
+		g.cancel()
+	}
+
+	return nil
+}
+
+func (g *Gateway) corsMiddleware(h http.Handler) http.Handler {
+	c := cors.New(cors.Options{
+		AllowedOrigins:   g.allowedOrigins,
+		AllowedMethods:   []string{http.MethodPost, http.MethodGet, http.MethodOptions},
+		AllowCredentials: true,
+		MaxAge:           600,
+		AllowedHeaders:   []string{"*"},
+	})
+	return c.Handler(h)
+}
+
+const swaggerDir = "proto/prysm/v1alpha1/"
+
+// SwaggerServer returns swagger specification files located under "/swagger/"
+func SwaggerServer() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if !strings.HasSuffix(r.URL.Path, ".swagger.json") {
+			log.Debugf("Not found: %s", r.URL.Path)
+			http.NotFound(w, r)
+			return
+		}
+
+		log.Debugf("Serving %s\n", r.URL.Path)
+		p := strings.TrimPrefix(r.URL.Path, "/swagger/")
+		p = path.Join(swaggerDir, p)
+		http.ServeFile(w, r, p)
+	}
+}
+
+// dial the gRPC server.
+func (g *Gateway) dial(ctx context.Context, network, addr string) (*grpc.ClientConn, error) {
+	switch network {
+	case "tcp":
+		return g.dialTCP(ctx, addr)
+	case "unix":
+		return g.dialUnix(ctx, addr)
+	default:
+		return nil, fmt.Errorf("unsupported network type %q", network)
+	}
+}
+
+// dialTCP creates a client connection via TCP.
+// "addr" must be a valid TCP address with a port number.
+func (g *Gateway) dialTCP(ctx context.Context, addr string) (*grpc.ClientConn, error) {
+	security := grpc.WithInsecure()
+	if len(g.remoteCert) > 0 {
+		creds, err := credentials.NewClientTLSFromFile(g.remoteCert, "")
+		if err != nil {
+			return nil, err
+		}
+		security = grpc.WithTransportCredentials(creds)
+	}
+	opts := []grpc.DialOption{
+		security,
+		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(g.maxCallRecvMsgSize))),
+	}
+
+	return grpc.DialContext(ctx, addr, opts...)
+}
+
+// dialUnix creates a client connection via a unix domain socket.
+// "addr" must be a valid path to the socket.
+func (g *Gateway) dialUnix(ctx context.Context, addr string) (*grpc.ClientConn, error) {
+	d := func(addr string, timeout time.Duration) (net.Conn, error) {
+		return net.DialTimeout("unix", addr, timeout)
+	}
+	f := func(ctx context.Context, addr string) (net.Conn, error) {
+		if deadline, ok := ctx.Deadline(); ok {
+			return d(addr, time.Until(deadline))
+		}
+		return d(addr, 0)
+	}
+	opts := []grpc.DialOption{
+		grpc.WithInsecure(),
+		grpc.WithContextDialer(f),
+		grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(g.maxCallRecvMsgSize))),
+	}
+	return grpc.DialContext(ctx, addr, opts...)
+}
+
+func (g *Gateway) registerApiMiddleware() {
+	proxy := &apimiddleware.ApiProxyMiddleware{
+		GatewayAddress:  g.gatewayAddr,
+		EndpointCreator: g.apiMiddlewareEndpointFactory,
+	}
+	log.Info("Starting API middleware")
+	proxy.Run(g.router)
+}

api/gateway/gateway_test.go

@@ -0,0 +1,120 @@
+package gateway
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/gorilla/mux"
+	"github.com/prysmaticlabs/prysm/api/gateway/apimiddleware"
+	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/flags"
+	"github.com/prysmaticlabs/prysm/testing/assert"
+	"github.com/prysmaticlabs/prysm/testing/require"
+	logTest "github.com/sirupsen/logrus/hooks/test"
+	"github.com/urfave/cli/v2"
+)
+
+type mockEndpointFactory struct {
+}
+
+func (*mockEndpointFactory) Paths() []string {
+	return []string{}
+}
+
+func (*mockEndpointFactory) Create(_ string) (*apimiddleware.Endpoint, error) {
+	return nil, nil
+}
+
+func (*mockEndpointFactory) IsNil() bool {
+	return false
+}
+
+func TestGateway_Customized(t *testing.T) {
+	r := mux.NewRouter()
+	cert := "cert"
+	origins := []string{"origin"}
+	size := uint64(100)
+	endpointFactory := &mockEndpointFactory{}
+
+	g := New(
+		context.Background(),
+		[]*PbMux{},
+		func(handler http.Handler, writer http.ResponseWriter, request *http.Request) {
+
+		},
+		"",
+		"",
+	).WithRouter(r).
+		WithRemoteCert(cert).
+		WithAllowedOrigins(origins).
+		WithMaxCallRecvMsgSize(size).
+		WithApiMiddleware(endpointFactory)
+
+	assert.Equal(t, r, g.router)
+	assert.Equal(t, cert, g.remoteCert)
+	require.Equal(t, 1, len(g.allowedOrigins))
+	assert.Equal(t, origins[0], g.allowedOrigins[0])
+	assert.Equal(t, size, g.maxCallRecvMsgSize)
+	assert.Equal(t, endpointFactory, g.apiMiddlewareEndpointFactory)
+}
+
+func TestGateway_StartStop(t *testing.T) {
+	hook := logTest.NewGlobal()
+
+	app := cli.App{}
+	set := flag.NewFlagSet("test", 0)
+	ctx := cli.NewContext(&app, set, nil)
+
+	gatewayPort := ctx.Int(flags.GRPCGatewayPort.Name)
+	gatewayHost := ctx.String(flags.GRPCGatewayHost.Name)
+	rpcHost := ctx.String(flags.RPCHost.Name)
+	selfAddress := fmt.Sprintf("%s:%d", rpcHost, ctx.Int(flags.RPCPort.Name))
+	gatewayAddress := fmt.Sprintf("%s:%d", gatewayHost, gatewayPort)
+
+	g := New(
+		ctx.Context,
+		[]*PbMux{},
+		func(handler http.Handler, writer http.ResponseWriter, request *http.Request) {
+
+		},
+		selfAddress,
+		gatewayAddress,
+	)
+
+	g.Start()
+	go func() {
+		require.LogsContain(t, hook, "Starting gRPC gateway")
+		require.LogsDoNotContain(t, hook, "Starting API middleware")
+	}()
+
+	err := g.Stop()
+	require.NoError(t, err)
+}
+
+func TestGateway_NilHandler_NotFoundHandlerRegistered(t *testing.T) {
+	app := cli.App{}
+	set := flag.NewFlagSet("test", 0)
+	ctx := cli.NewContext(&app, set, nil)
+
+	gatewayPort := ctx.Int(flags.GRPCGatewayPort.Name)
+	gatewayHost := ctx.String(flags.GRPCGatewayHost.Name)
+	rpcHost := ctx.String(flags.RPCHost.Name)
+	selfAddress := fmt.Sprintf("%s:%d", rpcHost, ctx.Int(flags.RPCPort.Name))
+	gatewayAddress := fmt.Sprintf("%s:%d", gatewayHost, gatewayPort)
+
+	g := New(
+		ctx.Context,
+		[]*PbMux{},
+		/* muxHandler */ nil,
+		selfAddress,
+		gatewayAddress,
+	)
+
+	writer := httptest.NewRecorder()
+	g.router.ServeHTTP(writer, &http.Request{Method: "GET", Host: "localhost", URL: &url.URL{Path: "/foo"}})
+	assert.Equal(t, http.StatusNotFound, writer.Code)
+}

api/gateway/log.go

@@ -0,0 +1,5 @@
+package gateway
+
+import "github.com/sirupsen/logrus"
+
+var log = logrus.WithField("prefix", "gateway")

api/grpc/BUILD.bazel

@@ -0,0 +1,30 @@
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "grpcutils.go",
+        "parameters.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/api/grpc",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_github_sirupsen_logrus//:go_default_library",
+        "@org_golang_google_grpc//:go_default_library",
+        "@org_golang_google_grpc//metadata:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["grpcutils_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//testing/assert:go_default_library",
+        "//testing/require:go_default_library",
+        "@com_github_grpc_ecosystem_grpc_gateway_v2//runtime:go_default_library",
+        "@com_github_sirupsen_logrus//hooks/test:go_default_library",
+        "@org_golang_google_grpc//:go_default_library",
+        "@org_golang_google_grpc//metadata:go_default_library",
+    ],
+)

api/grpc/grpcutils.go

@@ -0,0 +1,96 @@
+package grpc
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+)
+
+// LogRequests logs the gRPC backend as well as request duration when the log level is set to debug
+// or higher.
+func LogRequests(
+	ctx context.Context,
+	method string, req,
+	reply interface{},
+	cc *grpc.ClientConn,
+	invoker grpc.UnaryInvoker,
+	opts ...grpc.CallOption,
+) error {
+	// Shortcut when debug logging is not enabled.
+	if logrus.GetLevel() < logrus.DebugLevel {
+		return invoker(ctx, method, req, reply, cc, opts...)
+	}
+
+	var header metadata.MD
+	opts = append(
+		opts,
+		grpc.Header(&header),
+	)
+	start := time.Now()
+	err := invoker(ctx, method, req, reply, cc, opts...)
+	logrus.WithField("backend", header["x-backend"]).
+		WithField("method", method).WithField("duration", time.Since(start)).
+		Debug("gRPC request finished.")
+	return err
+}
+
+// LogStream prints the method at DEBUG level at the start of the stream.
+func LogStream(
+	ctx context.Context,
+	sd *grpc.StreamDesc,
+	conn *grpc.ClientConn,
+	method string,
+	streamer grpc.Streamer,
+	opts ...grpc.CallOption,
+) (grpc.ClientStream, error) {
+	// Shortcut when debug logging is not enabled.
+	if logrus.GetLevel() < logrus.DebugLevel {
+		return streamer(ctx, sd, conn, method, opts...)
+	}
+
+	var header metadata.MD
+	opts = append(
+		opts,
+		grpc.Header(&header),
+	)
+	strm, err := streamer(ctx, sd, conn, method, opts...)
+	logrus.WithField("backend", header["x-backend"]).
+		WithField("method", method).
+		Debug("gRPC stream started.")
+	return strm, err
+}
+
+// AppendHeaders parses the provided GRPC headers
+// and attaches them to the provided context.
+func AppendHeaders(parent context.Context, headers []string) context.Context {
+	for _, h := range headers {
+		if h != "" {
+			keyValue := strings.Split(h, "=")
+			if len(keyValue) < 2 {
+				logrus.Warnf("Incorrect gRPC header flag format. Skipping %v", keyValue[0])
+				continue
+			}
+			parent = metadata.AppendToOutgoingContext(parent, keyValue[0], strings.Join(keyValue[1:], "="))
+		}
+	}
+	return parent
+}
+
+// AppendCustomErrorHeader sets a CustomErrorMetadataKey gRPC header on the passed in context,
+// using the passed in error data as the header's value. The data is serialized as JSON.
+func AppendCustomErrorHeader(ctx context.Context, errorData interface{}) error {
+	j, err := json.Marshal(errorData)
+	if err != nil {
+		return fmt.Errorf("could not marshal error data into JSON: %w", err)
+	}
+	if err := grpc.SetHeader(ctx, metadata.Pairs(CustomErrorMetadataKey, string(j))); err != nil {
+		return fmt.Errorf("could not set custom error header: %w", err)
+	}
+	return nil
+}

api/grpc/grpcutils_test.go

@@ -0,0 +1,78 @@
+package grpc
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
+	"github.com/prysmaticlabs/prysm/testing/assert"
+	"github.com/prysmaticlabs/prysm/testing/require"
+	logTest "github.com/sirupsen/logrus/hooks/test"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+)
+
+type customErrorData struct {
+	Message string `json:"message"`
+}
+
+func TestAppendHeaders(t *testing.T) {
+	t.Run("one_header", func(t *testing.T) {
+		ctx := AppendHeaders(context.Background(), []string{"first=value1"})
+		md, ok := metadata.FromOutgoingContext(ctx)
+		require.Equal(t, true, ok, "Failed to read context metadata")
+		require.Equal(t, 1, md.Len(), "MetadataV0 contains wrong number of values")
+		assert.Equal(t, "value1", md.Get("first")[0])
+	})
+
+	t.Run("multiple_headers", func(t *testing.T) {
+		ctx := AppendHeaders(context.Background(), []string{"first=value1", "second=value2"})
+		md, ok := metadata.FromOutgoingContext(ctx)
+		require.Equal(t, true, ok, "Failed to read context metadata")
+		require.Equal(t, 2, md.Len(), "MetadataV0 contains wrong number of values")
+		assert.Equal(t, "value1", md.Get("first")[0])
+		assert.Equal(t, "value2", md.Get("second")[0])
+	})
+
+	t.Run("one_empty_header", func(t *testing.T) {
+		ctx := AppendHeaders(context.Background(), []string{"first=value1", ""})
+		md, ok := metadata.FromOutgoingContext(ctx)
+		require.Equal(t, true, ok, "Failed to read context metadata")
+		require.Equal(t, 1, md.Len(), "MetadataV0 contains wrong number of values")
+		assert.Equal(t, "value1", md.Get("first")[0])
+	})
+
+	t.Run("incorrect_header", func(t *testing.T) {
+		logHook := logTest.NewGlobal()
+		ctx := AppendHeaders(context.Background(), []string{"first=value1", "second"})
+		md, ok := metadata.FromOutgoingContext(ctx)
+		require.Equal(t, true, ok, "Failed to read context metadata")
+		require.Equal(t, 1, md.Len(), "MetadataV0 contains wrong number of values")
+		assert.Equal(t, "value1", md.Get("first")[0])
+		assert.LogsContain(t, logHook, "Skipping second")
+	})
+
+	t.Run("header_value_with_equal_sign", func(t *testing.T) {
+		ctx := AppendHeaders(context.Background(), []string{"first=value=1"})
+		md, ok := metadata.FromOutgoingContext(ctx)
+		require.Equal(t, true, ok, "Failed to read context metadata")
+		require.Equal(t, 1, md.Len(), "MetadataV0 contains wrong number of values")
+		assert.Equal(t, "value=1", md.Get("first")[0])
+	})
+}
+
+func TestAppendCustomErrorHeader(t *testing.T) {
+	stream := &runtime.ServerTransportStream{}
+	ctx := grpc.NewContextWithServerTransportStream(context.Background(), stream)
+	data := &customErrorData{Message: "foo"}
+	require.NoError(t, AppendCustomErrorHeader(ctx, data))
+	// The stream used in test setup sets the metadata key in lowercase.
+	value, ok := stream.Header()[strings.ToLower(CustomErrorMetadataKey)]
+	require.Equal(t, true, ok, "Failed to retrieve custom error metadata value")
+	expected, err := json.Marshal(data)
+	require.NoError(t, err)
+	assert.Equal(t, string(expected), value[0])
+
+}

api/grpc/parameters.go

@@ -0,0 +1,8 @@
+package grpc
+
+// CustomErrorMetadataKey is the name of the metadata key storing additional error information.
+// Metadata value is expected to be a byte-encoded JSON object.
+const CustomErrorMetadataKey = "Custom-Error"
+
+// HttpCodeMetadataKey is the key to use when setting custom HTTP status codes in gRPC metadata.
+const HttpCodeMetadataKey = "X-Http-Code"

api/pagination/BUILD.bazel

@@ -0,0 +1,22 @@
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["pagination.go"],
+    importpath = "github.com/prysmaticlabs/prysm/api/pagination",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//config/params:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["pagination_test.go"],
+    deps = [
+        ":go_default_library",
+        "//testing/assert:go_default_library",
+        "//testing/require:go_default_library",
+    ],
+)

api/pagination/pagination.go

@@ -0,0 +1,43 @@
+// Package pagination contains useful pagination-related helpers.
+package pagination
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/config/params"
+)
+
+// StartAndEndPage takes in the requested page token, wanted page size, total page size.
+// It returns start, end page and the next page token.
+func StartAndEndPage(pageToken string, pageSize, totalSize int) (int, int, string, error) {
+	if pageToken == "" {
+		pageToken = "0"
+	}
+	if pageSize == 0 {
+		pageSize = params.BeaconConfig().DefaultPageSize
+	}
+
+	token, err := strconv.Atoi(pageToken)
+	if err != nil {
+		return 0, 0, "", errors.Wrap(err, "could not convert page token")
+	}
+
+	// Start page can not be greater than set size.
+	start := token * pageSize
+	if start >= totalSize {
+		return 0, 0, "", fmt.Errorf("page start %d >= list %d", start, totalSize)
+	}
+
+	// End page can not go out of bound.
+	end := start + pageSize
+	nextPageToken := strconv.Itoa(token + 1)
+
+	if end >= totalSize {
+		end = totalSize
+		nextPageToken = "" // Return an empty next page token for the last page of a set
+	}
+
+	return start, end, nextPageToken, nil
+}

api/pagination/pagination_test.go

@@ -0,0 +1,87 @@
+package pagination_test
+
+import (
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/api/pagination"
+	"github.com/prysmaticlabs/prysm/testing/assert"
+	"github.com/prysmaticlabs/prysm/testing/require"
+)
+
+func TestStartAndEndPage(t *testing.T) {
+	tests := []struct {
+		token     string
+		pageSize  int
+		totalSize int
+		nextToken string
+		start     int
+		end       int
+	}{
+		{
+			token:     "0",
+			pageSize:  9,
+			totalSize: 100,
+			nextToken: "1",
+			start:     0,
+			end:       9,
+		},
+		{
+			token:     "10",
+			pageSize:  4,
+			totalSize: 100,
+			nextToken: "11",
+			start:     40,
+			end:       44,
+		},
+		{
+			token:     "100",
+			pageSize:  5,
+			totalSize: 1000,
+			nextToken: "101",
+			start:     500,
+			end:       505,
+		},
+		{
+			token:     "3",
+			pageSize:  33,
+			totalSize: 100,
+			nextToken: "",
+			start:     99,
+			end:       100,
+		},
+		{
+			token:     "34",
+			pageSize:  500,
+			totalSize: 17500,
+			nextToken: "",
+			start:     17000,
+			end:       17500,
+		},
+	}
+
+	for _, test := range tests {
+		start, end, next, err := pagination.StartAndEndPage(test.token, test.pageSize, test.totalSize)
+		require.NoError(t, err)
+		if test.start != start {
+			t.Errorf("expected start and computed start are not equal %d, %d", test.start, start)
+		}
+		if test.end != end {
+			t.Errorf("expected end and computed end are not equal %d, %d", test.end, end)
+		}
+		if test.nextToken != next {
+			t.Errorf("expected next token and computed next token are not equal %v, %v", test.nextToken, next)
+		}
+	}
+}
+
+func TestStartAndEndPage_CannotConvertPage(t *testing.T) {
+	wanted := "could not convert page token: strconv.Atoi: parsing"
+	_, _, _, err := pagination.StartAndEndPage("bad", 0, 0)
+	assert.ErrorContains(t, wanted, err)
+}
+
+func TestStartAndEndPage_ExceedsMaxPage(t *testing.T) {
+	wanted := "page start 0 >= list 0"
+	_, _, _, err := pagination.StartAndEndPage("", 0, 0)
+	assert.ErrorContains(t, wanted, err)
+}

async/BUILD.bazel

@@ -0,0 +1,33 @@
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "debounce.go",
+        "every.go",
+        "multilock.go",
+        "scatter.go",
+    ],
+    importpath = "github.com/prysmaticlabs/prysm/async",
+    visibility = ["//visibility:public"],
+    deps = ["@com_github_sirupsen_logrus//:go_default_library"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "benchmark_test.go",
+        "debounce_test.go",
+        "every_test.go",
+        "multilock_test.go",
+        "scatter_test.go",
+    ],
+    embed = [":go_default_library"],
+    deps = [
+        "//testing/assert:go_default_library",
+        "//testing/require:go_default_library",
+        "//testing/util:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+        "@com_github_stretchr_testify//assert:go_default_library",
+    ],
+)

async/abool/BUILD.bazel

@@ -0,0 +1,14 @@
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["abool.go"],
+    importpath = "github.com/prysmaticlabs/prysm/async/abool",
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["abool_test.go"],
+    embed = [":go_default_library"],
+)

async/abool/abool.go

@@ -0,0 +1,72 @@
+// Package abool provides atomic Boolean type for cleaner code and
+// better performance.
+// All rights reserved to https://github.com/tevino/abool.
+package abool
+
+import "sync/atomic"
+
+// New creates an AtomicBool with default set to false.
+func New() *AtomicBool {
+	return new(AtomicBool)
+}
+
+// NewBool creates an AtomicBool with given default value.
+func NewBool(ok bool) *AtomicBool {
+	ab := New()
+	if ok {
+		ab.Set()
+	}
+	return ab
+}
+
+// AtomicBool is an atomic Boolean.
+// Its methods are all atomic, thus safe to be called by multiple goroutines simultaneously.
+// Note: When embedding into a struct one should always use *AtomicBool to avoid copy.
+type AtomicBool int32
+
+// Set sets the Boolean to true.
+func (ab *AtomicBool) Set() {
+	atomic.StoreInt32((*int32)(ab), 1)
+}
+
+// UnSet sets the Boolean to false.
+func (ab *AtomicBool) UnSet() {
+	atomic.StoreInt32((*int32)(ab), 0)
+}
+
+// IsSet returns whether the Boolean is true.
+func (ab *AtomicBool) IsSet() bool {
+	return atomic.LoadInt32((*int32)(ab))&1 == 1
+}
+
+// IsNotSet returns whether the Boolean is false.
+func (ab *AtomicBool) IsNotSet() bool {
+	return !ab.IsSet()
+}
+
+// SetTo sets the boolean with given Boolean.
+func (ab *AtomicBool) SetTo(yes bool) {
+	if yes {
+		atomic.StoreInt32((*int32)(ab), 1)
+	} else {
+		atomic.StoreInt32((*int32)(ab), 0)
+	}
+}
+
+// Toggle inverts the Boolean then returns the value before inverting.
+func (ab *AtomicBool) Toggle() bool {
+	return atomic.AddInt32((*int32)(ab), 1)&1 == 0
+}
+
+// SetToIf sets the Boolean to new only if the Boolean matches the old.
+// Returns whether the set was done.
+func (ab *AtomicBool) SetToIf(old, curr bool) (set bool) {
+	var o, n int32
+	if old {
+		o = 1
+	}
+	if curr {
+		n = 1
+	}
+	return atomic.CompareAndSwapInt32((*int32)(ab), o, n)
+}

async/abool/abool_test.go

@@ -0,0 +1,298 @@
+// All rights reserved to https://github.com/tevino/abool.
+package abool
+
+import (
+	"math"
+	"sync"
+	"sync/atomic"
+	"testing"
+)
+
+func TestDefaultValue(t *testing.T) {
+	t.Parallel()
+	v := New()
+	if v.IsSet() {
+		t.Fatal("Empty value of AtomicBool should be false")
+	}
+
+	v = NewBool(true)
+	if !v.IsSet() {
+		t.Fatal("NewValue(true) should be true")
+	}
+
+	v = NewBool(false)
+	if v.IsSet() {
+		t.Fatal("NewValue(false) should be false")
+	}
+}
+
+func TestIsNotSet(t *testing.T) {
+	t.Parallel()
+	v := New()
+
+	if v.IsSet() == v.IsNotSet() {
+		t.Fatal("AtomicBool.IsNotSet() should be the opposite of IsSet()")
+	}
+}
+
+func TestSetUnSet(t *testing.T) {
+	t.Parallel()
+	v := New()
+
+	v.Set()
+	if !v.IsSet() {
+		t.Fatal("AtomicBool.Set() failed")
+	}
+
+	v.UnSet()
+	if v.IsSet() {
+		t.Fatal("AtomicBool.UnSet() failed")
+	}
+}
+
+func TestSetTo(t *testing.T) {
+	t.Parallel()
+	v := New()
+
+	v.SetTo(true)
+	if !v.IsSet() {
+		t.Fatal("AtomicBool.SetTo(true) failed")
+	}
+
+	v.SetTo(false)
+	if v.IsSet() {
+		t.Fatal("AtomicBool.SetTo(false) failed")
+	}
+
+	if set := v.SetToIf(true, false); set || v.IsSet() {
+		t.Fatal("AtomicBool.SetTo(true, false) failed")
+	}
+
+	if set := v.SetToIf(false, true); !set || !v.IsSet() {
+		t.Fatal("AtomicBool.SetTo(false, true) failed")
+	}
+}
+
+func TestToggle(t *testing.T) {
+	t.Parallel()
+	v := New()
+
+	_ = v.Toggle()
+	if !v.IsSet() {
+		t.Fatal("AtomicBool.Toggle() to true failed")
+	}
+
+	prev := v.Toggle()
+	if v.IsSet() == prev {
+		t.Fatal("AtomicBool.Toggle() to false failed")
+	}
+}
+
+func TestToogleMultipleTimes(t *testing.T) {
+	t.Parallel()
+
+	v := New()
+	pre := !v.IsSet()
+	for i := 0; i < 100; i++ {
+		v.SetTo(false)
+		for j := 0; j < i; j++ {
+			pre = v.Toggle()
+		}
+
+		expected := i%2 != 0
+		if v.IsSet() != expected {
+			t.Fatalf("AtomicBool.Toogle() doesn't work after %d calls, expected: %v, got %v", i, expected, v.IsSet())
+		}
+
+		if pre == v.IsSet() {
+			t.Fatalf("AtomicBool.Toogle() returned wrong value at the %dth calls, expected: %v, got %v", i, !v.IsSet(), pre)
+		}
+	}
+}
+
+func TestToogleAfterOverflow(t *testing.T) {
+	t.Parallel()
+
+	var value int32 = math.MaxInt32
+	v := (*AtomicBool)(&value)
+
+	valueBeforeToggle := *(*int32)(v)
+
+	// test first toggle after overflow
+	v.Toggle()
+	expected := math.MaxInt32%2 == 0
+	if v.IsSet() != expected {
+		t.Fatalf("AtomicBool.Toogle() doesn't work after overflow, expected: %v, got %v", expected, v.IsSet())
+	}
+
+	// make sure overflow happened
+	var valueAfterToggle = *(*int32)(v)
+	if valueAfterToggle >= valueBeforeToggle {
+		t.Fatalf("Overflow does not happen as expected, before %d, after: %d", valueBeforeToggle, valueAfterToggle)
+	}
+
+	// test second toggle after overflow
+	v.Toggle()
+	expected = !expected
+	if v.IsSet() != expected {
+		t.Fatalf("AtomicBool.Toogle() doesn't work after the second call after overflow, expected: %v, got %v", expected, v.IsSet())
+	}
+}
+
+func TestRace(t *testing.T) {
+	t.Parallel()
+
+	repeat := 10000
+	var wg sync.WaitGroup
+	wg.Add(repeat * 4)
+	v := New()
+
+	// Writer
+	go func() {
+		for i := 0; i < repeat; i++ {
+			v.Set()
+			wg.Done()
+		}
+	}()
+
+	// Reader
+	go func() {
+		for i := 0; i < repeat; i++ {
+			v.IsSet()
+			wg.Done()
+		}
+	}()
+
+	// Writer
+	go func() {
+		for i := 0; i < repeat; i++ {
+			v.UnSet()
+			wg.Done()
+		}
+	}()
+
+	// Reader And Writer
+	go func() {
+		for i := 0; i < repeat; i++ {
+			v.Toggle()
+			wg.Done()
+		}
+	}()
+
+	wg.Wait()
+}
+
+func ExampleAtomicBool() {
+	cond := New()             // default to false
+	cond.Set()                // Sets to true
+	cond.IsSet()              // Returns true
+	cond.UnSet()              // Sets to false
+	cond.IsNotSet()           // Returns true
+	cond.SetTo(true)          // Sets to whatever you want
+	cond.SetToIf(true, false) // Sets to `new` only if the Boolean matches the `old`, returns whether succeeded
+	cond.Toggle()             // Inverts the boolean then returns the value before inverting
+}
+
+// Benchmark Read
+
+func BenchmarkMutexRead(b *testing.B) {
+	var m sync.RWMutex
+	var v bool
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.RLock()
+		_ = v
+		m.RUnlock()
+	}
+}
+
+func BenchmarkAtomicValueRead(b *testing.B) {
+	var v atomic.Value
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_ = v.Load() != nil
+	}
+}
+
+func BenchmarkAtomicBoolRead(b *testing.B) {
+	v := New()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_ = v.IsSet()
+	}
+}
+
+// Benchmark Write
+
+func BenchmarkMutexWrite(b *testing.B) {
+	var m sync.RWMutex
+	var v bool
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.RLock()
+		v = true
+		m.RUnlock()
+	}
+	b.StopTimer()
+	_ = v
+}
+
+func BenchmarkAtomicValueWrite(b *testing.B) {
+	var v atomic.Value
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.Store(true)
+	}
+}
+
+func BenchmarkAtomicBoolWrite(b *testing.B) {
+	v := New()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.Set()
+	}
+}
+
+// Benchmark CAS
+
+func BenchmarkMutexCAS(b *testing.B) {
+	var m sync.RWMutex
+	var v bool
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.Lock()
+		if !v {
+			v = true
+		}
+		m.Unlock()
+	}
+}
+
+func BenchmarkAtomicBoolCAS(b *testing.B) {
+	v := New()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.SetToIf(false, true)
+	}
+}
+
+// Benchmark toggle
+
+func BenchmarkMutexToggle(b *testing.B) {
+	var m sync.RWMutex
+	var v bool
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.Lock()
+		v = !v
+		m.Unlock()
+	}
+}
+
+func BenchmarkAtomicBoolToggle(b *testing.B) {
+	v := New()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.Toggle()
+	}
+}

async/benchmark_test.go

@@ -0,0 +1,63 @@
+package async_test
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"sync"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/async"
+	"github.com/prysmaticlabs/prysm/testing/require"
+	log "github.com/sirupsen/logrus"
+)
+
+var input [][]byte
+
+const (
+	benchmarkElements    = 65536
+	benchmarkElementSize = 32
+	benchmarkHashRuns    = 128
+)
+
+func init() {
+	input = make([][]byte, benchmarkElements)
+	for i := 0; i < benchmarkElements; i++ {
+		input[i] = make([]byte, benchmarkElementSize)
+		_, err := rand.Read(input[i])
+		if err != nil {
+			log.WithError(err).Debug("Cannot read from rand")
+		}
+	}
+}
+
+// hash repeatedly hashes the data passed to it
+func hash(input [][]byte) [][]byte {
+	output := make([][]byte, len(input))
+	for i := range input {
+		copy(output, input)
+		for j := 0; j < benchmarkHashRuns; j++ {
+			hash := sha256.Sum256(output[i])
+			output[i] = hash[:]
+		}
+	}
+	return output
+}
+
+func BenchmarkHash(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		hash(input)
+	}
+}
+
+func BenchmarkHashMP(b *testing.B) {
+	output := make([][]byte, len(input))
+	for i := 0; i < b.N; i++ {
+		workerResults, err := async.Scatter(len(input), func(offset int, entries int, _ *sync.RWMutex) (interface{}, error) {
+			return hash(input[offset : offset+entries]), nil
+		})
+		require.NoError(b, err)
+		for _, result := range workerResults {
+			copy(output[result.Offset:], result.Extent.([][]byte))
+		}
+	}
+}

async/debounce.go

@@ -0,0 +1,45 @@
+package async
+
+import (
+	"context"
+	"time"
+)
+
+// Debounce events fired over a channel by a specified duration, ensuring no events
+// are handled until a certain interval of time has passed.
+func Debounce(ctx context.Context, interval time.Duration, eventsChan <-chan interface{}, handler func(interface{})) {
+	var timer *time.Timer
+	defer func() {
+		if timer != nil {
+			timer.Stop()
+		}
+	}()
+	for {
+		select {
+		// Wait until an event is triggered.
+		case event := <-eventsChan:
+			timer = time.NewTimer(interval)
+		loop:
+			for {
+				// If an event is received, wait the specified interval before calling the handler.
+				// If another event is received before the interval has passed, store
+				// it and reset the timer.
+				select {
+				case event = <-eventsChan:
+					// Reset timer.
+					timer.Stop()
+					timer = time.NewTimer(interval)
+				case <-timer.C:
+					// Stop the current timer, handle the request, and wait for more events.
+					timer.Stop()
+					handler(event)
+					break loop
+				case <-ctx.Done():
+					return
+				}
+			}
+		case <-ctx.Done():
+			return
+		}
+	}
+}

async/debounce_test.go

@@ -0,0 +1,116 @@
+package async_test
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/prysmaticlabs/prysm/async"
+	"github.com/prysmaticlabs/prysm/testing/assert"
+	"github.com/prysmaticlabs/prysm/testing/require"
+	"github.com/prysmaticlabs/prysm/testing/util"
+)
+
+func TestDebounce_NoEvents(t *testing.T) {
+	eventsChan := make(chan interface{}, 100)
+	ctx, cancel := context.WithCancel(context.Background())
+	interval := time.Second
+	timesHandled := 0
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		time.AfterFunc(interval, func() {
+			cancel()
+		})
+	}()
+	go func() {
+		async.Debounce(ctx, interval, eventsChan, func(event interface{}) {
+			timesHandled++
+		})
+		wg.Done()
+	}()
+	if util.WaitTimeout(wg, interval*2) {
+		t.Fatalf("Test should have exited by now, timed out")
+	}
+	assert.Equal(t, 0, timesHandled, "Wrong number of handled calls")
+}
+
+func TestDebounce_CtxClosing(t *testing.T) {
+	eventsChan := make(chan interface{}, 100)
+	ctx, cancel := context.WithCancel(context.Background())
+	interval := time.Second
+	timesHandled := 0
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		ticker := time.NewTicker(time.Millisecond * 100)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				eventsChan <- struct{}{}
+			}
+		}
+	}()
+	go func() {
+		time.AfterFunc(interval, func() {
+			cancel()
+		})
+	}()
+	go func() {
+		async.Debounce(ctx, interval, eventsChan, func(event interface{}) {
+			timesHandled++
+		})
+		wg.Done()
+	}()
+	if util.WaitTimeout(wg, interval*2) {
+		t.Fatalf("Test should have exited by now, timed out")
+	}
+	assert.Equal(t, 0, timesHandled, "Wrong number of handled calls")
+}
+
+func TestDebounce_SingleHandlerInvocation(t *testing.T) {
+	eventsChan := make(chan interface{}, 100)
+	ctx, cancel := context.WithCancel(context.Background())
+	interval := time.Second
+	timesHandled := 0
+	go async.Debounce(ctx, interval, eventsChan, func(event interface{}) {
+		timesHandled++
+	})
+	for i := 0; i < 100; i++ {
+		eventsChan <- struct{}{}
+	}
+	// We should expect 100 rapid fire changes to only have caused
+	// 1 handler to trigger after the debouncing period.
+	time.Sleep(interval * 2)
+	assert.Equal(t, 1, timesHandled, "Wrong number of handled calls")
+	cancel()
+}
+
+func TestDebounce_MultipleHandlerInvocation(t *testing.T) {
+	eventsChan := make(chan interface{}, 100)
+	ctx, cancel := context.WithCancel(context.Background())
+	interval := time.Second
+	timesHandled := 0
+	go async.Debounce(ctx, interval, eventsChan, func(event interface{}) {
+		timesHandled++
+	})
+	for i := 0; i < 100; i++ {
+		eventsChan <- struct{}{}
+	}
+	require.Equal(t, 0, timesHandled, "Events must prevent from handler execution")
+
+	// By this time the first event should be triggered.
+	time.Sleep(2 * time.Second)
+	assert.Equal(t, 1, timesHandled, "Wrong number of handled calls")
+
+	// Second event.
+	eventsChan <- struct{}{}
+	time.Sleep(2 * time.Second)
+	assert.Equal(t, 2, timesHandled, "Wrong number of handled calls")
+
+	cancel()
+}

async/event/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -6,13 +6,14 @@ go_library(
         "feed.go",
         "subscription.go",
     ],
-    importpath = "github.com/prysmaticlabs/prysm/shared/event",
+    importpath = "github.com/prysmaticlabs/prysm/async/event",
     visibility = ["//visibility:public"],
-    deps = ["//shared/mclockutil:go_default_library"],
+    deps = ["//time/mclock:go_default_library"],
 )
 
 go_test(
     name = "go_default_test",
+    size = "small",
     srcs = [
         "example_feed_test.go",
         "example_scope_test.go",
@@ -21,4 +22,8 @@ go_test(
         "subscription_test.go",
     ],
     embed = [":go_default_library"],
+    deps = [
+        "//testing/assert:go_default_library",
+        "//testing/require:go_default_library",
+    ],
 )

async/event/example_feed_test.go

@@ -19,7 +19,7 @@ package event_test
 import (
 	"fmt"
 
-	"github.com/prysmaticlabs/prysm/shared/event"
+	"github.com/prysmaticlabs/prysm/async/event"
 )
 
 func ExampleFeed_acknowledgedEvents() {

async/event/example_scope_test.go

@@ -20,7 +20,7 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/prysmaticlabs/prysm/shared/event"
+	"github.com/prysmaticlabs/prysm/async/event"
 )
 
 // This example demonstrates how SubscriptionScope can be used to control the lifetime of

async/event/example_subscription_test.go

@@ -19,7 +19,7 @@ package event_test
 import (
 	"fmt"
 
-	"github.com/prysmaticlabs/prysm/shared/event"
+	"github.com/prysmaticlabs/prysm/async/event"
 )
 
 func ExampleNewSubscription() {

async/event/feed.go

@@ -14,6 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
+// Package event contains an event feed implementation for process communication.
 package event
 
 import (
@@ -97,6 +98,11 @@ func (f *Feed) typecheck(typ reflect.Type) bool {
 		f.etype = typ
 		return true
 	}
+	// In the event the feed's type is an actual interface, we
+	// perform an interface conformance check here.
+	if f.etype.Kind() == reflect.Interface && typ.Implements(f.etype) {
+		return true
+	}
 	return f.etype == typ
 }
 
@@ -195,6 +201,7 @@ type feedSub struct {
 	err     chan error
 }
 
+// Unsubscribe remove feed subscription.
 func (sub *feedSub) Unsubscribe() {
 	sub.errOnce.Do(func() {
 		sub.feed.remove(sub)
@@ -202,6 +209,7 @@ func (sub *feedSub) Unsubscribe() {
 	})
 }
 
+// Err returns error channel.
 func (sub *feedSub) Err() <-chan error {
 	return sub.err
 }

async/event/feed_test.go

@@ -22,55 +22,47 @@ import (
 	"sync"
 	"testing"
 	"time"
+
+	"github.com/prysmaticlabs/prysm/testing/assert"
 )
 
 func TestFeedPanics(t *testing.T) {
 	{
 		var f Feed
-		f.Send(int(2))
-		want := feedTypeError{op: "Send", got: reflect.TypeOf(uint64(0)), want: reflect.TypeOf(int(0))}
-		if err := checkPanic(want, func() { f.Send(uint64(2)) }); err != nil {
-			t.Error(err)
-		}
+		f.Send(2)
+		want := feedTypeError{op: "Send", got: reflect.TypeOf(uint64(0)), want: reflect.TypeOf(0)}
+		assert.NoError(t, checkPanic(want, func() { f.Send(uint64(2)) }))
 	}
 	{
 		var f Feed
 		ch := make(chan int)
 		f.Subscribe(ch)
-		want := feedTypeError{op: "Send", got: reflect.TypeOf(uint64(0)), want: reflect.TypeOf(int(0))}
-		if err := checkPanic(want, func() { f.Send(uint64(2)) }); err != nil {
-			t.Error(err)
-		}
+		want := feedTypeError{op: "Send", got: reflect.TypeOf(uint64(0)), want: reflect.TypeOf(0)}
+		assert.NoError(t, checkPanic(want, func() { f.Send(uint64(2)) }))
 	}
 	{
 		var f Feed
-		f.Send(int(2))
+		f.Send(2)
 		want := feedTypeError{op: "Subscribe", got: reflect.TypeOf(make(chan uint64)), want: reflect.TypeOf(make(chan<- int))}
-		if err := checkPanic(want, func() { f.Subscribe(make(chan uint64)) }); err != nil {
-			t.Error(err)
-		}
+		assert.NoError(t, checkPanic(want, func() { f.Subscribe(make(chan uint64)) }))
 	}
 	{
 		var f Feed
-		if err := checkPanic(errBadChannel, func() { f.Subscribe(make(<-chan int)) }); err != nil {
-			t.Error(err)
-		}
+		assert.NoError(t, checkPanic(errBadChannel, func() { f.Subscribe(make(<-chan int)) }))
 	}
 	{
 		var f Feed
-		if err := checkPanic(errBadChannel, func() { f.Subscribe(int(0)) }); err != nil {
-			t.Error(err)
-		}
+		assert.NoError(t, checkPanic(errBadChannel, func() { f.Subscribe(0) }))
 	}
 }
 
 func checkPanic(want error, fn func()) (err error) {
 	defer func() {
-		panic := recover()
-		if panic == nil {
+		panicResult := recover()
+		if panicResult == nil {
 			err = fmt.Errorf("didn't panic")
-		} else if !reflect.DeepEqual(panic, want) {
-			err = fmt.Errorf("panicked with wrong error: got %q, want %q", panic, want)
+		} else if !reflect.DeepEqual(panicResult, want) {
+			err = fmt.Errorf("panicked with wrong error: got %q, want %q", panicResult, want)
 		}
 	}()
 	fn()
@@ -167,7 +159,7 @@ func TestFeedSubscribeSameChannel(t *testing.T) {
 	done.Wait()
 }
 
-func TestFeedSubscribeBlockedPost(t *testing.T) {
+func TestFeedSubscribeBlockedPost(_ *testing.T) {
 	var (
 		feed   Feed
 		nsends = 2000
@@ -200,7 +192,7 @@ func TestFeedSubscribeBlockedPost(t *testing.T) {
 	}
 }
 
-func TestFeedUnsubscribeBlockedPost(t *testing.T) {
+func TestFeedUnsubscribeBlockedPost(_ *testing.T) {
 	var (
 		feed   Feed
 		nsends = 200
@@ -237,7 +229,7 @@ func TestFeedUnsubscribeBlockedPost(t *testing.T) {
 
 // Checks that unsubscribing a channel during Send works even if that
 // channel has already been sent on.
-func TestFeedUnsubscribeSentChan(t *testing.T) {
+func TestFeedUnsubscribeSentChan(_ *testing.T) {
 	var (
 		feed Feed
 		ch1  = make(chan int)
@@ -283,22 +275,14 @@ func TestFeedUnsubscribeFromInbox(t *testing.T) {
 		sub2 = feed.Subscribe(ch1)
 		sub3 = feed.Subscribe(ch2)
 	)
-	if len(feed.inbox) != 3 {
-		t.Errorf("inbox length != 3 after subscribe")
-	}
-	if len(feed.sendCases) != 1 {
-		t.Errorf("sendCases is non-empty after unsubscribe")
-	}
+	assert.Equal(t, 3, len(feed.inbox))
+	assert.Equal(t, 1, len(feed.sendCases), "sendCases is non-empty after unsubscribe")
 
 	sub1.Unsubscribe()
 	sub2.Unsubscribe()
 	sub3.Unsubscribe()
-	if len(feed.inbox) != 0 {
-		t.Errorf("inbox is non-empty after unsubscribe")
-	}
-	if len(feed.sendCases) != 1 {
-		t.Errorf("sendCases is non-empty after unsubscribe")
-	}
+	assert.Equal(t, 0, len(feed.inbox), "Inbox is non-empty after unsubscribe")
+	assert.Equal(t, 1, len(feed.sendCases), "sendCases is non-empty after unsubscribe")
 }
 
 func BenchmarkFeedSend1000(b *testing.B) {
@@ -331,3 +315,193 @@ func BenchmarkFeedSend1000(b *testing.B) {
 	b.StopTimer()
 	done.Wait()
 }
+
+func TestFeed_Send(t *testing.T) {
+	tests := []struct {
+		name        string
+		evFeed      *Feed
+		testSetup   func(fd *Feed, t *testing.T, o interface{})
+		obj         interface{}
+		expectPanic bool
+	}{
+		{
+			name:   "normal struct",
+			evFeed: new(Feed),
+			testSetup: func(fd *Feed, t *testing.T, o interface{}) {
+				testChan := make(chan testFeedWithPointer, 1)
+				fd.Subscribe(testChan)
+			},
+			obj: testFeedWithPointer{
+				a: new(uint64),
+				b: new(string),
+			},
+			expectPanic: false,
+		},
+		{
+			name:   "un-implemented interface",
+			evFeed: new(Feed),
+			testSetup: func(fd *Feed, t *testing.T, o interface{}) {
+				testChan := make(chan testFeedIface, 1)
+				fd.Subscribe(testChan)
+			},
+			obj: testFeedWithPointer{
+				a: new(uint64),
+				b: new(string),
+			},
+			expectPanic: true,
+		},
+		{
+			name:   "semi-implemented interface",
+			evFeed: new(Feed),
+			testSetup: func(fd *Feed, t *testing.T, o interface{}) {
+				testChan := make(chan testFeedIface, 1)
+				fd.Subscribe(testChan)
+			},
+			obj: testFeed2{
+				a: 0,
+				b: "",
+				c: []byte{'A'},
+			},
+			expectPanic: true,
+		},
+		{
+			name:   "fully-implemented interface",
+			evFeed: new(Feed),
+			testSetup: func(fd *Feed, t *testing.T, o interface{}) {
+				testChan := make(chan testFeedIface)
+				// Make it unbuffered to allow message to
+				// pass through
+				go func() {
+					a := <-testChan
+					if !reflect.DeepEqual(a, o) {
+						t.Errorf("Got = %v, want = %v", a, o)
+					}
+				}()
+				fd.Subscribe(testChan)
+			},
+			obj: testFeed{
+				a: 0,
+				b: "",
+			},
+			expectPanic: false,
+		},
+		{
+			name:   "fully-implemented interface with additional methods",
+			evFeed: new(Feed),
+			testSetup: func(fd *Feed, t *testing.T, o interface{}) {
+				testChan := make(chan testFeedIface)
+				// Make it unbuffered to allow message to
+				// pass through
+				go func() {
+					a := <-testChan
+					if !reflect.DeepEqual(a, o) {
+						t.Errorf("Got = %v, want = %v", a, o)
+					}
+				}()
+				fd.Subscribe(testChan)
+			},
+			obj: testFeed3{
+				a: 0,
+				b: "",
+				c: []byte{'A'},
+				d: []byte{'B'},
+			},
+			expectPanic: false,
+		},
+		{
+			name:   "concrete types implementing the same interface",
+			evFeed: new(Feed),
+			testSetup: func(fd *Feed, t *testing.T, o interface{}) {
+				testChan := make(chan testFeed, 1)
+				// Make it unbuffered to allow message to
+				// pass through
+				go func() {
+					a := <-testChan
+					if !reflect.DeepEqual(a, o) {
+						t.Errorf("Got = %v, want = %v", a, o)
+					}
+				}()
+				fd.Subscribe(testChan)
+			},
+			obj: testFeed3{
+				a: 0,
+				b: "",
+				c: []byte{'A'},
+				d: []byte{'B'},
+			},
+			expectPanic: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			defer func() {
+				if r := recover(); r != nil {
+					if !tt.expectPanic {
+						t.Errorf("panic triggered when unexpected: %v", r)
+					}
+				} else {
+					if tt.expectPanic {
+						t.Error("panic not triggered when expected")
+					}
+				}
+			}()
+			tt.testSetup(tt.evFeed, t, tt.obj)
+			if gotNsent := tt.evFeed.Send(tt.obj); gotNsent != 1 {
+				t.Errorf("Send() = %v, want %v", gotNsent, 1)
+			}
+		})
+	}
+}
+
+// The following objects below are a collection of different
+// struct types to test with.
+type testFeed struct {
+	a uint64
+	b string
+}
+
+func (testFeed) method1() {
+
+}
+
+func (testFeed) method2() {
+
+}
+
+type testFeedWithPointer struct {
+	a *uint64
+	b *string
+}
+
+type testFeed2 struct {
+	a uint64
+	b string
+	c []byte
+}
+
+func (testFeed2) method1() {
+
+}
+
+type testFeed3 struct {
+	a    uint64
+	b    string
+	c, d []byte
+}
+
+func (testFeed3) method1() {
+
+}
+
+func (testFeed3) method2() {
+
+}
+
+func (testFeed3) method3() {
+
+}
+
+type testFeedIface interface {
+	method1()
+	method2()
+}

async/event/subscription.go

@@ -21,9 +21,13 @@ import (
 	"sync"
 	"time"
 
-	"github.com/prysmaticlabs/prysm/shared/mclockutil"
+	"github.com/prysmaticlabs/prysm/time/mclock"
 )
 
+// waitQuotient is divided against the max backoff time, in order to have N requests based on the full
+// request backoff time.
+const waitQuotient = 10
+
 // Subscription represents a stream of events. The carrier of the events is typically a
 // channel, but isn't part of the interface.
 //
@@ -70,6 +74,7 @@ type funcSub struct {
 	unsubscribed bool
 }
 
+// Unsubscribe unsubscribes from subscription.
 func (s *funcSub) Unsubscribe() {
 	s.mu.Lock()
 	if s.unsubscribed {
@@ -83,6 +88,7 @@ func (s *funcSub) Unsubscribe() {
 	<-s.err
 }
 
+// Err exposes error channel.
 func (s *funcSub) Err() <-chan error {
 	return s.err
 }
@@ -96,7 +102,7 @@ func (s *funcSub) Err() <-chan error {
 // based on the error rate, but will never exceed backoffMax.
 func Resubscribe(backoffMax time.Duration, fn ResubscribeFunc) Subscription {
 	s := &resubscribeSub{
-		waitTime:   backoffMax / 10,
+		waitTime:   backoffMax / waitQuotient,
 		backoffMax: backoffMax,
 		fn:         fn,
 		err:        make(chan error),
@@ -114,10 +120,11 @@ type resubscribeSub struct {
 	err                  chan error
 	unsub                chan struct{}
 	unsubOnce            sync.Once
-	lastTry              mclockutil.AbsTime
+	lastTry              mclock.AbsTime
 	waitTime, backoffMax time.Duration
 }
 
+// Unsubscribe unsubscribes from subscription.
 func (s *resubscribeSub) Unsubscribe() {
 	s.unsubOnce.Do(func() {
 		s.unsub <- struct{}{}
@@ -125,6 +132,7 @@ func (s *resubscribeSub) Unsubscribe() {
 	})
 }
 
+// Err exposes error channel.
 func (s *resubscribeSub) Err() <-chan error {
 	return s.err
 }
@@ -143,12 +151,12 @@ func (s *resubscribeSub) loop() {
 }
 
 func (s *resubscribeSub) subscribe() Subscription {
-	subscribed := make(chan error)
+	subscribed := make(chan error, 1)
 	var sub Subscription
 retry:
 	for {
-		s.lastTry = mclockutil.Now()
-		ctx, cancel := context.WithCancel(context.Background())
+		s.lastTry = mclock.Now()
+		ctx, cancel := context.WithCancel(context.TODO())
 		go func() {
 			rsub, err := s.fn(ctx)
 			sub = rsub
@@ -186,8 +194,8 @@ func (s *resubscribeSub) waitForError(sub Subscription) bool {
 }
 
 func (s *resubscribeSub) backoffWait() bool {
-	if time.Duration(mclockutil.Now()-s.lastTry) > s.backoffMax {
-		s.waitTime = s.backoffMax / 10
+	if time.Duration(mclock.Now()-s.lastTry) > s.backoffMax {
+		s.waitTime = s.backoffMax / waitQuotient
 	} else {
 		s.waitTime *= 2
 		if s.waitTime > s.backoffMax {
@@ -263,6 +271,7 @@ func (sc *SubscriptionScope) Count() int {
 	return len(sc.subs)
 }
 
+// Unsubscribe unsubscribes from subscription.
 func (s *scopeSub) Unsubscribe() {
 	s.s.Unsubscribe()
 	s.sc.mu.Lock()
@@ -270,6 +279,7 @@ func (s *scopeSub) Unsubscribe() {
 	delete(s.sc.subs, s)
 }
 
+// Err exposes error channel.
 func (s *scopeSub) Err() <-chan error {
 	return s.s.Err()
 }

async/event/subscription_test.go

@@ -19,8 +19,11 @@ package event
 import (
 	"context"
 	"errors"
+	"runtime"
 	"testing"
 	"time"
+
+	"github.com/prysmaticlabs/prysm/testing/require"
 )
 
 var errInts = errors.New("error in subscribeInts")
@@ -50,25 +53,17 @@ loop:
 	for want := 0; want < 10; want++ {
 		select {
 		case got := <-channel:
-			if got != want {
-				t.Fatalf("wrong int %d, want %d", got, want)
-			}
+			require.Equal(t, want, got)
 		case err := <-sub.Err():
-			if err != errInts {
-				t.Fatalf("wrong error: got %q, want %q", err, errInts)
-			}
-			if want != 2 {
-				t.Fatalf("got errInts at int %d, should be received at 2", want)
-			}
+			require.Equal(t, errInts, err)
+			require.Equal(t, 2, want)
 			break loop
 		}
 	}
 	sub.Unsubscribe()
 
 	err, ok := <-sub.Err()
-	if err != nil {
-		t.Fatal("got non-nil error after Unsubscribe")
-	}
+	require.NoError(t, err)
 	if ok {
 		t.Fatal("channel still open after Unsubscribe")
 	}
@@ -94,9 +89,7 @@ func TestResubscribe(t *testing.T) {
 	})
 
 	<-sub.Err()
-	if i != nfails {
-		t.Fatalf("resubscribe function called %d times, want %d times", i, nfails)
-	}
+	require.Equal(t, nfails, i)
 }
 
 func TestResubscribeAbort(t *testing.T) {
@@ -114,7 +107,20 @@ func TestResubscribeAbort(t *testing.T) {
 	})
 
 	sub.Unsubscribe()
-	if err := <-done; err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, <-done)
+}
+
+func TestResubscribeNonBlocking(t *testing.T) {
+	done := make(chan struct{})
+	sub := Resubscribe(0, func(ctx context.Context) (Subscription, error) {
+		<-done
+		return nil, nil
+	})
+
+	resub, ok := sub.(*resubscribeSub)
+	require.Equal(t, true, ok)
+	currNum := runtime.NumGoroutine()
+	resub.unsub <- struct{}{}
+	done <- struct{}{}
+	require.Equal(t, currNum-1, runtime.NumGoroutine())
 }

async/every.go

@@ -0,0 +1,31 @@
+// Package async includes helpers for scheduling runnable, periodic functions and contains useful helpers for converting multi-processor computation.
+package async
+
+import (
+	"context"
+	"reflect"
+	"runtime"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// RunEvery runs the provided command periodically.
+// It runs in a goroutine, and can be cancelled by finishing the supplied context.
+func RunEvery(ctx context.Context, period time.Duration, f func()) {
+	funcName := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
+	ticker := time.NewTicker(period)
+	go func() {
+		for {
+			select {
+			case <-ticker.C:
+				log.WithField("function", funcName).Trace("running")
+				f()
+			case <-ctx.Done():
+				log.WithField("function", funcName).Debug("context is closed, exiting")
+				ticker.Stop()
+				return
+			}
+		}
+	}()
+}

async/every_test.go

@@ -0,0 +1,39 @@
+package async_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/prysmaticlabs/prysm/async"
+)
+
+func TestEveryRuns(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	i := 0
+	async.RunEvery(ctx, 100*time.Millisecond, func() {
+		i++
+	})
+
+	// Sleep for a bit and ensure the value has increased.
+	time.Sleep(200 * time.Millisecond)
+
+	if i == 0 {
+		t.Error("Counter failed to increment with ticker")
+	}
+
+	cancel()
+
+	// Sleep for a bit to let the cancel take place.
+	time.Sleep(100 * time.Millisecond)
+
+	last := i
+
+	// Sleep for a bit and ensure the value has not increased.
+	time.Sleep(200 * time.Millisecond)
+
+	if i != last {
+		t.Error("Counter incremented after stop")
+	}
+}

async/multilock.go

@@ -0,0 +1,139 @@
+/*
+Copyright 2017 Albert Tedja
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package async
+
+import (
+	"runtime"
+	"sort"
+)
+
+var locks = struct {
+	lock chan byte
+	list map[string]chan byte
+}{
+	lock: make(chan byte, 1),
+	list: make(map[string]chan byte),
+}
+
+type Lock struct {
+	keys   []string
+	chans  []chan byte
+	lock   chan byte
+	unlock chan byte
+}
+
+func (lk *Lock) Lock() {
+	lk.lock <- 1
+
+	// get the channels and attempt to acquire them
+	lk.chans = make([]chan byte, 0, len(lk.keys))
+	for i := 0; i < len(lk.keys); {
+		ch := getChan(lk.keys[i])
+		_, ok := <-ch
+		if ok {
+			lk.chans = append(lk.chans, ch)
+			i++
+		}
+	}
+
+	lk.unlock <- 1
+}
+
+// Unlocks this lock. Must be called after Lock.
+// Can only be invoked if there is a previous call to Lock.
+func (lk *Lock) Unlock() {
+	<-lk.unlock
+
+	if lk.chans != nil {
+		for _, ch := range lk.chans {
+			ch <- 1
+		}
+		lk.chans = nil
+	}
+	// Clean unused channels after the unlock.
+	Clean()
+	<-lk.lock
+}
+
+// Temporarily unlocks, gives up the cpu time to other goroutine, and attempts to lock again.
+func (lk *Lock) Yield() {
+	lk.Unlock()
+	runtime.Gosched()
+	lk.Lock()
+}
+
+// Creates a new multilock for the specified keys
+func NewMultilock(locks ...string) *Lock {
+	if len(locks) == 0 {
+		return nil
+	}
+
+	locks = unique(locks)
+	sort.Strings(locks)
+	return &Lock{
+		keys:   locks,
+		lock:   make(chan byte, 1),
+		unlock: make(chan byte, 1),
+	}
+}
+
+// Cleans old unused locks. Returns removed keys.
+func Clean() []string {
+	locks.lock <- 1
+	defer func() { <-locks.lock }()
+
+	toDelete := make([]string, 0, len(locks.list))
+	for key, ch := range locks.list {
+		select {
+		case <-ch:
+			close(ch)
+			toDelete = append(toDelete, key)
+		default:
+		}
+	}
+
+	for _, del := range toDelete {
+		delete(locks.list, del)
+	}
+
+	return toDelete
+}
+
+// Create and get the channel for the specified key.
+func getChan(key string) chan byte {
+	locks.lock <- 1
+	defer func() { <-locks.lock }()
+
+	if locks.list[key] == nil {
+		locks.list[key] = make(chan byte, 1)
+		locks.list[key] <- 1
+	}
+	return locks.list[key]
+}
+
+// Return a new string with unique elements.
+func unique(arr []string) []string {
+	if arr == nil || len(arr) <= 1 {
+		return arr
+	}
+
+	found := map[string]bool{}
+	result := make([]string, 0, len(arr))
+	for _, v := range arr {
+		if !found[v] {
+			found[v] = true
+			result = append(result, v)
+		}
+	}
+	return result
+}

async/multilock_test.go

@@ -0,0 +1,342 @@
+/*
+Copyright 2017 Albert Tedja
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package async
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUnique(t *testing.T) {
+	var arr []string
+	assert := assert.New(t)
+
+	arr = []string{"a", "b", "c"}
+	assert.Equal(arr, unique(arr))
+
+	arr = []string{"a", "a", "a"}
+	assert.Equal([]string{"a"}, unique(arr))
+
+	arr = []string{"a", "a", "b"}
+	assert.Equal([]string{"a", "b"}, unique(arr))
+
+	arr = []string{"a", "b", "a"}
+	assert.Equal([]string{"a", "b"}, unique(arr))
+
+	arr = []string{"a", "b", "c", "b", "d"}
+	assert.Equal([]string{"a", "b", "c", "d"}, unique(arr))
+}
+
+func TestGetChan(t *testing.T) {
+	ch1 := getChan("a")
+	ch2 := getChan("aa")
+	ch3 := getChan("a")
+
+	assert := assert.New(t)
+	assert.NotEqual(ch1, ch2)
+	assert.Equal(ch1, ch3)
+}
+
+func TestLockUnlock(_ *testing.T) {
+	var wg sync.WaitGroup
+
+	wg.Add(5)
+
+	go func() {
+		lock := NewMultilock("dog", "cat", "owl")
+		lock.Lock()
+		defer lock.Unlock()
+
+		<-time.After(100 * time.Millisecond)
+		wg.Done()
+	}()
+
+	go func() {
+		lock := NewMultilock("cat", "dog", "bird")
+		lock.Lock()
+		defer lock.Unlock()
+
+		<-time.After(100 * time.Millisecond)
+		wg.Done()
+	}()
+
+	go func() {
+		lock := NewMultilock("cat", "bird", "owl")
+		lock.Lock()
+		defer lock.Unlock()
+
+		<-time.After(100 * time.Millisecond)
+		wg.Done()
+	}()
+
+	go func() {
+		lock := NewMultilock("bird", "owl", "snake")
+		lock.Lock()
+		defer lock.Unlock()
+
+		<-time.After(100 * time.Millisecond)
+		wg.Done()
+	}()
+
+	go func() {
+		lock := NewMultilock("owl", "snake")
+		lock.Lock()
+		defer lock.Unlock()
+
+		<-time.After(1 * time.Second)
+		wg.Done()
+	}()
+
+	wg.Wait()
+}
+
+func TestLockUnlock_CleansUnused(t *testing.T) {
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		lock := NewMultilock("dog", "cat", "owl")
+		lock.Lock()
+		assert.Equal(t, 3, len(locks.list))
+		defer lock.Unlock()
+
+		<-time.After(100 * time.Millisecond)
+		wg.Done()
+	}()
+	wg.Wait()
+	// We expect that unlocking completely cleared the locks list
+	// given all 3 lock keys were unused at time of unlock.
+	assert.Equal(t, 0, len(locks.list))
+}
+
+func TestLockUnlock_DoesNotCleanIfHeldElsewhere(t *testing.T) {
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		lock := NewMultilock("cat")
+		lock.Lock()
+		// We take 200 milliseconds to release the lock on "cat"
+		<-time.After(200 * time.Millisecond)
+		lock.Unlock()
+		// Assert that at the end of this goroutine, all locks are cleared.
+		assert.Equal(t, 0, len(locks.list))
+		wg.Done()
+	}()
+	go func() {
+		lock := NewMultilock("dog", "cat", "owl")
+		lock.Lock()
+		// We release the locks after 100 milliseconds, and check that "cat" is not
+		// cleared as a lock for it is still held by the previous goroutine.
+		<-time.After(100 * time.Millisecond)
+		lock.Unlock()
+		assert.Equal(t, 1, len(locks.list))
+		_, ok := locks.list["cat"]
+		assert.Equal(t, true, ok)
+		wg.Done()
+	}()
+	wg.Wait()
+	// We expect that at the end of this test, all locks are cleared.
+	assert.Equal(t, 0, len(locks.list))
+}
+
+func TestYield(t *testing.T) {
+	var wg sync.WaitGroup
+
+	wg.Add(2)
+	var resources = map[string]int{}
+
+	go func() {
+		lock := NewMultilock("A", "C")
+		lock.Lock()
+		defer lock.Unlock()
+
+		for resources["ac"] == 0 {
+			lock.Yield()
+		}
+		resources["dc"] = 10
+
+		wg.Done()
+	}()
+
+	go func() {
+		lock := NewMultilock("D", "C")
+		lock.Lock()
+		defer lock.Unlock()
+
+		resources["ac"] = 5
+		for resources["dc"] == 0 {
+			lock.Yield()
+		}
+
+		wg.Done()
+	}()
+
+	wg.Wait()
+
+	assert.Equal(t, 5, resources["ac"])
+	assert.Equal(t, 10, resources["dc"])
+}
+
+func TestClean(t *testing.T) {
+	var wg sync.WaitGroup
+
+	wg.Add(3)
+
+	// some goroutine that holds multiple locks
+	go1done := make(chan bool, 1)
+	go func() {
+	Loop:
+		for {
+			select {
+			case <-go1done:
+				break Loop
+			default:
+				lock := NewMultilock("A", "B", "C", "E", "Z")
+				lock.Lock()
+				<-time.After(30 * time.Millisecond)
+				lock.Unlock()
+			}
+		}
+		wg.Done()
+	}()
+
+	// another goroutine
+	go2done := make(chan bool, 1)
+	go func() {
+	Loop:
+		for {
+			select {
+			case <-go2done:
+				break Loop
+			default:
+				lock := NewMultilock("B", "C", "K", "L", "Z")
+				lock.Lock()
+				<-time.After(200 * time.Millisecond)
+				lock.Unlock()
+			}
+		}
+		wg.Done()
+	}()
+
+	// this one cleans up the locks every 100 ms
+	done := make(chan bool, 1)
+	go func() {
+		c := time.Tick(100 * time.Millisecond)
+	Loop:
+		for {
+			select {
+			case <-done:
+				break Loop
+			case <-c:
+				Clean()
+			}
+		}
+		wg.Done()
+	}()
+
+	<-time.After(2 * time.Second)
+	go1done <- true
+	go2done <- true
+	<-time.After(1 * time.Second)
+	done <- true
+	wg.Wait()
+	assert.Equal(t, []string{}, Clean())
+}
+
+func TestBankAccountProblem(t *testing.T) {
+	var wg sync.WaitGroup
+	wg.Add(3)
+
+	joe := 50.0
+	susan := 100.0
+
+	// withdraw $80 from joe, only if balance is sufficient
+	go func() {
+		lock := NewMultilock("joe")
+		lock.Lock()
+		defer lock.Unlock()
+
+		for joe < 80.0 {
+			lock.Yield()
+		}
+		joe -= 80.0
+
+		wg.Done()
+	}()
+
+	// transfer $200 from susan to joe, only if balance is sufficient
+	go func() {
+		lock := NewMultilock("joe", "susan")
+		lock.Lock()
+		defer lock.Unlock()
+
+		for susan < 200.0 {
+			lock.Yield()
+		}
+
+		susan -= 200.0
+		joe += 200.0
+
+		wg.Done()
+	}()
+
+	// susan deposit $300 to cover balance
+	go func() {
+		lock := NewMultilock("susan")
+		lock.Lock()
+		defer lock.Unlock()
+
+		susan += 300.0
+
+		wg.Done()
+	}()
+
+	wg.Wait()
+	assert.Equal(t, 170.0, joe)
+	assert.Equal(t, 200.0, susan)
+}
+
+func TestSyncCondCompatibility(t *testing.T) {
+	var wg sync.WaitGroup
+	wg.Add(2)
+	cond := sync.NewCond(NewMultilock("A", "C"))
+	var testValues = [3]string{"foo", "bar", "fizz!"}
+	sharedRsc := testValues[0]
+
+	go func() {
+		cond.L.Lock()
+		for sharedRsc == testValues[0] {
+			cond.Wait()
+		}
+		sharedRsc = testValues[2]
+		cond.Broadcast()
+		cond.L.Unlock()
+		wg.Done()
+	}()
+
+	go func() {
+		cond.L.Lock()
+		sharedRsc = testValues[1]
+		cond.Broadcast()
+		for sharedRsc == testValues[1] {
+			cond.Wait()
+		}
+		cond.L.Unlock()
+		wg.Done()
+	}()
+
+	wg.Wait()
+	assert.Equal(t, testValues[2], sharedRsc)
+}

async/scatter.go

@@ -0,0 +1,77 @@
+package async
+
+import (
+	"errors"
+	"runtime"
+	"sync"
+)
+
+// WorkerResults are the results of a scatter worker.
+type WorkerResults struct {
+	Offset int
+	Extent interface{}
+}
+
+// Scatter scatters a computation across multiple goroutines.
+// This breaks the task in to a number of chunks and executes those chunks in parallel with the function provided.
+// Results returned are collected and presented a a set of WorkerResults, which can be reassembled by the calling function.
+// Any error that occurs in the workers will be passed back to the calling function.
+func Scatter(inputLen int, sFunc func(int, int, *sync.RWMutex) (interface{}, error)) ([]*WorkerResults, error) {
+	if inputLen <= 0 {
+		return nil, errors.New("input length must be greater than 0")
+	}
+
+	chunkSize := calculateChunkSize(inputLen)
+	workers := inputLen / chunkSize
+	if inputLen%chunkSize != 0 {
+		workers++
+	}
+	resultCh := make(chan *WorkerResults, workers)
+	defer close(resultCh)
+	errorCh := make(chan error, workers)
+	defer close(errorCh)
+	mutex := new(sync.RWMutex)
+	for worker := 0; worker < workers; worker++ {
+		offset := worker * chunkSize
+		entries := chunkSize
+		if offset+entries > inputLen {
+			entries = inputLen - offset
+		}
+		go func(offset int, entries int) {
+			extent, err := sFunc(offset, entries, mutex)
+			if err != nil {
+				errorCh <- err
+			} else {
+				resultCh <- &WorkerResults{
+					Offset: offset,
+					Extent: extent,
+				}
+			}
+		}(offset, entries)
+	}
+
+	// Collect results from workers
+	results := make([]*WorkerResults, workers)
+	for i := 0; i < workers; i++ {
+		select {
+		case result := <-resultCh:
+			results[i] = result
+		case err := <-errorCh:
+			return nil, err
+		}
+	}
+	return results, nil
+}
+
+// calculateChunkSize calculates a suitable chunk size for the purposes of parallelisation.
+func calculateChunkSize(items int) int {
+	// Start with a simple even split
+	chunkSize := items / runtime.GOMAXPROCS(0)
+
+	// Add 1 if we have leftovers (or if we have fewer items than processors).
+	if chunkSize == 0 || items%chunkSize != 0 {
+		chunkSize++
+	}
+
+	return chunkSize
+}

async/scatter_test.go

@@ -0,0 +1,108 @@
+package async_test
+
+import (
+	"errors"
+	"sync"
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/async"
+	"github.com/prysmaticlabs/prysm/testing/assert"
+	"github.com/prysmaticlabs/prysm/testing/require"
+)
+
+func TestDouble(t *testing.T) {
+	tests := []struct {
+		name     string
+		inValues int
+		err      error
+	}{
+		{
+			name:     "0",
+			inValues: 0,
+			err:      errors.New("input length must be greater than 0"),
+		},
+		{
+			name:     "1",
+			inValues: 1,
+		},
+		{
+			name:     "1023",
+			inValues: 1023,
+		},
+		{
+			name:     "1024",
+			inValues: 1024,
+		},
+		{
+			name:     "1025",
+			inValues: 1025,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			inValues := make([]int, test.inValues)
+			for i := 0; i < test.inValues; i++ {
+				inValues[i] = i
+			}
+			outValues := make([]int, test.inValues)
+			workerResults, err := async.Scatter(len(inValues), func(offset int, entries int, _ *sync.RWMutex) (interface{}, error) {
+				extent := make([]int, entries)
+				for i := 0; i < entries; i++ {
+					extent[i] = inValues[offset+i] * 2
+				}
+				return extent, nil
+			})
+			if test.err != nil {
+				assert.ErrorContains(t, test.err.Error(), err)
+			} else {
+				require.NoError(t, err)
+				for _, result := range workerResults {
+					copy(outValues[result.Offset:], result.Extent.([]int))
+				}
+
+				for i := 0; i < test.inValues; i++ {
+					require.Equal(t, inValues[i]*2, outValues[i], "Outvalue at %d mismatch", i)
+				}
+			}
+		})
+	}
+}
+
+func TestMutex(t *testing.T) {
+	totalRuns := 1048576
+	val := 0
+	_, err := async.Scatter(totalRuns, func(offset int, entries int, mu *sync.RWMutex) (interface{}, error) {
+		for i := 0; i < entries; i++ {
+			mu.Lock()
+			val++
+			mu.Unlock()
+		}
+		return nil, nil
+	})
+	require.NoError(t, err)
+
+	if val != totalRuns {
+		t.Fatalf("Unexpected value: expected \"%v\", found \"%v\"", totalRuns, val)
+	}
+}
+
+func TestError(t *testing.T) {
+	totalRuns := 1024
+	val := 0
+	_, err := async.Scatter(totalRuns, func(offset int, entries int, mu *sync.RWMutex) (interface{}, error) {
+		for i := 0; i < entries; i++ {
+			mu.Lock()
+			val++
+			if val == 1011 {
+				mu.Unlock()
+				return nil, errors.New("bad number")
+			}
+			mu.Unlock()
+		}
+		return nil, nil
+	})
+	if err == nil {
+		t.Fatalf("Missing expected error")
+	}
+}

bazel.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# This script serves as a wrapper around bazel to limit the scope of environment variables that
+# may change the action output. Using this script should result in a higher cache hit ratio for
+# cached actions with a more heremtic build.
+
+env -i \
+ PATH=/usr/bin:/bin \
+ HOME="$HOME" \
+ GOOGLE_APPLICATION_CREDENTIALS="$GOOGLE_APPLICATION_CREDENTIALS" \
+ bazel "$@"

beacon-chain/BUILD.bazel

@@ -1,67 +1,19 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
-load("@io_bazel_rules_docker//go:image.bzl", "go_image")
-load("@io_bazel_rules_docker//container:container.bzl", "container_push")
+load("//tools:target_migration.bzl", "moved_targets")
 
-go_library(
-    name = "go_default_library",
-    srcs = [
-        "main.go",
-        "usage.go",
-    ],
-    importpath = "github.com/prysmaticlabs/prysm/beacon-chain",
-    visibility = ["//beacon-chain:__subpackages__"],
-    deps = [
-        "//beacon-chain/node:go_default_library",
-        "//beacon-chain/utils:go_default_library",
-        "//shared/cmd:go_default_library",
-        "//shared/debug:go_default_library",
-        "//shared/featureconfig:go_default_library",
-        "//shared/version:go_default_library",
-        "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_urfave_cli//:go_default_library",
-        "@com_github_x_cray_logrus_prefixed_formatter//:go_default_library",
+moved_targets(
+    [
+        ":push_images_debug",
+        ":push_images_alpine",
+        ":push_images",
+        ":image_bundle_debug",
+        ":image_debug",
+        ":image_bundle_alpine",
+        ":image_bundle",
+        ":image_with_creation_time",
+        ":image_alpine",
+        ":image",
+        ":go_default_test",
+        ":beacon-chain",
     ],
-)
-
-go_image(
-    name = "image",
-    srcs = [
-        "main.go",
-        "usage.go",
-    ],
-    goarch = "amd64",
-    goos = "linux",
-    importpath = "github.com/prysmaticlabs/prysm/beacon-chain",
-    race = "off",
-    static = "off",  # Static enabled binary seems to cause issues with DNS lookup with cgo.
-    tags = ["manual"],
-    visibility = ["//visibility:private"],
-    deps = [
-        "//beacon-chain/node:go_default_library",
-        "//beacon-chain/utils:go_default_library",
-        "//shared/cmd:go_default_library",
-        "//shared/debug:go_default_library",
-        "//shared/featureconfig:go_default_library",
-        "//shared/version:go_default_library",
-        "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_urfave_cli//:go_default_library",
-        "@com_github_x_cray_logrus_prefixed_formatter//:go_default_library",
-    ],
-)
-
-container_push(
-    name = "push_image",
-    format = "Docker",
-    image = ":image",
-    registry = "gcr.io",
-    repository = "prysmaticlabs/prysm/beacon-chain",
-    tag = "latest",
-    tags = ["manual"],
-    visibility = ["//visibility:private"],
-)
-
-go_binary(
-    name = "beacon-chain",
-    embed = [":go_default_library"],
-    visibility = ["//beacon-chain:__subpackages__"],
+    "//cmd/beacon-chain",
 )

beacon-chain/README.md

@@ -1,10 +1,9 @@
 # Prysmatic Labs Beacon Chain Implementation
 
-This is the main project folder for the beacon chain implementation of Ethereum Serenity in Golang by [Prysmatic Labs](https://prysmaticlabs.com). Before you begin, check out our [Contribution Guidelines](https://github.com/prysmaticlabs/prysm/blob/master/CONTRIBUTING.md) and join our active chat room on Discord or Gitter below:
+This is the main project folder for the beacon chain implementation of Ethereum written in Go by [Prysmatic Labs](https://prysmaticlabs.com).
 
-[![Discord](https://user-images.githubusercontent.com/7288322/34471967-1df7808a-efbb-11e7-9088-ed0b04151291.png)](https://discord.gg/KSA7rPr)
-[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/prysmaticlabs/prysm?badge&utm_medium=badge&utm_campaign=pr-badge)
+You can also read our main [README](https://github.com/prysmaticlabs/prysm/blob/master/README.md) and join our active chat room on Discord.
 
-Also, read the latest beacon chain [design spec](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md), this design spec serves as a source of truth for the beacon chain implementation we follow at prysmatic labs.
-Check out the [FAQs](https://notes.ethereum.org/9MMuzWeFTTSg-3Tz_YeiBA?view). Refer this page on [why](http://email.mg2.substack.com/c/eJwlj9GOhCAMRb9G3jRQQPGBh5mM8xsbhKrsDGIAM9m_X9xN2qZtbpt7rCm4xvSjj5gLOTOmL-809CMbKXFaOKakIl4DZYr2AGyQIGjHOnWH22OiYnoIxmDijaBhhS6fcy7GvjobA9m0mSXOcnZq5GBqLkilXBZhBsus5ZK89VbKkRt-a-BZI6DzZ7iur1lQ953KJ9bemnxgahuQU9XJu6pFPdu8meT8vragzEjpMCwMGLlgLo6h5z1JumQTu4IJd4v15xqMf_8ZLP_Y1bSLdbnrD-LL71i2Kj7DLxaWWF4)
-we are combining sharding and casper together.
+[![Discord](https://user-images.githubusercontent.com/7288322/34471967-1df7808a-efbb-11e7-9088-ed0b04151291.png)](https://discord.gg/CTYGPUJ)
+
+Also, read the official beacon chain [specification](https://github.com/ethereum/consensus-specs/blob/master/specs/phase0/beacon-chain.md), this design spec serves as a source of truth for the beacon chain implementation we follow at Prysmatic Labs.

beacon-chain/attestation/BUILD.bazel

@@ -1,45 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
-
-go_library(
-    name = "go_default_library",
-    srcs = [
-        "service.go",
-        "vote_metrics.go",
-    ],
-    importpath = "github.com/prysmaticlabs/prysm/beacon-chain/attestation",
-    visibility = ["//beacon-chain:__subpackages__"],
-    deps = [
-        "//beacon-chain/cache:go_default_library",
-        "//beacon-chain/core/helpers:go_default_library",
-        "//beacon-chain/core/state:go_default_library",
-        "//beacon-chain/db:go_default_library",
-        "//proto/beacon/p2p/v1:go_default_library",
-        "//shared/bitutil:go_default_library",
-        "//shared/bytesutil:go_default_library",
-        "//shared/event:go_default_library",
-        "//shared/hashutil:go_default_library",
-        "//shared/messagehandler:go_default_library",
-        "//shared/params:go_default_library",
-        "@com_github_gogo_protobuf//proto:go_default_library",
-        "@com_github_prometheus_client_golang//prometheus:go_default_library",
-        "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
-        "@com_github_sirupsen_logrus//:go_default_library",
-    ],
-)
-
-go_test(
-    name = "go_default_test",
-    srcs = ["service_test.go"],
-    embed = [":go_default_library"],
-    deps = [
-        "//beacon-chain/cache:go_default_library",
-        "//beacon-chain/internal:go_default_library",
-        "//proto/beacon/p2p/v1:go_default_library",
-        "//shared/bytesutil:go_default_library",
-        "//shared/hashutil:go_default_library",
-        "//shared/params:go_default_library",
-        "//shared/testutil:go_default_library",
-        "@com_github_sirupsen_logrus//:go_default_library",
-        "@com_github_sirupsen_logrus//hooks/test:go_default_library",
-    ],
-)

beacon-chain/attestation/service.go

@@ -1,333 +0,0 @@
-// Package attestation defines the life-cycle and status of single and aggregated attestation.
-package attestation
-
-import (
-	"context"
-	"fmt"
-	"sort"
-	"sync"
-
-	"github.com/gogo/protobuf/proto"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bitutil"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/event"
-	"github.com/prysmaticlabs/prysm/shared/hashutil"
-	handler "github.com/prysmaticlabs/prysm/shared/messagehandler"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/sirupsen/logrus"
-)
-
-var log = logrus.WithField("prefix", "attestation")
-var committeeCache = cache.NewCommitteesCache()
-
-// TargetHandler provides an interface for fetching latest attestation targets
-// and updating attestations in batches.
-type TargetHandler interface {
-	LatestAttestationTarget(state *pb.BeaconState, validatorIndex uint64) (*pb.AttestationTarget, error)
-	BatchUpdateLatestAttestation(ctx context.Context, atts []*pb.Attestation) error
-}
-
-type attestationStore struct {
-	sync.RWMutex
-	m map[[48]byte]*pb.Attestation
-}
-
-// Service represents a service that handles the internal
-// logic of managing single and aggregated attestation.
-type Service struct {
-	ctx          context.Context
-	cancel       context.CancelFunc
-	beaconDB     *db.BeaconDB
-	incomingFeed *event.Feed
-	incomingChan chan *pb.Attestation
-	// store is the mapping of individual
-	// validator's public key to it's latest attestation.
-	store attestationStore
-}
-
-// Config options for the service.
-type Config struct {
-	BeaconDB *db.BeaconDB
-}
-
-// NewAttestationService instantiates a new service instance that will
-// be registered into a running beacon node.
-func NewAttestationService(ctx context.Context, cfg *Config) *Service {
-	ctx, cancel := context.WithCancel(ctx)
-	return &Service{
-		ctx:          ctx,
-		cancel:       cancel,
-		beaconDB:     cfg.BeaconDB,
-		incomingFeed: new(event.Feed),
-		incomingChan: make(chan *pb.Attestation, params.BeaconConfig().DefaultBufferSize),
-		store:        attestationStore{m: make(map[[48]byte]*pb.Attestation)},
-	}
-}
-
-// Start an attestation service's main event loop.
-func (a *Service) Start() {
-	log.Info("Starting service")
-	go a.attestationPool()
-}
-
-// Stop the Attestation service's main event loop and associated goroutines.
-func (a *Service) Stop() error {
-	defer a.cancel()
-	log.Info("Stopping service")
-	return nil
-}
-
-// Status always returns nil.
-// TODO(1201): Add service health checks.
-func (a *Service) Status() error {
-	return nil
-}
-
-// IncomingAttestationFeed returns a feed that any service can send incoming p2p attestations into.
-// The attestation service will subscribe to this feed in order to relay incoming attestations.
-func (a *Service) IncomingAttestationFeed() *event.Feed {
-	return a.incomingFeed
-}
-
-// LatestAttestationTarget returns the target block that the validator index attested to,
-// the highest slotNumber attestation in attestation pool gets returned.
-//
-// Spec pseudocode definition:
-//	Let `get_latest_attestation_target(store: Store, validator_index: ValidatorIndex) ->
-//		BeaconBlock` be the target block in the attestation
-//		`get_latest_attestation(store, validator_index)`.
-func (a *Service) LatestAttestationTarget(beaconState *pb.BeaconState, index uint64) (*pb.AttestationTarget, error) {
-	if index >= uint64(len(beaconState.ValidatorRegistry)) {
-		return nil, fmt.Errorf("invalid validator index %d", index)
-	}
-	validator := beaconState.ValidatorRegistry[index]
-
-	pubKey := bytesutil.ToBytes48(validator.Pubkey)
-	a.store.RLock()
-	defer a.store.RUnlock()
-	if _, exists := a.store.m[pubKey]; !exists {
-		return nil, nil
-	}
-
-	attestation := a.store.m[pubKey]
-	if attestation == nil {
-		return nil, nil
-	}
-	targetRoot := bytesutil.ToBytes32(attestation.Data.BeaconBlockRootHash32)
-	if !a.beaconDB.HasBlock(targetRoot) {
-		return nil, nil
-	}
-
-	return a.beaconDB.AttestationTarget(targetRoot)
-}
-
-// attestationPool takes an newly received attestation from sync service
-// and updates attestation pool.
-func (a *Service) attestationPool() {
-	incomingSub := a.incomingFeed.Subscribe(a.incomingChan)
-	defer incomingSub.Unsubscribe()
-	for {
-		select {
-		case <-a.ctx.Done():
-			log.Debug("Attestation pool closed, exiting goroutine")
-			return
-		// Listen for a newly received incoming attestation from the sync service.
-		case attestations := <-a.incomingChan:
-			handler.SafelyHandleMessage(a.ctx, a.handleAttestation, attestations)
-		}
-	}
-}
-
-func (a *Service) handleAttestation(ctx context.Context, msg proto.Message) error {
-	attestation := msg.(*pb.Attestation)
-	if err := a.UpdateLatestAttestation(ctx, attestation); err != nil {
-		return fmt.Errorf("could not update attestation pool: %v", err)
-	}
-	return nil
-}
-
-// UpdateLatestAttestation inputs an new attestation and checks whether
-// the attesters who submitted this attestation with the higher slot number
-// have been noted in the attestation pool. If not, it updates the
-// attestation pool with attester's public key to attestation.
-func (a *Service) UpdateLatestAttestation(ctx context.Context, attestation *pb.Attestation) error {
-	totalAttestationSeen.Inc()
-
-	// Potential improvement, instead of getting the state,
-	// we could get a mapping of validator index to public key.
-	beaconState, err := a.beaconDB.HeadState(ctx)
-	if err != nil {
-		return err
-	}
-	head, err := a.beaconDB.ChainHead()
-	if err != nil {
-		return err
-	}
-	headRoot, err := hashutil.HashBeaconBlock(head)
-	if err != nil {
-		return err
-	}
-	return a.updateAttestation(ctx, headRoot, beaconState, attestation)
-}
-
-// BatchUpdateLatestAttestation updates multiple attestations and adds them into the attestation store
-// if they are valid.
-func (a *Service) BatchUpdateLatestAttestation(ctx context.Context, attestations []*pb.Attestation) error {
-
-	if attestations == nil {
-		return nil
-	}
-	// Potential improvement, instead of getting the state,
-	// we could get a mapping of validator index to public key.
-	beaconState, err := a.beaconDB.HeadState(ctx)
-	if err != nil {
-		return err
-	}
-	head, err := a.beaconDB.ChainHead()
-	if err != nil {
-		return err
-	}
-	headRoot, err := hashutil.HashBeaconBlock(head)
-	if err != nil {
-		return err
-	}
-
-	attestations = a.sortAttestations(attestations)
-
-	for _, attestation := range attestations {
-		if err := a.updateAttestation(ctx, headRoot, beaconState, attestation); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// InsertAttestationIntoStore locks the store, inserts the attestation, then
-// unlocks the store again. This method may be used by external services
-// in testing to populate the attestation store.
-func (a *Service) InsertAttestationIntoStore(pubkey [48]byte, att *pb.Attestation) {
-	a.store.Lock()
-	defer a.store.Unlock()
-	a.store.m[pubkey] = att
-}
-
-func (a *Service) updateAttestation(ctx context.Context, headRoot [32]byte, beaconState *pb.BeaconState,
-	attestation *pb.Attestation) error {
-	totalAttestationSeen.Inc()
-
-	slot := attestation.Data.Slot
-	var committee []uint64
-	var cachedCommittees *cache.CommitteesInSlot
-	var err error
-
-	for beaconState.Slot < slot {
-		beaconState, err = state.ExecuteStateTransition(
-			ctx, beaconState, nil /* block */, headRoot, &state.TransitionConfig{},
-		)
-		if err != nil {
-			return fmt.Errorf("could not execute head transition: %v", err)
-		}
-	}
-
-	cachedCommittees, err = committeeCache.CommitteesInfoBySlot(slot)
-	if err != nil {
-		return err
-	}
-	if cachedCommittees == nil {
-		crosslinkCommittees, err := helpers.CrosslinkCommitteesAtSlot(beaconState, slot, false /* registryChange */)
-		if err != nil {
-			return err
-		}
-		cachedCommittees = helpers.ToCommitteeCache(slot, crosslinkCommittees)
-		if err := committeeCache.AddCommittees(cachedCommittees); err != nil {
-			return err
-		}
-	}
-
-	// Find committee for shard.
-	for _, v := range cachedCommittees.Committees {
-		if v.Shard == attestation.Data.Shard {
-			committee = v.Committee
-			break
-		}
-	}
-
-	log.WithFields(logrus.Fields{
-		"attestationSlot":    attestation.Data.Slot - params.BeaconConfig().GenesisSlot,
-		"attestationShard":   attestation.Data.Shard,
-		"committeesShard":    cachedCommittees.Committees[0].Shard,
-		"committeesList":     cachedCommittees.Committees[0].Committee,
-		"lengthOfCommittees": len(cachedCommittees.Committees),
-	}).Debug("Updating latest attestation")
-
-	// The participation bitfield from attestation is represented in bytes,
-	// here we multiply by 8 to get an accurate validator count in bits.
-	bitfield := attestation.AggregationBitfield
-	totalBits := len(bitfield) * 8
-
-	// Check each bit of participation bitfield to find out which
-	// attester has submitted new attestation.
-	// This is has O(n) run time and could be optimized down the line.
-	for i := 0; i < totalBits; i++ {
-		bitSet, err := bitutil.CheckBit(bitfield, i)
-		if err != nil {
-			return err
-		}
-		if !bitSet {
-			continue
-		}
-
-		if i >= len(committee) {
-			log.Errorf("Bitfield points to an invalid index in the committee: bitfield %08b", bitfield)
-			continue
-		}
-
-		if int(committee[i]) >= len(beaconState.ValidatorRegistry) {
-			log.Errorf("Index doesn't exist in validator registry: index %d", committee[i])
-		}
-
-		// If the attestation came from this attester. We use the slot committee to find the
-		// validator's actual index.
-		pubkey := bytesutil.ToBytes48(beaconState.ValidatorRegistry[committee[i]].Pubkey)
-		newAttestationSlot := attestation.Data.Slot
-		currentAttestationSlot := uint64(0)
-		a.store.Lock()
-		defer a.store.Unlock()
-		if _, exists := a.store.m[pubkey]; exists {
-			currentAttestationSlot = a.store.m[pubkey].Data.Slot
-		}
-		// If the attestation is newer than this attester's one in pool.
-		if newAttestationSlot > currentAttestationSlot {
-			a.store.m[pubkey] = attestation
-
-			log.WithFields(
-				logrus.Fields{
-					"attestationSlot": attestation.Data.Slot - params.BeaconConfig().GenesisSlot,
-					"justifiedEpoch":  attestation.Data.JustifiedEpoch - params.BeaconConfig().GenesisEpoch,
-				},
-			).Debug("Attestation store updated")
-
-			blockRoot := bytesutil.ToBytes32(attestation.Data.BeaconBlockRootHash32)
-			votedBlock, err := a.beaconDB.Block(blockRoot)
-			if err != nil {
-				return err
-			}
-			reportVoteMetrics(committee[i], votedBlock)
-		}
-	}
-	return nil
-}
-
-// sortAttestations sorts attestations by their slot number in ascending order.
-func (a *Service) sortAttestations(attestations []*pb.Attestation) []*pb.Attestation {
-	sort.SliceStable(attestations, func(i, j int) bool {
-		return attestations[i].Data.Slot < attestations[j].Data.Slot
-	})
-
-	return attestations
-}

beacon-chain/attestation/service_test.go

@@ -1,446 +0,0 @@
-package attestation
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"reflect"
-	"strings"
-	"testing"
-
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	"github.com/prysmaticlabs/prysm/beacon-chain/internal"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/hashutil"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/prysmaticlabs/prysm/shared/testutil"
-	"github.com/sirupsen/logrus"
-	logTest "github.com/sirupsen/logrus/hooks/test"
-)
-
-func init() {
-	logrus.SetLevel(logrus.DebugLevel)
-}
-
-var _ = TargetHandler(&Service{})
-
-func TestUpdateLatestAttestation_UpdatesLatest(t *testing.T) {
-	beaconDB := internal.SetupDB(t)
-	defer internal.TeardownDB(t, beaconDB)
-	ctx := context.Background()
-
-	var validators []*pb.Validator
-	for i := 0; i < 64; i++ {
-		validators = append(validators, &pb.Validator{
-			Pubkey:          []byte{byte(i)},
-			ActivationEpoch: params.BeaconConfig().GenesisEpoch,
-			ExitEpoch:       params.BeaconConfig().GenesisEpoch + 10,
-		})
-	}
-
-	beaconState := &pb.BeaconState{
-		Slot:              params.BeaconConfig().GenesisSlot + 1,
-		ValidatorRegistry: validators,
-	}
-	block := &pb.BeaconBlock{
-		Slot: params.BeaconConfig().GenesisSlot + 1,
-	}
-	if err := beaconDB.SaveBlock(block); err != nil {
-		t.Fatal(err)
-	}
-	if err := beaconDB.UpdateChainHead(ctx, block, beaconState); err != nil {
-		t.Fatal(err)
-	}
-	service := NewAttestationService(context.Background(), &Config{BeaconDB: beaconDB})
-
-	attestation := &pb.Attestation{
-		AggregationBitfield: []byte{0x80},
-		Data: &pb.AttestationData{
-			Slot:  params.BeaconConfig().GenesisSlot + 1,
-			Shard: 1,
-		},
-	}
-
-	if err := service.UpdateLatestAttestation(ctx, attestation); err != nil {
-		t.Fatalf("could not update latest attestation: %v", err)
-	}
-	pubkey := bytesutil.ToBytes48([]byte{byte(3)})
-	if service.store.m[pubkey].Data.Slot !=
-		attestation.Data.Slot {
-		t.Errorf("Incorrect slot stored, wanted: %d, got: %d",
-			attestation.Data.Slot, service.store.m[pubkey].Data.Slot)
-	}
-
-	beaconState = &pb.BeaconState{
-		Slot:              params.BeaconConfig().GenesisSlot + 36,
-		ValidatorRegistry: validators,
-	}
-	if err := beaconDB.UpdateChainHead(ctx, block, beaconState); err != nil {
-		t.Fatalf("could not save state: %v", err)
-	}
-
-	attestation.Data.Slot = params.BeaconConfig().GenesisSlot + 36
-	attestation.Data.Shard = 36
-	if err := service.UpdateLatestAttestation(ctx, attestation); err != nil {
-		t.Fatalf("could not update latest attestation: %v", err)
-	}
-	if service.store.m[pubkey].Data.Slot !=
-		attestation.Data.Slot {
-		t.Errorf("Incorrect slot stored, wanted: %d, got: %d",
-			attestation.Data.Slot, service.store.m[pubkey].Data.Slot)
-	}
-}
-
-func TestAttestationPool_UpdatesAttestationPool(t *testing.T) {
-	beaconDB := internal.SetupDB(t)
-	defer internal.TeardownDB(t, beaconDB)
-	ctx := context.Background()
-
-	var validators []*pb.Validator
-	for i := 0; i < 64; i++ {
-		validators = append(validators, &pb.Validator{
-			Pubkey:          []byte{byte(i)},
-			ActivationEpoch: params.BeaconConfig().GenesisEpoch,
-			ExitEpoch:       params.BeaconConfig().GenesisEpoch + 10,
-		})
-	}
-	beaconState := &pb.BeaconState{
-		Slot:              params.BeaconConfig().GenesisSlot + 1,
-		ValidatorRegistry: validators,
-	}
-	block := &pb.BeaconBlock{
-		Slot: params.BeaconConfig().GenesisSlot + 1,
-	}
-	if err := beaconDB.SaveBlock(block); err != nil {
-		t.Fatal(err)
-	}
-	if err := beaconDB.UpdateChainHead(ctx, block, beaconState); err != nil {
-		t.Fatal(err)
-	}
-
-	service := NewAttestationService(context.Background(), &Config{BeaconDB: beaconDB})
-	attestation := &pb.Attestation{
-		AggregationBitfield: []byte{0x80},
-		Data: &pb.AttestationData{
-			Slot:  params.BeaconConfig().GenesisSlot + 1,
-			Shard: 1,
-		},
-	}
-
-	if err := service.handleAttestation(context.Background(), attestation); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestLatestAttestationTarget_CantGetAttestation(t *testing.T) {
-	beaconDB := internal.SetupDB(t)
-	defer internal.TeardownDB(t, beaconDB)
-	ctx := context.Background()
-
-	if err := beaconDB.SaveState(ctx, &pb.BeaconState{
-		ValidatorRegistry: []*pb.Validator{{}},
-	}); err != nil {
-		t.Fatalf("could not save state: %v", err)
-	}
-	service := NewAttestationService(context.Background(), &Config{BeaconDB: beaconDB})
-	headState, err := beaconDB.HeadState(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	index := uint64(100)
-	want := fmt.Sprintf("invalid validator index %d", index)
-	if _, err := service.LatestAttestationTarget(headState, index); !strings.Contains(err.Error(), want) {
-		t.Errorf("Wanted error to contain %s, received %v", want, err)
-	}
-}
-
-func TestLatestAttestationTarget_ReturnsLatestAttestedBlock(t *testing.T) {
-	beaconDB := internal.SetupDB(t)
-	defer internal.TeardownDB(t, beaconDB)
-	ctx := context.Background()
-
-	pubKey := []byte{'A'}
-	if err := beaconDB.SaveState(ctx, &pb.BeaconState{
-		ValidatorRegistry: []*pb.Validator{{Pubkey: pubKey}},
-	}); err != nil {
-		t.Fatalf("could not save state: %v", err)
-	}
-
-	block := &pb.BeaconBlock{Slot: 999}
-	if err := beaconDB.SaveBlock(block); err != nil {
-		t.Fatalf("could not save block: %v", err)
-	}
-	blockRoot, err := hashutil.HashBeaconBlock(block)
-	if err != nil {
-		log.Fatalf("could not hash block: %v", err)
-	}
-	if err := beaconDB.SaveAttestationTarget(ctx, &pb.AttestationTarget{
-		Slot:       block.Slot,
-		BlockRoot:  blockRoot[:],
-		ParentRoot: []byte{},
-	}); err != nil {
-		log.Fatalf("could not save att target: %v", err)
-	}
-
-	service := NewAttestationService(context.Background(), &Config{BeaconDB: beaconDB})
-
-	attestation := &pb.Attestation{
-		Data: &pb.AttestationData{
-			BeaconBlockRootHash32: blockRoot[:],
-		}}
-	pubKey48 := bytesutil.ToBytes48(pubKey)
-	service.store.m[pubKey48] = attestation
-
-	headState, err := beaconDB.HeadState(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	latestAttestedTarget, err := service.LatestAttestationTarget(headState, 0)
-	if err != nil {
-		t.Fatalf("Could not get latest attestation: %v", err)
-	}
-
-	if !bytes.Equal(blockRoot[:], latestAttestedTarget.BlockRoot) {
-		t.Errorf("Wanted: %v, got: %v", blockRoot[:], latestAttestedTarget.BlockRoot)
-	}
-}
-
-func TestUpdateLatestAttestation_CacheEnabledAndMiss(t *testing.T) {
-
-	beaconDB := internal.SetupDB(t)
-	defer internal.TeardownDB(t, beaconDB)
-	ctx := context.Background()
-
-	var validators []*pb.Validator
-	for i := 0; i < 64; i++ {
-		validators = append(validators, &pb.Validator{
-			Pubkey:          []byte{byte(i)},
-			ActivationEpoch: params.BeaconConfig().GenesisEpoch,
-			ExitEpoch:       params.BeaconConfig().GenesisEpoch + 10,
-		})
-	}
-
-	beaconState := &pb.BeaconState{
-		Slot:              params.BeaconConfig().GenesisSlot + 1,
-		ValidatorRegistry: validators,
-	}
-	block := &pb.BeaconBlock{
-		Slot: params.BeaconConfig().GenesisSlot + 1,
-	}
-	if err := beaconDB.SaveBlock(block); err != nil {
-		t.Fatal(err)
-	}
-	if err := beaconDB.UpdateChainHead(ctx, block, beaconState); err != nil {
-		t.Fatal(err)
-	}
-	service := NewAttestationService(context.Background(), &Config{BeaconDB: beaconDB})
-
-	attestation := &pb.Attestation{
-		AggregationBitfield: []byte{0x80},
-		Data: &pb.AttestationData{
-			Slot:  params.BeaconConfig().GenesisSlot + 1,
-			Shard: 1,
-		},
-	}
-
-	if err := service.UpdateLatestAttestation(ctx, attestation); err != nil {
-		t.Fatalf("could not update latest attestation: %v", err)
-	}
-	pubkey := bytesutil.ToBytes48([]byte{byte(3)})
-	if service.store.m[pubkey].Data.Slot !=
-		attestation.Data.Slot {
-		t.Errorf("Incorrect slot stored, wanted: %d, got: %d",
-			attestation.Data.Slot, service.store.m[pubkey].Data.Slot)
-	}
-
-	attestation.Data.Slot = params.BeaconConfig().GenesisSlot + 36
-	attestation.Data.Shard = 36
-
-	beaconState = &pb.BeaconState{
-		Slot:              params.BeaconConfig().GenesisSlot + 36,
-		ValidatorRegistry: validators,
-	}
-	if err := beaconDB.UpdateChainHead(ctx, block, beaconState); err != nil {
-		t.Fatalf("could not save state: %v", err)
-	}
-
-	if err := service.UpdateLatestAttestation(ctx, attestation); err != nil {
-		t.Fatalf("could not update latest attestation: %v", err)
-	}
-	if service.store.m[pubkey].Data.Slot !=
-		attestation.Data.Slot {
-		t.Errorf("Incorrect slot stored, wanted: %d, got: %d",
-			attestation.Data.Slot, service.store.m[pubkey].Data.Slot)
-	}
-
-	// Verify the committee for attestation's data slot was cached.
-	fetchedCommittees, err := committeeCache.CommitteesInfoBySlot(attestation.Data.Slot)
-	if err != nil {
-		t.Fatal(err)
-	}
-	wantedCommittee := []uint64{38}
-	if !reflect.DeepEqual(wantedCommittee, fetchedCommittees.Committees[0].Committee) {
-		t.Errorf(
-			"Result indices was an unexpected value. Wanted %d, got %d",
-			wantedCommittee,
-			fetchedCommittees.Committees[0].Committee,
-		)
-	}
-}
-
-func TestUpdateLatestAttestation_CacheEnabledAndHit(t *testing.T) {
-
-	var validators []*pb.Validator
-	for i := 0; i < 64; i++ {
-		validators = append(validators, &pb.Validator{
-			Pubkey:          []byte{byte(i)},
-			ActivationEpoch: params.BeaconConfig().GenesisEpoch,
-			ExitEpoch:       params.BeaconConfig().GenesisEpoch + 10,
-		})
-	}
-
-	beaconDB := internal.SetupDB(t)
-	defer internal.TeardownDB(t, beaconDB)
-	ctx := context.Background()
-
-	beaconState := &pb.BeaconState{
-		Slot:              params.BeaconConfig().GenesisSlot + 2,
-		ValidatorRegistry: validators,
-	}
-	block := &pb.BeaconBlock{
-		Slot: params.BeaconConfig().GenesisSlot + 2,
-	}
-	if err := beaconDB.SaveBlock(block); err != nil {
-		t.Fatal(err)
-	}
-	if err := beaconDB.UpdateChainHead(ctx, block, beaconState); err != nil {
-		t.Fatal(err)
-	}
-
-	service := NewAttestationService(context.Background(), &Config{BeaconDB: beaconDB})
-
-	slot := params.BeaconConfig().GenesisSlot + 2
-	shard := uint64(3)
-	index := uint64(4)
-	attestation := &pb.Attestation{
-		AggregationBitfield: []byte{0x80},
-		Data: &pb.AttestationData{
-			Slot:  slot,
-			Shard: shard,
-		},
-	}
-
-	csInSlot := &cache.CommitteesInSlot{
-		Slot: slot,
-		Committees: []*cache.CommitteeInfo{
-			{Shard: shard, Committee: []uint64{index, 999}},
-		}}
-
-	if err := committeeCache.AddCommittees(csInSlot); err != nil {
-		t.Fatal(err)
-	}
-
-	if err := service.UpdateLatestAttestation(ctx, attestation); err != nil {
-		t.Fatalf("could not update latest attestation: %v", err)
-	}
-	pubkey := bytesutil.ToBytes48([]byte{byte(index)})
-	if err := service.UpdateLatestAttestation(ctx, attestation); err != nil {
-		t.Fatalf("could not update latest attestation: %v", err)
-	}
-
-	if service.store.m[pubkey].Data.Slot !=
-		attestation.Data.Slot {
-		t.Errorf("Incorrect slot stored, wanted: %d, got: %d",
-			attestation.Data.Slot, service.store.m[pubkey].Data.Slot)
-	}
-}
-
-func TestUpdateLatestAttestation_InvalidIndex(t *testing.T) {
-	beaconDB := internal.SetupDB(t)
-	hook := logTest.NewGlobal()
-	defer internal.TeardownDB(t, beaconDB)
-	ctx := context.Background()
-
-	var validators []*pb.Validator
-	for i := 0; i < 64; i++ {
-		validators = append(validators, &pb.Validator{
-			Pubkey:          []byte{byte(i)},
-			ActivationEpoch: params.BeaconConfig().GenesisEpoch,
-			ExitEpoch:       params.BeaconConfig().GenesisEpoch + 10,
-		})
-	}
-
-	beaconState := &pb.BeaconState{
-		Slot:              params.BeaconConfig().GenesisSlot + 1,
-		ValidatorRegistry: validators,
-	}
-	block := &pb.BeaconBlock{
-		Slot: params.BeaconConfig().GenesisSlot + 1,
-	}
-	if err := beaconDB.SaveBlock(block); err != nil {
-		t.Fatal(err)
-	}
-	if err := beaconDB.UpdateChainHead(ctx, block, beaconState); err != nil {
-		t.Fatal(err)
-	}
-	service := NewAttestationService(context.Background(), &Config{BeaconDB: beaconDB})
-	attestation := &pb.Attestation{
-		AggregationBitfield: []byte{0xC0},
-		Data: &pb.AttestationData{
-			Slot:  params.BeaconConfig().GenesisSlot + 1,
-			Shard: 1,
-		},
-	}
-
-	if err := service.UpdateLatestAttestation(ctx, attestation); err != nil {
-		t.Fatalf("could not update latest attestation: %v", err)
-	}
-	testutil.AssertLogsContain(t, hook, "Bitfield points to an invalid index in the committee")
-}
-
-func TestUpdateLatestAttestation_BatchUpdate(t *testing.T) {
-	beaconDB := internal.SetupDB(t)
-	defer internal.TeardownDB(t, beaconDB)
-	ctx := context.Background()
-
-	var validators []*pb.Validator
-	for i := 0; i < 64; i++ {
-		validators = append(validators, &pb.Validator{
-			Pubkey:          []byte{byte(i)},
-			ActivationEpoch: params.BeaconConfig().GenesisEpoch,
-			ExitEpoch:       params.BeaconConfig().GenesisEpoch + 10,
-		})
-	}
-
-	beaconState := &pb.BeaconState{
-		Slot:              params.BeaconConfig().GenesisSlot + 1,
-		ValidatorRegistry: validators,
-	}
-	block := &pb.BeaconBlock{
-		Slot: params.BeaconConfig().GenesisSlot + 1,
-	}
-	if err := beaconDB.SaveBlock(block); err != nil {
-		t.Fatal(err)
-	}
-	if err := beaconDB.UpdateChainHead(ctx, block, beaconState); err != nil {
-		t.Fatal(err)
-	}
-	service := NewAttestationService(context.Background(), &Config{BeaconDB: beaconDB})
-	attestations := make([]*pb.Attestation, 0)
-	for i := 0; i < 10; i++ {
-		attestations = append(attestations, &pb.Attestation{
-			AggregationBitfield: []byte{0x80},
-			Data: &pb.AttestationData{
-				Slot:  params.BeaconConfig().GenesisSlot + 1,
-				Shard: 1,
-			},
-		})
-	}
-
-	if err := service.BatchUpdateLatestAttestation(ctx, attestations); err != nil {
-		t.Fatalf("could not update latest attestation: %v", err)
-	}
-}

beacon-chain/attestation/vote_metrics.go

@@ -1,34 +0,0 @@
-package attestation
-
-import (
-	"strconv"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/params"
-)
-
-var (
-	validatorLastVoteGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Name: "validators_last_vote",
-		Help: "Votes of validators, updated when there's a new attestation",
-	}, []string{
-		"validatorIndex",
-	})
-	totalAttestationSeen = promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "total_seen_attestations",
-		Help: "Total number of attestations seen by the validators",
-	})
-)
-
-func reportVoteMetrics(index uint64, block *pb.BeaconBlock) {
-	// Don't update vote metrics if the incoming block is nil.
-	if block == nil {
-		return
-	}
-
-	s := params.BeaconConfig().GenesisSlot
-	validatorLastVoteGauge.WithLabelValues(
-		"v" + strconv.Itoa(int(index))).Set(float64(block.Slot - s))
-}

beacon-chain/blockchain/BUILD.bazel

@@ -1,75 +1,181 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
     srcs = [
-        "block_processing.go",
-        "fork_choice.go",
+        "chain_info.go",
+        "head.go",
+        "head_sync_committee_info.go",
+        "info.go",
+        "init_sync_process_block.go",
+        "log.go",
+        "metrics.go",
+        "process_attestation.go",
+        "process_attestation_helpers.go",
+        "process_block.go",
+        "process_block_helpers.go",
+        "receive_attestation.go",
+        "receive_block.go",
         "service.go",
+        "weak_subjectivity_checks.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/blockchain",
-    visibility = ["//beacon-chain:__subpackages__"],
+    visibility = [
+        "//beacon-chain:__subpackages__",
+        "//testing/fuzz:__pkg__",
+    ],
     deps = [
-        "//beacon-chain/attestation:go_default_library",
+        "//async:go_default_library",
+        "//async/event:go_default_library",
         "//beacon-chain/cache:go_default_library",
-        "//beacon-chain/core/blocks:go_default_library",
+        "//beacon-chain/cache/depositcache:go_default_library",
+        "//beacon-chain/core:go_default_library",
+        "//beacon-chain/core/altair:go_default_library",
+        "//beacon-chain/core/epoch/precompute:go_default_library",
+        "//beacon-chain/core/feed:go_default_library",
+        "//beacon-chain/core/feed/state:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
-        "//beacon-chain/core/state:go_default_library",
-        "//beacon-chain/core/validators:go_default_library",
+        "//beacon-chain/core/signing:go_default_library",
+        "//beacon-chain/core/transition:go_default_library",
         "//beacon-chain/db:go_default_library",
-        "//beacon-chain/operations:go_default_library",
+        "//beacon-chain/db/filters:go_default_library",
+        "//beacon-chain/forkchoice:go_default_library",
+        "//beacon-chain/forkchoice/protoarray:go_default_library",
+        "//beacon-chain/operations/attestations:go_default_library",
+        "//beacon-chain/operations/slashings:go_default_library",
+        "//beacon-chain/operations/voluntaryexits:go_default_library",
+        "//beacon-chain/p2p:go_default_library",
         "//beacon-chain/powchain:go_default_library",
-        "//proto/beacon/p2p/v1:go_default_library",
-        "//proto/beacon/rpc/v1:go_default_library",
-        "//shared/bytesutil:go_default_library",
-        "//shared/event:go_default_library",
-        "//shared/featureconfig:go_default_library",
-        "//shared/hashutil:go_default_library",
-        "//shared/p2p:go_default_library",
-        "//shared/params:go_default_library",
+        "//beacon-chain/state:go_default_library",
+        "//beacon-chain/state/stategen:go_default_library",
+        "//cmd/beacon-chain/flags:go_default_library",
+        "//config/features:go_default_library",
+        "//config/params:go_default_library",
+        "//crypto/bls:go_default_library",
+        "//encoding/bytesutil:go_default_library",
+        "//monitoring/tracing:go_default_library",
+        "//proto/eth/v1:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
+        "//proto/prysm/v1alpha1/attestation:go_default_library",
+        "//proto/prysm/v1alpha1/block:go_default_library",
+        "//runtime/version:go_default_library",
+        "//time:go_default_library",
+        "//time/slots:go_default_library",
+        "@com_github_emicklei_dot//:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
         "@com_github_prometheus_client_golang//prometheus:go_default_library",
         "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
+        "@com_github_prysmaticlabs_eth2_types//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@io_opencensus_go//trace:go_default_library",
     ],
 )
 
-go_test(
+test_suite(
     name = "go_default_test",
+    tests = [
+        ":go_raceoff_test",
+        ":go_raceon_test",
+    ],
+)
+
+go_test(
+    name = "go_raceoff_test",
+    size = "medium",
     srcs = [
-        "block_processing_test.go",
-        "fork_choice_reorg_test.go",
-        "fork_choice_test.go",
+        "blockchain_test.go",
+        "chain_info_test.go",
+        "checktags_test.go",
+        "head_sync_committee_info_test.go",
+        "head_test.go",
+        "info_test.go",
+        "init_test.go",
+        "log_test.go",
+        "metrics_test.go",
+        "process_attestation_test.go",
+        "process_block_test.go",
+        "receive_attestation_test.go",
+        "receive_block_test.go",
         "service_test.go",
+        "weak_subjectivity_checks_test.go",
     ],
     embed = [":go_default_library"],
+    gotags = ["develop"],
     deps = [
-        "//beacon-chain/attestation:go_default_library",
-        "//beacon-chain/cache:go_default_library",
+        "//async/event:go_default_library",
+        "//beacon-chain/blockchain/testing:go_default_library",
+        "//beacon-chain/cache/depositcache:go_default_library",
         "//beacon-chain/core/blocks:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
-        "//beacon-chain/core/state:go_default_library",
-        "//beacon-chain/core/validators:go_default_library",
+        "//beacon-chain/core/transition:go_default_library",
         "//beacon-chain/db:go_default_library",
-        "//beacon-chain/internal:go_default_library",
+        "//beacon-chain/db/testing:go_default_library",
+        "//beacon-chain/p2p:go_default_library",
         "//beacon-chain/powchain:go_default_library",
-        "//proto/beacon/p2p/v1:go_default_library",
-        "//proto/beacon/rpc/v1:go_default_library",
-        "//shared/bls:go_default_library",
-        "//shared/bytesutil:go_default_library",
-        "//shared/event:go_default_library",
-        "//shared/featureconfig:go_default_library",
-        "//shared/forkutil:go_default_library",
-        "//shared/hashutil:go_default_library",
-        "//shared/p2p:go_default_library",
-        "//shared/params:go_default_library",
-        "//shared/testutil:go_default_library",
-        "//shared/trieutil:go_default_library",
+        "//beacon-chain/state/stateutil:go_default_library",
+        "//beacon-chain/state/v1:go_default_library",
+        "//config/params:go_default_library",
+        "//encoding/bytesutil:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
+        "//proto/prysm/v1alpha1/wrapper:go_default_library",
+        "//testing/assert:go_default_library",
+        "//testing/require:go_default_library",
+        "//testing/util:go_default_library",
         "@com_github_ethereum_go_ethereum//:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_ethereum_go_ethereum//core/types:go_default_library",
-        "@com_github_gogo_protobuf//proto:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",
+        "@in_gopkg_d4l3k_messagediff_v1//:go_default_library",
+        "@org_golang_google_protobuf//proto:go_default_library",
+        "@org_golang_x_net//context:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_raceon_test",
+    srcs = [
+        "chain_info_norace_test.go",
+        "checktags_test.go",
+        "init_test.go",
+        "receive_block_test.go",
+        "service_norace_test.go",
+    ],
+    embed = [":go_default_library"],
+    gc_goopts = [
+        # Go 1.14 enables checkptr by default when building with -race or -msan. There is a pointer
+        # issue in boltdb, so must disable checkptr at compile time. This flag can be removed once
+        # the project is migrated to etcd's version of boltdb and the issue has been fixed.
+        # See: https://github.com/etcd-io/bbolt/issues/187.
+        "-d=checkptr=0",
+    ],
+    gotags = ["develop"],
+    race = "on",
+    tags = ["race_on"],
+    deps = [
+        "//async/event:go_default_library",
+        "//beacon-chain/blockchain/testing:go_default_library",
+        "//beacon-chain/cache/depositcache:go_default_library",
+        "//beacon-chain/core/blocks:go_default_library",
+        "//beacon-chain/core/helpers:go_default_library",
+        "//beacon-chain/core/transition:go_default_library",
+        "//beacon-chain/db:go_default_library",
+        "//beacon-chain/db/testing:go_default_library",
+        "//beacon-chain/p2p:go_default_library",
+        "//beacon-chain/powchain:go_default_library",
+        "//config/params:go_default_library",
+        "//encoding/bytesutil:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
+        "//proto/prysm/v1alpha1/wrapper:go_default_library",
+        "//testing/assert:go_default_library",
+        "//testing/require:go_default_library",
+        "//testing/util:go_default_library",
+        "@com_github_ethereum_go_ethereum//:go_default_library",
+        "@com_github_ethereum_go_ethereum//common:go_default_library",
+        "@com_github_ethereum_go_ethereum//core/types:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+        "@com_github_sirupsen_logrus//hooks/test:go_default_library",
+        "@org_golang_google_protobuf//proto:go_default_library",
+        "@org_golang_x_net//context:go_default_library",
     ],
 )

beacon-chain/blockchain/block_processing.go

@@ -1,343 +0,0 @@
-package blockchain
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"fmt"
-
-	b "github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/validators"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	pbrpc "github.com/prysmaticlabs/prysm/proto/beacon/rpc/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/event"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"github.com/prysmaticlabs/prysm/shared/hashutil"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/sirupsen/logrus"
-	"go.opencensus.io/trace"
-)
-
-// BlockReceiver interface defines the methods in the blockchain service which
-// directly receives a new block from other services and applies the full processing pipeline.
-type BlockReceiver interface {
-	CanonicalBlockFeed() *event.Feed
-	ReceiveBlock(ctx context.Context, block *pb.BeaconBlock) (*pb.BeaconState, error)
-	IsCanonical(slot uint64, hash []byte) bool
-	CanonicalBlock(slot uint64) (*pb.BeaconBlock, error)
-	RecentCanonicalRoots(count uint64) []*pbrpc.BlockRoot
-}
-
-// BlockProcessor defines a common interface for methods useful for directly applying state transitions
-// to beacon blocks and generating a new beacon state from the Ethereum 2.0 core primitives.
-type BlockProcessor interface {
-	VerifyBlockValidity(ctx context.Context, block *pb.BeaconBlock, beaconState *pb.BeaconState) error
-	ApplyBlockStateTransition(ctx context.Context, block *pb.BeaconBlock, beaconState *pb.BeaconState) (*pb.BeaconState, error)
-	CleanupBlockOperations(ctx context.Context, block *pb.BeaconBlock) error
-}
-
-// BlockFailedProcessingErr represents a block failing a state transition function.
-type BlockFailedProcessingErr struct {
-	err error
-}
-
-func (b *BlockFailedProcessingErr) Error() string {
-	return fmt.Sprintf("block failed processing: %v", b.err)
-}
-
-// ReceiveBlock is a function that defines the operations that are preformed on
-// any block that is received from p2p layer or rpc. It performs the following actions: It checks the block to see
-// 1. Verify a block passes pre-processing conditions
-// 2. Save and broadcast the block via p2p to other peers
-// 3. Apply the block state transition function and account for skip slots.
-// 4. Process and cleanup any block operations, such as attestations and deposits, which would need to be
-//    either included or flushed from the beacon node's runtime.
-func (c *ChainService) ReceiveBlock(ctx context.Context, block *pb.BeaconBlock) (*pb.BeaconState, error) {
-	c.receiveBlockLock.Lock()
-	defer c.receiveBlockLock.Unlock()
-	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ReceiveBlock")
-	defer span.End()
-	parentRoot := bytesutil.ToBytes32(block.ParentRootHash32)
-	parent, err := c.beaconDB.Block(parentRoot)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get parent block: %v", err)
-	}
-	if parent == nil {
-		return nil, errors.New("parent does not exist in DB")
-	}
-	beaconState, err := c.beaconDB.HistoricalStateFromSlot(ctx, parent.Slot)
-	if err != nil {
-		return nil, fmt.Errorf("could not retrieve beacon state: %v", err)
-	}
-	saveLatestBlock := beaconState.LatestBlock
-
-	blockRoot, err := hashutil.HashBeaconBlock(block)
-	if err != nil {
-		return nil, fmt.Errorf("could not hash beacon block")
-	}
-	// We first verify the block's basic validity conditions.
-	if err := c.VerifyBlockValidity(ctx, block, beaconState); err != nil {
-		return beaconState, fmt.Errorf("block with slot %d is not ready for processing: %v", block.Slot, err)
-	}
-
-	// We save the block to the DB and broadcast it to our peers.
-	if err := c.SaveAndBroadcastBlock(ctx, block); err != nil {
-		return beaconState, fmt.Errorf(
-			"could not save and broadcast beacon block with slot %d: %v",
-			block.Slot-params.BeaconConfig().GenesisSlot, err,
-		)
-	}
-
-	log.WithField("slotNumber", block.Slot-params.BeaconConfig().GenesisSlot).Info(
-		"Executing state transition")
-
-	// We then apply the block state transition accordingly to obtain the resulting beacon state.
-	beaconState, err = c.ApplyBlockStateTransition(ctx, block, beaconState)
-	if err != nil {
-		switch err.(type) {
-		case *BlockFailedProcessingErr:
-			// If the block fails processing, we mark it as blacklisted and delete it from our DB.
-			c.beaconDB.MarkEvilBlockHash(blockRoot)
-			if err := c.beaconDB.DeleteBlock(block); err != nil {
-				return nil, fmt.Errorf("could not delete bad block from db: %v", err)
-			}
-			return beaconState, err
-		default:
-			return beaconState, fmt.Errorf("could not apply block state transition: %v", err)
-		}
-	}
-
-	log.WithFields(logrus.Fields{
-		"slotNumber":   block.Slot - params.BeaconConfig().GenesisSlot,
-		"currentEpoch": helpers.SlotToEpoch(block.Slot) - params.BeaconConfig().GenesisEpoch,
-	}).Info("State transition complete")
-
-	// Check state root
-	if featureconfig.FeatureConfig().EnableCheckBlockStateRoot {
-		// Calc state hash with previous block
-		beaconState.LatestBlock = saveLatestBlock
-		stateRoot, err := hashutil.HashProto(beaconState)
-		if err != nil {
-			return nil, fmt.Errorf("could not hash beacon state: %v", err)
-		}
-		beaconState.LatestBlock = block
-		if !bytes.Equal(block.StateRootHash32, stateRoot[:]) {
-			return nil, fmt.Errorf("beacon state root is not equal to block state root: %#x != %#x", stateRoot, block.StateRootHash32)
-		}
-	}
-
-	// We process the block's contained deposits, attestations, and other operations
-	// and that may need to be stored or deleted from the beacon node's persistent storage.
-	if err := c.CleanupBlockOperations(ctx, block); err != nil {
-		return beaconState, fmt.Errorf("could not process block deposits, attestations, and other operations: %v", err)
-	}
-
-	log.WithField("slot", block.Slot-params.BeaconConfig().GenesisSlot).Info("Finished processing beacon block")
-	return beaconState, nil
-}
-
-// ApplyBlockStateTransition runs the Ethereum 2.0 state transition function
-// to produce a new beacon state and also accounts for skip slots occurring.
-//
-//  def apply_block_state_transition(block):
-//  	# process skipped slots
-// 		while (state.slot < block.slot - 1):
-//      	state = slot_state_transition(state, block=None)
-//
-//		# process slot with block
-//		state = slot_state_transition(state, block)
-//
-//		# check state root
-//      if block.state_root == hash(state):
-//			return state, error
-//		else:
-//			return nil, error  # or throw or whatever
-//
-func (c *ChainService) ApplyBlockStateTransition(
-	ctx context.Context, block *pb.BeaconBlock, beaconState *pb.BeaconState,
-) (*pb.BeaconState, error) {
-	// Retrieve the last processed beacon block's hash root.
-	headRoot, err := c.ChainHeadRoot()
-	if err != nil {
-		return beaconState, fmt.Errorf("could not retrieve chain head root: %v", err)
-	}
-
-	// Check for skipped slots.
-	numSkippedSlots := 0
-	for beaconState.Slot < block.Slot-1 {
-		beaconState, err = c.runStateTransition(ctx, headRoot, nil, beaconState)
-		if err != nil {
-			return beaconState, err
-		}
-		numSkippedSlots++
-	}
-	if numSkippedSlots > 0 {
-		log.Warnf("Processed %d skipped slots", numSkippedSlots)
-	}
-
-	beaconState, err = c.runStateTransition(ctx, headRoot, block, beaconState)
-	if err != nil {
-		return beaconState, err
-	}
-	return beaconState, nil
-}
-
-// VerifyBlockValidity cross-checks the block against the pre-processing conditions from
-// Ethereum 2.0, namely:
-//   The parent block with root block.parent_root has been processed and accepted.
-//   The node has processed its state up to slot, block.slot - 1.
-//   The Ethereum 1.0 block pointed to by the state.processed_pow_receipt_root has been processed and accepted.
-//   The node's local clock time is greater than or equal to state.genesis_time + block.slot * SECONDS_PER_SLOT.
-func (c *ChainService) VerifyBlockValidity(
-	ctx context.Context,
-	block *pb.BeaconBlock,
-	beaconState *pb.BeaconState,
-) error {
-	if block.Slot == params.BeaconConfig().GenesisSlot {
-		return fmt.Errorf("cannot process a genesis block: received block with slot %d",
-			block.Slot-params.BeaconConfig().GenesisSlot)
-	}
-	powBlockFetcher := c.web3Service.Client().BlockByHash
-	if err := b.IsValidBlock(ctx, beaconState, block,
-		c.beaconDB.HasBlock, powBlockFetcher, c.genesisTime); err != nil {
-		return fmt.Errorf("block does not fulfill pre-processing conditions %v", err)
-	}
-	return nil
-}
-
-// SaveAndBroadcastBlock stores the block in persistent storage and then broadcasts it to
-// peers via p2p. Blocks which have already been saved are not processed again via p2p, which is why
-// the order of operations is important in this function to prevent infinite p2p loops.
-func (c *ChainService) SaveAndBroadcastBlock(ctx context.Context, block *pb.BeaconBlock) error {
-	blockRoot, err := hashutil.HashBeaconBlock(block)
-	if err != nil {
-		return fmt.Errorf("could not tree hash incoming block: %v", err)
-	}
-	if err := c.beaconDB.SaveBlock(block); err != nil {
-		return fmt.Errorf("failed to save block: %v", err)
-	}
-	if err := c.beaconDB.SaveAttestationTarget(ctx, &pb.AttestationTarget{
-		Slot:       block.Slot,
-		BlockRoot:  blockRoot[:],
-		ParentRoot: block.ParentRootHash32,
-	}); err != nil {
-		return fmt.Errorf("failed to save attestation target: %v", err)
-	}
-	// Announce the new block to the network.
-	c.p2p.Broadcast(ctx, &pb.BeaconBlockAnnounce{
-		Hash:       blockRoot[:],
-		SlotNumber: block.Slot,
-	})
-	return nil
-}
-
-// CleanupBlockOperations processes and cleans up any block operations relevant to the beacon node
-// such as attestations, exits, and deposits. We update the latest seen attestation by validator
-// in the local node's runtime, cleanup and remove pending deposits which have been included in the block
-// from our node's local cache, and process validator exits and more.
-func (c *ChainService) CleanupBlockOperations(ctx context.Context, block *pb.BeaconBlock) error {
-	// Forward processed block to operation pool to remove individual operation from DB.
-	if c.opsPoolService.IncomingProcessedBlockFeed().Send(block) == 0 {
-		log.Error("Sent processed block to no subscribers")
-	}
-
-	if err := c.attsService.BatchUpdateLatestAttestation(ctx, block.Body.Attestations); err != nil {
-		return fmt.Errorf("failed to update latest attestation for store: %v", err)
-	}
-
-	// Remove pending deposits from the deposit queue.
-	for _, dep := range block.Body.Deposits {
-		c.beaconDB.RemovePendingDeposit(ctx, dep)
-	}
-	return nil
-}
-
-// runStateTransition executes the Ethereum 2.0 core state transition for the beacon chain and
-// updates important checkpoints and local persistent data during epoch transitions. It serves as a wrapper
-// around the more low-level, core state transition function primitive.
-func (c *ChainService) runStateTransition(
-	ctx context.Context,
-	headRoot [32]byte,
-	block *pb.BeaconBlock,
-	beaconState *pb.BeaconState,
-) (*pb.BeaconState, error) {
-	newState, err := state.ExecuteStateTransition(
-		ctx,
-		beaconState,
-		block,
-		headRoot,
-		&state.TransitionConfig{
-			VerifySignatures: false, // We disable signature verification for now.
-			Logging:          true,  // We enable logging in this state transition call.
-		},
-	)
-	if err != nil {
-		return beaconState, &BlockFailedProcessingErr{err}
-	}
-	log.WithField(
-		"slotsSinceGenesis", newState.Slot-params.BeaconConfig().GenesisSlot,
-	).Info("Slot transition successfully processed")
-
-	if block != nil {
-		log.WithField(
-			"slotsSinceGenesis", newState.Slot-params.BeaconConfig().GenesisSlot,
-		).Info("Block transition successfully processed")
-
-		// Save Historical States.
-		if err := c.beaconDB.SaveHistoricalState(ctx, beaconState); err != nil {
-			return nil, fmt.Errorf("could not save historical state: %v", err)
-		}
-	}
-
-	if helpers.IsEpochEnd(newState.Slot) {
-		// Save activated validators of this epoch to public key -> index DB.
-		if err := c.saveValidatorIdx(newState); err != nil {
-			return newState, fmt.Errorf("could not save validator index: %v", err)
-		}
-		// Delete exited validators of this epoch to public key -> index DB.
-		if err := c.deleteValidatorIdx(newState); err != nil {
-			return newState, fmt.Errorf("could not delete validator index: %v", err)
-		}
-		// Update FFG checkpoints in DB.
-		if err := c.updateFFGCheckPts(ctx, newState); err != nil {
-			return newState, fmt.Errorf("could not update FFG checkpts: %v", err)
-		}
-		log.WithField(
-			"SlotsSinceGenesis", newState.Slot-params.BeaconConfig().GenesisSlot,
-		).Info("Epoch transition successfully processed")
-	}
-	return newState, nil
-}
-
-// saveValidatorIdx saves the validators public key to index mapping in DB, these
-// validators were activated from current epoch. After it saves, current epoch key
-// is deleted from ActivatedValidators mapping.
-func (c *ChainService) saveValidatorIdx(state *pb.BeaconState) error {
-	activatedValidators := validators.ActivatedValFromEpoch(helpers.CurrentEpoch(state) + 1)
-	for _, idx := range activatedValidators {
-		pubKey := state.ValidatorRegistry[idx].Pubkey
-		if err := c.beaconDB.SaveValidatorIndex(pubKey, int(idx)); err != nil {
-			return fmt.Errorf("could not save validator index: %v", err)
-		}
-	}
-	validators.DeleteActivatedVal(helpers.CurrentEpoch(state))
-	return nil
-}
-
-// deleteValidatorIdx deletes the validators public key to index mapping in DB, the
-// validators were exited from current epoch. After it deletes, current epoch key
-// is deleted from ExitedValidators mapping.
-func (c *ChainService) deleteValidatorIdx(state *pb.BeaconState) error {
-	exitedValidators := validators.ExitedValFromEpoch(helpers.CurrentEpoch(state) + 1)
-	for _, idx := range exitedValidators {
-		pubKey := state.ValidatorRegistry[idx].Pubkey
-		if err := c.beaconDB.DeleteValidatorIndex(pubKey); err != nil {
-			return fmt.Errorf("could not delete validator index: %v", err)
-		}
-	}
-	validators.DeleteExitedVal(helpers.CurrentEpoch(state))
-	return nil
-}

beacon-chain/blockchain/block_processing_test.go

@@ -1,840 +0,0 @@
-package blockchain
-
-import (
-	"context"
-	"encoding/binary"
-	"math/big"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/prysmaticlabs/prysm/beacon-chain/attestation"
-	b "github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
-	v "github.com/prysmaticlabs/prysm/beacon-chain/core/validators"
-	"github.com/prysmaticlabs/prysm/beacon-chain/internal"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/featureconfig"
-	"github.com/prysmaticlabs/prysm/shared/hashutil"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/prysmaticlabs/prysm/shared/testutil"
-	"github.com/prysmaticlabs/prysm/shared/trieutil"
-	logTest "github.com/sirupsen/logrus/hooks/test"
-)
-
-// Ensure ChainService implements interfaces.
-var _ = BlockProcessor(&ChainService{})
-
-func initBlockStateRoot(t *testing.T, block *pb.BeaconBlock, chainService *ChainService) {
-	parentRoot := bytesutil.ToBytes32(block.ParentRootHash32)
-	parent, err := chainService.beaconDB.Block(parentRoot)
-	if err != nil {
-		t.Fatal(err)
-	}
-	beaconState, err := chainService.beaconDB.HistoricalStateFromSlot(context.Background(), parent.Slot)
-	if err != nil {
-		t.Fatalf("Unable to retrieve state %v", err)
-	}
-	saveLatestBlock := beaconState.LatestBlock
-
-	computedState, err := chainService.ApplyBlockStateTransition(context.Background(), block, beaconState)
-	if err != nil {
-		t.Fatalf("could not apply block state transition: %v", err)
-	}
-
-	computedState.LatestBlock = saveLatestBlock
-	stateRoot, err := hashutil.HashProto(computedState)
-	if err != nil {
-		t.Fatalf("could not tree hash state: %v", err)
-	}
-	block.StateRootHash32 = stateRoot[:]
-	t.Logf("state root after block: %#x", stateRoot)
-}
-
-func TestReceiveBlock_FaultyPOWChain(t *testing.T) {
-	db := internal.SetupDB(t)
-	defer internal.TeardownDB(t, db)
-	chainService := setupBeaconChain(t, db, nil)
-	unixTime := uint64(time.Now().Unix())
-	deposits, _ := setupInitialDeposits(t, 100)
-	if err := db.InitializeState(context.Background(), unixTime, deposits, &pb.Eth1Data{}); err != nil {
-		t.Fatalf("Could not initialize beacon state to disk: %v", err)
-	}
-
-	if err := SetSlotInState(chainService, 1); err != nil {
-		t.Fatal(err)
-	}
-
-	parentBlock := &pb.BeaconBlock{
-		Slot: 1,
-	}
-
-	parentRoot, err := hashutil.HashBeaconBlock(parentBlock)
-	if err != nil {
-		t.Fatalf("Unable to tree hash block %v", err)
-	}
-
-	if err := chainService.beaconDB.SaveBlock(parentBlock); err != nil {
-		t.Fatalf("Unable to save block %v", err)
-	}
-
-	block := &pb.BeaconBlock{
-		Slot:             2,
-		ParentRootHash32: parentRoot[:],
-		Eth1Data: &pb.Eth1Data{
-			DepositRootHash32: []byte("a"),
-			BlockHash32:       []byte("b"),
-		},
-	}
-
-	if err := chainService.beaconDB.SaveBlock(block); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := chainService.ReceiveBlock(context.Background(), block); err == nil {
-		t.Errorf("Expected receive block to fail, received nil: %v", err)
-	}
-}
-
-func TestReceiveBlock_ProcessCorrectly(t *testing.T) {
-	hook := logTest.NewGlobal()
-	db := internal.SetupDB(t)
-	defer internal.TeardownDB(t, db)
-	ctx := context.Background()
-
-	chainService := setupBeaconChain(t, db, nil)
-	deposits, privKeys := setupInitialDeposits(t, 100)
-	eth1Data := &pb.Eth1Data{
-		DepositRootHash32: []byte{},
-		BlockHash32:       []byte{},
-	}
-	beaconState, err := state.GenesisBeaconState(deposits, 0, eth1Data)
-	if err != nil {
-		t.Fatalf("Can't generate genesis state: %v", err)
-	}
-	stateRoot, err := hashutil.HashProto(beaconState)
-	if err != nil {
-		t.Fatalf("Could not tree hash state: %v", err)
-	}
-	if err := db.SaveHistoricalState(ctx, beaconState); err != nil {
-		t.Fatal(err)
-	}
-
-	genesis := b.NewGenesisBlock([]byte{})
-	if err := chainService.beaconDB.SaveBlock(genesis); err != nil {
-		t.Fatalf("Could not save block to db: %v", err)
-	}
-	parentHash, err := hashutil.HashBeaconBlock(genesis)
-	if err != nil {
-		t.Fatalf("Unable to get tree hash root of canonical head: %v", err)
-	}
-	if err := chainService.beaconDB.UpdateChainHead(ctx, genesis, beaconState); err != nil {
-		t.Fatal(err)
-	}
-
-	beaconState.Slot++
-	randaoReveal := createRandaoReveal(t, beaconState, privKeys)
-
-	block := &pb.BeaconBlock{
-		Slot:             beaconState.Slot,
-		StateRootHash32:  stateRoot[:],
-		ParentRootHash32: parentHash[:],
-		RandaoReveal:     randaoReveal,
-		Eth1Data: &pb.Eth1Data{
-			DepositRootHash32: []byte("a"),
-			BlockHash32:       []byte("b"),
-		},
-		Body: &pb.BeaconBlockBody{
-			Attestations: nil,
-		},
-	}
-
-	initBlockStateRoot(t, block, chainService)
-
-	if err := chainService.beaconDB.SaveJustifiedBlock(block); err != nil {
-		t.Fatal(err)
-	}
-	if err := chainService.beaconDB.SaveFinalizedBlock(block); err != nil {
-		t.Fatal(err)
-	}
-	if err := chainService.beaconDB.SaveBlock(block); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := chainService.ReceiveBlock(context.Background(), block); err != nil {
-		t.Errorf("Block failed processing: %v", err)
-	}
-
-	testutil.AssertLogsContain(t, hook, "Finished processing beacon block")
-}
-
-func TestReceiveBlock_UsesParentBlockState(t *testing.T) {
-	hook := logTest.NewGlobal()
-	db := internal.SetupDB(t)
-	defer internal.TeardownDB(t, db)
-	ctx := context.Background()
-
-	chainService := setupBeaconChain(t, db, nil)
-	deposits, _ := setupInitialDeposits(t, 100)
-	eth1Data := &pb.Eth1Data{
-		DepositRootHash32: []byte{},
-		BlockHash32:       []byte{},
-	}
-	beaconState, err := state.GenesisBeaconState(deposits, 0, eth1Data)
-	if err != nil {
-		t.Fatalf("Can't generate genesis state: %v", err)
-	}
-	if err := chainService.beaconDB.SaveHistoricalState(ctx, beaconState); err != nil {
-		t.Fatal(err)
-	}
-	stateRoot, err := hashutil.HashProto(beaconState)
-	if err != nil {
-		t.Fatalf("Could not tree hash state: %v", err)
-	}
-
-	parentHash, genesisBlock := setupGenesisBlock(t, chainService)
-	if err := chainService.beaconDB.UpdateChainHead(ctx, genesisBlock, beaconState); err != nil {
-		t.Fatal(err)
-	}
-
-	// We ensure the block uses the right state parent if its ancestor is not block.Slot-1.
-	block := &pb.BeaconBlock{
-		Slot:             beaconState.Slot + 4,
-		StateRootHash32:  stateRoot[:],
-		ParentRootHash32: parentHash[:],
-		RandaoReveal:     []byte{},
-		Eth1Data: &pb.Eth1Data{
-			DepositRootHash32: []byte("a"),
-			BlockHash32:       []byte("b"),
-		},
-		Body: &pb.BeaconBlockBody{
-			Attestations: nil,
-		},
-	}
-	initBlockStateRoot(t, block, chainService)
-	if err := chainService.beaconDB.SaveBlock(block); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := chainService.ReceiveBlock(context.Background(), block); err != nil {
-		t.Errorf("Block failed processing: %v", err)
-	}
-	testutil.AssertLogsContain(t, hook, "Finished processing beacon block")
-}
-
-func TestReceiveBlock_DeletesBadBlock(t *testing.T) {
-	featureconfig.InitFeatureConfig(&featureconfig.FeatureFlagConfig{
-		EnableCheckBlockStateRoot: false,
-	})
-	db := internal.SetupDB(t)
-	defer internal.TeardownDB(t, db)
-	ctx := context.Background()
-
-	chainService := setupBeaconChain(t, db, nil)
-	deposits, _ := setupInitialDeposits(t, 100)
-	eth1Data := &pb.Eth1Data{
-		DepositRootHash32: []byte{},
-		BlockHash32:       []byte{},
-	}
-	beaconState, err := state.GenesisBeaconState(deposits, 0, eth1Data)
-	if err != nil {
-		t.Fatalf("Can't generate genesis state: %v", err)
-	}
-	stateRoot, err := hashutil.HashProto(beaconState)
-	if err != nil {
-		t.Fatalf("Could not tree hash state: %v", err)
-	}
-	if err := chainService.beaconDB.SaveHistoricalState(ctx, beaconState); err != nil {
-		t.Fatal(err)
-	}
-
-	parentHash, genesisBlock := setupGenesisBlock(t, chainService)
-	if err := chainService.beaconDB.UpdateChainHead(ctx, genesisBlock, beaconState); err != nil {
-		t.Fatal(err)
-	}
-
-	beaconState.Slot++
-
-	block := &pb.BeaconBlock{
-		Slot:             beaconState.Slot,
-		StateRootHash32:  stateRoot[:],
-		ParentRootHash32: parentHash[:],
-		RandaoReveal:     []byte{},
-		Eth1Data: &pb.Eth1Data{
-			DepositRootHash32: []byte("a"),
-			BlockHash32:       []byte("b"),
-		},
-		Body: &pb.BeaconBlockBody{
-			Attestations: []*pb.Attestation{
-				{
-					Data: &pb.AttestationData{
-						JustifiedEpoch: params.BeaconConfig().GenesisSlot * 100,
-					},
-				},
-			},
-		},
-	}
-
-	blockRoot, err := hashutil.HashBeaconBlock(block)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	_, err = chainService.ReceiveBlock(context.Background(), block)
-	switch err.(type) {
-	case *BlockFailedProcessingErr:
-		t.Log("Block failed processing as expected")
-	default:
-		t.Errorf("Unexpected block processing error: %v", err)
-	}
-
-	savedBlock, err := db.Block(blockRoot)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if savedBlock != nil {
-		t.Errorf("Expected bad block to have been deleted, received: %v", savedBlock)
-	}
-	// We also verify the block has been blacklisted.
-	if !db.IsEvilBlockHash(blockRoot) {
-		t.Error("Expected block root to have been blacklisted")
-	}
-	featureconfig.InitFeatureConfig(&featureconfig.FeatureFlagConfig{
-		EnableCheckBlockStateRoot: true,
-	})
-}
-
-func TestReceiveBlock_CheckBlockStateRoot_GoodState(t *testing.T) {
-	hook := logTest.NewGlobal()
-	db := internal.SetupDB(t)
-	defer internal.TeardownDB(t, db)
-	ctx := context.Background()
-
-	attsService := attestation.NewAttestationService(
-		context.Background(),
-		&attestation.Config{BeaconDB: db})
-	chainService := setupBeaconChain(t, db, attsService)
-	deposits, privKeys := setupInitialDeposits(t, 100)
-	eth1Data := &pb.Eth1Data{
-		DepositRootHash32: []byte{},
-		BlockHash32:       []byte{},
-	}
-	beaconState, err := state.GenesisBeaconState(deposits, 0, eth1Data)
-	if err != nil {
-		t.Fatalf("Can't generate genesis state: %v", err)
-	}
-	if err := chainService.beaconDB.SaveHistoricalState(ctx, beaconState); err != nil {
-		t.Fatal(err)
-	}
-	parentHash, genesisBlock := setupGenesisBlock(t, chainService)
-	beaconState.Slot++
-	if err := chainService.beaconDB.UpdateChainHead(ctx, genesisBlock, beaconState); err != nil {
-		t.Fatal(err)
-	}
-
-	beaconState.Slot++
-	goodStateBlock := &pb.BeaconBlock{
-		Slot:             beaconState.Slot,
-		ParentRootHash32: parentHash[:],
-		RandaoReveal:     createRandaoReveal(t, beaconState, privKeys),
-		Body:             &pb.BeaconBlockBody{},
-	}
-	beaconState.Slot--
-	initBlockStateRoot(t, goodStateBlock, chainService)
-
-	if err := chainService.beaconDB.SaveBlock(goodStateBlock); err != nil {
-		t.Fatal(err)
-	}
-
-	_, err = chainService.ReceiveBlock(context.Background(), goodStateBlock)
-	if err != nil {
-		t.Fatalf("error exists for good block %v", err)
-	}
-	testutil.AssertLogsContain(t, hook, "Executing state transition")
-}
-
-func TestReceiveBlock_CheckBlockStateRoot_BadState(t *testing.T) {
-	db := internal.SetupDB(t)
-	defer internal.TeardownDB(t, db)
-	chainService := setupBeaconChain(t, db, nil)
-	deposits, privKeys := setupInitialDeposits(t, 100)
-	ctx := context.Background()
-	eth1Data := &pb.Eth1Data{
-		DepositRootHash32: []byte{},
-		BlockHash32:       []byte{},
-	}
-	beaconState, err := state.GenesisBeaconState(deposits, 0, eth1Data)
-	if err != nil {
-		t.Fatalf("Can't generate genesis state: %v", err)
-	}
-	if err := chainService.beaconDB.SaveHistoricalState(ctx, beaconState); err != nil {
-		t.Fatal(err)
-	}
-	parentHash, genesisBlock := setupGenesisBlock(t, chainService)
-	beaconState.Slot++
-	if err := chainService.beaconDB.UpdateChainHead(ctx, genesisBlock, beaconState); err != nil {
-		t.Fatal(err)
-	}
-
-	beaconState.Slot++
-	invalidStateBlock := &pb.BeaconBlock{
-		Slot:             beaconState.Slot,
-		StateRootHash32:  []byte{'b', 'a', 'd', ' ', 'h', 'a', 's', 'h'},
-		ParentRootHash32: parentHash[:],
-		RandaoReveal:     createRandaoReveal(t, beaconState, privKeys),
-		Body:             &pb.BeaconBlockBody{},
-	}
-	beaconState.Slot--
-
-	_, err = chainService.ReceiveBlock(context.Background(), invalidStateBlock)
-	if err == nil {
-		t.Fatal("no error for wrong block state root")
-	}
-	if !strings.Contains(err.Error(), "beacon state root is not equal to block state root: ") {
-		t.Fatal(err)
-	}
-}
-
-func TestReceiveBlock_RemovesPendingDeposits(t *testing.T) {
-	hook := logTest.NewGlobal()
-	db := internal.SetupDB(t)
-	defer internal.TeardownDB(t, db)
-	ctx := context.Background()
-
-	attsService := attestation.NewAttestationService(
-		context.Background(),
-		&attestation.Config{BeaconDB: db})
-	chainService := setupBeaconChain(t, db, attsService)
-	deposits, privKeys := setupInitialDeposits(t, 100)
-	eth1Data := &pb.Eth1Data{
-		DepositRootHash32: []byte{},
-		BlockHash32:       []byte{},
-	}
-	beaconState, err := state.GenesisBeaconState(deposits, 0, eth1Data)
-	if err != nil {
-		t.Fatalf("Can't generate genesis state: %v", err)
-	}
-	if err := chainService.beaconDB.SaveJustifiedState(beaconState); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveFinalizedState(beaconState); err != nil {
-		t.Fatal(err)
-	}
-
-	stateRoot, err := hashutil.HashProto(beaconState)
-	if err != nil {
-		t.Fatalf("Could not tree hash state: %v", err)
-	}
-	parentHash, genesisBlock := setupGenesisBlock(t, chainService)
-	beaconState.Slot++
-	if err := chainService.beaconDB.UpdateChainHead(ctx, genesisBlock, beaconState); err != nil {
-		t.Fatal(err)
-	}
-
-	currentSlot := params.BeaconConfig().GenesisSlot
-	randaoReveal := createRandaoReveal(t, beaconState, privKeys)
-
-	pendingDeposits := []*pb.Deposit{
-		createPreChainStartDeposit(t, []byte{'F'}, beaconState.DepositIndex),
-	}
-	pendingDepositsData := make([][]byte, len(pendingDeposits))
-	for i, pd := range pendingDeposits {
-		pendingDepositsData[i] = pd.DepositData
-	}
-	depositTrie, err := trieutil.GenerateTrieFromItems(pendingDepositsData, int(params.BeaconConfig().DepositContractTreeDepth))
-	if err != nil {
-		t.Fatalf("Could not generate deposit trie: %v", err)
-	}
-	for i := range pendingDeposits {
-		pendingDeposits[i].MerkleTreeIndex = 0
-		proof, err := depositTrie.MerkleProof(int(pendingDeposits[i].MerkleTreeIndex))
-		if err != nil {
-			t.Fatalf("Could not generate proof: %v", err)
-		}
-		pendingDeposits[i].MerkleProofHash32S = proof
-	}
-	depositRoot := depositTrie.Root()
-	beaconState.LatestEth1Data.DepositRootHash32 = depositRoot[:]
-	if err := db.SaveHistoricalState(context.Background(), beaconState); err != nil {
-		t.Fatal(err)
-	}
-
-	block := &pb.BeaconBlock{
-		Slot:             currentSlot + 1,
-		StateRootHash32:  stateRoot[:],
-		ParentRootHash32: parentHash[:],
-		RandaoReveal:     randaoReveal,
-		Eth1Data: &pb.Eth1Data{
-			DepositRootHash32: []byte("a"),
-			BlockHash32:       []byte("b"),
-		},
-		Body: &pb.BeaconBlockBody{
-			Deposits: pendingDeposits,
-		},
-	}
-
-	beaconState.Slot--
-	beaconState.DepositIndex = 0
-	if err := chainService.beaconDB.SaveState(ctx, beaconState); err != nil {
-		t.Fatal(err)
-	}
-	initBlockStateRoot(t, block, chainService)
-
-	blockRoot, err := hashutil.HashBeaconBlock(block)
-	if err != nil {
-		log.Fatalf("could not hash block: %v", err)
-	}
-
-	if err := chainService.beaconDB.SaveJustifiedBlock(block); err != nil {
-		t.Fatal(err)
-	}
-	if err := chainService.beaconDB.SaveFinalizedBlock(block); err != nil {
-		t.Fatal(err)
-	}
-
-	for _, dep := range pendingDeposits {
-		db.InsertPendingDeposit(chainService.ctx, dep, big.NewInt(0))
-	}
-
-	if len(db.PendingDeposits(chainService.ctx, nil)) != len(pendingDeposits) || len(pendingDeposits) == 0 {
-		t.Fatalf("Expected %d pending deposits", len(pendingDeposits))
-	}
-
-	beaconState.Slot--
-	if err := chainService.beaconDB.SaveState(ctx, beaconState); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveHistoricalState(context.Background(), beaconState); err != nil {
-		t.Fatal(err)
-	}
-	computedState, err := chainService.ReceiveBlock(context.Background(), block)
-	if err != nil {
-		t.Fatal(err)
-	}
-	for i := 0; i < len(beaconState.ValidatorRegistry); i++ {
-		pubKey := bytesutil.ToBytes48(beaconState.ValidatorRegistry[i].Pubkey)
-		attsService.InsertAttestationIntoStore(pubKey, &pb.Attestation{
-			Data: &pb.AttestationData{
-				BeaconBlockRootHash32: blockRoot[:],
-			}},
-		)
-	}
-	if err := chainService.ApplyForkChoiceRule(context.Background(), block, computedState); err != nil {
-		t.Fatal(err)
-	}
-
-	if len(db.PendingDeposits(chainService.ctx, nil)) != 0 {
-		t.Fatalf("Expected 0 pending deposits, but there are %+v", db.PendingDeposits(chainService.ctx, nil))
-	}
-	testutil.AssertLogsContain(t, hook, "Executing state transition")
-}
-
-// Scenario graph: http://bit.ly/2K1k2KZ
-//
-//digraph G {
-//    rankdir=LR;
-//    node [shape="none"];
-//
-//    subgraph blocks {
-//        rankdir=LR;
-//        node [shape="box"];
-//        a->b;
-//        b->c;
-//        c->e;
-//        c->f;
-//        f->g;
-//        e->h;
-//    }
-//
-//    { rank=same; 1; a;}
-//    { rank=same; 2; b;}
-//    { rank=same; 3; c;}
-//    { rank=same; 5; e;}
-//    { rank=same; 6; f;}
-//    { rank=same; 7; g;}
-//    { rank=same; 8; h;}
-//
-//    1->2->3->4->5->6->7->8->9[arrowhead=none];
-//}
-func TestReceiveBlock_OnChainSplit(t *testing.T) {
-	// The scenario to test is that we think that the canonical head is block H
-	// and then we receive block G. We don't have block F, so we request it. Then
-	// we process F, the G. The expected behavior is that we load the historical
-	// state from slot 3 where the common ancestor block C is present.
-
-	db := internal.SetupDB(t)
-	defer internal.TeardownDB(t, db)
-	ctx := context.Background()
-
-	chainService := setupBeaconChain(t, db, nil)
-	deposits, privKeys := setupInitialDeposits(t, 100)
-	eth1Data := &pb.Eth1Data{
-		DepositRootHash32: []byte{},
-		BlockHash32:       []byte{},
-	}
-	beaconState, err := state.GenesisBeaconState(deposits, 0, eth1Data)
-	if err != nil {
-		t.Fatalf("Can't generate genesis state: %v", err)
-	}
-	stateRoot, err := hashutil.HashProto(beaconState)
-	if err != nil {
-		t.Fatalf("Could not tree hash state: %v", err)
-	}
-	parentHash, genesisBlock := setupGenesisBlock(t, chainService)
-	if err := db.UpdateChainHead(ctx, genesisBlock, beaconState); err != nil {
-		t.Fatal(err)
-	}
-	if err := db.SaveFinalizedState(beaconState); err != nil {
-		t.Fatal(err)
-	}
-	genesisSlot := params.BeaconConfig().GenesisSlot
-
-	// Top chain slots (see graph)
-	blockSlots := []uint64{1, 2, 3, 5, 8}
-	for _, slot := range blockSlots {
-		block := &pb.BeaconBlock{
-			Slot:             genesisSlot + slot,
-			StateRootHash32:  stateRoot[:],
-			ParentRootHash32: parentHash[:],
-			RandaoReveal:     createRandaoReveal(t, beaconState, privKeys),
-			Body:             &pb.BeaconBlockBody{},
-		}
-		initBlockStateRoot(t, block, chainService)
-		computedState, err := chainService.ReceiveBlock(ctx, block)
-		if err != nil {
-			t.Fatal(err)
-		}
-		stateRoot, err = hashutil.HashProto(computedState)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if err = db.SaveBlock(block); err != nil {
-			t.Fatal(err)
-		}
-		if err = db.UpdateChainHead(ctx, block, computedState); err != nil {
-			t.Fatal(err)
-		}
-		parentHash, err = hashutil.HashBeaconBlock(block)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// Common ancestor is block at slot 3
-	commonAncestor, err := db.BlockBySlot(ctx, genesisSlot+3)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	parentHash, err = hashutil.HashBeaconBlock(commonAncestor)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	beaconState, err = db.HistoricalStateFromSlot(ctx, commonAncestor.Slot)
-	if err != nil {
-		t.Fatal(err)
-	}
-	stateRoot, err = hashutil.HashProto(beaconState)
-	if err != nil {
-		t.Fatal(err)
-	}
-	// Then we receive the block `f` from slot 6
-	blockF := &pb.BeaconBlock{
-		Slot:             genesisSlot + 6,
-		ParentRootHash32: parentHash[:],
-		StateRootHash32:  stateRoot[:],
-		RandaoReveal:     createRandaoReveal(t, beaconState, privKeys),
-		Body:             &pb.BeaconBlockBody{},
-	}
-	initBlockStateRoot(t, blockF, chainService)
-
-	computedState, err := chainService.ReceiveBlock(ctx, blockF)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	stateRoot, err = hashutil.HashProto(computedState)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err := db.SaveBlock(blockF); err != nil {
-		t.Fatal(err)
-	}
-
-	parentHash, err = hashutil.HashBeaconBlock(blockF)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Then we apply block `g` from slot 7
-	blockG := &pb.BeaconBlock{
-		Slot:             genesisSlot + 7,
-		ParentRootHash32: parentHash[:],
-		StateRootHash32:  stateRoot[:],
-		RandaoReveal:     createRandaoReveal(t, computedState, privKeys),
-		Body:             &pb.BeaconBlockBody{},
-	}
-	initBlockStateRoot(t, blockG, chainService)
-
-	computedState, err = chainService.ReceiveBlock(ctx, blockG)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if computedState.Slot != blockG.Slot {
-		t.Errorf("Unexpect state slot %d, wanted %d", computedState.Slot, blockG.Slot)
-	}
-}
-
-func TestIsBlockReadyForProcessing_ValidBlock(t *testing.T) {
-	db := internal.SetupDB(t)
-	defer internal.TeardownDB(t, db)
-	ctx := context.Background()
-
-	chainService := setupBeaconChain(t, db, nil)
-	unixTime := uint64(time.Now().Unix())
-	deposits, privKeys := setupInitialDeposits(t, 100)
-	if err := db.InitializeState(context.Background(), unixTime, deposits, &pb.Eth1Data{}); err != nil {
-		t.Fatalf("Could not initialize beacon state to disk: %v", err)
-	}
-	beaconState, err := db.HeadState(ctx)
-	if err != nil {
-		t.Fatalf("Can't get genesis state: %v", err)
-	}
-	block := &pb.BeaconBlock{
-		ParentRootHash32: []byte{'a'},
-	}
-
-	if err := chainService.VerifyBlockValidity(ctx, block, beaconState); err == nil {
-		t.Fatal("block processing succeeded despite block having no parent saved")
-	}
-
-	beaconState.Slot = params.BeaconConfig().GenesisSlot + 10
-
-	stateRoot, err := hashutil.HashProto(beaconState)
-	if err != nil {
-		t.Fatalf("Could not tree hash state: %v", err)
-	}
-	genesis := b.NewGenesisBlock([]byte{})
-	if err := chainService.beaconDB.SaveBlock(genesis); err != nil {
-		t.Fatalf("cannot save block: %v", err)
-	}
-	parentRoot, err := hashutil.HashBeaconBlock(genesis)
-	if err != nil {
-		t.Fatalf("unable to get root of canonical head: %v", err)
-	}
-
-	beaconState.LatestEth1Data = &pb.Eth1Data{
-		DepositRootHash32: []byte{2},
-		BlockHash32:       []byte{3},
-	}
-	beaconState.Slot = params.BeaconConfig().GenesisSlot
-
-	currentSlot := params.BeaconConfig().GenesisSlot + 1
-	attestationSlot := params.BeaconConfig().GenesisSlot
-
-	randaoReveal := createRandaoReveal(t, beaconState, privKeys)
-	block2 := &pb.BeaconBlock{
-		Slot:             currentSlot,
-		StateRootHash32:  stateRoot[:],
-		ParentRootHash32: parentRoot[:],
-		RandaoReveal:     randaoReveal,
-		Eth1Data: &pb.Eth1Data{
-			DepositRootHash32: []byte("a"),
-			BlockHash32:       []byte("b"),
-		},
-		Body: &pb.BeaconBlockBody{
-			Attestations: []*pb.Attestation{{
-				AggregationBitfield: []byte{128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-					0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
-				Data: &pb.AttestationData{
-					Slot:                     attestationSlot,
-					JustifiedBlockRootHash32: parentRoot[:],
-				},
-			}},
-		},
-	}
-
-	if err := chainService.VerifyBlockValidity(ctx, block2, beaconState); err != nil {
-		t.Fatalf("block processing failed despite being a valid block: %v", err)
-	}
-}
-
-func TestDeleteValidatorIdx_DeleteWorks(t *testing.T) {
-	db := internal.SetupDB(t)
-	defer internal.TeardownDB(t, db)
-	epoch := uint64(2)
-	v.InsertActivatedVal(epoch+1, []uint64{0, 1, 2})
-	v.InsertExitedVal(epoch+1, []uint64{0, 2})
-	var validators []*pb.Validator
-	for i := 0; i < 3; i++ {
-		pubKeyBuf := make([]byte, params.BeaconConfig().BLSPubkeyLength)
-		binary.PutUvarint(pubKeyBuf, uint64(i))
-		validators = append(validators, &pb.Validator{
-			Pubkey: pubKeyBuf,
-		})
-	}
-	state := &pb.BeaconState{
-		ValidatorRegistry: validators,
-		Slot:              epoch * params.BeaconConfig().SlotsPerEpoch,
-	}
-	chainService := setupBeaconChain(t, db, nil)
-	if err := chainService.saveValidatorIdx(state); err != nil {
-		t.Fatalf("Could not save validator idx: %v", err)
-	}
-	if err := chainService.deleteValidatorIdx(state); err != nil {
-		t.Fatalf("Could not delete validator idx: %v", err)
-	}
-	wantedIdx := uint64(1)
-	idx, err := chainService.beaconDB.ValidatorIndex(validators[wantedIdx].Pubkey)
-	if err != nil {
-		t.Fatalf("Could not get validator index: %v", err)
-	}
-	if wantedIdx != idx {
-		t.Errorf("Wanted: %d, got: %d", wantedIdx, idx)
-	}
-
-	wantedIdx = uint64(2)
-	if chainService.beaconDB.HasValidator(validators[wantedIdx].Pubkey) {
-		t.Errorf("Validator index %d should have been deleted", wantedIdx)
-	}
-	if v.ExitedValFromEpoch(epoch) != nil {
-		t.Errorf("Activated validators mapping for epoch %d still there", epoch)
-	}
-}
-
-func TestSaveValidatorIdx_SaveRetrieveWorks(t *testing.T) {
-	db := internal.SetupDB(t)
-	defer internal.TeardownDB(t, db)
-	epoch := uint64(1)
-	v.InsertActivatedVal(epoch+1, []uint64{0, 1, 2})
-	var validators []*pb.Validator
-	for i := 0; i < 3; i++ {
-		pubKeyBuf := make([]byte, params.BeaconConfig().BLSPubkeyLength)
-		binary.PutUvarint(pubKeyBuf, uint64(i))
-		validators = append(validators, &pb.Validator{
-			Pubkey: pubKeyBuf,
-		})
-	}
-	state := &pb.BeaconState{
-		ValidatorRegistry: validators,
-		Slot:              epoch * params.BeaconConfig().SlotsPerEpoch,
-	}
-	chainService := setupBeaconChain(t, db, nil)
-	if err := chainService.saveValidatorIdx(state); err != nil {
-		t.Fatalf("Could not save validator idx: %v", err)
-	}
-
-	wantedIdx := uint64(2)
-	idx, err := chainService.beaconDB.ValidatorIndex(validators[wantedIdx].Pubkey)
-	if err != nil {
-		t.Fatalf("Could not get validator index: %v", err)
-	}
-	if wantedIdx != idx {
-		t.Errorf("Wanted: %d, got: %d", wantedIdx, idx)
-	}
-
-	if v.ActivatedValFromEpoch(epoch) != nil {
-		t.Errorf("Activated validators mapping for epoch %d still there", epoch)
-	}
-}

beacon-chain/blockchain/blockchain_test.go

@@ -0,0 +1,15 @@
+package blockchain
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/sirupsen/logrus"
+)
+
+func TestMain(m *testing.M) {
+	logrus.SetLevel(logrus.DebugLevel)
+	logrus.SetOutput(ioutil.Discard)
+
+	m.Run()
+}

beacon-chain/blockchain/chain_info.go

@@ -0,0 +1,314 @@
+package blockchain
+
+import (
+	"context"
+	"time"
+
+	types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/config/params"
+	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block"
+	"go.opencensus.io/trace"
+)
+
+// ChainInfoFetcher defines a common interface for methods in blockchain service which
+// directly retrieves chain info related data.
+type ChainInfoFetcher interface {
+	HeadFetcher
+	FinalizationFetcher
+	GenesisFetcher
+	CanonicalFetcher
+	ForkFetcher
+	TimeFetcher
+	HeadDomainFetcher
+}
+
+// TimeFetcher retrieves the Ethereum consensus data that's related to time.
+type TimeFetcher interface {
+	GenesisTime() time.Time
+	CurrentSlot() types.Slot
+}
+
+// GenesisFetcher retrieves the Ethereum consensus data related to its genesis.
+type GenesisFetcher interface {
+	GenesisValidatorRoot() [32]byte
+}
+
+// HeadFetcher defines a common interface for methods in blockchain service which
+// directly retrieves head related data.
+type HeadFetcher interface {
+	HeadSlot() types.Slot
+	HeadRoot(ctx context.Context) ([]byte, error)
+	HeadBlock(ctx context.Context) (block.SignedBeaconBlock, error)
+	HeadState(ctx context.Context) (state.BeaconState, error)
+	HeadValidatorsIndices(ctx context.Context, epoch types.Epoch) ([]types.ValidatorIndex, error)
+	HeadSeed(ctx context.Context, epoch types.Epoch) ([32]byte, error)
+	HeadGenesisValidatorRoot() [32]byte
+	HeadETH1Data() *ethpb.Eth1Data
+	HeadPublicKeyToValidatorIndex(ctx context.Context, pubKey [48]byte) (types.ValidatorIndex, bool)
+	HeadValidatorIndexToPublicKey(ctx context.Context, index types.ValidatorIndex) ([48]byte, error)
+	ProtoArrayStore() *protoarray.Store
+	ChainHeads() ([][32]byte, []types.Slot)
+	HeadSyncCommitteeFetcher
+	HeadDomainFetcher
+}
+
+// ForkFetcher retrieves the current fork information of the Ethereum beacon chain.
+type ForkFetcher interface {
+	CurrentFork() *ethpb.Fork
+}
+
+// CanonicalFetcher retrieves the current chain's canonical information.
+type CanonicalFetcher interface {
+	IsCanonical(ctx context.Context, blockRoot [32]byte) (bool, error)
+	VerifyBlkDescendant(ctx context.Context, blockRoot [32]byte) error
+}
+
+// FinalizationFetcher defines a common interface for methods in blockchain service which
+// directly retrieves finalization and justification related data.
+type FinalizationFetcher interface {
+	FinalizedCheckpt() *ethpb.Checkpoint
+	CurrentJustifiedCheckpt() *ethpb.Checkpoint
+	PreviousJustifiedCheckpt() *ethpb.Checkpoint
+}
+
+// FinalizedCheckpt returns the latest finalized checkpoint from head state.
+func (s *Service) FinalizedCheckpt() *ethpb.Checkpoint {
+	if s.finalizedCheckpt == nil {
+		return &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	}
+
+	return ethpb.CopyCheckpoint(s.finalizedCheckpt)
+}
+
+// CurrentJustifiedCheckpt returns the current justified checkpoint from head state.
+func (s *Service) CurrentJustifiedCheckpt() *ethpb.Checkpoint {
+	if s.justifiedCheckpt == nil {
+		return &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	}
+
+	return ethpb.CopyCheckpoint(s.justifiedCheckpt)
+}
+
+// PreviousJustifiedCheckpt returns the previous justified checkpoint from head state.
+func (s *Service) PreviousJustifiedCheckpt() *ethpb.Checkpoint {
+	if s.prevJustifiedCheckpt == nil {
+		return &ethpb.Checkpoint{Root: params.BeaconConfig().ZeroHash[:]}
+	}
+
+	return ethpb.CopyCheckpoint(s.prevJustifiedCheckpt)
+}
+
+// HeadSlot returns the slot of the head of the chain.
+func (s *Service) HeadSlot() types.Slot {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	if !s.hasHeadState() {
+		return 0
+	}
+
+	return s.headSlot()
+}
+
+// HeadRoot returns the root of the head of the chain.
+func (s *Service) HeadRoot(ctx context.Context) ([]byte, error) {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	if s.headRoot() != params.BeaconConfig().ZeroHash {
+		r := s.headRoot()
+		return r[:], nil
+	}
+
+	b, err := s.cfg.BeaconDB.HeadBlock(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if b == nil || b.IsNil() {
+		return params.BeaconConfig().ZeroHash[:], nil
+	}
+
+	r, err := b.Block().HashTreeRoot()
+	if err != nil {
+		return nil, err
+	}
+
+	return r[:], nil
+}
+
+// HeadBlock returns the head block of the chain.
+// If the head is nil from service struct,
+// it will attempt to get the head block from DB.
+func (s *Service) HeadBlock(ctx context.Context) (block.SignedBeaconBlock, error) {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	if s.hasHeadState() {
+		return s.headBlock(), nil
+	}
+
+	return s.cfg.BeaconDB.HeadBlock(ctx)
+}
+
+// HeadState returns the head state of the chain.
+// If the head is nil from service struct,
+// it will attempt to get the head state from DB.
+func (s *Service) HeadState(ctx context.Context) (state.BeaconState, error) {
+	ctx, span := trace.StartSpan(ctx, "blockChain.HeadState")
+	defer span.End()
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	ok := s.hasHeadState()
+	span.AddAttributes(trace.BoolAttribute("cache_hit", ok))
+
+	if ok {
+		return s.headState(ctx), nil
+	}
+
+	return s.cfg.StateGen.StateByRoot(ctx, s.headRoot())
+}
+
+// HeadValidatorsIndices returns a list of active validator indices from the head view of a given epoch.
+func (s *Service) HeadValidatorsIndices(ctx context.Context, epoch types.Epoch) ([]types.ValidatorIndex, error) {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	if !s.hasHeadState() {
+		return []types.ValidatorIndex{}, nil
+	}
+	return helpers.ActiveValidatorIndices(ctx, s.headState(ctx), epoch)
+}
+
+// HeadSeed returns the seed from the head view of a given epoch.
+func (s *Service) HeadSeed(ctx context.Context, epoch types.Epoch) ([32]byte, error) {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	if !s.hasHeadState() {
+		return [32]byte{}, nil
+	}
+
+	return helpers.Seed(s.headState(ctx), epoch, params.BeaconConfig().DomainBeaconAttester)
+}
+
+// HeadGenesisValidatorRoot returns genesis validator root of the head state.
+func (s *Service) HeadGenesisValidatorRoot() [32]byte {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	if !s.hasHeadState() {
+		return [32]byte{}
+	}
+
+	return s.headGenesisValidatorRoot()
+}
+
+// HeadETH1Data returns the eth1data of the current head state.
+func (s *Service) HeadETH1Data() *ethpb.Eth1Data {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	if !s.hasHeadState() {
+		return &ethpb.Eth1Data{}
+	}
+	return s.head.state.Eth1Data()
+}
+
+// ProtoArrayStore returns the proto array store object.
+func (s *Service) ProtoArrayStore() *protoarray.Store {
+	return s.cfg.ForkChoiceStore.Store()
+}
+
+// GenesisTime returns the genesis time of beacon chain.
+func (s *Service) GenesisTime() time.Time {
+	return s.genesisTime
+}
+
+// GenesisValidatorRoot returns the genesis validator
+// root of the chain.
+func (s *Service) GenesisValidatorRoot() [32]byte {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	if !s.hasHeadState() {
+		return [32]byte{}
+	}
+	return bytesutil.ToBytes32(s.head.state.GenesisValidatorRoot())
+}
+
+// CurrentFork retrieves the latest fork information of the beacon chain.
+func (s *Service) CurrentFork() *ethpb.Fork {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+
+	if !s.hasHeadState() {
+		return &ethpb.Fork{
+			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
+			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
+		}
+	}
+	return s.head.state.Fork()
+}
+
+// IsCanonical returns true if the input block root is part of the canonical chain.
+func (s *Service) IsCanonical(ctx context.Context, blockRoot [32]byte) (bool, error) {
+	// If the block has been finalized, the block will always be part of the canonical chain.
+	if s.cfg.BeaconDB.IsFinalizedBlock(ctx, blockRoot) {
+		return true, nil
+	}
+
+	// If the block has not been finalized, check fork choice store to see if the block is canonical
+	return s.cfg.ForkChoiceStore.IsCanonical(blockRoot), nil
+}
+
+// ChainHeads returns all possible chain heads (leaves of fork choice tree).
+// Heads roots and heads slots are returned.
+func (s *Service) ChainHeads() ([][32]byte, []types.Slot) {
+	nodes := s.ProtoArrayStore().Nodes()
+
+	// Deliberate choice to not preallocate space for below.
+	// Heads cant be more than 2-3 in the worst case where pre-allocation will be 64 to begin with.
+	headsRoots := make([][32]byte, 0)
+	headsSlots := make([]types.Slot, 0)
+
+	nonExistentNode := ^uint64(0)
+	for _, node := range nodes {
+		// Possible heads have no children.
+		if node.BestDescendant() == nonExistentNode && node.BestChild() == nonExistentNode {
+			headsRoots = append(headsRoots, node.Root())
+			headsSlots = append(headsSlots, node.Slot())
+		}
+	}
+
+	return headsRoots, headsSlots
+}
+
+// HeadPublicKeyToValidatorIndex returns the validator index of the `pubkey` in current head state.
+func (s *Service) HeadPublicKeyToValidatorIndex(ctx context.Context, pubKey [48]byte) (types.ValidatorIndex, bool) {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+	if !s.hasHeadState() {
+		return 0, false
+	}
+	return s.headState(ctx).ValidatorIndexByPubkey(pubKey)
+}
+
+// HeadValidatorIndexToPublicKey returns the pubkey of the validator `index`  in current head state.
+func (s *Service) HeadValidatorIndexToPublicKey(_ context.Context, index types.ValidatorIndex) ([48]byte, error) {
+	s.headLock.RLock()
+	defer s.headLock.RUnlock()
+	if !s.hasHeadState() {
+		return [48]byte{}, nil
+	}
+	v, err := s.headValidatorAtIndex(index)
+	if err != nil {
+		return [48]byte{}, err
+	}
+	return v.PublicKey(), nil
+}

beacon-chain/blockchain/chain_info_norace_test.go

@@ -0,0 +1,61 @@
+package blockchain
+
+import (
+	"context"
+	"testing"
+
+	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper"
+	"github.com/prysmaticlabs/prysm/testing/require"
+)
+
+func TestHeadSlot_DataRace(t *testing.T) {
+	beaconDB := testDB.SetupDB(t)
+	s := &Service{
+		cfg: &Config{BeaconDB: beaconDB},
+	}
+	go func() {
+		require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
+	}()
+	s.HeadSlot()
+}
+
+func TestHeadRoot_DataRace(t *testing.T) {
+	beaconDB := testDB.SetupDB(t)
+	s := &Service{
+		cfg:  &Config{BeaconDB: beaconDB, StateGen: stategen.New(beaconDB)},
+		head: &head{root: [32]byte{'A'}},
+	}
+	go func() {
+		require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
+	}()
+	_, err := s.HeadRoot(context.Background())
+	require.NoError(t, err)
+}
+
+func TestHeadBlock_DataRace(t *testing.T) {
+	beaconDB := testDB.SetupDB(t)
+	s := &Service{
+		cfg:  &Config{BeaconDB: beaconDB, StateGen: stategen.New(beaconDB)},
+		head: &head{block: wrapper.WrappedPhase0SignedBeaconBlock(&ethpb.SignedBeaconBlock{})},
+	}
+	go func() {
+		require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
+	}()
+	_, err := s.HeadBlock(context.Background())
+	require.NoError(t, err)
+}
+
+func TestHeadState_DataRace(t *testing.T) {
+	beaconDB := testDB.SetupDB(t)
+	s := &Service{
+		cfg: &Config{BeaconDB: beaconDB, StateGen: stategen.New(beaconDB)},
+	}
+	go func() {
+		require.NoError(t, s.saveHead(context.Background(), [32]byte{}))
+	}()
+	_, err := s.HeadState(context.Background())
+	require.NoError(t, err)
+}

beacon-chain/blockchain/chain_info_test.go

@@ -0,0 +1,356 @@
+package blockchain
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	v1 "github.com/prysmaticlabs/prysm/beacon-chain/state/v1"
+	"github.com/prysmaticlabs/prysm/config/params"
+	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper"
+	"github.com/prysmaticlabs/prysm/testing/assert"
+	"github.com/prysmaticlabs/prysm/testing/require"
+	"github.com/prysmaticlabs/prysm/testing/util"
+	"google.golang.org/protobuf/proto"
+)
+
+// Ensure Service implements chain info interface.
+var _ ChainInfoFetcher = (*Service)(nil)
+var _ TimeFetcher = (*Service)(nil)
+var _ ForkFetcher = (*Service)(nil)
+
+func TestFinalizedCheckpt_Nil(t *testing.T) {
+	beaconDB := testDB.SetupDB(t)
+	c := setupBeaconChain(t, beaconDB)
+	assert.DeepEqual(t, params.BeaconConfig().ZeroHash[:], c.FinalizedCheckpt().Root, "Incorrect pre chain start value")
+}
+
+func TestHeadRoot_Nil(t *testing.T) {
+	beaconDB := testDB.SetupDB(t)
+	c := setupBeaconChain(t, beaconDB)
+	headRoot, err := c.HeadRoot(context.Background())
+	require.NoError(t, err)
+	assert.DeepEqual(t, params.BeaconConfig().ZeroHash[:], headRoot, "Incorrect pre chain start value")
+}
+
+func TestFinalizedCheckpt_CanRetrieve(t *testing.T) {
+	beaconDB := testDB.SetupDB(t)
+
+	cp := &ethpb.Checkpoint{Epoch: 5, Root: bytesutil.PadTo([]byte("foo"), 32)}
+	c := setupBeaconChain(t, beaconDB)
+	c.finalizedCheckpt = cp
+
+	assert.Equal(t, cp.Epoch, c.FinalizedCheckpt().Epoch, "Unexpected finalized epoch")
+}
+
+func TestFinalizedCheckpt_GenesisRootOk(t *testing.T) {
+	beaconDB := testDB.SetupDB(t)
+
+	genesisRoot := [32]byte{'A'}
+	cp := &ethpb.Checkpoint{Root: genesisRoot[:]}
+	c := setupBeaconChain(t, beaconDB)
+	c.finalizedCheckpt = cp
+	c.genesisRoot = genesisRoot
+	assert.DeepEqual(t, c.genesisRoot[:], c.FinalizedCheckpt().Root)
+}
+
+func TestCurrentJustifiedCheckpt_CanRetrieve(t *testing.T) {
+	beaconDB := testDB.SetupDB(t)
+
+	c := setupBeaconChain(t, beaconDB)
+	assert.Equal(t, params.BeaconConfig().ZeroHash, bytesutil.ToBytes32(c.CurrentJustifiedCheckpt().Root), "Unexpected justified epoch")
+	cp := &ethpb.Checkpoint{Epoch: 6, Root: bytesutil.PadTo([]byte("foo"), 32)}
+	c.justifiedCheckpt = cp
+	assert.Equal(t, cp.Epoch, c.CurrentJustifiedCheckpt().Epoch, "Unexpected justified epoch")
+}
+
+func TestJustifiedCheckpt_GenesisRootOk(t *testing.T) {
+	beaconDB := testDB.SetupDB(t)
+
+	c := setupBeaconChain(t, beaconDB)
+	genesisRoot := [32]byte{'B'}
+	cp := &ethpb.Checkpoint{Root: genesisRoot[:]}
+	c.justifiedCheckpt = cp
+	c.genesisRoot = genesisRoot
+	assert.DeepEqual(t, c.genesisRoot[:], c.CurrentJustifiedCheckpt().Root)
+}
+
+func TestPreviousJustifiedCheckpt_CanRetrieve(t *testing.T) {
+	beaconDB := testDB.SetupDB(t)
+
+	cp := &ethpb.Checkpoint{Epoch: 7, Root: bytesutil.PadTo([]byte("foo"), 32)}
+	c := setupBeaconChain(t, beaconDB)
+	assert.Equal(t, params.BeaconConfig().ZeroHash, bytesutil.ToBytes32(c.CurrentJustifiedCheckpt().Root), "Unexpected justified epoch")
+	c.prevJustifiedCheckpt = cp
+	assert.Equal(t, cp.Epoch, c.PreviousJustifiedCheckpt().Epoch, "Unexpected previous justified epoch")
+}
+
+func TestPrevJustifiedCheckpt_GenesisRootOk(t *testing.T) {
+	beaconDB := testDB.SetupDB(t)
+
+	genesisRoot := [32]byte{'C'}
+	cp := &ethpb.Checkpoint{Root: genesisRoot[:]}
+	c := setupBeaconChain(t, beaconDB)
+	c.prevJustifiedCheckpt = cp
+	c.genesisRoot = genesisRoot
+	assert.DeepEqual(t, c.genesisRoot[:], c.PreviousJustifiedCheckpt().Root)
+}
+
+func TestHeadSlot_CanRetrieve(t *testing.T) {
+	c := &Service{}
+	s, err := v1.InitializeFromProto(&ethpb.BeaconState{})
+	require.NoError(t, err)
+	c.head = &head{slot: 100, state: s}
+	assert.Equal(t, types.Slot(100), c.HeadSlot())
+}
+
+func TestHeadRoot_CanRetrieve(t *testing.T) {
+	c := &Service{}
+	c.head = &head{root: [32]byte{'A'}}
+	r, err := c.HeadRoot(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, [32]byte{'A'}, bytesutil.ToBytes32(r))
+}
+
+func TestHeadRoot_UseDB(t *testing.T) {
+	beaconDB := testDB.SetupDB(t)
+	c := &Service{cfg: &Config{BeaconDB: beaconDB}}
+	c.head = &head{root: params.BeaconConfig().ZeroHash}
+	b := util.NewBeaconBlock()
+	br, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, beaconDB.SaveBlock(context.Background(), wrapper.WrappedPhase0SignedBeaconBlock(b)))
+	require.NoError(t, beaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: br[:]}))
+	require.NoError(t, beaconDB.SaveHeadBlockRoot(context.Background(), br))
+	r, err := c.HeadRoot(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, br, bytesutil.ToBytes32(r))
+}
+
+func TestHeadBlock_CanRetrieve(t *testing.T) {
+	b := util.NewBeaconBlock()
+	b.Block.Slot = 1
+	s, err := v1.InitializeFromProto(&ethpb.BeaconState{})
+	require.NoError(t, err)
+	c := &Service{}
+	c.head = &head{block: wrapper.WrappedPhase0SignedBeaconBlock(b), state: s}
+
+	recevied, err := c.HeadBlock(context.Background())
+	require.NoError(t, err)
+	assert.DeepEqual(t, b, recevied.Proto(), "Incorrect head block received")
+}
+
+func TestHeadState_CanRetrieve(t *testing.T) {
+	s, err := v1.InitializeFromProto(&ethpb.BeaconState{Slot: 2, GenesisValidatorsRoot: params.BeaconConfig().ZeroHash[:]})
+	require.NoError(t, err)
+	c := &Service{}
+	c.head = &head{state: s}
+	headState, err := c.HeadState(context.Background())
+	require.NoError(t, err)
+	assert.DeepEqual(t, headState.InnerStateUnsafe(), s.InnerStateUnsafe(), "Incorrect head state received")
+}
+
+func TestGenesisTime_CanRetrieve(t *testing.T) {
+	c := &Service{genesisTime: time.Unix(999, 0)}
+	wanted := time.Unix(999, 0)
+	assert.Equal(t, wanted, c.GenesisTime(), "Did not get wanted genesis time")
+}
+
+func TestCurrentFork_CanRetrieve(t *testing.T) {
+	f := &ethpb.Fork{Epoch: 999}
+	s, err := v1.InitializeFromProto(&ethpb.BeaconState{Fork: f})
+	require.NoError(t, err)
+	c := &Service{}
+	c.head = &head{state: s}
+	if !proto.Equal(c.CurrentFork(), f) {
+		t.Error("Received incorrect fork version")
+	}
+}
+
+func TestCurrentFork_NilHeadSTate(t *testing.T) {
+	f := &ethpb.Fork{
+		PreviousVersion: params.BeaconConfig().GenesisForkVersion,
+		CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
+	}
+	c := &Service{}
+	if !proto.Equal(c.CurrentFork(), f) {
+		t.Error("Received incorrect fork version")
+	}
+}
+
+func TestGenesisValidatorRoot_CanRetrieve(t *testing.T) {
+	// Should not panic if head state is nil.
+	c := &Service{}
+	assert.Equal(t, [32]byte{}, c.GenesisValidatorRoot(), "Did not get correct genesis validator root")
+
+	s, err := v1.InitializeFromProto(&ethpb.BeaconState{GenesisValidatorsRoot: []byte{'a'}})
+	require.NoError(t, err)
+	c.head = &head{state: s}
+	assert.Equal(t, [32]byte{'a'}, c.GenesisValidatorRoot(), "Did not get correct genesis validator root")
+}
+
+func TestHeadETH1Data_Nil(t *testing.T) {
+	beaconDB := testDB.SetupDB(t)
+	c := setupBeaconChain(t, beaconDB)
+	assert.DeepEqual(t, &ethpb.Eth1Data{}, c.HeadETH1Data(), "Incorrect pre chain start value")
+}
+
+func TestHeadETH1Data_CanRetrieve(t *testing.T) {
+	d := &ethpb.Eth1Data{DepositCount: 999}
+	s, err := v1.InitializeFromProto(&ethpb.BeaconState{Eth1Data: d})
+	require.NoError(t, err)
+	c := &Service{}
+	c.head = &head{state: s}
+	if !proto.Equal(c.HeadETH1Data(), d) {
+		t.Error("Received incorrect eth1 data")
+	}
+}
+
+func TestIsCanonical_Ok(t *testing.T) {
+	ctx := context.Background()
+	beaconDB := testDB.SetupDB(t)
+	c := setupBeaconChain(t, beaconDB)
+
+	blk := util.NewBeaconBlock()
+	blk.Block.Slot = 0
+	root, err := blk.Block.HashTreeRoot()
+	require.NoError(t, err)
+	require.NoError(t, beaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(blk)))
+	require.NoError(t, beaconDB.SaveGenesisBlockRoot(ctx, root))
+	can, err := c.IsCanonical(ctx, root)
+	require.NoError(t, err)
+	assert.Equal(t, true, can)
+
+	can, err = c.IsCanonical(ctx, [32]byte{'a'})
+	require.NoError(t, err)
+	assert.Equal(t, false, can)
+}
+
+func TestService_HeadValidatorsIndices(t *testing.T) {
+	s, _ := util.DeterministicGenesisState(t, 10)
+	c := &Service{}
+
+	c.head = &head{}
+	indices, err := c.HeadValidatorsIndices(context.Background(), 0)
+	require.NoError(t, err)
+	require.Equal(t, 0, len(indices))
+
+	c.head = &head{state: s}
+	indices, err = c.HeadValidatorsIndices(context.Background(), 0)
+	require.NoError(t, err)
+	require.Equal(t, 10, len(indices))
+}
+
+func TestService_HeadSeed(t *testing.T) {
+	s, _ := util.DeterministicGenesisState(t, 1)
+	c := &Service{}
+	seed, err := helpers.Seed(s, 0, params.BeaconConfig().DomainBeaconAttester)
+	require.NoError(t, err)
+
+	c.head = &head{}
+	root, err := c.HeadSeed(context.Background(), 0)
+	require.NoError(t, err)
+	require.Equal(t, [32]byte{}, root)
+
+	c.head = &head{state: s}
+	root, err = c.HeadSeed(context.Background(), 0)
+	require.NoError(t, err)
+	require.DeepEqual(t, seed, root)
+}
+
+func TestService_HeadGenesisValidatorRoot(t *testing.T) {
+	s, _ := util.DeterministicGenesisState(t, 1)
+	c := &Service{}
+
+	c.head = &head{}
+	root := c.HeadGenesisValidatorRoot()
+	require.Equal(t, [32]byte{}, root)
+
+	c.head = &head{state: s}
+	root = c.HeadGenesisValidatorRoot()
+	require.DeepEqual(t, root[:], s.GenesisValidatorRoot())
+}
+
+func TestService_ProtoArrayStore(t *testing.T) {
+	c := &Service{cfg: &Config{ForkChoiceStore: protoarray.New(0, 0, [32]byte{})}}
+	p := c.ProtoArrayStore()
+	require.Equal(t, 0, int(p.FinalizedEpoch()))
+}
+
+func TestService_ChainHeads(t *testing.T) {
+	ctx := context.Background()
+	c := &Service{cfg: &Config{ForkChoiceStore: protoarray.New(0, 0, [32]byte{})}}
+	require.NoError(t, c.cfg.ForkChoiceStore.ProcessBlock(ctx, 100, [32]byte{'a'}, [32]byte{}, [32]byte{}, 0, 0))
+	require.NoError(t, c.cfg.ForkChoiceStore.ProcessBlock(ctx, 101, [32]byte{'b'}, [32]byte{'a'}, [32]byte{}, 0, 0))
+	require.NoError(t, c.cfg.ForkChoiceStore.ProcessBlock(ctx, 102, [32]byte{'c'}, [32]byte{'b'}, [32]byte{}, 0, 0))
+	require.NoError(t, c.cfg.ForkChoiceStore.ProcessBlock(ctx, 103, [32]byte{'d'}, [32]byte{}, [32]byte{}, 0, 0))
+	require.NoError(t, c.cfg.ForkChoiceStore.ProcessBlock(ctx, 104, [32]byte{'e'}, [32]byte{'b'}, [32]byte{}, 0, 0))
+
+	roots, slots := c.ChainHeads()
+	require.DeepEqual(t, [][32]byte{{'c'}, {'d'}, {'e'}}, roots)
+	require.DeepEqual(t, []types.Slot{102, 103, 104}, slots)
+}
+
+func TestService_HeadPublicKeyToValidatorIndex(t *testing.T) {
+	s, _ := util.DeterministicGenesisState(t, 10)
+	c := &Service{}
+	c.head = &head{state: s}
+
+	_, e := c.HeadPublicKeyToValidatorIndex(context.Background(), [48]byte{})
+	require.Equal(t, false, e)
+
+	v, err := s.ValidatorAtIndex(0)
+	require.NoError(t, err)
+
+	i, e := c.HeadPublicKeyToValidatorIndex(context.Background(), bytesutil.ToBytes48(v.PublicKey))
+	require.Equal(t, true, e)
+	require.Equal(t, types.ValidatorIndex(0), i)
+}
+
+func TestService_HeadPublicKeyToValidatorIndexNil(t *testing.T) {
+	c := &Service{}
+	c.head = nil
+
+	idx, e := c.HeadPublicKeyToValidatorIndex(context.Background(), [48]byte{})
+	require.Equal(t, false, e)
+	require.Equal(t, types.ValidatorIndex(0), idx)
+
+	c.head = &head{state: nil}
+	i, e := c.HeadPublicKeyToValidatorIndex(context.Background(), [48]byte{})
+	require.Equal(t, false, e)
+	require.Equal(t, types.ValidatorIndex(0), i)
+}
+
+func TestService_HeadValidatorIndexToPublicKey(t *testing.T) {
+	s, _ := util.DeterministicGenesisState(t, 10)
+	c := &Service{}
+	c.head = &head{state: s}
+
+	p, err := c.HeadValidatorIndexToPublicKey(context.Background(), 0)
+	require.NoError(t, err)
+
+	v, err := s.ValidatorAtIndex(0)
+	require.NoError(t, err)
+
+	require.Equal(t, bytesutil.ToBytes48(v.PublicKey), p)
+}
+
+func TestService_HeadValidatorIndexToPublicKeyNil(t *testing.T) {
+	c := &Service{}
+	c.head = nil
+
+	p, err := c.HeadValidatorIndexToPublicKey(context.Background(), 0)
+	require.NoError(t, err)
+	require.Equal(t, [48]byte{}, p)
+
+	c.head = &head{state: nil}
+	p, err = c.HeadValidatorIndexToPublicKey(context.Background(), 0)
+	require.NoError(t, err)
+	require.Equal(t, [48]byte{}, p)
+}

beacon-chain/blockchain/checktags_test.go

@@ -0,0 +1,7 @@
+// +build !develop
+
+package blockchain
+
+func init() {
+	log.Fatal("Tests in this package require extra build tag: re-run with `-tags develop`")
+}

beacon-chain/blockchain/fork_choice.go

@@ -1,470 +0,0 @@
-package blockchain
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
-	"github.com/prysmaticlabs/prysm/beacon-chain/db"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/bytesutil"
-	"github.com/prysmaticlabs/prysm/shared/hashutil"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/sirupsen/logrus"
-	"go.opencensus.io/trace"
-)
-
-var (
-	reorgCount = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "reorg_counter",
-		Help: "The number of chain reorganization events that have happened in the fork choice rule",
-	})
-)
-var blkAncestorCache = cache.NewBlockAncestorCache()
-
-// ForkChoice interface defines the methods for applying fork choice rule
-// operations to the blockchain.
-type ForkChoice interface {
-	ApplyForkChoiceRule(ctx context.Context, block *pb.BeaconBlock, computedState *pb.BeaconState) error
-}
-
-// updateFFGCheckPts checks whether the existing FFG check points saved in DB
-// are not older than the ones just processed in state. If it's older, we update
-// the db with the latest FFG check points, both justification and finalization.
-func (c *ChainService) updateFFGCheckPts(ctx context.Context, state *pb.BeaconState) error {
-	lastJustifiedSlot := helpers.StartSlot(state.JustifiedEpoch)
-	savedJustifiedBlock, err := c.beaconDB.JustifiedBlock()
-	if err != nil {
-		return err
-	}
-	// If the last processed justification slot in state is greater than
-	// the slot of justified block saved in DB.
-	if lastJustifiedSlot > savedJustifiedBlock.Slot {
-		// Retrieve the new justified block from DB using the new justified slot and save it.
-		newJustifiedBlock, err := c.beaconDB.BlockBySlot(ctx, lastJustifiedSlot)
-		if err != nil {
-			return err
-		}
-		// If the new justified slot is a skip slot in db then we keep getting it's ancestors
-		// until we can get a block.
-		lastAvailBlkSlot := lastJustifiedSlot
-		for newJustifiedBlock == nil {
-			log.WithField("slot", lastAvailBlkSlot-params.BeaconConfig().GenesisSlot).Debug("Missing block in DB, looking one slot back")
-			lastAvailBlkSlot--
-			newJustifiedBlock, err = c.beaconDB.BlockBySlot(ctx, lastAvailBlkSlot)
-			if err != nil {
-				return err
-			}
-		}
-
-		// Fetch justified state from historical states db.
-		newJustifiedState, err := c.beaconDB.HistoricalStateFromSlot(ctx, newJustifiedBlock.Slot)
-		if err != nil {
-			return err
-		}
-		if err := c.beaconDB.SaveJustifiedBlock(newJustifiedBlock); err != nil {
-			return err
-		}
-		if err := c.beaconDB.SaveJustifiedState(newJustifiedState); err != nil {
-			return err
-		}
-	}
-
-	lastFinalizedSlot := helpers.StartSlot(state.FinalizedEpoch)
-	savedFinalizedBlock, err := c.beaconDB.FinalizedBlock()
-	// If the last processed finalized slot in state is greater than
-	// the slot of finalized block saved in DB.
-	if err != nil {
-		return err
-	}
-	if lastFinalizedSlot > savedFinalizedBlock.Slot {
-		// Retrieve the new finalized block from DB using the new finalized slot and save it.
-		newFinalizedBlock, err := c.beaconDB.BlockBySlot(ctx, lastFinalizedSlot)
-		if err != nil {
-			return err
-		}
-		// If the new finalized slot is a skip slot in db then we keep getting it's ancestors
-		// until we can get a block.
-		lastAvailBlkSlot := lastFinalizedSlot
-		for newFinalizedBlock == nil {
-			log.WithField("slot", lastAvailBlkSlot-params.BeaconConfig().GenesisSlot).Debug("Missing block in DB, looking one slot back")
-			lastAvailBlkSlot--
-			newFinalizedBlock, err = c.beaconDB.BlockBySlot(ctx, lastAvailBlkSlot)
-			if err != nil {
-				return err
-			}
-		}
-
-		// Generate the new finalized state with using new finalized block and
-		// save it.
-		newFinalizedState, err := c.beaconDB.HistoricalStateFromSlot(ctx, lastFinalizedSlot)
-		if err != nil {
-			return err
-		}
-		if err := c.beaconDB.SaveFinalizedBlock(newFinalizedBlock); err != nil {
-			return err
-		}
-		if err := c.beaconDB.SaveFinalizedState(newFinalizedState); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// ApplyForkChoiceRule determines the current beacon chain head using LMD
-// GHOST as a block-vote weighted function to select a canonical head in
-// Ethereum Serenity. The inputs are the the recently processed block and its
-// associated state.
-func (c *ChainService) ApplyForkChoiceRule(
-	ctx context.Context,
-	block *pb.BeaconBlock,
-	postState *pb.BeaconState,
-) error {
-	ctx, span := trace.StartSpan(ctx, "beacon-chain.blockchain.ApplyForkChoiceRule")
-	defer span.End()
-	log.Info("Applying LMD-GHOST Fork Choice Rule")
-
-	justifiedState, err := c.beaconDB.JustifiedState()
-	if err != nil {
-		return fmt.Errorf("could not retrieve justified state: %v", err)
-	}
-	attestationTargets, err := c.attestationTargets(justifiedState)
-	if err != nil {
-		return fmt.Errorf("could not retrieve attestation target: %v", err)
-	}
-	justifiedHead, err := c.beaconDB.JustifiedBlock()
-	if err != nil {
-		return fmt.Errorf("could not retrieve justified head: %v", err)
-	}
-
-	newHead, err := c.lmdGhost(ctx, justifiedHead, justifiedState, attestationTargets)
-	if err != nil {
-		return fmt.Errorf("could not run fork choice: %v", err)
-	}
-	newHeadRoot, err := hashutil.HashBeaconBlock(newHead)
-	if err != nil {
-		return fmt.Errorf("could not hash head block: %v", err)
-	}
-	c.canonicalBlocksLock.Lock()
-	defer c.canonicalBlocksLock.Unlock()
-	c.canonicalBlocks[newHead.Slot] = newHeadRoot[:]
-
-	currentHead, err := c.beaconDB.ChainHead()
-	if err != nil {
-		return fmt.Errorf("could not retrieve chain head: %v", err)
-	}
-
-	isDescendant, err := c.isDescendant(currentHead, newHead)
-	if err != nil {
-		return fmt.Errorf("could not check if block is descendant: %v", err)
-	}
-
-	newState := postState
-	if !isDescendant {
-		log.Warnf("Reorg happened, last head at slot %d, new head block at slot %d",
-			currentHead.Slot-params.BeaconConfig().GenesisSlot, newHead.Slot-params.BeaconConfig().GenesisSlot)
-
-		// Only regenerate head state if there was a reorg.
-		newState, err = c.beaconDB.HistoricalStateFromSlot(ctx, newHead.Slot)
-		if err != nil {
-			return fmt.Errorf("could not gen state: %v", err)
-		}
-
-		for revertedSlot := currentHead.Slot; revertedSlot > newHead.Slot; revertedSlot-- {
-			delete(c.canonicalBlocks, revertedSlot)
-		}
-		reorgCount.Inc()
-	}
-
-	// If we receive forked blocks.
-	if newHead.Slot != newState.Slot {
-		newState, err = c.beaconDB.HistoricalStateFromSlot(ctx, newHead.Slot)
-		if err != nil {
-			return fmt.Errorf("could not gen state: %v", err)
-		}
-	}
-
-	if err := c.beaconDB.UpdateChainHead(ctx, newHead, newState); err != nil {
-		return fmt.Errorf("failed to update chain: %v", err)
-	}
-	h, err := hashutil.HashBeaconBlock(newHead)
-	if err != nil {
-		return fmt.Errorf("could not hash head: %v", err)
-	}
-	log.WithFields(logrus.Fields{
-		"headRoot":  fmt.Sprintf("%#x", bytesutil.Trunc(h[:])),
-		"headSlot":  newHead.Slot - params.BeaconConfig().GenesisSlot,
-		"stateSlot": newState.Slot - params.BeaconConfig().GenesisSlot,
-	}).Info("Chain head block and state updated")
-
-	return nil
-}
-
-// lmdGhost applies the Latest Message Driven, Greediest Heaviest Observed Sub-Tree
-// fork-choice rule defined in the Ethereum Serenity specification for the beacon chain.
-//
-// Spec pseudocode definition:
-//	def lmd_ghost(store: Store, start_state: BeaconState, start_block: BeaconBlock) -> BeaconBlock:
-//    """
-//    Execute the LMD-GHOST algorithm to find the head ``BeaconBlock``.
-//    """
-//    validators = start_state.validator_registry
-//    active_validator_indices = get_active_validator_indices(validators, slot_to_epoch(start_state.slot))
-//    attestation_targets = [
-//        (validator_index, get_latest_attestation_target(store, validator_index))
-//        for validator_index in active_validator_indices
-//    ]
-//
-//    def get_vote_count(block: BeaconBlock) -> int:
-//        return sum(
-//            get_effective_balance(start_state.validator_balances[validator_index]) // FORK_CHOICE_BALANCE_INCREMENT
-//            for validator_index, target in attestation_targets
-//            if get_ancestor(store, target, block.slot) == block
-//        )
-//
-//    head = start_block
-//    while 1:
-//        children = get_children(store, head)
-//        if len(children) == 0:
-//            return head
-//        head = max(children, key=get_vote_count)
-func (c *ChainService) lmdGhost(
-	ctx context.Context,
-	startBlock *pb.BeaconBlock,
-	startState *pb.BeaconState,
-	voteTargets map[uint64]*pb.AttestationTarget,
-) (*pb.BeaconBlock, error) {
-	highestSlot := c.beaconDB.HighestBlockSlot()
-	head := startBlock
-	for {
-		children, err := c.blockChildren(ctx, head, highestSlot)
-		if err != nil {
-			return nil, fmt.Errorf("could not fetch block children: %v", err)
-		}
-		if len(children) == 0 {
-			return head, nil
-		}
-		maxChild := children[0]
-
-		maxChildVotes, err := VoteCount(maxChild, startState, voteTargets, c.beaconDB)
-		if err != nil {
-			return nil, fmt.Errorf("unable to determine vote count for block: %v", err)
-		}
-		for i := 1; i < len(children); i++ {
-			candidateChildVotes, err := VoteCount(children[i], startState, voteTargets, c.beaconDB)
-			if err != nil {
-				return nil, fmt.Errorf("unable to determine vote count for block: %v", err)
-			}
-			maxChildRoot, err := hashutil.HashBeaconBlock(maxChild)
-			if err != nil {
-				return nil, err
-			}
-			candidateChildRoot, err := hashutil.HashBeaconBlock(children[i])
-			if err != nil {
-				return nil, err
-			}
-			if candidateChildVotes > maxChildVotes ||
-				(candidateChildVotes == maxChildVotes && bytesutil.LowerThan(maxChildRoot[:], candidateChildRoot[:])) {
-				maxChild = children[i]
-			}
-		}
-		head = maxChild
-	}
-}
-
-// blockChildren returns the child blocks of the given block up to a given
-// highest slot.
-//
-// ex:
-//       /- C - E
-// A - B - D - F
-//       \- G
-// Input: B. Output: [C, D, G]
-//
-// Spec pseudocode definition:
-//	get_children(store: Store, block: BeaconBlock) -> List[BeaconBlock]
-//		returns the child blocks of the given block.
-func (c *ChainService) blockChildren(ctx context.Context, block *pb.BeaconBlock, highestSlot uint64) ([]*pb.BeaconBlock, error) {
-	var children []*pb.BeaconBlock
-
-	currentRoot, err := hashutil.HashBeaconBlock(block)
-	if err != nil {
-		return nil, fmt.Errorf("could not tree hash incoming block: %v", err)
-	}
-	startSlot := block.Slot + 1
-	for i := startSlot; i <= highestSlot; i++ {
-		block, err := c.beaconDB.BlockBySlot(ctx, i)
-		if err != nil {
-			return nil, fmt.Errorf("could not get block by slot: %v", err)
-		}
-		// Continue if there's a skip block.
-		if block == nil {
-			continue
-		}
-
-		parentRoot := bytesutil.ToBytes32(block.ParentRootHash32)
-		if currentRoot == parentRoot {
-			children = append(children, block)
-		}
-	}
-	return children, nil
-}
-
-// isDescendant checks if the new head block is a descendant block of the current head.
-func (c *ChainService) isDescendant(currentHead *pb.BeaconBlock, newHead *pb.BeaconBlock) (bool, error) {
-	currentHeadRoot, err := hashutil.HashBeaconBlock(currentHead)
-	if err != nil {
-		return false, nil
-	}
-	for newHead.Slot > currentHead.Slot {
-		if bytesutil.ToBytes32(newHead.ParentRootHash32) == currentHeadRoot {
-			return true, nil
-		}
-		newHead, err = c.beaconDB.Block(bytesutil.ToBytes32(newHead.ParentRootHash32))
-		if err != nil {
-			return false, err
-		}
-		if newHead == nil {
-			return false, nil
-		}
-	}
-	return false, nil
-}
-
-// attestationTargets retrieves the list of attestation targets since last finalized epoch,
-// each attestation target consists of validator index and its attestation target (i.e. the block
-// which the validator attested to)
-func (c *ChainService) attestationTargets(state *pb.BeaconState) (map[uint64]*pb.AttestationTarget, error) {
-	indices := helpers.ActiveValidatorIndices(state.ValidatorRegistry, helpers.CurrentEpoch(state))
-	attestationTargets := make(map[uint64]*pb.AttestationTarget)
-	for i, index := range indices {
-		target, err := c.attsService.LatestAttestationTarget(state, index)
-		if err != nil {
-			return nil, fmt.Errorf("could not retrieve attestation target: %v", err)
-		}
-		if target == nil {
-			continue
-		}
-		attestationTargets[uint64(i)] = target
-	}
-	return attestationTargets, nil
-}
-
-// VoteCount determines the number of votes on a beacon block by counting the number
-// of target blocks that have such beacon block as a common ancestor.
-//
-// Spec pseudocode definition:
-//  def get_vote_count(block: BeaconBlock) -> int:
-//        return sum(
-//            get_effective_balance(start_state.validator_balances[validator_index]) // FORK_CHOICE_BALANCE_INCREMENT
-//            for validator_index, target in attestation_targets
-//            if get_ancestor(store, target, block.slot) == block
-//        )
-func VoteCount(block *pb.BeaconBlock, state *pb.BeaconState, targets map[uint64]*pb.AttestationTarget, beaconDB *db.BeaconDB) (int, error) {
-	balances := 0
-	var ancestorRoot []byte
-	var err error
-
-	blockRoot, err := hashutil.HashBeaconBlock(block)
-	if err != nil {
-		return 0, err
-	}
-
-	for validatorIndex, target := range targets {
-		ancestorRoot, err = cachedAncestor(target, block.Slot, beaconDB)
-		if err != nil {
-			return 0, err
-		}
-		// This covers the following case, we start at B5, and want to process B6 and B7
-		// B6 can be processed, B7 can not be processed because it's pointed to the
-		// block older than current block 5.
-		// B4 - B5 - B6
-		//   \ - - - - - B7
-		if ancestorRoot == nil {
-			continue
-		}
-
-		if bytes.Equal(blockRoot[:], ancestorRoot) {
-			balances += int(helpers.EffectiveBalance(state, validatorIndex))
-		}
-	}
-	return balances, nil
-}
-
-// BlockAncestor obtains the ancestor at of a block at a certain slot.
-//
-// Spec pseudocode definition:
-//  def get_ancestor(store: Store, block: BeaconBlock, slot: Slot) -> BeaconBlock:
-//    """
-//    Get the ancestor of ``block`` with slot number ``slot``; return ``None`` if not found.
-//    """
-//    if block.slot == slot:
-//        return block
-//    elif block.slot < slot:
-//        return None
-//    else:
-//        return get_ancestor(store, store.get_parent(block), slot)
-func BlockAncestor(targetBlock *pb.AttestationTarget, slot uint64, beaconDB *db.BeaconDB) ([]byte, error) {
-	if targetBlock.Slot == slot {
-		return targetBlock.BlockRoot[:], nil
-	}
-	if targetBlock.Slot < slot {
-		return nil, nil
-	}
-	parentRoot := bytesutil.ToBytes32(targetBlock.ParentRoot)
-	parent, err := beaconDB.Block(parentRoot)
-	if err != nil {
-		return nil, fmt.Errorf("could not get parent block: %v", err)
-	}
-	if parent == nil {
-		return nil, fmt.Errorf("parent block does not exist: %v", err)
-	}
-	newTarget := &pb.AttestationTarget{
-		Slot:       parent.Slot,
-		BlockRoot:  parentRoot[:],
-		ParentRoot: parent.ParentRootHash32,
-	}
-	return BlockAncestor(newTarget, slot, beaconDB)
-}
-
-// cachedAncestor retrieves the cached ancestor target from block ancestor cache,
-// if it's not there it looks up the block tree get it and cache it.
-func cachedAncestor(target *pb.AttestationTarget, height uint64, beaconDB *db.BeaconDB) ([]byte, error) {
-	// check if the ancestor block of from a given block height was cached.
-	cachedAncestorInfo, err := blkAncestorCache.AncestorBySlot(target.BlockRoot, height)
-	if err != nil {
-		return nil, nil
-	}
-	if cachedAncestorInfo != nil {
-		return cachedAncestorInfo.Target.BlockRoot, nil
-	}
-
-	ancestorRoot, err := BlockAncestor(target, height, beaconDB)
-	if err != nil {
-		return nil, err
-	}
-	ancestor, err := beaconDB.Block(bytesutil.ToBytes32(ancestorRoot))
-	if err != nil {
-		return nil, err
-	}
-	if ancestor == nil {
-		return nil, nil
-	}
-	ancestorTarget := &pb.AttestationTarget{
-		Slot:       ancestor.Slot,
-		BlockRoot:  ancestorRoot,
-		ParentRoot: ancestor.ParentRootHash32,
-	}
-	if err := blkAncestorCache.AddBlockAncestor(&cache.AncestorInfo{
-		Height: height,
-		Hash:   target.BlockRoot,
-		Target: ancestorTarget,
-	}); err != nil {
-		return nil, err
-	}
-	return ancestorRoot, nil
-}

beacon-chain/blockchain/fork_choice_reorg_test.go

@@ -1,225 +0,0 @@
-package blockchain
-
-import (
-	"context"
-	"fmt"
-	"testing"
-
-	"github.com/gogo/protobuf/proto"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
-	"github.com/prysmaticlabs/prysm/beacon-chain/internal"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
-	"github.com/prysmaticlabs/prysm/shared/hashutil"
-	"github.com/prysmaticlabs/prysm/shared/params"
-	"github.com/prysmaticlabs/prysm/shared/testutil"
-	logTest "github.com/sirupsen/logrus/hooks/test"
-)
-
-type mockAttestationHandler struct {
-	targets map[uint64]*pb.AttestationTarget
-}
-
-func (m *mockAttestationHandler) LatestAttestationTarget(beaconState *pb.BeaconState, idx uint64) (*pb.AttestationTarget, error) {
-	return m.targets[idx], nil
-}
-
-func (m *mockAttestationHandler) BatchUpdateLatestAttestation(ctx context.Context, atts []*pb.Attestation) error {
-	return nil
-}
-
-func TestApplyForkChoice_ChainSplitReorg(t *testing.T) {
-	hook := logTest.NewGlobal()
-	beaconDB := internal.SetupDB(t)
-	defer internal.TeardownDB(t, beaconDB)
-
-	ctx := context.Background()
-	deposits, _ := setupInitialDeposits(t, 100)
-	eth1Data := &pb.Eth1Data{
-		DepositRootHash32: []byte{},
-		BlockHash32:       []byte{},
-	}
-	justifiedState, err := state.GenesisBeaconState(deposits, 0, eth1Data)
-	if err != nil {
-		t.Fatalf("Can't generate genesis state: %v", err)
-	}
-
-	chainService := setupBeaconChain(t, beaconDB, nil)
-
-	// Construct a forked chain that looks as follows:
-	//    /------B1 ----B3 ----- B5 (current head)
-	// B0 --B2 -------------B4
-	blocks, roots := constructForkedChain(t, justifiedState)
-
-	// We then setup a canonical chain of the following blocks:
-	// B0->B1->B3->B5.
-	if err := chainService.beaconDB.SaveBlock(blocks[0]); err != nil {
-		t.Fatal(err)
-	}
-	justifiedState.LatestBlock = blocks[0]
-	if err := chainService.beaconDB.SaveJustifiedState(justifiedState); err != nil {
-		t.Fatal(err)
-	}
-	if err := chainService.beaconDB.SaveJustifiedBlock(blocks[0]); err != nil {
-		t.Fatal(err)
-	}
-	if err := chainService.beaconDB.UpdateChainHead(ctx, blocks[0], justifiedState); err != nil {
-		t.Fatal(err)
-	}
-	canonicalBlockIndices := []int{1, 3, 5}
-	postState := proto.Clone(justifiedState).(*pb.BeaconState)
-	for _, canonicalIndex := range canonicalBlockIndices {
-		postState, err = chainService.ApplyBlockStateTransition(ctx, blocks[canonicalIndex], postState)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if err := chainService.beaconDB.SaveBlock(blocks[canonicalIndex]); err != nil {
-			t.Fatal(err)
-		}
-		if err := chainService.beaconDB.UpdateChainHead(ctx, blocks[canonicalIndex], postState); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	chainHead, err := chainService.beaconDB.ChainHead()
-	if err != nil {
-		t.Fatal(err)
-	}
-	if chainHead.Slot != justifiedState.Slot+5 {
-		t.Errorf(
-			"Expected chain head with slot %d, received %d",
-			justifiedState.Slot+5-params.BeaconConfig().GenesisSlot,
-			chainHead.Slot-params.BeaconConfig().GenesisSlot,
-		)
-	}
-
-	// We then save forked blocks and their historical states (but do not update chain head).
-	// The fork is from B0->B2->B4.
-	forkedBlockIndices := []int{2, 4}
-	forkState := proto.Clone(justifiedState).(*pb.BeaconState)
-	for _, forkIndex := range forkedBlockIndices {
-		forkState, err = chainService.ApplyBlockStateTransition(ctx, blocks[forkIndex], forkState)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if err := chainService.beaconDB.SaveBlock(blocks[forkIndex]); err != nil {
-			t.Fatal(err)
-		}
-		if err := chainService.beaconDB.SaveHistoricalState(ctx, forkState); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// Give the block from the forked chain, B4, the most votes.
-	voteTargets := make(map[uint64]*pb.AttestationTarget)
-	voteTargets[0] = &pb.AttestationTarget{
-		Slot:       blocks[5].Slot,
-		BlockRoot:  roots[5][:],
-		ParentRoot: blocks[5].ParentRootHash32,
-	}
-	for i := 1; i < len(deposits); i++ {
-		voteTargets[uint64(i)] = &pb.AttestationTarget{
-			Slot:       blocks[4].Slot,
-			BlockRoot:  roots[4][:],
-			ParentRoot: blocks[4].ParentRootHash32,
-		}
-	}
-	attHandler := &mockAttestationHandler{
-		targets: voteTargets,
-	}
-	chainService.attsService = attHandler
-
-	block4State, err := chainService.beaconDB.HistoricalStateFromSlot(ctx, blocks[4].Slot)
-	if err != nil {
-		t.Fatal(err)
-	}
-	// Applying the fork choice rule should reorg to B4 successfully.
-	if err := chainService.ApplyForkChoiceRule(ctx, blocks[4], block4State); err != nil {
-		t.Fatal(err)
-	}
-
-	newHead, err := chainService.beaconDB.ChainHead()
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !proto.Equal(newHead, blocks[4]) {
-		t.Errorf(
-			"Expected chain head %v, received %v",
-			blocks[4],
-			newHead,
-		)
-	}
-	want := fmt.Sprintf(
-		"Reorg happened, last head at slot %d, new head block at slot %d",
-		blocks[5].Slot-params.BeaconConfig().GenesisSlot, blocks[4].Slot-params.BeaconConfig().GenesisSlot,
-	)
-	testutil.AssertLogsContain(t, hook, want)
-}
-
-func constructForkedChain(t *testing.T, beaconState *pb.BeaconState) ([]*pb.BeaconBlock, [][32]byte) {
-	// Construct the following chain:
-	//    /------B1 ----B3 ----- B5 (current head)
-	// B0 --B2 -------------B4
-	blocks := make([]*pb.BeaconBlock, 6)
-	roots := make([][32]byte, 6)
-	var err error
-	blocks[0] = &pb.BeaconBlock{
-		Slot:             beaconState.Slot,
-		ParentRootHash32: []byte{'A'},
-		Body:             &pb.BeaconBlockBody{},
-	}
-	roots[0], err = hashutil.HashBeaconBlock(blocks[0])
-	if err != nil {
-		t.Fatalf("Could not hash block: %v", err)
-	}
-
-	blocks[1] = &pb.BeaconBlock{
-		Slot:             beaconState.Slot + 2,
-		ParentRootHash32: roots[0][:],
-		Body:             &pb.BeaconBlockBody{},
-	}
-	roots[1], err = hashutil.HashBeaconBlock(blocks[1])
-	if err != nil {
-		t.Fatalf("Could not hash block: %v", err)
-	}
-
-	blocks[2] = &pb.BeaconBlock{
-		Slot:             beaconState.Slot + 1,
-		ParentRootHash32: roots[0][:],
-		Body:             &pb.BeaconBlockBody{},
-	}
-	roots[2], err = hashutil.HashBeaconBlock(blocks[2])
-	if err != nil {
-		t.Fatalf("Could not hash block: %v", err)
-	}
-
-	blocks[3] = &pb.BeaconBlock{
-		Slot:             beaconState.Slot + 3,
-		ParentRootHash32: roots[1][:],
-		Body:             &pb.BeaconBlockBody{},
-	}
-	roots[3], err = hashutil.HashBeaconBlock(blocks[3])
-	if err != nil {
-		t.Fatalf("Could not hash block: %v", err)
-	}
-
-	blocks[4] = &pb.BeaconBlock{
-		Slot:             beaconState.Slot + 4,
-		ParentRootHash32: roots[2][:],
-		Body:             &pb.BeaconBlockBody{},
-	}
-	roots[4], err = hashutil.HashBeaconBlock(blocks[4])
-	if err != nil {
-		t.Fatalf("Could not hash block: %v", err)
-	}
-
-	blocks[5] = &pb.BeaconBlock{
-		Slot:             beaconState.Slot + 5,
-		ParentRootHash32: roots[3][:],
-		Body:             &pb.BeaconBlockBody{},
-	}
-	roots[5], err = hashutil.HashBeaconBlock(blocks[5])
-	if err != nil {
-		t.Fatalf("Could not hash block: %v", err)
-	}
-	return blocks, roots
-}

beacon-chain/blockchain/head.go

@@ -0,0 +1,412 @@
+package blockchain
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
+	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/config/features"
+	"github.com/prysmaticlabs/prysm/config/params"
+	"github.com/prysmaticlabs/prysm/encoding/bytesutil"
+	ethpbv1 "github.com/prysmaticlabs/prysm/proto/eth/v1"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/block"
+	"github.com/prysmaticlabs/prysm/time/slots"
+	"github.com/sirupsen/logrus"
+	"go.opencensus.io/trace"
+)
+
+// This defines the current chain service's view of head.
+type head struct {
+	slot  types.Slot              // current head slot.
+	root  [32]byte                // current head root.
+	block block.SignedBeaconBlock // current head block.
+	state state.BeaconState       // current head state.
+}
+
+// Determined the head from the fork choice service and saves its new data
+// (head root, head block, and head state) to the local service cache.
+func (s *Service) updateHead(ctx context.Context, balances []uint64) error {
+	ctx, span := trace.StartSpan(ctx, "blockChain.updateHead")
+	defer span.End()
+
+	// To get the proper head update, a node first checks its best justified
+	// can become justified. This is designed to prevent bounce attack and
+	// ensure head gets its best justified info.
+	if s.bestJustifiedCheckpt.Epoch > s.justifiedCheckpt.Epoch {
+		s.justifiedCheckpt = s.bestJustifiedCheckpt
+		if err := s.cacheJustifiedStateBalances(ctx, bytesutil.ToBytes32(s.justifiedCheckpt.Root)); err != nil {
+			return err
+		}
+	}
+
+	// Get head from the fork choice service.
+	f := s.finalizedCheckpt
+	j := s.justifiedCheckpt
+	// To get head before the first justified epoch, the fork choice will start with genesis root
+	// instead of zero hashes.
+	headStartRoot := bytesutil.ToBytes32(j.Root)
+	if headStartRoot == params.BeaconConfig().ZeroHash {
+		headStartRoot = s.genesisRoot
+	}
+
+	// In order to process head, fork choice store requires justified info.
+	// If the fork choice store is missing justified block info, a node should
+	// re-initiate fork choice store using the latest justified info.
+	// This recovers a fatal condition and should not happen in run time.
+	if !s.cfg.ForkChoiceStore.HasNode(headStartRoot) {
+		jb, err := s.cfg.BeaconDB.Block(ctx, headStartRoot)
+		if err != nil {
+			return err
+		}
+		s.cfg.ForkChoiceStore = protoarray.New(j.Epoch, f.Epoch, bytesutil.ToBytes32(f.Root))
+		if err := s.insertBlockToForkChoiceStore(ctx, jb.Block(), headStartRoot, f, j); err != nil {
+			return err
+		}
+	}
+
+	headRoot, err := s.cfg.ForkChoiceStore.Head(ctx, j.Epoch, headStartRoot, balances, f.Epoch)
+	if err != nil {
+		return err
+	}
+
+	// Save head to the local service cache.
+	return s.saveHead(ctx, headRoot)
+}
+
+// This saves head info to the local service cache, it also saves the
+// new head root to the DB.
+func (s *Service) saveHead(ctx context.Context, headRoot [32]byte) error {
+	ctx, span := trace.StartSpan(ctx, "blockChain.saveHead")
+	defer span.End()
+
+	// Do nothing if head hasn't changed.
+	r, err := s.HeadRoot(ctx)
+	if err != nil {
+		return err
+	}
+	if headRoot == bytesutil.ToBytes32(r) {
+		return nil
+	}
+
+	// If the head state is not available, just return nil.
+	// There's nothing to cache
+	if !s.cfg.BeaconDB.HasStateSummary(ctx, headRoot) {
+		return nil
+	}
+
+	// Get the new head block from DB.
+	newHeadBlock, err := s.cfg.BeaconDB.Block(ctx, headRoot)
+	if err != nil {
+		return err
+	}
+	if newHeadBlock == nil || newHeadBlock.IsNil() || newHeadBlock.Block().IsNil() {
+		return errors.New("cannot save nil head block")
+	}
+
+	// Get the new head state from cached state or DB.
+	newHeadState, err := s.cfg.StateGen.StateByRoot(ctx, headRoot)
+	if err != nil {
+		return errors.Wrap(err, "could not retrieve head state in DB")
+	}
+	if newHeadState == nil || newHeadState.IsNil() {
+		return errors.New("cannot save nil head state")
+	}
+
+	// A chain re-org occurred, so we fire an event notifying the rest of the services.
+	headSlot := s.HeadSlot()
+	newHeadSlot := newHeadBlock.Block().Slot()
+	oldHeadRoot := s.headRoot()
+	oldStateRoot := s.headBlock().Block().StateRoot()
+	newStateRoot := newHeadBlock.Block().StateRoot()
+	if bytesutil.ToBytes32(newHeadBlock.Block().ParentRoot()) != bytesutil.ToBytes32(r) {
+		log.WithFields(logrus.Fields{
+			"newSlot": fmt.Sprintf("%d", newHeadSlot),
+			"oldSlot": fmt.Sprintf("%d", headSlot),
+		}).Debug("Chain reorg occurred")
+		absoluteSlotDifference := slots.AbsoluteValueSlotDifference(newHeadSlot, headSlot)
+		s.cfg.StateNotifier.StateFeed().Send(&feed.Event{
+			Type: statefeed.Reorg,
+			Data: &ethpbv1.EventChainReorg{
+				Slot:         newHeadSlot,
+				Depth:        absoluteSlotDifference,
+				OldHeadBlock: oldHeadRoot[:],
+				NewHeadBlock: headRoot[:],
+				OldHeadState: oldStateRoot,
+				NewHeadState: newStateRoot,
+				Epoch:        core.SlotToEpoch(newHeadSlot),
+			},
+		})
+
+		if err := s.saveOrphanedAtts(ctx, bytesutil.ToBytes32(r)); err != nil {
+			return err
+		}
+
+		reorgCount.Inc()
+	}
+
+	// Cache the new head info.
+	s.setHead(headRoot, newHeadBlock, newHeadState)
+
+	// Save the new head root to DB.
+	if err := s.cfg.BeaconDB.SaveHeadBlockRoot(ctx, headRoot); err != nil {
+		return errors.Wrap(err, "could not save head root in DB")
+	}
+
+	// Forward an event capturing a new chain head over a common event feed
+	// done in a goroutine to avoid blocking the critical runtime main routine.
+	go func() {
+		if err := s.notifyNewHeadEvent(newHeadSlot, newHeadState, newStateRoot, headRoot[:]); err != nil {
+			log.WithError(err).Error("Could not notify event feed of new chain head")
+		}
+	}()
+
+	return nil
+}
+
+// This gets called to update canonical root mapping. It does not save head block
+// root in DB. With the inception of initial-sync-cache-state flag, it uses finalized
+// check point as anchors to resume sync therefore head is no longer needed to be saved on per slot basis.
+func (s *Service) saveHeadNoDB(ctx context.Context, b block.SignedBeaconBlock, r [32]byte, hs state.BeaconState) error {
+	if err := helpers.VerifyNilBeaconBlock(b); err != nil {
+		return err
+	}
+	cachedHeadRoot, err := s.HeadRoot(ctx)
+	if err != nil {
+		return errors.Wrap(err, "could not get head root from cache")
+	}
+	if bytes.Equal(r[:], cachedHeadRoot) {
+		return nil
+	}
+
+	s.setHeadInitialSync(r, b.Copy(), hs)
+	return nil
+}
+
+// This sets head view object which is used to track the head slot, root, block and state.
+func (s *Service) setHead(root [32]byte, block block.SignedBeaconBlock, state state.BeaconState) {
+	s.headLock.Lock()
+	defer s.headLock.Unlock()
+
+	// This does a full copy of the block and state.
+	s.head = &head{
+		slot:  block.Block().Slot(),
+		root:  root,
+		block: block.Copy(),
+		state: state.Copy(),
+	}
+}
+
+// This sets head view object which is used to track the head slot, root, block and state. The method
+// assumes that state being passed into the method will not be modified by any other alternate
+// caller which holds the state's reference.
+func (s *Service) setHeadInitialSync(root [32]byte, block block.SignedBeaconBlock, state state.BeaconState) {
+	s.headLock.Lock()
+	defer s.headLock.Unlock()
+
+	// This does a full copy of the block only.
+	s.head = &head{
+		slot:  block.Block().Slot(),
+		root:  root,
+		block: block.Copy(),
+		state: state,
+	}
+}
+
+// This returns the head slot.
+// This is a lock free version.
+func (s *Service) headSlot() types.Slot {
+	return s.head.slot
+}
+
+// This returns the head root.
+// It does a full copy on head root for immutability.
+// This is a lock free version.
+func (s *Service) headRoot() [32]byte {
+	if s.head == nil {
+		return params.BeaconConfig().ZeroHash
+	}
+
+	return s.head.root
+}
+
+// This returns the head block.
+// It does a full copy on head block for immutability.
+// This is a lock free version.
+func (s *Service) headBlock() block.SignedBeaconBlock {
+	return s.head.block.Copy()
+}
+
+// This returns the head state.
+// It does a full copy on head state for immutability.
+// This is a lock free version.
+func (s *Service) headState(ctx context.Context) state.BeaconState {
+	ctx, span := trace.StartSpan(ctx, "blockChain.headState")
+	defer span.End()
+
+	return s.head.state.Copy()
+}
+
+// This returns the genesis validator root of the head state.
+// This is a lock free version.
+func (s *Service) headGenesisValidatorRoot() [32]byte {
+	return bytesutil.ToBytes32(s.head.state.GenesisValidatorRoot())
+}
+
+// This returns the validator referenced by the provided index in
+// the head state.
+// This is a lock free version.
+func (s *Service) headValidatorAtIndex(index types.ValidatorIndex) (state.ReadOnlyValidator, error) {
+	return s.head.state.ValidatorAtIndexReadOnly(index)
+}
+
+// Returns true if head state exists.
+// This is the lock free version.
+func (s *Service) hasHeadState() bool {
+	return s.head != nil && s.head.state != nil
+}
+
+// This caches justified state balances to be used for fork choice.
+func (s *Service) cacheJustifiedStateBalances(ctx context.Context, justifiedRoot [32]byte) error {
+	if err := s.cfg.BeaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
+		return err
+	}
+
+	s.clearInitSyncBlocks()
+
+	var justifiedState state.BeaconState
+	var err error
+	if justifiedRoot == s.genesisRoot {
+		justifiedState, err = s.cfg.BeaconDB.GenesisState(ctx)
+		if err != nil {
+			return err
+		}
+	} else {
+		justifiedState, err = s.cfg.StateGen.StateByRoot(ctx, justifiedRoot)
+		if err != nil {
+			return err
+		}
+	}
+	if justifiedState == nil || justifiedState.IsNil() {
+		return errors.New("justified state can't be nil")
+	}
+
+	epoch := core.CurrentEpoch(justifiedState)
+
+	justifiedBalances := make([]uint64, justifiedState.NumValidators())
+	if err := justifiedState.ReadFromEveryValidator(func(idx int, val state.ReadOnlyValidator) error {
+		if helpers.IsActiveValidatorUsingTrie(val, epoch) {
+			justifiedBalances[idx] = val.EffectiveBalance()
+		} else {
+			justifiedBalances[idx] = 0
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	s.justifiedBalancesLock.Lock()
+	defer s.justifiedBalancesLock.Unlock()
+	s.justifiedBalances = justifiedBalances
+	return nil
+}
+
+func (s *Service) getJustifiedBalances() []uint64 {
+	s.justifiedBalancesLock.RLock()
+	defer s.justifiedBalancesLock.RUnlock()
+	return s.justifiedBalances
+}
+
+// Notifies a common event feed of a new chain head event. Called right after a new
+// chain head is determined, set, and saved to disk.
+func (s *Service) notifyNewHeadEvent(
+	newHeadSlot types.Slot,
+	newHeadState state.BeaconState,
+	newHeadStateRoot,
+	newHeadRoot []byte,
+) error {
+	previousDutyDependentRoot := s.genesisRoot[:]
+	currentDutyDependentRoot := s.genesisRoot[:]
+
+	var previousDutyEpoch types.Epoch
+	currentDutyEpoch := core.SlotToEpoch(newHeadSlot)
+	if currentDutyEpoch > 0 {
+		previousDutyEpoch = currentDutyEpoch.Sub(1)
+	}
+	currentDutySlot, err := core.StartSlot(currentDutyEpoch)
+	if err != nil {
+		return errors.Wrap(err, "could not get duty slot")
+	}
+	previousDutySlot, err := core.StartSlot(previousDutyEpoch)
+	if err != nil {
+		return errors.Wrap(err, "could not get duty slot")
+	}
+	if currentDutySlot > 0 {
+		currentDutyDependentRoot, err = helpers.BlockRootAtSlot(newHeadState, currentDutySlot-1)
+		if err != nil {
+			return errors.Wrap(err, "could not get duty dependent root")
+		}
+	}
+	if previousDutySlot > 0 {
+		previousDutyDependentRoot, err = helpers.BlockRootAtSlot(newHeadState, previousDutySlot-1)
+		if err != nil {
+			return errors.Wrap(err, "could not get duty dependent root")
+		}
+	}
+	s.cfg.StateNotifier.StateFeed().Send(&feed.Event{
+		Type: statefeed.NewHead,
+		Data: &ethpbv1.EventHead{
+			Slot:                      newHeadSlot,
+			Block:                     newHeadRoot,
+			State:                     newHeadStateRoot,
+			EpochTransition:           core.IsEpochEnd(newHeadSlot),
+			PreviousDutyDependentRoot: previousDutyDependentRoot,
+			CurrentDutyDependentRoot:  currentDutyDependentRoot,
+		},
+	})
+	return nil
+}
+
+// This saves the attestations inside the beacon block with respect to root `orphanedRoot` back into the
+// attestation pool. It also filters out the attestations that is one epoch older as a
+// defense so invalid attestations don't flow into the attestation pool.
+func (s *Service) saveOrphanedAtts(ctx context.Context, orphanedRoot [32]byte) error {
+	if !features.Get().CorrectlyInsertOrphanedAtts {
+		return nil
+	}
+
+	orphanedBlk, err := s.cfg.BeaconDB.Block(ctx, orphanedRoot)
+	if err != nil {
+		return err
+	}
+
+	if orphanedBlk == nil || orphanedBlk.IsNil() {
+		return errors.New("orphaned block can't be nil")
+	}
+
+	for _, a := range orphanedBlk.Block().Body().Attestations() {
+		// Is the attestation one epoch older.
+		if a.Data.Slot+params.BeaconConfig().SlotsPerEpoch < s.CurrentSlot() {
+			continue
+		}
+		if helpers.IsAggregated(a) {
+			if err := s.cfg.AttPool.SaveAggregatedAttestation(a); err != nil {
+				return err
+			}
+		} else {
+			if err := s.cfg.AttPool.SaveUnaggregatedAttestation(a); err != nil {
+				return err
+			}
+		}
+		saveOrphanedAttCount.Inc()
+	}
+
+	return nil
+}

beacon-chain/blockchain/head_sync_committee_info.go

@@ -0,0 +1,174 @@
+package blockchain
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/async"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/altair"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/signing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/transition"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/config/params"
+	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
+)
+
+// Initialize the state cache for sync committees.
+var syncCommitteeHeadStateCache = cache.NewSyncCommitteeHeadState()
+
+// HeadSyncCommitteeFetcher is the interface that wraps the head sync committee related functions.
+// The head sync committee functions return callers sync committee indices and public keys with respect to current head state.
+type HeadSyncCommitteeFetcher interface {
+	HeadSyncCommitteeIndices(ctx context.Context, index types.ValidatorIndex, slot types.Slot) ([]types.CommitteeIndex, error)
+	HeadSyncCommitteePubKeys(ctx context.Context, slot types.Slot, committeeIndex types.CommitteeIndex) ([][]byte, error)
+}
+
+// HeadDomainFetcher is the interface that wraps the head sync domain related functions.
+// The head sync committee domain functions return callers domain data with respect to slot and head state.
+type HeadDomainFetcher interface {
+	HeadSyncCommitteeDomain(ctx context.Context, slot types.Slot) ([]byte, error)
+	HeadSyncSelectionProofDomain(ctx context.Context, slot types.Slot) ([]byte, error)
+	HeadSyncContributionProofDomain(ctx context.Context, slot types.Slot) ([]byte, error)
+}
+
+// HeadSyncCommitteeDomain returns the head sync committee domain using current head state advanced up to `slot`.
+func (s *Service) HeadSyncCommitteeDomain(ctx context.Context, slot types.Slot) ([]byte, error) {
+	return s.domainWithHeadState(ctx, slot, params.BeaconConfig().DomainSyncCommittee)
+}
+
+// HeadSyncSelectionProofDomain returns the head sync committee domain using current head state advanced up to `slot`.
+func (s *Service) HeadSyncSelectionProofDomain(ctx context.Context, slot types.Slot) ([]byte, error) {
+	return s.domainWithHeadState(ctx, slot, params.BeaconConfig().DomainSyncCommitteeSelectionProof)
+}
+
+// HeadSyncContributionProofDomain returns the head sync committee domain using current head state advanced up to `slot`.
+func (s *Service) HeadSyncContributionProofDomain(ctx context.Context, slot types.Slot) ([]byte, error) {
+	return s.domainWithHeadState(ctx, slot, params.BeaconConfig().DomainContributionAndProof)
+}
+
+// HeadSyncCommitteeIndices returns the sync committee index position using the head state. Input `slot` is taken in consideration
+// where validator's duty for `slot - 1` is used for block inclusion in `slot`. That means when a validator is at epoch boundary
+// across EPOCHS_PER_SYNC_COMMITTEE_PERIOD then the valiator will be considered using next period sync committee.
+//
+// Spec definition:
+// Being assigned to a sync committee for a given slot means that the validator produces and broadcasts signatures for slot - 1 for inclusion in slot.
+// This means that when assigned to an epoch sync committee signatures must be produced and broadcast for slots on range
+// [compute_start_slot_at_epoch(epoch) - 1, compute_start_slot_at_epoch(epoch) + SLOTS_PER_EPOCH - 1)
+// rather than for the range
+// [compute_start_slot_at_epoch(epoch), compute_start_slot_at_epoch(epoch) + SLOTS_PER_EPOCH)
+func (s *Service) HeadSyncCommitteeIndices(ctx context.Context, index types.ValidatorIndex, slot types.Slot) ([]types.CommitteeIndex, error) {
+	nextSlotEpoch := core.SlotToEpoch(slot + 1)
+	currentEpoch := core.SlotToEpoch(slot)
+
+	switch {
+	case core.SyncCommitteePeriod(nextSlotEpoch) == core.SyncCommitteePeriod(currentEpoch):
+		return s.headCurrentSyncCommitteeIndices(ctx, index, slot)
+	// At sync committee period boundary, validator should sample the next epoch sync committee.
+	case core.SyncCommitteePeriod(nextSlotEpoch) == core.SyncCommitteePeriod(currentEpoch)+1:
+		return s.headNextSyncCommitteeIndices(ctx, index, slot)
+	default:
+		// Impossible condition.
+		return nil, errors.New("could get calculate sync subcommittee based on the period")
+	}
+}
+
+// headCurrentSyncCommitteeIndices returns the input validator `index`'s position indices in the current sync committee with respect to `slot`.
+// Head state advanced up to `slot` is used for calculation.
+func (s *Service) headCurrentSyncCommitteeIndices(ctx context.Context, index types.ValidatorIndex, slot types.Slot) ([]types.CommitteeIndex, error) {
+	headState, err := s.getSyncCommitteeHeadState(ctx, slot)
+	if err != nil {
+		return nil, err
+	}
+	return helpers.CurrentPeriodSyncSubcommitteeIndices(headState, index)
+}
+
+// headNextSyncCommitteeIndices returns the input validator `index`'s position indices in the next sync committee with respect to `slot`.
+// Head state advanced up to `slot` is used for calculation.
+func (s *Service) headNextSyncCommitteeIndices(ctx context.Context, index types.ValidatorIndex, slot types.Slot) ([]types.CommitteeIndex, error) {
+	headState, err := s.getSyncCommitteeHeadState(ctx, slot)
+	if err != nil {
+		return nil, err
+	}
+	return helpers.NextPeriodSyncSubcommitteeIndices(headState, index)
+}
+
+// HeadSyncCommitteePubKeys returns the head sync committee public keys with respect to `slot` and subcommittee index `committeeIndex`.
+// Head state advanced up to `slot` is used for calculation.
+func (s *Service) HeadSyncCommitteePubKeys(ctx context.Context, slot types.Slot, committeeIndex types.CommitteeIndex) ([][]byte, error) {
+	headState, err := s.getSyncCommitteeHeadState(ctx, slot)
+	if err != nil {
+		return nil, err
+	}
+
+	nextSlotEpoch := core.SlotToEpoch(headState.Slot() + 1)
+	currEpoch := core.SlotToEpoch(headState.Slot())
+
+	var syncCommittee *ethpb.SyncCommittee
+	if currEpoch == nextSlotEpoch || core.SyncCommitteePeriod(currEpoch) == core.SyncCommitteePeriod(nextSlotEpoch) {
+		syncCommittee, err = headState.CurrentSyncCommittee()
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		syncCommittee, err = headState.NextSyncCommittee()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return altair.SyncSubCommitteePubkeys(syncCommittee, committeeIndex)
+}
+
+// returns calculated domain using input `domain` and `slot`.
+func (s *Service) domainWithHeadState(ctx context.Context, slot types.Slot, domain [4]byte) ([]byte, error) {
+	headState, err := s.getSyncCommitteeHeadState(ctx, slot)
+	if err != nil {
+		return nil, err
+	}
+	return signing.Domain(headState.Fork(), core.SlotToEpoch(headState.Slot()), domain, headState.GenesisValidatorRoot())
+}
+
+// returns the head state that is advanced up to `slot`. It utilizes the cache `syncCommitteeHeadState` by retrieving using `slot` as key.
+// For the cache miss, it processes head state up to slot and fill the cache with `slot` as key.
+func (s *Service) getSyncCommitteeHeadState(ctx context.Context, slot types.Slot) (state.BeaconState, error) {
+	var headState state.BeaconState
+	var err error
+	mLock := async.NewMultilock(fmt.Sprintf("%s-%d", "syncHeadState", slot))
+	mLock.Lock()
+	defer mLock.Unlock()
+
+	// If there's already a head state exists with the request slot, we don't need to process slots.
+	cachedState, err := syncCommitteeHeadStateCache.Get(slot)
+	switch {
+	case err == nil:
+		syncHeadStateHit.Inc()
+		headState = cachedState
+		return headState, nil
+	case errors.Is(err, cache.ErrNotFound):
+		headState, err = s.HeadState(ctx)
+		if err != nil {
+			return nil, err
+		}
+		if headState == nil || headState.IsNil() {
+			return nil, errors.New("nil state")
+		}
+		if slot > headState.Slot() {
+			headState, err = transition.ProcessSlots(ctx, headState, slot)
+			if err != nil {
+				return nil, err
+			}
+		}
+		syncHeadStateMiss.Inc()
+		err = syncCommitteeHeadStateCache.Put(slot, headState)
+		return headState, err
+	default:
+		// In the event, we encounter another error
+		// we return it.
+		return nil, err
+	}
+}

beacon-chain/blockchain/head_sync_committee_info_test.go

@@ -0,0 +1,176 @@
+package blockchain
+
+import (
+	"context"
+	"testing"
+
+	types "github.com/prysmaticlabs/eth2-types"
+	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/signing"
+	dbtest "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/config/params"
+	"github.com/prysmaticlabs/prysm/testing/require"
+	"github.com/prysmaticlabs/prysm/testing/util"
+)
+
+func TestService_headSyncCommitteeFetcher_Errors(t *testing.T) {
+	beaconDB := dbtest.SetupDB(t)
+	c := &Service{
+		cfg: &Config{
+			StateGen: stategen.New(beaconDB),
+		},
+	}
+	c.head = &head{}
+	_, err := c.headCurrentSyncCommitteeIndices(context.Background(), types.ValidatorIndex(0), types.Slot(0))
+	require.ErrorContains(t, "nil state", err)
+
+	_, err = c.headNextSyncCommitteeIndices(context.Background(), types.ValidatorIndex(0), types.Slot(0))
+	require.ErrorContains(t, "nil state", err)
+
+	_, err = c.HeadSyncCommitteePubKeys(context.Background(), types.Slot(0), types.CommitteeIndex(0))
+	require.ErrorContains(t, "nil state", err)
+}
+
+func TestService_HeadDomainFetcher_Errors(t *testing.T) {
+	beaconDB := dbtest.SetupDB(t)
+	c := &Service{
+		cfg: &Config{
+			StateGen: stategen.New(beaconDB),
+		},
+	}
+	c.head = &head{}
+	_, err := c.HeadSyncCommitteeDomain(context.Background(), types.Slot(0))
+	require.ErrorContains(t, "nil state", err)
+
+	_, err = c.HeadSyncSelectionProofDomain(context.Background(), types.Slot(0))
+	require.ErrorContains(t, "nil state", err)
+
+	_, err = c.HeadSyncSelectionProofDomain(context.Background(), types.Slot(0))
+	require.ErrorContains(t, "nil state", err)
+}
+
+func TestService_HeadSyncCommitteeIndices(t *testing.T) {
+	s, _ := util.DeterministicGenesisStateAltair(t, params.BeaconConfig().TargetCommitteeSize)
+	c := &Service{}
+	c.head = &head{state: s}
+
+	// Current period
+	slot := 2*uint64(params.BeaconConfig().EpochsPerSyncCommitteePeriod)*uint64(params.BeaconConfig().SlotsPerEpoch) + 1
+	a, err := c.HeadSyncCommitteeIndices(context.Background(), 0, types.Slot(slot))
+	require.NoError(t, err)
+
+	// Current period where slot-2 across EPOCHS_PER_SYNC_COMMITTEE_PERIOD
+	slot = 3*uint64(params.BeaconConfig().EpochsPerSyncCommitteePeriod)*uint64(params.BeaconConfig().SlotsPerEpoch) - 2
+	b, err := c.HeadSyncCommitteeIndices(context.Background(), 0, types.Slot(slot))
+	require.NoError(t, err)
+	require.DeepEqual(t, a, b)
+
+	// Next period where slot-1 across EPOCHS_PER_SYNC_COMMITTEE_PERIOD
+	slot = 3*uint64(params.BeaconConfig().EpochsPerSyncCommitteePeriod)*uint64(params.BeaconConfig().SlotsPerEpoch) - 1
+	b, err = c.HeadSyncCommitteeIndices(context.Background(), 0, types.Slot(slot))
+	require.NoError(t, err)
+	require.DeepNotEqual(t, a, b)
+}
+
+func TestService_headCurrentSyncCommitteeIndices(t *testing.T) {
+	s, _ := util.DeterministicGenesisStateAltair(t, params.BeaconConfig().TargetCommitteeSize)
+	c := &Service{}
+	c.head = &head{state: s}
+
+	// Process slot up to `EpochsPerSyncCommitteePeriod` so it can `ProcessSyncCommitteeUpdates`.
+	slot := uint64(params.BeaconConfig().EpochsPerSyncCommitteePeriod)*uint64(params.BeaconConfig().SlotsPerEpoch) + 1
+	indices, err := c.headCurrentSyncCommitteeIndices(context.Background(), 0, types.Slot(slot))
+	require.NoError(t, err)
+
+	// NextSyncCommittee becomes CurrentSyncCommittee so it should be empty by default.
+	require.Equal(t, 0, len(indices))
+}
+
+func TestService_headNextSyncCommitteeIndices(t *testing.T) {
+	s, _ := util.DeterministicGenesisStateAltair(t, params.BeaconConfig().TargetCommitteeSize)
+	c := &Service{}
+	c.head = &head{state: s}
+
+	// Process slot up to `EpochsPerSyncCommitteePeriod` so it can `ProcessSyncCommitteeUpdates`.
+	slot := uint64(params.BeaconConfig().EpochsPerSyncCommitteePeriod)*uint64(params.BeaconConfig().SlotsPerEpoch) + 1
+	indices, err := c.headNextSyncCommitteeIndices(context.Background(), 0, types.Slot(slot))
+	require.NoError(t, err)
+
+	// NextSyncCommittee should be be empty after `ProcessSyncCommitteeUpdates`. Validator should get indices.
+	require.NotEqual(t, 0, len(indices))
+}
+
+func TestService_HeadSyncCommitteePubKeys(t *testing.T) {
+	s, _ := util.DeterministicGenesisStateAltair(t, params.BeaconConfig().TargetCommitteeSize)
+	c := &Service{}
+	c.head = &head{state: s}
+
+	// Process slot up to 2 * `EpochsPerSyncCommitteePeriod` so it can run `ProcessSyncCommitteeUpdates` twice.
+	slot := uint64(2*params.BeaconConfig().EpochsPerSyncCommitteePeriod)*uint64(params.BeaconConfig().SlotsPerEpoch) + 1
+	pubkeys, err := c.HeadSyncCommitteePubKeys(context.Background(), types.Slot(slot), 0)
+	require.NoError(t, err)
+
+	// Any subcommittee should match the subcommittee size.
+	subCommitteeSize := params.BeaconConfig().SyncCommitteeSize / params.BeaconConfig().SyncCommitteeSubnetCount
+	require.Equal(t, int(subCommitteeSize), len(pubkeys))
+}
+
+func TestService_HeadSyncCommitteeDomain(t *testing.T) {
+	s, _ := util.DeterministicGenesisStateAltair(t, params.BeaconConfig().TargetCommitteeSize)
+	c := &Service{}
+	c.head = &head{state: s}
+
+	wanted, err := signing.Domain(s.Fork(), core.SlotToEpoch(s.Slot()), params.BeaconConfig().DomainSyncCommittee, s.GenesisValidatorRoot())
+	require.NoError(t, err)
+
+	d, err := c.HeadSyncCommitteeDomain(context.Background(), 0)
+	require.NoError(t, err)
+
+	require.DeepEqual(t, wanted, d)
+}
+
+func TestService_HeadSyncContributionProofDomain(t *testing.T) {
+	s, _ := util.DeterministicGenesisStateAltair(t, params.BeaconConfig().TargetCommitteeSize)
+	c := &Service{}
+	c.head = &head{state: s}
+
+	wanted, err := signing.Domain(s.Fork(), core.SlotToEpoch(s.Slot()), params.BeaconConfig().DomainContributionAndProof, s.GenesisValidatorRoot())
+	require.NoError(t, err)
+
+	d, err := c.HeadSyncContributionProofDomain(context.Background(), 0)
+	require.NoError(t, err)
+
+	require.DeepEqual(t, wanted, d)
+}
+
+func TestService_HeadSyncSelectionProofDomain(t *testing.T) {
+	s, _ := util.DeterministicGenesisStateAltair(t, params.BeaconConfig().TargetCommitteeSize)
+	c := &Service{}
+	c.head = &head{state: s}
+
+	wanted, err := signing.Domain(s.Fork(), core.SlotToEpoch(s.Slot()), params.BeaconConfig().DomainSyncCommitteeSelectionProof, s.GenesisValidatorRoot())
+	require.NoError(t, err)
+
+	d, err := c.HeadSyncSelectionProofDomain(context.Background(), 0)
+	require.NoError(t, err)
+
+	require.DeepEqual(t, wanted, d)
+}
+
+func TestSyncCommitteeHeadStateCache_RoundTrip(t *testing.T) {
+	c := syncCommitteeHeadStateCache
+	t.Cleanup(func() {
+		syncCommitteeHeadStateCache = cache.NewSyncCommitteeHeadState()
+	})
+	beaconState, _ := util.DeterministicGenesisStateAltair(t, 100)
+	require.NoError(t, beaconState.SetSlot(100))
+	cachedState, err := c.Get(101)
+	require.ErrorContains(t, cache.ErrNotFound.Error(), err)
+	require.Equal(t, nil, cachedState)
+	require.NoError(t, c.Put(101, beaconState))
+	cachedState, err = c.Get(101)
+	require.NoError(t, err)
+	require.DeepEqual(t, beaconState, cachedState)
+}

beacon-chain/blockchain/head_test.go

@@ -0,0 +1,267 @@
+package blockchain
+
+import (
+	"bytes"
+	"context"
+	"testing"
+	"time"
+
+	types "github.com/prysmaticlabs/eth2-types"
+	mock "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core"
+	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
+	"github.com/prysmaticlabs/prysm/config/features"
+	"github.com/prysmaticlabs/prysm/config/params"
+	ethpbv1 "github.com/prysmaticlabs/prysm/proto/eth/v1"
+	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
+	"github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1/wrapper"
+	"github.com/prysmaticlabs/prysm/testing/assert"
+	"github.com/prysmaticlabs/prysm/testing/require"
+	"github.com/prysmaticlabs/prysm/testing/util"
+	logTest "github.com/sirupsen/logrus/hooks/test"
+)
+
+func TestSaveHead_Same(t *testing.T) {
+	beaconDB := testDB.SetupDB(t)
+	service := setupBeaconChain(t, beaconDB)
+
+	r := [32]byte{'A'}
+	service.head = &head{slot: 0, root: r}
+
+	require.NoError(t, service.saveHead(context.Background(), r))
+	assert.Equal(t, types.Slot(0), service.headSlot(), "Head did not stay the same")
+	assert.Equal(t, r, service.headRoot(), "Head did not stay the same")
+}
+
+func TestSaveHead_Different(t *testing.T) {
+	ctx := context.Background()
+	beaconDB := testDB.SetupDB(t)
+	service := setupBeaconChain(t, beaconDB)
+
+	util.NewBeaconBlock()
+	oldBlock := wrapper.WrappedPhase0SignedBeaconBlock(
+		util.NewBeaconBlock(),
+	)
+	require.NoError(t, service.cfg.BeaconDB.SaveBlock(context.Background(), oldBlock))
+	oldRoot, err := oldBlock.Block().HashTreeRoot()
+	require.NoError(t, err)
+	service.head = &head{
+		slot:  0,
+		root:  oldRoot,
+		block: oldBlock,
+	}
+
+	newHeadSignedBlock := util.NewBeaconBlock()
+	newHeadSignedBlock.Block.Slot = 1
+	newHeadBlock := newHeadSignedBlock.Block
+
+	require.NoError(t, service.cfg.BeaconDB.SaveBlock(context.Background(), wrapper.WrappedPhase0SignedBeaconBlock(newHeadSignedBlock)))
+	newRoot, err := newHeadBlock.HashTreeRoot()
+	require.NoError(t, err)
+	headState, err := util.NewBeaconState()
+	require.NoError(t, err)
+	require.NoError(t, headState.SetSlot(1))
+	require.NoError(t, service.cfg.BeaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Slot: 1, Root: newRoot[:]}))
+	require.NoError(t, service.cfg.BeaconDB.SaveState(context.Background(), headState, newRoot))
+	require.NoError(t, service.saveHead(context.Background(), newRoot))
+
+	assert.Equal(t, types.Slot(1), service.HeadSlot(), "Head did not change")
+
+	cachedRoot, err := service.HeadRoot(context.Background())
+	require.NoError(t, err)
+	assert.DeepEqual(t, cachedRoot, newRoot[:], "Head did not change")
+	assert.DeepEqual(t, newHeadSignedBlock, service.headBlock().Proto(), "Head did not change")
+	assert.DeepSSZEqual(t, headState.CloneInnerState(), service.headState(ctx).CloneInnerState(), "Head did not change")
+}
+
+func TestSaveHead_Different_Reorg(t *testing.T) {
+	ctx := context.Background()
+	hook := logTest.NewGlobal()
+	beaconDB := testDB.SetupDB(t)
+	service := setupBeaconChain(t, beaconDB)
+
+	oldBlock := wrapper.WrappedPhase0SignedBeaconBlock(
+		util.NewBeaconBlock(),
+	)
+	require.NoError(t, service.cfg.BeaconDB.SaveBlock(context.Background(), oldBlock))
+	oldRoot, err := oldBlock.Block().HashTreeRoot()
+	require.NoError(t, err)
+	service.head = &head{
+		slot:  0,
+		root:  oldRoot,
+		block: oldBlock,
+	}
+
+	reorgChainParent := [32]byte{'B'}
+	newHeadSignedBlock := util.NewBeaconBlock()
+	newHeadSignedBlock.Block.Slot = 1
+	newHeadSignedBlock.Block.ParentRoot = reorgChainParent[:]
+	newHeadBlock := newHeadSignedBlock.Block
+
+	require.NoError(t, service.cfg.BeaconDB.SaveBlock(context.Background(), wrapper.WrappedPhase0SignedBeaconBlock(newHeadSignedBlock)))
+	newRoot, err := newHeadBlock.HashTreeRoot()
+	require.NoError(t, err)
+	headState, err := util.NewBeaconState()
+	require.NoError(t, err)
+	require.NoError(t, headState.SetSlot(1))
+	require.NoError(t, service.cfg.BeaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Slot: 1, Root: newRoot[:]}))
+	require.NoError(t, service.cfg.BeaconDB.SaveState(context.Background(), headState, newRoot))
+	require.NoError(t, service.saveHead(context.Background(), newRoot))
+
+	assert.Equal(t, types.Slot(1), service.HeadSlot(), "Head did not change")
+
+	cachedRoot, err := service.HeadRoot(context.Background())
+	require.NoError(t, err)
+	if !bytes.Equal(cachedRoot, newRoot[:]) {
+		t.Error("Head did not change")
+	}
+	assert.DeepEqual(t, newHeadSignedBlock, service.headBlock().Proto(), "Head did not change")
+	assert.DeepSSZEqual(t, headState.CloneInnerState(), service.headState(ctx).CloneInnerState(), "Head did not change")
+	require.LogsContain(t, hook, "Chain reorg occurred")
+}
+
+func TestCacheJustifiedStateBalances_CanCache(t *testing.T) {
+	beaconDB := testDB.SetupDB(t)
+	service := setupBeaconChain(t, beaconDB)
+
+	state, _ := util.DeterministicGenesisState(t, 100)
+	r := [32]byte{'a'}
+	require.NoError(t, service.cfg.BeaconDB.SaveStateSummary(context.Background(), &ethpb.StateSummary{Root: r[:]}))
+	require.NoError(t, service.cfg.BeaconDB.SaveState(context.Background(), state, r))
+	require.NoError(t, service.cacheJustifiedStateBalances(context.Background(), r))
+	require.DeepEqual(t, service.getJustifiedBalances(), state.Balances(), "Incorrect justified balances")
+}
+
+func TestUpdateHead_MissingJustifiedRoot(t *testing.T) {
+	beaconDB := testDB.SetupDB(t)
+	service := setupBeaconChain(t, beaconDB)
+
+	b := util.NewBeaconBlock()
+	require.NoError(t, service.cfg.BeaconDB.SaveBlock(context.Background(), wrapper.WrappedPhase0SignedBeaconBlock(b)))
+	r, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+
+	service.justifiedCheckpt = &ethpb.Checkpoint{Root: r[:]}
+	service.finalizedCheckpt = &ethpb.Checkpoint{}
+	service.bestJustifiedCheckpt = &ethpb.Checkpoint{}
+
+	require.NoError(t, service.updateHead(context.Background(), []uint64{}))
+}
+
+func Test_notifyNewHeadEvent(t *testing.T) {
+	t.Run("genesis_state_root", func(t *testing.T) {
+		bState, _ := util.DeterministicGenesisState(t, 10)
+		notifier := &mock.MockStateNotifier{RecordEvents: true}
+		srv := &Service{
+			cfg: &Config{
+				StateNotifier: notifier,
+			},
+			genesisRoot: [32]byte{1},
+		}
+		newHeadStateRoot := [32]byte{2}
+		newHeadRoot := [32]byte{3}
+		err := srv.notifyNewHeadEvent(1, bState, newHeadStateRoot[:], newHeadRoot[:])
+		require.NoError(t, err)
+		events := notifier.ReceivedEvents()
+		require.Equal(t, 1, len(events))
+
+		eventHead, ok := events[0].Data.(*ethpbv1.EventHead)
+		require.Equal(t, true, ok)
+		wanted := &ethpbv1.EventHead{
+			Slot:                      1,
+			Block:                     newHeadRoot[:],
+			State:                     newHeadStateRoot[:],
+			EpochTransition:           false,
+			PreviousDutyDependentRoot: srv.genesisRoot[:],
+			CurrentDutyDependentRoot:  srv.genesisRoot[:],
+		}
+		require.DeepSSZEqual(t, wanted, eventHead)
+	})
+	t.Run("non_genesis_values", func(t *testing.T) {
+		bState, _ := util.DeterministicGenesisState(t, 10)
+		notifier := &mock.MockStateNotifier{RecordEvents: true}
+		genesisRoot := [32]byte{1}
+		srv := &Service{
+			cfg: &Config{
+				StateNotifier: notifier,
+			},
+			genesisRoot: genesisRoot,
+		}
+		epoch1Start, err := core.StartSlot(1)
+		require.NoError(t, err)
+		epoch2Start, err := core.StartSlot(1)
+		require.NoError(t, err)
+		require.NoError(t, bState.SetSlot(epoch1Start))
+
+		newHeadStateRoot := [32]byte{2}
+		newHeadRoot := [32]byte{3}
+		err = srv.notifyNewHeadEvent(epoch2Start, bState, newHeadStateRoot[:], newHeadRoot[:])
+		require.NoError(t, err)
+		events := notifier.ReceivedEvents()
+		require.Equal(t, 1, len(events))
+
+		eventHead, ok := events[0].Data.(*ethpbv1.EventHead)
+		require.Equal(t, true, ok)
+		wanted := &ethpbv1.EventHead{
+			Slot:                      epoch2Start,
+			Block:                     newHeadRoot[:],
+			State:                     newHeadStateRoot[:],
+			EpochTransition:           false,
+			PreviousDutyDependentRoot: genesisRoot[:],
+			CurrentDutyDependentRoot:  make([]byte, 32),
+		}
+		require.DeepSSZEqual(t, wanted, eventHead)
+	})
+}
+
+func TestSaveOrphanedAtts(t *testing.T) {
+	resetCfg := features.InitWithReset(&features.Flags{
+		CorrectlyInsertOrphanedAtts: true,
+	})
+	defer resetCfg()
+
+	genesis, keys := util.DeterministicGenesisState(t, 64)
+	b, err := util.GenerateFullBlock(genesis, keys, util.DefaultBlockGenConfig(), 1)
+	assert.NoError(t, err)
+	r, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	beaconDB := testDB.SetupDB(t)
+	service := setupBeaconChain(t, beaconDB)
+	service.genesisTime = time.Now()
+
+	require.NoError(t, service.cfg.BeaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b)))
+	require.NoError(t, service.saveOrphanedAtts(ctx, r))
+
+	require.Equal(t, len(b.Block.Body.Attestations), service.cfg.AttPool.AggregatedAttestationCount())
+	savedAtts := service.cfg.AttPool.AggregatedAttestations()
+	atts := b.Block.Body.Attestations
+	require.DeepSSZEqual(t, atts, savedAtts)
+}
+
+func TestSaveOrphanedAtts_CanFilter(t *testing.T) {
+	resetCfg := features.InitWithReset(&features.Flags{
+		CorrectlyInsertOrphanedAtts: true,
+	})
+	defer resetCfg()
+
+	genesis, keys := util.DeterministicGenesisState(t, 64)
+	b, err := util.GenerateFullBlock(genesis, keys, util.DefaultBlockGenConfig(), 1)
+	assert.NoError(t, err)
+	r, err := b.Block.HashTreeRoot()
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	beaconDB := testDB.SetupDB(t)
+	service := setupBeaconChain(t, beaconDB)
+	service.genesisTime = time.Now().Add(time.Duration(-1*int64(params.BeaconConfig().SlotsPerEpoch+1)*int64(params.BeaconConfig().SecondsPerSlot)) * time.Second)
+
+	require.NoError(t, service.cfg.BeaconDB.SaveBlock(ctx, wrapper.WrappedPhase0SignedBeaconBlock(b)))
+	require.NoError(t, service.saveOrphanedAtts(ctx, r))
+
+	require.Equal(t, 0, service.cfg.AttPool.AggregatedAttestationCount())
+	savedAtts := service.cfg.AttPool.AggregatedAttestations()
+	atts := b.Block.Body.Attestations
+	require.DeepNotSSZEqual(t, atts, savedAtts)
+}