Bazel: cleanup .bazelrc file (#12059)

* Reorganize bazelrc and intro new flags

* move cross compilation toolchain into its own bazelrc

* Restore build_tests_only

.bazelrc

@@ -1,9 +1,9 @@
-# Print warnings for tests with inappropriate test size or timeout.
+# Import bazelrc presets
-test --test_verbose_timeout_warnings
+import %workspace%/build/bazelrc/convenience.bazelrc
-
+import %workspace%/build/bazelrc/correctness.bazelrc
-# Only build test targets when running bazel test //...
+import %workspace%/build/bazelrc/cross.bazelrc
-test --build_tests_only
+import %workspace%/build/bazelrc/debug.bazelrc
-test --test_output=errors
+import %workspace%/build/bazelrc/performance.bazelrc
 
 # E2E run with debug gotag
 test:e2e --define gotags=debug
@@ -11,21 +11,9 @@ test:e2e --define gotags=debug
 # Clearly indicate that coverage is enabled to disable certain nogo checks.
 coverage --define=coverage_enabled=1
 
-# Fix for rules_docker. See: https://github.com/bazelbuild/rules_docker/issues/842
-build --host_force_python=PY2
-run --host_force_python=PY2
-
-# Networking is blocked for tests by default, add "requires-network" tag to your test if networking
-# is required within the sandbox. Network sandboxing only works on linux.
-build --sandbox_default_allow_network=false
-
 # Stamp binaries with git information
 build --workspace_status_command=./hack/workspace_status.sh
 
-# Prevent PATH changes from rebuilding when switching from IDE to command line.
-build --incompatible_strict_action_env
-run --incompatible_strict_action_env
-
 build --define blst_disabled=false
 run --define blst_disabled=false
 
@@ -68,42 +56,6 @@ build:cgo_symbolizer --define=USE_CGO_SYMBOLIZER=true
 build:cgo_symbolizer -c dbg
 build:cgo_symbolizer --define=gotags=cgosymbolizer_enabled
 
-# multi-arch cross-compiling toolchain configs:
------------------------------------------------
-build:cross --crosstool_top=@prysm_toolchains//:multiarch_toolchain
-build:cross --host_platform=@io_bazel_rules_go//go/toolchain:linux_amd64
-build:cross --host_crosstool_top=@prysm_toolchains//:hostonly_toolchain
-
-# linux_amd64 config for cross compiler toolchain, not strictly necessary since host/exec env is amd64
-build:linux_amd64 --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64_cgo
-
-# osx_amd64 config for cross compiler toolchain
-build:osx_amd64 --config=cross
-build:osx_amd64 --platforms=@io_bazel_rules_go//go/toolchain:darwin_amd64_cgo
-build:osx_amd64 --compiler=osxcross
-
-# osx_arm64 config for cross compiler toolchain
-build:osx_arm64 --config=cross
-build:osx_arm64 --platforms=@io_bazel_rules_go//go/toolchain:darwin_arm64_cgo
-build:osx_arm64 --compiler=osxcross
-build:osx_arm64 --cpu=aarch64
-
-# windows
-build:windows_amd64 --config=cross
-build:windows_amd64  --platforms=@io_bazel_rules_go//go/toolchain:windows_amd64_cgo
-build:windows_amd64 --compiler=mingw-w64
-
-# linux_arm64 conifg for cross compiler toolchain
-build:linux_arm64 --config=cross
-build:linux_arm64 --platforms=@io_bazel_rules_go//go/toolchain:linux_arm64_cgo
-build:linux_arm64 --copt=-funsafe-math-optimizations
-build:linux_arm64 --copt=-ftree-vectorize
-build:linux_arm64 --copt=-fomit-frame-pointer
-build:linux_arm64 --cpu=aarch64
-build:linux_arm64 --compiler=clang
-build:linux_arm64 --copt=-march=armv8-a
-
-
 # toolchain build debug configs
 #------------------------------
 build:debug --sandbox_debug
@@ -111,123 +63,5 @@ build:debug --toolchain_resolution_debug
 build:debug --verbose_failures
 build:debug -s
 
-# windows debug
-build:windows_amd64_debug --config=windows_amd64
-build:windows_amd64_debug --config=debug
-
-# osx_amd64 debug config
-build:osx_amd64_debug --config=debug
-build:osx_amd64_debug --config=osx_amd64
-
-# osx_arm64 debug config
-build:osx_arm64_debug --config=debug
-build:osx_arm64_debug --config=osx_arm64
-
-# linux_arm64_debug
-build:linux_arm64_debug --config=linux_arm64
-build:linux_arm64_debug --config=debug
-
-# linux_amd64_debug
-build:linux_amd64_debug --config=linux_amd64
-build:linux_amd64_debug --config=debug
-
-
-# Docker Sandbox Configs
-#-----------------------
-# Note all docker sandbox configs must run from a linux x86_64 host
-# build:docker-sandbox --experimental_docker_image=gcr.io/prysmaticlabs/rbe-worker:latest
-build:docker-sandbox --spawn_strategy=docker --strategy=Javac=docker --genrule_strategy=docker
-build:docker-sandbox --define=EXECUTOR=remote
-build:docker-sandbox --experimental_docker_verbose
-build:docker-sandbox --experimental_enable_docker_sandbox
-build:docker-sandbox --crosstool_top=@rbe_ubuntu_clang//cc:toolchain 
-build:docker-sandbox --host_javabase=@rbe_ubuntu_clang//java:jdk 
-build:docker-sandbox --javabase=@rbe_ubuntu_clang//java:jdk 
-build:docker-sandbox --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 
-build:docker-sandbox --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 
-build:docker-sandbox --extra_execution_platforms=@rbe_ubuntu_clang//config:platform 
-build:docker-sandbox --host_platform=@rbe_ubuntu_clang//config:platform 
-build:docker-sandbox --platforms=@rbe_ubuntu_clang//config:platform
-build:docker-sandbox --extra_toolchains=@prysm_toolchains//:cc-toolchain-multiarch
-
-# windows_amd64 docker sandbox build config
-build:windows_amd64_docker --config=docker-sandbox --config=windows_amd64
-build:windows_amd64_docker_debug --config=windows_amd64_docker --config=debug
-
-# osx_amd64 docker sandbox build config
-build:osx_amd64_docker --config=docker-sandbox --config=osx_amd64
-build:osx_amd64_docker_debug --config=osx_amd64_docker --config=debug
-
-# osx_arm64 docker sandbox build config
-build:osx_arm64_docker --config=docker-sandbox --config=osx_arm64
-build:osx_arm64_docker_debug --config=osx_arm64_docker --config=debug
-
-# linux_arm64 docker sandbox build config
-build:linux_arm64_docker --config=docker-sandbox --config=linux_arm64
-build:linux_arm64_docker_debug --config=linux_arm64_docker --config=debug
-
-# linux_amd64 docker sandbox build config
-build:linux_amd64_docker --config=docker-sandbox --config=linux_amd64
-build:linux_amd64_docker_debug --config=linux_amd64_docker --config=debug
-
-
-# Remote Build Execution
-#-----------------------
-# Originally from https://github.com/bazelbuild/bazel-toolchains/blob/master/bazelrc/bazel-2.0.0.bazelrc
-#
-# Depending on how many machines are in the remote execution instance, setting
-# this higher can make builds faster by allowing more jobs to run in parallel.
-# Setting it too high can result in jobs that timeout, however, while waiting
-# for a remote machine to execute them.
-build:remote --jobs=50
-
-# Set several flags related to specifying the platform, toolchain and java
-# properties.
-# These flags should only be used as is for the rbe-ubuntu16-04 container
-# and need to be adapted to work with other toolchain containers.
-build:remote --host_javabase=@rbe_ubuntu_clang//java:jdk
-build:remote --javabase=@rbe_ubuntu_clang//java:jdk
-build:remote --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
-build:remote --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8
-build:remote --crosstool_top=@rbe_ubuntu_clang//cc:toolchain
-build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
-# Platform flags:
-# The toolchain container used for execution is defined in the target indicated
-# by "extra_execution_platforms", "host_platform" and "platforms".
-# More about platforms: https://docs.bazel.build/versions/master/platforms.html
-build:remote --extra_toolchains=@rbe_ubuntu_clang//config:cc-toolchain
-build:remote --extra_execution_platforms=@rbe_ubuntu_clang//config:platform
-build:remote --host_platform=@rbe_ubuntu_clang//config:platform
-build:remote --platforms=@rbe_ubuntu_clang//config:platform
-
-# Starting with Bazel 0.27.0 strategies do not need to be explicitly
-# defined. See https://github.com/bazelbuild/bazel/issues/7480
-build:remote --define=EXECUTOR=remote
-
-# Enable remote execution so actions are performed on the remote systems.
-# build:remote --remote_executor=grpcs://remotebuildexecution.googleapis.com
-
-# Enforce stricter environment rules, which eliminates some non-hermetic
-# behavior and therefore improves both the remote cache hit rate and the
-# correctness and repeatability of the build.
-build:remote --incompatible_strict_action_env=true
-
-# Set a higher timeout value, just in case.
-build:remote --remote_timeout=3600
-
-# Enable authentication. This will pick up application default credentials by
-# default. You can use --google_credentials=some_file.json to use a service
-# account credential instead.
-# build:remote --google_default_credentials=true
-
-# Enable build without the bytes
-# See: https://github.com/bazelbuild/bazel/issues/6862
-build:remote --experimental_remote_download_outputs=toplevel --experimental_inmemory_jdeps_files --experimental_inmemory_dotd_files
-
-build:remote --remote_local_fallback
-
-# Ignore GoStdLib with remote caching
-build --modify_execution_info='GoStdlib.*=+no-remote-cache'
-
 # Set bazel gotag
 build --define gotags=bazel

build/bazelrc/convenience.bazelrc

@@ -0,0 +1,33 @@
+# Attempt to build & test every target whose prerequisites were successfully built.
+# Docs: https://bazel.build/docs/user-manual#keep-going
+build --keep_going
+test  --keep_going
+
+# Output test errors to stderr so users don't have to `cat` or open test failure log files when test
+# fail. This makes the log noiser in exchange for reducing the time-to-feedback on test failures for
+# users.
+# Docs: https://bazel.build/docs/user-manual#test-output
+test --test_output=errors
+
+# Show the output files created by builds that requested more than one target. This helps users
+# locate the build outputs in more cases
+# Docs: https://bazel.build/docs/user-manual#show-result
+build --show_result=20
+
+# Bazel picks up host-OS-specific config lines from bazelrc files. For example, if the host OS is
+# Linux and you run bazel build, Bazel picks up lines starting with build:linux. Supported OS
+# identifiers are `linux`, `macos`, `windows`, `freebsd`, and `openbsd`. Enabling this flag is
+# equivalent to using `--config=linux` on Linux, `--config=windows` on Windows, etc.
+# Docs: https://bazel.build/reference/command-line-reference#flag--enable_platform_specific_config
+common --enable_platform_specific_config
+
+# Output a heap dump if an OOM is thrown during a Bazel invocation
+# (including OOMs due to `--experimental_oom_more_eagerly_threshold`).
+# The dump will be written to `<output_base>/<invocation_id>.heapdump.hprof`.
+# You may need to configure CI to capture this artifact and upload for later use.
+# Docs: https://bazel.build/reference/command-line-reference#flag--heap_dump_on_oom
+build --heap_dump_on_oom
+
+# Change the behavior of bazel test to not bother building targets which aren't dependencies of the
+# tests. Matches some developer expectations.
+test --build_tests_only

build/bazelrc/correctness.bazelrc

@@ -0,0 +1,63 @@
+# Do not upload locally executed action results to the remote cache.
+# This should be the default for local builds so local builds cannot poison the remote cache.
+# It should be flipped to `--remote_upload_local_results` on CI
+# by using `--bazelrc=.aspect/bazelrc/ci.bazelrc`.
+# Docs: https://bazel.build/reference/command-line-reference#flag--remote_upload_local_results
+build --noremote_upload_local_results
+
+# Don't allow network access for build actions in the sandbox.
+# Ensures that you don't accidentally make non-hermetic actions/tests which depend on remote
+# services.
+# Developers should tag targets with `tags=["requires-network"]` to opt-out of the enforcement.
+# Docs: https://bazel.build/reference/command-line-reference#flag--sandbox_default_allow_network
+build --sandbox_default_allow_network=false
+test --sandbox_default_allow_network=false
+
+# Warn if a test's timeout is significantly longer than the test's actual execution time.
+# Bazel's default for test_timeout is medium (5 min), but most tests should instead be short (1 min).
+# While a test's timeout should be set such that it is not flaky, a test that has a highly
+# over-generous timeout can hide real problems that crop up unexpectedly.
+# For instance, a test that normally executes in a minute or two should not have a timeout of
+# ETERNAL or LONG as these are much, much too generous.
+# Docs: https://bazel.build/docs/user-manual#test-verbose-timeout-warnings
+test --test_verbose_timeout_warnings
+
+# Allow the Bazel server to check directory sources for changes. Ensures that the Bazel server
+# notices when a directory changes, if you have a directory listed in the srcs of some target.
+# Recommended when using
+# [copy_directory](https://github.com/aspect-build/bazel-lib/blob/main/docs/copy_directory.md) and
+# [rules_js](https://github.com/aspect-build/rules_js) since npm package are source directories
+# inputs to copy_directory actions.
+# Docs: https://bazel.build/reference/command-line-reference#flag--host_jvm_args
+startup --host_jvm_args=-DBAZEL_TRACK_SOURCE_DIRECTORIES=1
+
+# Allow exclusive tests to run in the sandbox. Fixes a bug where Bazel doesn't enable sandboxing for
+# tests with `tags=["exclusive"]`.
+# Docs: https://bazel.build/reference/command-line-reference#flag--incompatible_exclusive_test_sandboxed
+test --incompatible_exclusive_test_sandboxed
+
+# Use a static value for `PATH` and does not inherit `LD_LIBRARY_PATH`. Doesn't let environment
+# variables like `PATH` sneak into the build, which can cause massive cache misses when they change.
+# Use `--action_env=ENV_VARIABLE` if you want to inherit specific environment variables from the
+# client, but note that doing so can prevent cross-user caching if a shared cache is used.
+# Docs: https://bazel.build/reference/command-line-reference#flag--incompatible_strict_action_env
+build --incompatible_strict_action_env
+
+# Propagate tags from a target declaration to the actions' execution requirements.
+# Ensures that tags applied in your BUILD file, like `tags=["no-remote"]`
+# get propagated to actions created by the rule.
+# Without this option, you rely on rules authors to manually check the tags you passed
+# and apply relevant ones to the actions they create.
+# See https://github.com/bazelbuild/bazel/issues/8830 for details.
+# Docs: https://bazel.build/reference/command-line-reference#flag--experimental_allow_tags_propagation
+build --experimental_allow_tags_propagation
+fetch --experimental_allow_tags_propagation
+query --experimental_allow_tags_propagation
+
+# Do not automatically create `__init__.py` files in the runfiles of Python targets. Fixes the wrong
+# default that comes from Google's internal monorepo by using `__init__.py` to delimit a Python
+# package. Precisely, when a `py_binary` or `py_test` target has `legacy_create_init` set to `auto (the
+# default), it is treated as false if and only if this flag is set. See
+# https://github.com/bazelbuild/bazel/issues/10076.
+# Docs: https://bazel.build/reference/command-line-reference#flag--incompatible_default_to_explicit_init_py
+build --incompatible_default_to_explicit_init_py

build/bazelrc/cross.bazelrc

@@ -0,0 +1,74 @@
+# multi-arch cross-compiling toolchain configs
+-----------------------------------------------
+
+build:cross --crosstool_top=@prysm_toolchains//:multiarch_toolchain
+build:cross --host_platform=@io_bazel_rules_go//go/toolchain:linux_amd64
+build:cross --host_crosstool_top=@prysm_toolchains//:hostonly_toolchain
+
+# linux_amd64 config for cross compiler toolchain, not strictly necessary since host/exec env is amd64
+build:linux_amd64 --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64_cgo
+
+# osx_amd64 config for cross compiler toolchain
+build:osx_amd64 --config=cross
+build:osx_amd64 --platforms=@io_bazel_rules_go//go/toolchain:darwin_amd64_cgo
+build:osx_amd64 --compiler=osxcross
+
+# osx_arm64 config for cross compiler toolchain
+build:osx_arm64 --config=cross
+build:osx_arm64 --platforms=@io_bazel_rules_go//go/toolchain:darwin_arm64_cgo
+build:osx_arm64 --compiler=osxcross
+build:osx_arm64 --cpu=aarch64
+
+# windows
+build:windows_amd64 --config=cross
+build:windows_amd64  --platforms=@io_bazel_rules_go//go/toolchain:windows_amd64_cgo
+build:windows_amd64 --compiler=mingw-w64
+
+# linux_arm64 conifg for cross compiler toolchain
+build:linux_arm64 --config=cross
+build:linux_arm64 --platforms=@io_bazel_rules_go//go/toolchain:linux_arm64_cgo
+build:linux_arm64 --copt=-funsafe-math-optimizations
+build:linux_arm64 --copt=-ftree-vectorize
+build:linux_arm64 --copt=-fomit-frame-pointer
+build:linux_arm64 --cpu=aarch64
+build:linux_arm64 --compiler=clang
+build:linux_arm64 --copt=-march=armv8-a
+
+
+# Docker Sandbox Configs
+#-----------------------
+# Note all docker sandbox configs must run from a linux x86_64 host
+# build:docker-sandbox --experimental_docker_image=gcr.io/prysmaticlabs/rbe-worker:latest
+build:docker-sandbox --spawn_strategy=docker --strategy=Javac=docker --genrule_strategy=docker
+build:docker-sandbox --define=EXECUTOR=remote
+build:docker-sandbox --experimental_docker_verbose
+build:docker-sandbox --experimental_enable_docker_sandbox
+build:docker-sandbox --crosstool_top=@rbe_ubuntu_clang//cc:toolchain 
+build:docker-sandbox --host_javabase=@rbe_ubuntu_clang//java:jdk 
+build:docker-sandbox --javabase=@rbe_ubuntu_clang//java:jdk 
+build:docker-sandbox --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 
+build:docker-sandbox --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 
+build:docker-sandbox --extra_execution_platforms=@rbe_ubuntu_clang//config:platform 
+build:docker-sandbox --host_platform=@rbe_ubuntu_clang//config:platform 
+build:docker-sandbox --platforms=@rbe_ubuntu_clang//config:platform
+build:docker-sandbox --extra_toolchains=@prysm_toolchains//:cc-toolchain-multiarch
+
+# windows_amd64 docker sandbox build config
+build:windows_amd64_docker --config=docker-sandbox --config=windows_amd64
+build:windows_amd64_docker_debug --config=windows_amd64_docker --config=debug
+
+# osx_amd64 docker sandbox build config
+build:osx_amd64_docker --config=docker-sandbox --config=osx_amd64
+build:osx_amd64_docker_debug --config=osx_amd64_docker --config=debug
+
+# osx_arm64 docker sandbox build config
+build:osx_arm64_docker --config=docker-sandbox --config=osx_arm64
+build:osx_arm64_docker_debug --config=osx_arm64_docker --config=debug
+
+# linux_arm64 docker sandbox build config
+build:linux_arm64_docker --config=docker-sandbox --config=linux_arm64
+build:linux_arm64_docker_debug --config=linux_arm64_docker --config=debug
+
+# linux_amd64 docker sandbox build config
+build:linux_amd64_docker --config=docker-sandbox --config=linux_amd64
+build:linux_amd64_docker_debug --config=linux_amd64_docker --config=debug

build/bazelrc/debug.bazelrc

@@ -0,0 +1,19 @@
+############################################################
+# Use `bazel test --config=debug` to enable these settings #
+############################################################
+
+# Stream stdout/stderr output from each test in real-time.
+# Docs: https://bazel.build/docs/user-manual#test-output
+test:debug --test_output=streamed
+
+# Run one test at a time.
+# Docs: https://bazel.build/reference/command-line-reference#flag--test_strategy
+test:debug --test_strategy=exclusive
+
+# Prevent long running tests from timing out.
+# Docs: https://bazel.build/docs/user-manual#test-timeout
+test:debug --test_timeout=9999
+
+# Always run tests even if they have cached results.
+# Docs: https://bazel.build/docs/user-manual#cache-test-results
+test:debug --nocache_test_results

build/bazelrc/performance.bazelrc

@@ -0,0 +1,44 @@
+# Merkle tree calculations will be memoized to improve the remote cache hit checking speed. The
+# memory foot print of the cache is controlled by `--experimental_remote_merkle_tree_cache_size`.
+# Docs: https://bazel.build/reference/command-line-reference#flag--experimental_remote_merkle_tree_cache
+build --experimental_remote_merkle_tree_cache
+query --experimental_remote_merkle_tree_cache
+
+# The number of Merkle trees to memoize to improve the remote cache hit checking speed. Even though
+# the cache is automatically pruned according to Java's handling of soft references, out-of-memory
+# errors can occur if set too high. If set to 0 the cache size is unlimited. Optimal value varies
+# depending on project's size.
+# Docs: https://bazel.build/reference/command-line-reference#flag--experimental_remote_merkle_tree_cache_size
+build --experimental_remote_merkle_tree_cache_size=1000
+query --experimental_remote_merkle_tree_cache_size=1000
+
+# Speed up all builds by not checking if output files have been modified. Lets you make changes to
+# the output tree without triggering a build for local debugging. For example, you can modify
+# [rules_js](https://github.com/aspect-build/rules_js) 3rd party npm packages in the output tree
+# when local debugging.
+# Docs: https://github.com/bazelbuild/bazel/blob/1af61b21df99edc2fc66939cdf14449c2661f873/src/main/java/com/google/devtools/build/lib/pkgcache/PackageOptions.java#L185
+build --noexperimental_check_output_files
+fetch --noexperimental_check_output_files
+query --noexperimental_check_output_files
+
+# Don't apply `--noremote_upload_local_results` and `--noremote_accept_cached` to the disk cache.
+# If you have both `--noremote_upload_local_results` and `--disk_cache`, then this fixes a bug where
+# Bazel doesn't write to the local disk cache as it treats as a remote cache.
+# Docs: https://bazel.build/reference/command-line-reference#flag--incompatible_remote_results_ignore_disk
+build --incompatible_remote_results_ignore_disk
+
+# Directories used by sandboxed non-worker execution may be reused to avoid unnecessary setup costs.
+# Save time on Sandbox creation and deletion when many of the same kind of action run during the
+# build.
+# No longer experimental in Bazel 6: https://github.com/bazelbuild/bazel/commit/c1a95501a5611878e5cc43a3cc531f2b9e47835b
+# Docs: https://bazel.build/reference/command-line-reference#flag--reuse_sandbox_directories
+build --experimental_reuse_sandbox_directories
+
+# Some actions are always IO-intensive but require little compute. It's wasteful to put the output
+# in the remote cache, it just saturates the network and fills the cache storage causing earlier
+# evictions. It's also not worth sending them for remote execution.
+# For actions like PackageTar and GoStdLib actions it's usually faster to just re-run the work locally every time.
+# You'll have to look at an execution log to figure out what other action mnemonics you care about.
+# In some cases you may need to patch rulesets to add a mnemonic to actions that don't have one.
+# https://bazel.build/reference/command-line-reference#flag--modify_execution_info
+build --modify_execution_info='PackageTar=+no-remote,GoStdlib.*=+no-remote'