Add fulu spec tests for sanity and reward (#15285)

* Add fulu spec tests for sanity and reward

* Delete setting electra fork epoch to 0

Co-authored-by: Preston Van Loon <pvanloon@offchainlabs.com>

---------

Co-authored-by: Preston Van Loon <pvanloon@offchainlabs.com>

changelog/tt_melon.md

@@ -0,0 +1,3 @@
+### Added
+
+- sanity and rewards spec tests for fulu

testing/spectest/mainnet/fulu/rewards/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@prysm//tools/go:def.bzl", "go_test")
+
+go_test(
+    name = "go_default_test",
+    size = "small",
+    srcs = ["rewards_test.go"],
+    data = glob(["*.yaml"]) + [
+        "@consensus_spec_tests_mainnet//:test_data",
+    ],
+    tags = ["spectest"],
+    deps = ["//testing/spectest/shared/fulu/rewards:go_default_library"],
+)

testing/spectest/mainnet/fulu/rewards/rewards_test.go

@@ -0,0 +1,11 @@
+package rewards
+
+import (
+	"testing"
+
+	"github.com/OffchainLabs/prysm/v6/testing/spectest/shared/fulu/rewards"
+)
+
+func TestMainnet_Fulu_Rewards(t *testing.T) {
+	rewards.RunPrecomputeRewardsAndPenaltiesTests(t, "mainnet")
+}

testing/spectest/mainnet/fulu/sanity/BUILD.bazel

@@ -0,0 +1,16 @@
+load("@prysm//tools/go:def.bzl", "go_test")
+
+go_test(
+    name = "go_default_test",
+    size = "medium",
+    timeout = "short",
+    srcs = [
+        "blocks_test.go",
+        "slots_test.go",
+    ],
+    data = glob(["*.yaml"]) + [
+        "@consensus_spec_tests_mainnet//:test_data",
+    ],
+    tags = ["spectest"],
+    deps = ["//testing/spectest/shared/fulu/sanity:go_default_library"],
+)

testing/spectest/mainnet/fulu/sanity/blocks_test.go

@@ -0,0 +1,11 @@
+package sanity
+
+import (
+	"testing"
+
+	"github.com/OffchainLabs/prysm/v6/testing/spectest/shared/fulu/sanity"
+)
+
+func TestMainnet_Fulu_Sanity_Blocks(t *testing.T) {
+	sanity.RunBlockProcessingTest(t, "mainnet", "sanity/blocks/pyspec_tests")
+}

testing/spectest/mainnet/fulu/sanity/slots_test.go

@@ -0,0 +1,11 @@
+package sanity
+
+import (
+	"testing"
+
+	"github.com/OffchainLabs/prysm/v6/testing/spectest/shared/fulu/sanity"
+)
+
+func TestMainnet_Fulu_Sanity_Slots(t *testing.T) {
+	sanity.RunSlotProcessingTests(t, "mainnet")
+}

testing/spectest/minimal/fulu/rewards/BUILD.bazel

@@ -0,0 +1,13 @@
+load("@prysm//tools/go:def.bzl", "go_test")
+
+go_test(
+    name = "go_default_test",
+    size = "small",
+    srcs = ["rewards_test.go"],
+    data = glob(["*.yaml"]) + [
+        "@consensus_spec_tests_minimal//:test_data",
+    ],
+    eth_network = "minimal",
+    tags = ["spectest"],
+    deps = ["//testing/spectest/shared/fulu/rewards:go_default_library"],
+)

testing/spectest/minimal/fulu/rewards/rewards_test.go

@@ -0,0 +1,11 @@
+package rewards
+
+import (
+	"testing"
+
+	"github.com/OffchainLabs/prysm/v6/testing/spectest/shared/fulu/rewards"
+)
+
+func TestMinimal_Fulu_Rewards(t *testing.T) {
+	rewards.RunPrecomputeRewardsAndPenaltiesTests(t, "minimal")
+}

testing/spectest/minimal/fulu/sanity/BUILD.bazel

@@ -0,0 +1,17 @@
+load("@prysm//tools/go:def.bzl", "go_test")
+
+go_test(
+    name = "go_default_test",
+    size = "medium",
+    timeout = "short",
+    srcs = [
+        "blocks_test.go",
+        "slots_test.go",
+    ],
+    data = glob(["*.yaml"]) + [
+        "@consensus_spec_tests_minimal//:test_data",
+    ],
+    eth_network = "minimal",
+    tags = ["spectest"],
+    deps = ["//testing/spectest/shared/fulu/sanity:go_default_library"],
+)

testing/spectest/minimal/fulu/sanity/blocks_test.go

@@ -0,0 +1,11 @@
+package sanity
+
+import (
+	"testing"
+
+	"github.com/OffchainLabs/prysm/v6/testing/spectest/shared/fulu/sanity"
+)
+
+func TestMinimal_Fulu_Sanity_Blocks(t *testing.T) {
+	sanity.RunBlockProcessingTest(t, "minimal", "sanity/blocks/pyspec_tests")
+}

testing/spectest/minimal/fulu/sanity/slots_test.go

@@ -0,0 +1,11 @@
+package sanity
+
+import (
+	"testing"
+
+	"github.com/OffchainLabs/prysm/v6/testing/spectest/shared/fulu/sanity"
+)
+
+func TestMinimal_Fulu_Sanity_Slots(t *testing.T) {
+	sanity.RunSlotProcessingTests(t, "minimal")
+}

testing/spectest/shared/fulu/rewards/BUILD.bazel

@@ -0,0 +1,19 @@
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    testonly = True,
+    srcs = ["rewards_penalties.go"],
+    importpath = "github.com/OffchainLabs/prysm/v6/testing/spectest/shared/fulu/rewards",
+    visibility = ["//testing/spectest:__subpackages__"],
+    deps = [
+        "//beacon-chain/core/electra:go_default_library",
+        "//beacon-chain/core/helpers:go_default_library",
+        "//beacon-chain/state/state-native:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
+        "//testing/require:go_default_library",
+        "//testing/spectest/utils:go_default_library",
+        "//testing/util:go_default_library",
+        "@com_github_golang_snappy//:go_default_library",
+    ],
+)

testing/spectest/shared/fulu/rewards/rewards_penalties.go

@@ -0,0 +1,134 @@
+package rewards
+
+import (
+	"context"
+	"encoding/binary"
+	"fmt"
+	"path"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/OffchainLabs/prysm/v6/beacon-chain/core/electra"
+	"github.com/OffchainLabs/prysm/v6/beacon-chain/core/helpers"
+	state_native "github.com/OffchainLabs/prysm/v6/beacon-chain/state/state-native"
+	ethpb "github.com/OffchainLabs/prysm/v6/proto/prysm/v1alpha1"
+	"github.com/OffchainLabs/prysm/v6/testing/require"
+	"github.com/OffchainLabs/prysm/v6/testing/spectest/utils"
+	"github.com/OffchainLabs/prysm/v6/testing/util"
+	"github.com/golang/snappy"
+)
+
+// Delta contains list of rewards and penalties.
+type Delta struct {
+	Rewards   []uint64 `json:"rewards"`
+	Penalties []uint64 `json:"penalties"`
+}
+
+// unmarshalSSZ deserializes specs data into a simple aggregating container.
+func (d *Delta) unmarshalSSZ(buf []byte) error {
+	offset1 := binary.LittleEndian.Uint32(buf[:4])
+	offset2 := binary.LittleEndian.Uint32(buf[4:8])
+
+	for i := uint32(0); i < offset2-offset1; i += 8 {
+		d.Rewards = append(d.Rewards, binary.LittleEndian.Uint64(buf[offset1+i:offset1+i+8]))
+		d.Penalties = append(d.Penalties, binary.LittleEndian.Uint64(buf[offset2+i:offset2+i+8]))
+	}
+	return nil
+}
+
+// RunPrecomputeRewardsAndPenaltiesTests executes "rewards/{basic, leak, random}" tests.
+func RunPrecomputeRewardsAndPenaltiesTests(t *testing.T, config string) {
+	require.NoError(t, utils.SetConfig(t, config))
+
+	_, testsFolderPath := utils.TestFolders(t, config, "fulu", "rewards")
+	testTypes, err := util.BazelListDirectories(testsFolderPath)
+	require.NoError(t, err)
+
+	for _, testType := range testTypes {
+		testFolders, testsFolderPath := utils.TestFolders(t, config, "fulu", fmt.Sprintf("rewards/%s/pyspec_tests", testType))
+		for _, folder := range testFolders {
+			helpers.ClearCache()
+			t.Run(fmt.Sprintf("%v/%v", testType, folder.Name()), func(t *testing.T) {
+				folderPath := path.Join(testsFolderPath, folder.Name())
+				runPrecomputeRewardsAndPenaltiesTest(t, folderPath)
+			})
+		}
+	}
+}
+
+func runPrecomputeRewardsAndPenaltiesTest(t *testing.T, testFolderPath string) {
+	ctx := context.Background()
+	preBeaconStateFile, err := util.BazelFileBytes(path.Join(testFolderPath, "pre.ssz_snappy"))
+	require.NoError(t, err)
+	preBeaconStateSSZ, err := snappy.Decode(nil /* dst */, preBeaconStateFile)
+	require.NoError(t, err, "Failed to decompress")
+	preBeaconStateBase := &ethpb.BeaconStateElectra{}
+	require.NoError(t, preBeaconStateBase.UnmarshalSSZ(preBeaconStateSSZ), "Failed to unmarshal")
+	preBeaconState, err := state_native.InitializeFromProtoElectra(preBeaconStateBase)
+	require.NoError(t, err)
+
+	vp, bp, err := electra.InitializePrecomputeValidators(ctx, preBeaconState)
+	require.NoError(t, err)
+
+	vp, bp, err = electra.ProcessEpochParticipation(ctx, preBeaconState, bp, vp)
+	require.NoError(t, err)
+
+	activeBal, targetPrevious, targetCurrent, err := preBeaconState.UnrealizedCheckpointBalances()
+	require.NoError(t, err)
+	require.Equal(t, bp.ActiveCurrentEpoch, activeBal)
+	require.Equal(t, bp.CurrentEpochTargetAttested, targetCurrent)
+	require.Equal(t, bp.PrevEpochTargetAttested, targetPrevious)
+
+	deltas, err := electra.AttestationsDelta(preBeaconState, bp, vp)
+	require.NoError(t, err)
+
+	rewards := make([]uint64, len(deltas))
+	penalties := make([]uint64, len(deltas))
+	for i, d := range deltas {
+		rewards[i] = d.HeadReward + d.SourceReward + d.TargetReward
+		penalties[i] = d.SourcePenalty + d.TargetPenalty + d.InactivityPenalty
+	}
+
+	totalSpecTestRewards := make([]uint64, len(rewards))
+	totalSpecTestPenalties := make([]uint64, len(penalties))
+
+	// Fetch delta files. i.e. source_deltas.ssz_snappy, etc.
+	testfiles, err := util.BazelListFiles(path.Join(testFolderPath))
+	require.NoError(t, err)
+	deltaFiles := make([]string, 0, len(testfiles))
+	for _, tf := range testfiles {
+		if strings.Contains(tf, "deltas") {
+			deltaFiles = append(deltaFiles, tf)
+		}
+	}
+	if len(deltaFiles) == 0 {
+		t.Fatal("No delta files")
+	}
+
+	for _, dFile := range deltaFiles {
+		sourceFile, err := util.BazelFileBytes(path.Join(testFolderPath, dFile))
+		require.NoError(t, err)
+		sourceSSZ, err := snappy.Decode(nil /* dst */, sourceFile)
+		require.NoError(t, err, "Failed to decompress")
+		d := &Delta{}
+		require.NoError(t, d.unmarshalSSZ(sourceSSZ), "Failed to unmarshal")
+		for i, reward := range d.Rewards {
+			totalSpecTestRewards[i] += reward
+		}
+		for i, penalty := range d.Penalties {
+			totalSpecTestPenalties[i] += penalty
+		}
+	}
+
+	if !reflect.DeepEqual(rewards, totalSpecTestRewards) {
+		t.Error("Rewards don't match")
+		t.Log(rewards)
+		t.Log(totalSpecTestRewards)
+	}
+	if !reflect.DeepEqual(penalties, totalSpecTestPenalties) {
+		t.Error("Penalties don't match")
+		t.Log(penalties)
+		t.Log(totalSpecTestPenalties)
+	}
+}

testing/spectest/shared/fulu/sanity/BUILD.bazel

@@ -0,0 +1,29 @@
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    testonly = True,
+    srcs = [
+        "block_processing.go",
+        "block_processing.yaml.go",
+        "slot_processing.go",
+    ],
+    importpath = "github.com/OffchainLabs/prysm/v6/testing/spectest/shared/fulu/sanity",
+    visibility = ["//testing/spectest:__subpackages__"],
+    deps = [
+        "//beacon-chain/core/helpers:go_default_library",
+        "//beacon-chain/core/transition:go_default_library",
+        "//beacon-chain/state:go_default_library",
+        "//beacon-chain/state/state-native:go_default_library",
+        "//consensus-types/blocks:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
+        "//testing/require:go_default_library",
+        "//testing/spectest/utils:go_default_library",
+        "//testing/util:go_default_library",
+        "@com_github_golang_snappy//:go_default_library",
+        "@com_github_google_go_cmp//cmp:go_default_library",
+        "@io_bazel_rules_go//go/tools/bazel:go_default_library",
+        "@org_golang_google_protobuf//proto:go_default_library",
+        "@org_golang_google_protobuf//testing/protocmp:go_default_library",
+    ],
+)

testing/spectest/shared/fulu/sanity/block_processing.go

@@ -0,0 +1,113 @@
+package sanity
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/OffchainLabs/prysm/v6/beacon-chain/core/helpers"
+	"github.com/OffchainLabs/prysm/v6/beacon-chain/core/transition"
+	"github.com/OffchainLabs/prysm/v6/beacon-chain/state"
+	state_native "github.com/OffchainLabs/prysm/v6/beacon-chain/state/state-native"
+	"github.com/OffchainLabs/prysm/v6/consensus-types/blocks"
+	ethpb "github.com/OffchainLabs/prysm/v6/proto/prysm/v1alpha1"
+	"github.com/OffchainLabs/prysm/v6/testing/require"
+	"github.com/OffchainLabs/prysm/v6/testing/spectest/utils"
+	"github.com/OffchainLabs/prysm/v6/testing/util"
+	"github.com/bazelbuild/rules_go/go/tools/bazel"
+	"github.com/golang/snappy"
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/testing/protocmp"
+)
+
+func init() {
+	transition.SkipSlotCache.Disable()
+}
+
+// RunBlockProcessingTest executes "sanity/blocks" tests.
+func RunBlockProcessingTest(t *testing.T, config, folderPath string) {
+	require.NoError(t, utils.SetConfig(t, config))
+
+	testFolders, testsFolderPath := utils.TestFolders(t, config, "fulu", folderPath)
+	for _, folder := range testFolders {
+		t.Run(folder.Name(), func(t *testing.T) {
+			helpers.ClearCache()
+			preBeaconStateFile, err := util.BazelFileBytes(testsFolderPath, folder.Name(), "pre.ssz_snappy")
+			require.NoError(t, err)
+			preBeaconStateSSZ, err := snappy.Decode(nil /* dst */, preBeaconStateFile)
+			require.NoError(t, err, "Failed to decompress")
+			beaconStateBase := &ethpb.BeaconStateElectra{}
+			require.NoError(t, beaconStateBase.UnmarshalSSZ(preBeaconStateSSZ), "Failed to unmarshal")
+			beaconState, err := state_native.InitializeFromProtoElectra(beaconStateBase)
+			require.NoError(t, err)
+
+			file, err := util.BazelFileBytes(testsFolderPath, folder.Name(), "meta.yaml")
+			require.NoError(t, err)
+
+			metaYaml := &Config{}
+			require.NoError(t, utils.UnmarshalYaml(file, metaYaml), "Failed to Unmarshal")
+
+			var transitionError error
+			var processedState state.BeaconState
+			var ok bool
+			for i := 0; i < metaYaml.BlocksCount; i++ {
+				filename := fmt.Sprintf("blocks_%d.ssz_snappy", i)
+				blockFile, err := util.BazelFileBytes(testsFolderPath, folder.Name(), filename)
+				require.NoError(t, err)
+				blockSSZ, err := snappy.Decode(nil /* dst */, blockFile)
+				require.NoError(t, err, "Failed to decompress")
+				block := &ethpb.SignedBeaconBlockElectra{}
+				require.NoError(t, block.UnmarshalSSZ(blockSSZ), "Failed to unmarshal")
+				wsb, err := blocks.NewSignedBeaconBlock(block)
+				require.NoError(t, err)
+				processedState, transitionError = transition.ExecuteStateTransition(context.Background(), beaconState, wsb)
+				if transitionError != nil {
+					break
+				}
+				beaconState, ok = processedState.(*state_native.BeaconState)
+				require.Equal(t, true, ok)
+			}
+
+			// If the post.ssz is not present, it means the test should fail on our end.
+			postSSZFilepath, readError := bazel.Runfile(path.Join(testsFolderPath, folder.Name(), "post.ssz_snappy"))
+			postSSZExists := true
+			if readError != nil && strings.Contains(readError.Error(), "could not locate file") {
+				postSSZExists = false
+			} else if readError != nil {
+				t.Fatal(readError)
+			}
+
+			if postSSZExists {
+				if transitionError != nil {
+					t.Errorf("Unexpected error: %v", transitionError)
+				}
+
+				postBeaconStateFile, err := os.ReadFile(postSSZFilepath) // #nosec G304
+				require.NoError(t, err)
+				postBeaconStateSSZ, err := snappy.Decode(nil /* dst */, postBeaconStateFile)
+				require.NoError(t, err, "Failed to decompress")
+
+				postBeaconState := &ethpb.BeaconStateElectra{}
+				require.NoError(t, postBeaconState.UnmarshalSSZ(postBeaconStateSSZ), "Failed to unmarshal")
+				pbState, err := state_native.ProtobufBeaconStateElectra(beaconState.ToProtoUnsafe())
+				require.NoError(t, err)
+				if !proto.Equal(pbState, postBeaconState) {
+					t.Log(cmp.Diff(postBeaconState, pbState, protocmp.Transform()))
+					t.Fatal("Post state does not match expected")
+				}
+			} else {
+				// Note: This doesn't test anything worthwhile. It essentially tests
+				// that *any* error has occurred, not any specific error.
+				if transitionError == nil {
+					t.Fatal("Did not fail when expected")
+				}
+				t.Logf("Expected failure; failure reason = %v", transitionError)
+				return
+			}
+		})
+	}
+}

testing/spectest/shared/fulu/sanity/block_processing.yaml.go

@@ -0,0 +1,6 @@
+package sanity
+
+// Config --
+type Config struct {
+	BlocksCount int `json:"blocks_count"`
+}

testing/spectest/shared/fulu/sanity/slot_processing.go

@@ -0,0 +1,61 @@
+package sanity
+
+import (
+	"context"
+	"strconv"
+	"testing"
+
+	"github.com/OffchainLabs/prysm/v6/beacon-chain/core/transition"
+	state_native "github.com/OffchainLabs/prysm/v6/beacon-chain/state/state-native"
+	ethpb "github.com/OffchainLabs/prysm/v6/proto/prysm/v1alpha1"
+	"github.com/OffchainLabs/prysm/v6/testing/require"
+	"github.com/OffchainLabs/prysm/v6/testing/spectest/utils"
+	"github.com/OffchainLabs/prysm/v6/testing/util"
+	"github.com/golang/snappy"
+	"google.golang.org/protobuf/proto"
+)
+
+func init() {
+	transition.SkipSlotCache.Disable()
+}
+
+// RunSlotProcessingTests executes "sanity/slots" tests.
+func RunSlotProcessingTests(t *testing.T, config string) {
+	require.NoError(t, utils.SetConfig(t, config))
+
+	testFolders, testsFolderPath := utils.TestFolders(t, config, "fulu", "sanity/slots/pyspec_tests")
+
+	for _, folder := range testFolders {
+		t.Run(folder.Name(), func(t *testing.T) {
+			preBeaconStateFile, err := util.BazelFileBytes(testsFolderPath, folder.Name(), "pre.ssz_snappy")
+			require.NoError(t, err)
+			preBeaconStateSSZ, err := snappy.Decode(nil /* dst */, preBeaconStateFile)
+			require.NoError(t, err, "Failed to decompress")
+			base := &ethpb.BeaconStateElectra{}
+			require.NoError(t, base.UnmarshalSSZ(preBeaconStateSSZ), "Failed to unmarshal")
+			beaconState, err := state_native.InitializeFromProtoElectra(base)
+			require.NoError(t, err)
+
+			file, err := util.BazelFileBytes(testsFolderPath, folder.Name(), "slots.yaml")
+			require.NoError(t, err)
+			fileStr := string(file)
+			slotsCount, err := strconv.ParseUint(fileStr[:len(fileStr)-5], 10, 64)
+			require.NoError(t, err)
+
+			postBeaconStateFile, err := util.BazelFileBytes(testsFolderPath, folder.Name(), "post.ssz_snappy")
+			require.NoError(t, err)
+			postBeaconStateSSZ, err := snappy.Decode(nil /* dst */, postBeaconStateFile)
+			require.NoError(t, err, "Failed to decompress")
+			postBeaconState := &ethpb.BeaconStateElectra{}
+			require.NoError(t, postBeaconState.UnmarshalSSZ(postBeaconStateSSZ), "Failed to unmarshal")
+			postState, err := transition.ProcessSlots(context.Background(), beaconState, beaconState.Slot().Add(slotsCount))
+			require.NoError(t, err)
+
+			pbState, err := state_native.ProtobufBeaconStateElectra(postState.ToProto())
+			require.NoError(t, err)
+			if !proto.Equal(pbState, postBeaconState) {
+				t.Fatal("Did not receive expected post state")
+			}
+		})
+	}
+}