payload attestation pool

beacon-chain/operations/payloadattestation/BUILD.bazel

@@ -0,0 +1,32 @@
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["pool.go"],
+    importpath = "github.com/OffchainLabs/prysm/v7/beacon-chain/operations/payloadattestation",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//consensus-types/primitives:go_default_library",
+        "//crypto/bls:go_default_library",
+        "//crypto/hash:go_default_library",
+        "//encoding/bytesutil:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
+        "@com_github_prysmaticlabs_go_bitfield//:go_default_library",
+        "@org_golang_google_protobuf//proto:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["pool_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//consensus-types/primitives:go_default_library",
+        "//crypto/bls:go_default_library",
+        "//encoding/bytesutil:go_default_library",
+        "//proto/prysm/v1alpha1:go_default_library",
+        "//testing/assert:go_default_library",
+        "//testing/require:go_default_library",
+    ],
+)

beacon-chain/operations/payloadattestation/pool.go

@@ -0,0 +1,179 @@
+package payloadattestation
+
+import (
+	"sync"
+
+	"github.com/OffchainLabs/go-bitfield"
+	"github.com/OffchainLabs/prysm/v7/consensus-types/primitives"
+	"github.com/OffchainLabs/prysm/v7/crypto/bls"
+	"github.com/OffchainLabs/prysm/v7/crypto/hash"
+	"github.com/OffchainLabs/prysm/v7/encoding/bytesutil"
+	ethpb "github.com/OffchainLabs/prysm/v7/proto/prysm/v1alpha1"
+	"github.com/pkg/errors"
+	"google.golang.org/protobuf/proto"
+)
+
+var errNilPayloadAttestationMessage = errors.New("nil payload attestation message")
+
+// PoolManager maintains pending payload attestations.
+// This pool is used by proposers to insert payload attestations into new blocks.
+type PoolManager interface {
+	// PendingPayloadAttestations returns all pending aggregated payload attestations.
+	// If a slot is provided, only attestations for that slot are returned.
+	PendingPayloadAttestations(slot ...primitives.Slot) []*ethpb.PayloadAttestation
+	// InsertPayloadAttestation inserts or aggregates a payload attestation
+	// message into the pool. The idx parameter is the PTC committee index
+	// of the validator (position in the bitvector).
+	InsertPayloadAttestation(msg *ethpb.PayloadAttestationMessage, idx uint64) error
+	// Seen returns true if the PTC committee index has already been seen
+	// for the given PayloadAttestationData.
+	Seen(data *ethpb.PayloadAttestationData, idx uint64) bool
+	// MarkIncluded removes the attestation matching the given data from the pool.
+	MarkIncluded(att *ethpb.PayloadAttestation)
+}
+
+// Pool is a concrete implementation of PoolManager.
+// Keyed by hash of PayloadAttestationData; stores aggregated PayloadAttestation.
+type Pool struct {
+	lock    sync.RWMutex
+	pending map[[32]byte]*ethpb.PayloadAttestation
+}
+
+// NewPool returns an initialized pool.
+func NewPool() *Pool {
+	return &Pool{
+		pending: make(map[[32]byte]*ethpb.PayloadAttestation),
+	}
+}
+
+// PendingPayloadAttestations returns all pending payload attestations.
+// If a slot filter is provided, only attestations for that slot are returned.
+func (p *Pool) PendingPayloadAttestations(slot ...primitives.Slot) []*ethpb.PayloadAttestation {
+	p.lock.RLock()
+	defer p.lock.RUnlock()
+
+	result := make([]*ethpb.PayloadAttestation, 0, len(p.pending))
+	for _, att := range p.pending {
+		if len(slot) > 0 && att.Data.Slot != slot[0] {
+			continue
+		}
+		result = append(result, att)
+	}
+	return result
+}
+
+// InsertPayloadAttestation inserts a payload attestation message into the pool,
+// aggregating it with any existing attestation that shares the same PayloadAttestationData.
+// The idx parameter is the PTC committee index used to set the aggregation bit.
+func (p *Pool) InsertPayloadAttestation(msg *ethpb.PayloadAttestationMessage, idx uint64) error {
+	if msg == nil || msg.Data == nil {
+		return errNilPayloadAttestationMessage
+	}
+
+	key, err := dataKey(msg.Data)
+	if err != nil {
+		return errors.Wrap(err, "could not compute data key")
+	}
+
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	existing, ok := p.pending[key]
+	if !ok {
+		p.pending[key] = messageToPayloadAttestation(msg, idx)
+		return nil
+	}
+
+	if existing.AggregationBits.BitAt(idx) {
+		return nil
+	}
+
+	sig, err := aggregateSigFromMessage(existing, msg)
+	if err != nil {
+		return errors.Wrap(err, "could not aggregate signatures")
+	}
+	existing.Signature = sig
+	existing.AggregationBits.SetBitAt(idx, true)
+	return nil
+}
+
+// Seen returns true if the PTC committee index has already been seen
+// for the given PayloadAttestationData.
+func (p *Pool) Seen(data *ethpb.PayloadAttestationData, idx uint64) bool {
+	if data == nil {
+		return false
+	}
+
+	key, err := dataKey(data)
+	if err != nil {
+		return false
+	}
+
+	p.lock.RLock()
+	defer p.lock.RUnlock()
+
+	existing, ok := p.pending[key]
+	if !ok {
+		return false
+	}
+	return existing.AggregationBits.BitAt(idx)
+}
+
+// MarkIncluded removes the attestation with matching data from the pool.
+func (p *Pool) MarkIncluded(att *ethpb.PayloadAttestation) {
+	if att == nil || att.Data == nil {
+		return
+	}
+
+	key, err := dataKey(att.Data)
+	if err != nil {
+		return
+	}
+
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	delete(p.pending, key)
+}
+
+// messageToPayloadAttestation creates a PayloadAttestation with a single
+// aggregated bit from the passed PayloadAttestationMessage.
+func messageToPayloadAttestation(msg *ethpb.PayloadAttestationMessage, idx uint64) *ethpb.PayloadAttestation {
+	bits := bitfield.NewBitvector512()
+	bits.SetBitAt(idx, true)
+	data := &ethpb.PayloadAttestationData{
+		BeaconBlockRoot:   bytesutil.SafeCopyBytes(msg.Data.BeaconBlockRoot),
+		Slot:              msg.Data.Slot,
+		PayloadPresent:    msg.Data.PayloadPresent,
+		BlobDataAvailable: msg.Data.BlobDataAvailable,
+	}
+	return &ethpb.PayloadAttestation{
+		AggregationBits: bits,
+		Data:            data,
+		Signature:       bytesutil.SafeCopyBytes(msg.Signature),
+	}
+}
+
+// aggregateSigFromMessage returns the aggregated signature by combining the
+// existing aggregated signature with the message's signature.
+func aggregateSigFromMessage(aggregated *ethpb.PayloadAttestation, message *ethpb.PayloadAttestationMessage) ([]byte, error) {
+	aggSig, err := bls.SignatureFromBytesNoValidation(aggregated.Signature)
+	if err != nil {
+		return nil, err
+	}
+	sig, err := bls.SignatureFromBytesNoValidation(message.Signature)
+	if err != nil {
+		return nil, err
+	}
+	return bls.AggregateSignatures([]bls.Signature{aggSig, sig}).Marshal(), nil
+}
+
+// dataKey computes a deterministic key for PayloadAttestationData
+// by hashing its serialized form.
+func dataKey(data *ethpb.PayloadAttestationData) ([32]byte, error) {
+	enc, err := proto.Marshal(data)
+	if err != nil {
+		return [32]byte{}, err
+	}
+	return hash.Hash(enc), nil
+}

beacon-chain/operations/payloadattestation/pool_test.go

@@ -0,0 +1,291 @@
+package payloadattestation
+
+import (
+	"testing"
+
+	"github.com/OffchainLabs/prysm/v7/consensus-types/primitives"
+	"github.com/OffchainLabs/prysm/v7/crypto/bls"
+	"github.com/OffchainLabs/prysm/v7/encoding/bytesutil"
+	ethpb "github.com/OffchainLabs/prysm/v7/proto/prysm/v1alpha1"
+	"github.com/OffchainLabs/prysm/v7/testing/assert"
+	"github.com/OffchainLabs/prysm/v7/testing/require"
+)
+
+func TestPool_PendingPayloadAttestations(t *testing.T) {
+	t.Run("empty pool", func(t *testing.T) {
+		pool := NewPool()
+		atts := pool.PendingPayloadAttestations()
+		assert.Equal(t, 0, len(atts))
+	})
+
+	t.Run("non-empty pool", func(t *testing.T) {
+		pool := NewPool()
+		sig := bls.NewAggregateSignature().Marshal()
+		msg1 := &ethpb.PayloadAttestationMessage{
+			ValidatorIndex: 0,
+			Data: &ethpb.PayloadAttestationData{
+				BeaconBlockRoot:   make([]byte, 32),
+				Slot:              1,
+				PayloadPresent:    true,
+				BlobDataAvailable: false,
+			},
+			Signature: sig,
+		}
+		msg2 := &ethpb.PayloadAttestationMessage{
+			ValidatorIndex: 1,
+			Data: &ethpb.PayloadAttestationData{
+				BeaconBlockRoot:   make([]byte, 32),
+				Slot:              2,
+				PayloadPresent:    false,
+				BlobDataAvailable: true,
+			},
+			Signature: sig,
+		}
+		require.NoError(t, pool.InsertPayloadAttestation(msg1, 0))
+		require.NoError(t, pool.InsertPayloadAttestation(msg2, 1))
+		atts := pool.PendingPayloadAttestations()
+		assert.Equal(t, 2, len(atts))
+	})
+
+	t.Run("filter by slot", func(t *testing.T) {
+		pool := NewPool()
+		sig := bls.NewAggregateSignature().Marshal()
+		msg1 := &ethpb.PayloadAttestationMessage{
+			ValidatorIndex: 0,
+			Data: &ethpb.PayloadAttestationData{
+				BeaconBlockRoot:   make([]byte, 32),
+				Slot:              1,
+				PayloadPresent:    true,
+				BlobDataAvailable: false,
+			},
+			Signature: sig,
+		}
+		msg2 := &ethpb.PayloadAttestationMessage{
+			ValidatorIndex: 1,
+			Data: &ethpb.PayloadAttestationData{
+				BeaconBlockRoot:   make([]byte, 32),
+				Slot:              2,
+				PayloadPresent:    false,
+				BlobDataAvailable: true,
+			},
+			Signature: sig,
+		}
+		require.NoError(t, pool.InsertPayloadAttestation(msg1, 0))
+		require.NoError(t, pool.InsertPayloadAttestation(msg2, 1))
+
+		atts := pool.PendingPayloadAttestations(primitives.Slot(1))
+		assert.Equal(t, 1, len(atts))
+		assert.Equal(t, primitives.Slot(1), atts[0].Data.Slot)
+
+		atts = pool.PendingPayloadAttestations(primitives.Slot(2))
+		assert.Equal(t, 1, len(atts))
+		assert.Equal(t, primitives.Slot(2), atts[0].Data.Slot)
+
+		atts = pool.PendingPayloadAttestations(primitives.Slot(99))
+		assert.Equal(t, 0, len(atts))
+	})
+}
+
+func TestPool_InsertPayloadAttestation(t *testing.T) {
+	t.Run("nil message", func(t *testing.T) {
+		pool := NewPool()
+		err := pool.InsertPayloadAttestation(nil, 0)
+		require.ErrorContains(t, "nil payload attestation message", err)
+	})
+
+	t.Run("nil data", func(t *testing.T) {
+		pool := NewPool()
+		err := pool.InsertPayloadAttestation(&ethpb.PayloadAttestationMessage{}, 0)
+		require.ErrorContains(t, "nil payload attestation message", err)
+	})
+
+	t.Run("insert creates new entry with correct aggregation bit", func(t *testing.T) {
+		pool := NewPool()
+		sig := bls.NewAggregateSignature().Marshal()
+		idx := uint64(5)
+		msg := &ethpb.PayloadAttestationMessage{
+			ValidatorIndex: 0,
+			Data: &ethpb.PayloadAttestationData{
+				BeaconBlockRoot:   make([]byte, 32),
+				Slot:              1,
+				PayloadPresent:    true,
+				BlobDataAvailable: false,
+			},
+			Signature: sig,
+		}
+		require.NoError(t, pool.InsertPayloadAttestation(msg, idx))
+		atts := pool.PendingPayloadAttestations()
+		require.Equal(t, 1, len(atts))
+		assert.Equal(t, true, atts[0].AggregationBits.BitAt(idx))
+		assert.Equal(t, false, atts[0].AggregationBits.BitAt(idx+1))
+	})
+
+	t.Run("duplicate index is no-op", func(t *testing.T) {
+		pool := NewPool()
+		sig := bls.NewAggregateSignature().Marshal()
+		idx := uint64(3)
+		msg := &ethpb.PayloadAttestationMessage{
+			ValidatorIndex: 0,
+			Data: &ethpb.PayloadAttestationData{
+				BeaconBlockRoot:   make([]byte, 32),
+				Slot:              1,
+				PayloadPresent:    true,
+				BlobDataAvailable: false,
+			},
+			Signature: sig,
+		}
+		require.NoError(t, pool.InsertPayloadAttestation(msg, idx))
+		firstSig := bytesutil.SafeCopyBytes(pool.PendingPayloadAttestations()[0].Signature)
+
+		require.NoError(t, pool.InsertPayloadAttestation(msg, idx))
+		atts := pool.PendingPayloadAttestations()
+		require.Equal(t, 1, len(atts))
+		assert.DeepEqual(t, firstSig, atts[0].Signature)
+	})
+
+	t.Run("aggregates different indices", func(t *testing.T) {
+		pool := NewPool()
+		sig := bls.NewAggregateSignature().Marshal()
+		root := make([]byte, 32)
+		root[0] = 'r'
+		data := &ethpb.PayloadAttestationData{
+			BeaconBlockRoot:   root,
+			Slot:              1,
+			PayloadPresent:    true,
+			BlobDataAvailable: false,
+		}
+		msg1 := &ethpb.PayloadAttestationMessage{
+			ValidatorIndex: 0,
+			Data:           data,
+			Signature:      sig,
+		}
+		msg2 := &ethpb.PayloadAttestationMessage{
+			ValidatorIndex: 1,
+			Data:           data,
+			Signature:      sig,
+		}
+
+		require.NoError(t, pool.InsertPayloadAttestation(msg1, 5))
+		require.NoError(t, pool.InsertPayloadAttestation(msg2, 7))
+
+		atts := pool.PendingPayloadAttestations()
+		require.Equal(t, 1, len(atts))
+		assert.Equal(t, true, atts[0].AggregationBits.BitAt(5))
+		assert.Equal(t, true, atts[0].AggregationBits.BitAt(7))
+		assert.Equal(t, false, atts[0].AggregationBits.BitAt(6))
+	})
+
+	t.Run("different data creates separate entries", func(t *testing.T) {
+		pool := NewPool()
+		sig := bls.NewAggregateSignature().Marshal()
+		msg1 := &ethpb.PayloadAttestationMessage{
+			ValidatorIndex: 0,
+			Data: &ethpb.PayloadAttestationData{
+				BeaconBlockRoot:   make([]byte, 32),
+				Slot:              1,
+				PayloadPresent:    true,
+				BlobDataAvailable: false,
+			},
+			Signature: sig,
+		}
+		msg2 := &ethpb.PayloadAttestationMessage{
+			ValidatorIndex: 1,
+			Data: &ethpb.PayloadAttestationData{
+				BeaconBlockRoot:   make([]byte, 32),
+				Slot:              1,
+				PayloadPresent:    false, // different
+				BlobDataAvailable: false,
+			},
+			Signature: sig,
+		}
+		require.NoError(t, pool.InsertPayloadAttestation(msg1, 0))
+		require.NoError(t, pool.InsertPayloadAttestation(msg2, 1))
+		atts := pool.PendingPayloadAttestations()
+		assert.Equal(t, 2, len(atts))
+	})
+}
+
+func TestPool_Seen(t *testing.T) {
+	pool := NewPool()
+	sig := bls.NewAggregateSignature().Marshal()
+	data := &ethpb.PayloadAttestationData{
+		BeaconBlockRoot:   make([]byte, 32),
+		Slot:              1,
+		PayloadPresent:    true,
+		BlobDataAvailable: false,
+	}
+
+	assert.Equal(t, false, pool.Seen(data, 5))
+
+	msg := &ethpb.PayloadAttestationMessage{
+		ValidatorIndex: 0,
+		Data:           data,
+		Signature:      sig,
+	}
+	require.NoError(t, pool.InsertPayloadAttestation(msg, 5))
+
+	assert.Equal(t, true, pool.Seen(data, 5))
+	assert.Equal(t, false, pool.Seen(data, 6))
+	assert.Equal(t, false, pool.Seen(nil, 5))
+}
+
+func TestPool_MarkIncluded(t *testing.T) {
+	t.Run("mark included removes from pool", func(t *testing.T) {
+		pool := NewPool()
+		sig := bls.NewAggregateSignature().Marshal()
+		msg := &ethpb.PayloadAttestationMessage{
+			ValidatorIndex: 0,
+			Data: &ethpb.PayloadAttestationData{
+				BeaconBlockRoot:   make([]byte, 32),
+				Slot:              1,
+				PayloadPresent:    true,
+				BlobDataAvailable: false,
+			},
+			Signature: sig,
+		}
+		require.NoError(t, pool.InsertPayloadAttestation(msg, 0))
+		assert.Equal(t, 1, len(pool.PendingPayloadAttestations()))
+
+		pool.MarkIncluded(&ethpb.PayloadAttestation{
+			Data: &ethpb.PayloadAttestationData{
+				BeaconBlockRoot:   make([]byte, 32),
+				Slot:              1,
+				PayloadPresent:    true,
+				BlobDataAvailable: false,
+			},
+		})
+		assert.Equal(t, 0, len(pool.PendingPayloadAttestations()))
+	})
+
+	t.Run("mark included with non-matching data does nothing", func(t *testing.T) {
+		pool := NewPool()
+		sig := bls.NewAggregateSignature().Marshal()
+		msg := &ethpb.PayloadAttestationMessage{
+			ValidatorIndex: 0,
+			Data: &ethpb.PayloadAttestationData{
+				BeaconBlockRoot:   make([]byte, 32),
+				Slot:              1,
+				PayloadPresent:    true,
+				BlobDataAvailable: false,
+			},
+			Signature: sig,
+		}
+		require.NoError(t, pool.InsertPayloadAttestation(msg, 0))
+
+		pool.MarkIncluded(&ethpb.PayloadAttestation{
+			Data: &ethpb.PayloadAttestationData{
+				BeaconBlockRoot:   make([]byte, 32),
+				Slot:              999,
+				PayloadPresent:    true,
+				BlobDataAvailable: false,
+			},
+		})
+		assert.Equal(t, 1, len(pool.PendingPayloadAttestations()))
+	})
+
+	t.Run("mark included with nil is safe", func(t *testing.T) {
+		pool := NewPool()
+		pool.MarkIncluded(nil)
+		pool.MarkIncluded(&ethpb.PayloadAttestation{})
+	})
+}

changelog/jtraglia-add-specrefs-readme.md

@@ -1,3 +0,0 @@
-### Added
-
-- Added README for maintaining specrefs.

changelog/jtraglia-nightly-reftests-with-run-id.md

@@ -1,3 +0,0 @@
-### Added
-
-- The ability to download the nightly reference tests from a specific day.

changelog/syjn99_docs-ssz-ql.md

@@ -1,3 +0,0 @@
-### Ignored
-
-- Add handy documentation for SSZ Query package (`encoding/ssz/query`).

encoding/ssz/query/doc.md

@@ -1,190 +0,0 @@
-# SSZ Query Package
-
-The `encoding/ssz/query` package provides a system for analyzing and querying SSZ ([Simple Serialize](https://github.com/ethereum/consensus-specs/blob/master/ssz/simple-serialize.md)) data structures, as well as generating Merkle proofs from them. It enables runtime analysis of SSZ-serialized Go objects with reflection, path-based queries through nested structures, generalized index calculation, and Merkle proof generation.
-
-This package is designed to be generic. It operates on arbitrary SSZ-serialized Go values at runtime, so the same query/proof machinery applies equally to any SSZ type, including the BeaconState/BeaconBlock.
-
-## Usage Example
-
-```go
-// 1. Analyze an SSZ object
-block := &ethpb.BeaconBlock{...}
-info, err := query.AnalyzeObject(block)
-
-// 2. Parse a path
-path, err := query.ParsePath(".body.attestations[0].data.slot")
-
-// 3. Get the generalized index
-gindex, err := query.GetGeneralizedIndexFromPath(info, path)
-
-// 4. Generate a Merkle proof
-proof, err := info.Prove(gindex)
-
-// 5. Get offset and length to slice the SSZ-encoded bytes
-sszBytes, _ := block.MarshalSSZ()
-_, offset, length, err := query.CalculateOffsetAndLength(info, path)
-// slotBytes contains the SSZ-encoded value at the queried path
-slotBytes := sszBytes[offset : offset+length]
-```
-
-## Exported API
-
-The main exported API consists of:
-
-```go
-// AnalyzeObject analyzes an SSZ object and returns its structural information
-func AnalyzeObject(obj SSZObject) (*SszInfo, error)
-
-// ParsePath parses a path string like ".field1.field2[0].field3"
-func ParsePath(rawPath string) (Path, error)
-
-// CalculateOffsetAndLength computes byte offset and length for a path within an SSZ object
-func CalculateOffsetAndLength(sszInfo *SszInfo, path Path) (*SszInfo, uint64, uint64, error)
-
-// GetGeneralizedIndexFromPath calculates the generalized index for a given path
-func GetGeneralizedIndexFromPath(info *SszInfo, path Path) (uint64, error)
-
-// Prove generates a Merkle proof for a target generalized index
-func (s *SszInfo) Prove(gindex uint64) (*fastssz.Proof, error)
-```
-
-## Type System
-
-### SSZ Types
-
-The package now supports [all standard SSZ types](https://github.com/ethereum/consensus-specs/blob/master/ssz/simple-serialize.md#typing) except `ProgressiveList`, `ProgressiveContainer`, `ProgressiveBitlist`, `Union`, and `CompatibleUnion`.
-
-### Core Data Structures
-
-#### `SszInfo`
-
-The `SszInfo` structure contains complete structural metadata for an SSZ type:
-
-```go
-type SszInfo struct {
-   sszType       SszType           // SSZ Type classification
-   typ           reflect.Type      // Go reflect.Type
-   source        SSZObject         // Original SSZObject reference. Mostly used for reusing SSZ methods like `HashTreeRoot`.
-   isVariable    bool              // True if contains variable-size fields
-
-   // Composite types have corresponding metadata. Other fields would be nil except for the current type.
-   containerInfo *containerInfo
-   listInfo      *listInfo
-   vectorInfo    *vectorInfo
-   bitlistInfo   *bitlistInfo
-   bitvectorInfo *bitvectorInfo
-}
-```
-
-#### `Path`
-
-The `Path` structure represents navigation paths through SSZ structures. It supports accessing a field by field name, accessing an element by index (list/vector type), and finding the length of homogenous collection types. The `ParsePath` function parses a raw string into a `Path` instance, which is commonly used in other APIs like `CalculateOffsetAndLength` and `GetGeneralizedIndexFromPath`.
-
-```go
-type Path struct {
-   Length   bool           // Flag for length queries (e.g., len(.field))
-   Elements []PathElement  // Sequence of field accesses and indices
-}
-
-type PathElement struct {
-   Name  string  // Field name
-   Index *uint64 // list/vector index (nil if not an index access)
-}
-```
-
-## Implementation Details
-
-### Type Analysis (`analyzer.go`)
-
-The `AnalyzeObject` function performs recursive type introspection using Go reflection:
-
-1. **Type Inspection** - Examines Go `reflect.Value` to determine SSZ type
-   - Basic types (`uint8`, `uint16`, `uint32`, `uint64`, `bool`): `SSZType` constants
-   - Slices: Determined from struct tags (`ssz-size` for vectors, `ssz-max` for lists). There is a related [write-up](https://hackmd.io/@junsong/H101DKnwxl) regarding struct tags.
-   - Structs: Analyzed as Containers with field ordering from JSON tags
-   - Pointers: Dereferenced automatically
-
-2. **Variable-Length Population** - Determines actual sizes at runtime
-   - For lists: Iterates elements, caches sizes for variable-element lists
-   - For containers: Recursively populates variable fields, adjusts offsets
-   - For bitlists: Decodes bit length from bitvector
-
-3. **Offset Calculation** - Computes byte positions within serialized data
-   - Fixed-size fields: Offset = sum of preceding field sizes
-   - Variable-size fields: Offset stored as 4-byte pointer entries
-
-### Path Parsing (`path.go`)
-
-The `ParsePath` function parses path strings with the following rules:
-
-- **Dot notation**: `.field1.field2` for field access
-- **Array indexing**: `[0]`, `[42]` for element access
-- **Length queries**: `len(.field)` for list/vector lengths
-- **Character set**: Only `[A-Za-z0-9._\[\]\(\)]` allowed
-
-Example:
-```go
-path, _ := ParsePath(".nested.array_field[5].inner_field")
-// Returns: Path{
-//   Elements: [
-//     PathElement{Name: "nested"},
-//     PathElement{Name: "array_field", Index: <Pointer to uint64(5)>},
-//     PathElement{Name: "inner_field"}
-//   ]
-// }
-```
-
-### Generalized Index Calculation (`generalized_index.go`)
-
-The generalized index is a tree position identifier. This package follows the [Ethereum consensus-specs](https://github.com/ethereum/consensus-specs/blob/master/ssz/merkle-proofs.md#generalized-merkle-tree-index) to calculate the generalized index.
-
-### Merkle Proof Generation (`merkle_proof.go`, `proof_collector.go`)
-
-The `Prove` method generates Merkle proofs using a single-sweep merkleization algorithm:
-
-#### Algorithm Overview
-
-**Key Terms:**
-
-- **Target gindex** (generalized index): The position of the SSZ element you want to prove, expressed as a generalized Merkle tree index. Stored in `Proof.Index`.
-  - Note: The generalized index for root is 1.
-- **Registered gindices**: The set of tree positions whose node hashes must be captured during merkleization in order to later assemble the proof.
-- **Sibling node**: The node that shares the same parent as another node.
-- **Leaf value**: The 32-byte hash of the target node (the node being proven). Stored in `Proof.Leaf`.
-
-**Phases:**
-
-1. **Registration Phase** (`addTarget`)
-> Goal: determine exactly which sibling hashes are needed for the proof.
-
-   - Record the target gindex as the proof target.
-   - Starting from the target node, walk the Merkle tree from the leaf (target gindex) to the root (gindex = 1).
-   - At each step:
-     - Compute and register the sibling gindex (`i XOR 1`) as “must collect”.
-     - Move to the parent (`i = i/2`).
-   - This produces the full set of registered gindices (the sibling nodes on the target-to-root path).
-
-2. **Merkleization Phase** (`merkleize`)
-> Goal: recursively merkleize the tree and capture the needed hashes.
-
-   - Recursively traverse the SSZ structure and compute Merkle tree node hashes from leaves to root.
-   - Whenever the traversal computes a node whose gindex is in registered gindices, store that node’s hash for later proof construction.
-
-3. **Proof Assembly Phase** (`toProof`)
-> Goal: create the final `fastssz.Proof` object in the correct format and order.
-
-```go
-// Proof represents a merkle proof against a general index.
-type Proof struct {
-	Index  int
-	Leaf   []byte
-	Hashes [][]byte
-}
-```
-
-   - Set `Proof.Index` to the target gindex.
-   - Set `Proof.Leaf` to the 32-byte hash of the target node.
-   - Build `Proof.Hashes` by walking from the target node up to (but not including) the root:
-     - At node `i`, append the stored hash for the sibling (`i XOR 1`).
-     - Move to the parent (`i = i/2`).
-   - The resulting `Proof.Hashes` is ordered from the target level upward, containing one sibling hash per tree level on the path to the root.

specrefs/README.md

@@ -1,35 +0,0 @@
-# Specification References
-
-This directory contains specification reference tracking files managed by
-[ethspecify](https://github.com/jtraglia/ethspecify).
-
-## Installation
-
-Install `ethspecify` with the following command:
-
-```bash
-pipx install ethspecify
-```
-
-> [!NOTE]
-> You can run `ethspecify <cmd>` in the `specrefs` directory or
-> `ethspecify <cmd> --path=specrefs` from the project's root directory.
-
-## Maintenance
-
-When adding support for a new specification version, follow these steps:
-
-0. Change directory into the `specrefs` directory.
-1. Update the version in `.ethspecify.yml` configuration.
-2. Run `ethspecify process` to update/populate specrefs.
-3. Run `ethspecify check` to check specrefs.
-4. If there are errors, use the error message as a guide to fix the issue. If
-   there are new specrefs with empty sources, implement/locate each item and
-   update each specref source list. If you choose not to implement an item,
-   add an exception to the appropriate section the the `.ethspecify.yml`
-   configuration.
-5. Repeat steps 3 and 4 until `ethspecify check` passes.
-6. Run `git diff` to view updated specrefs. If an object/function/etc has
-   changed, make the necessary updates to the implementation.
-7. Lastly, in the project's root directory, run `act -j check-specrefs` to
-   ensure everything is correct.

testing/spectest/README.md

@@ -21,14 +21,10 @@ There are tests for mainnet and minimal config, so for each config we will add a
 
 ## Running nightly spectests
 
-Since [PR 15312](https://github.com/OffchainLabs/prysm/pull/15312), Prysm has support to download "nightly" spectests from github via a starlark rule configuration by environment variable.
-Set `--repo_env=CONSENSUS_SPEC_TESTS_VERSION=nightly` or `--repo_env=CONSENSUS_SPEC_TESTS_VERSION=nightly-<run_id>` when running spectest to download the "nightly" spectests.
-Note: A GITHUB_TOKEN environment variable is required to be set. The github token does not need to be associated with your main account; it can be from a "burner account". And the token does not need to be a fine-grained token; it can be a classic token.
+Since [PR 15312](https://github.com/OffchainLabs/prysm/pull/15312), Prysm has support to download "nightly" spectests from github via a starlark rule configuration by environment variable. 
+Set `--repo_env=CONSENSUS_SPEC_TESTS_VERSION=nightly` when running spectest to download the "nightly" spectests. 
+Note: A GITHUB_TOKEN environment variable is required to be set. The github token must be a [fine grained token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token).
 
 ```
 bazel test //... --test_tag_filters=spectest --repo_env=CONSENSUS_SPEC_TESTS_VERSION=nightly
 ```
-
-```
-bazel test //... --test_tag_filters=spectest --repo_env=CONSENSUS_SPEC_TESTS_VERSION=nightly-21422848633
-```

tools/download_spectests.bzl

@@ -1,6 +1,5 @@
 # bazel build @consensus_spec_tests//:test_data
 # bazel build @consensus_spec_tests//:test_data --repo_env=CONSENSUS_SPEC_TESTS_VERSION=nightly
-# bazel build @consensus_spec_tests//:test_data --repo_env=CONSENSUS_SPEC_TESTS_VERSION=nightly-<run_id>
 
 def _get_redirected_url(repository_ctx, url, headers):
     if not repository_ctx.which("curl"):
@@ -25,7 +24,7 @@ def _impl(repository_ctx):
     version = repository_ctx.getenv("CONSENSUS_SPEC_TESTS_VERSION") or repository_ctx.attr.version
     token = repository_ctx.getenv("GITHUB_TOKEN") or ""
 
-    if version == "nightly" or version.startswith("nightly-"):
+    if version == "nightly":
         print("Downloading nightly tests")
         if not token:
             fail("Error GITHUB_TOKEN is not set")
@@ -35,22 +34,16 @@ def _impl(repository_ctx):
             "Accept": "application/vnd.github+json",
         }
 
-        if version.startswith("nightly-"):
-            run_id = version.split("nightly-", 1)[1]
-            if not run_id:
-                fail("Error invalid run id")
-        else:
-            repository_ctx.download(
-                "https://api.github.com/repos/%s/actions/workflows/%s/runs?branch=%s&status=success&per_page=1"
-                    % (repository_ctx.attr.repo, repository_ctx.attr.workflow, repository_ctx.attr.branch),
-                headers = headers,
-                output = "runs.json"
-            )
+        repository_ctx.download(
+            "https://api.github.com/repos/%s/actions/workflows/%s/runs?branch=%s&status=success&per_page=1"
+                % (repository_ctx.attr.repo, repository_ctx.attr.workflow, repository_ctx.attr.branch),
+            headers = headers,
+            output = "runs.json"
+        )
 
-            run_id = json.decode(repository_ctx.read("runs.json"))["workflow_runs"][0]["id"]
-            repository_ctx.delete("runs.json")
+        run_id = json.decode(repository_ctx.read("runs.json"))["workflow_runs"][0]["id"]
+        repository_ctx.delete("runs.json")
 
-        print("Run id:", run_id)
         repository_ctx.download(
             "https://api.github.com/repos/%s/actions/runs/%s/artifacts"
                 % (repository_ctx.attr.repo, run_id),
@@ -115,8 +108,8 @@ consensus_spec_tests = repository_rule(
         "version": attr.string(mandatory = True),
         "flavors": attr.string_dict(mandatory = True),
         "repo": attr.string(default = "ethereum/consensus-specs"),
-        "workflow": attr.string(default = "nightly-reftests.yml"),
-        "branch": attr.string(default = "master"),
+        "workflow": attr.string(default = "generate_vectors.yml"),
+        "branch": attr.string(default = "dev"),
         "release_url_template": attr.string(default = "https://github.com/ethereum/consensus-specs/releases/download/%s"),
     },
 )