Implement all needed KZG wrappers for peerDAS in the kzg package. (#15186)

* Implement all needed KZG wrappers for peerDAS in the `kzg` package.

This way, If we need to change the KZG backend, the only package to
modify is the `kzg` package.

* `.bazelrc`: Add `build --compilation_mode=opt`

* Remove --compilation_mode=opt, use supranational blst headers.

* Fix Terence's comment.

* Fix Terence`s comments.

---------

Co-authored-by: Preston Van Loon <preston@pvl.dev>

beacon-chain/blockchain/kzg/BUILD.bazel

@@ -3,15 +3,19 @@ load("@prysm//tools/go:def.bzl", "go_library", "go_test")
 go_library(
     name = "go_default_library",
     srcs = [
+        "kzg.go",
         "trusted_setup.go",
         "validation.go",
     ],
-    embedsrcs = ["trusted_setup.json"],
+    embedsrcs = ["trusted_setup_4096.json"],
     importpath = "github.com/OffchainLabs/prysm/v6/beacon-chain/blockchain/kzg",
     visibility = ["//visibility:public"],
     deps = [
         "//consensus-types/blocks:go_default_library",
         "@com_github_crate_crypto_go_kzg_4844//:go_default_library",
+        "@com_github_ethereum_c_kzg_4844//bindings/go:go_default_library",
+        "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
+        "@com_github_ethereum_go_ethereum//crypto/kzg4844:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
     ],
 )

beacon-chain/blockchain/kzg/kzg.go

@@ -0,0 +1,143 @@
+package kzg
+
+import (
+	"github.com/pkg/errors"
+
+	ckzg4844 "github.com/ethereum/c-kzg-4844/v2/bindings/go"
+	"github.com/ethereum/go-ethereum/crypto/kzg4844"
+)
+
+// BytesPerBlob is the number of bytes in a single blob.
+const BytesPerBlob = ckzg4844.BytesPerBlob
+
+// Blob represents a serialized chunk of data.
+type Blob [BytesPerBlob]byte
+
+// BytesPerCell is the number of bytes in a single cell.
+const BytesPerCell = ckzg4844.BytesPerCell
+
+// Cell represents a chunk of an encoded Blob.
+type Cell [BytesPerCell]byte
+
+// Commitment represent a KZG commitment to a Blob.
+type Commitment [48]byte
+
+// Proof represents a KZG proof that attests to the validity of a Blob or parts of it.
+type Proof [48]byte
+
+// Bytes48 is a 48-byte array.
+type Bytes48 = ckzg4844.Bytes48
+
+// Bytes32 is a 32-byte array.
+type Bytes32 = ckzg4844.Bytes32
+
+// CellsAndProofs represents the Cells and Proofs corresponding to a single blob.
+type CellsAndProofs struct {
+	Cells  []Cell
+	Proofs []Proof
+}
+
+// BlobToKZGCommitment computes a KZG commitment from a given blob.
+func BlobToKZGCommitment(blob *Blob) (Commitment, error) {
+	var kzgBlob kzg4844.Blob
+	copy(kzgBlob[:], blob[:])
+
+	commitment, err := kzg4844.BlobToCommitment(&kzgBlob)
+	if err != nil {
+		return Commitment{}, err
+	}
+
+	return Commitment(commitment), nil
+}
+
+// ComputeCells computes the (extended) cells from a given blob.
+func ComputeCells(blob *Blob) ([]Cell, error) {
+	var ckzgBlob ckzg4844.Blob
+	copy(ckzgBlob[:], blob[:])
+
+	ckzgCells, err := ckzg4844.ComputeCells(&ckzgBlob)
+	if err != nil {
+		return nil, errors.Wrap(err, "compute cells")
+	}
+
+	cells := make([]Cell, len(ckzgCells))
+	for i := range ckzgCells {
+		cells[i] = Cell(ckzgCells[i])
+	}
+
+	return cells, nil
+}
+
+// ComputeBlobKZGProof computes the blob KZG proof from a given blob and its commitment.
+func ComputeBlobKZGProof(blob *Blob, commitment Commitment) (Proof, error) {
+	var kzgBlob kzg4844.Blob
+	copy(kzgBlob[:], blob[:])
+
+	proof, err := kzg4844.ComputeBlobProof(&kzgBlob, kzg4844.Commitment(commitment))
+	if err != nil {
+		return [48]byte{}, err
+	}
+	return Proof(proof), nil
+}
+
+// ComputeCellsAndKZGProofs computes the cells and cells KZG proofs from a given blob.
+func ComputeCellsAndKZGProofs(blob *Blob) (CellsAndProofs, error) {
+	var ckzgBlob ckzg4844.Blob
+	copy(ckzgBlob[:], blob[:])
+
+	ckzgCells, ckzgProofs, err := ckzg4844.ComputeCellsAndKZGProofs(&ckzgBlob)
+	if err != nil {
+		return CellsAndProofs{}, err
+	}
+
+	return makeCellsAndProofs(ckzgCells[:], ckzgProofs[:])
+}
+
+// VerifyCellKZGProofBatch verifies the KZG proofs for a given slice of commitments, cells indices, cells and proofs.
+// Note: It is way more efficient to call once this function with big slices than calling it multiple times with small slices.
+func VerifyCellKZGProofBatch(commitmentsBytes []Bytes48, cellIndices []uint64, cells []Cell, proofsBytes []Bytes48) (bool, error) {
+	// Convert `Cell` type to `ckzg4844.Cell`
+	ckzgCells := make([]ckzg4844.Cell, len(cells))
+
+	for i := range cells {
+		ckzgCells[i] = ckzg4844.Cell(cells[i])
+	}
+
+	return ckzg4844.VerifyCellKZGProofBatch(commitmentsBytes, cellIndices, ckzgCells, proofsBytes)
+}
+
+// RecoverCellsAndKZGProofs recovers the complete cells and KZG proofs from a given set of cell indices and partial cells.
+func RecoverCellsAndKZGProofs(cellIndices []uint64, partialCells []Cell) (CellsAndProofs, error) {
+	// Convert `Cell` type to `ckzg4844.Cell`
+	ckzgPartialCells := make([]ckzg4844.Cell, len(partialCells))
+	for i := range partialCells {
+		ckzgPartialCells[i] = ckzg4844.Cell(partialCells[i])
+	}
+
+	ckzgCells, ckzgProofs, err := ckzg4844.RecoverCellsAndKZGProofs(cellIndices, ckzgPartialCells)
+	if err != nil {
+		return CellsAndProofs{}, errors.Wrap(err, "recover cells and KZG proofs")
+	}
+
+	return makeCellsAndProofs(ckzgCells[:], ckzgProofs[:])
+}
+
+// makeCellsAndProofs converts cells/proofs to the CellsAndProofs type defined in this package.
+func makeCellsAndProofs(ckzgCells []ckzg4844.Cell, ckzgProofs []ckzg4844.KZGProof) (CellsAndProofs, error) {
+	if len(ckzgCells) != len(ckzgProofs) {
+		return CellsAndProofs{}, errors.New("different number of cells/proofs")
+	}
+
+	cells := make([]Cell, 0, len(ckzgCells))
+	proofs := make([]Proof, 0, len(ckzgProofs))
+
+	for i := range ckzgCells {
+		cells = append(cells, Cell(ckzgCells[i]))
+		proofs = append(proofs, Proof(ckzgProofs[i]))
+	}
+
+	return CellsAndProofs{
+		Cells:  cells,
+		Proofs: proofs,
+	}, nil
+}

beacon-chain/blockchain/kzg/trusted_setup.go

@@ -5,24 +5,69 @@ import (
 	"encoding/json"
 
 	GoKZG "github.com/crate-crypto/go-kzg-4844"
+	CKZG "github.com/ethereum/c-kzg-4844/v2/bindings/go"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/pkg/errors"
 )
 
 var (
-	//go:embed trusted_setup.json
+	// https://github.com/ethereum/consensus-specs/blob/dev/presets/mainnet/trusted_setups/trusted_setup_4096.json
+	//go:embed trusted_setup_4096.json
 	embeddedTrustedSetup []byte // 1.2Mb
 	kzgContext           *GoKZG.Context
+	kzgLoaded            bool
 )
 
+type TrustedSetup struct {
+	G1Monomial [GoKZG.ScalarsPerBlob]GoKZG.G1CompressedHexStr `json:"g1_monomial"`
+	G1Lagrange [GoKZG.ScalarsPerBlob]GoKZG.G1CompressedHexStr `json:"g1_lagrange"`
+	G2Monomial [65]GoKZG.G2CompressedHexStr                   `json:"g2_monomial"`
+}
+
 func Start() error {
-	parsedSetup := GoKZG.JSONTrustedSetup{}
-	err := json.Unmarshal(embeddedTrustedSetup, &parsedSetup)
+	trustedSetup := &TrustedSetup{}
+	err := json.Unmarshal(embeddedTrustedSetup, trustedSetup)
 	if err != nil {
 		return errors.Wrap(err, "could not parse trusted setup JSON")
 	}
-	kzgContext, err = GoKZG.NewContext4096(&parsedSetup)
+
+	kzgContext, err = GoKZG.NewContext4096(&GoKZG.JSONTrustedSetup{
+		SetupG2:         trustedSetup.G2Monomial[:],
+		SetupG1Lagrange: trustedSetup.G1Lagrange,
+	})
 	if err != nil {
 		return errors.Wrap(err, "could not initialize go-kzg context")
 	}
+
+	// Length of a G1 point, converted from hex to binary.
+	g1MonomialBytes := make([]byte, len(trustedSetup.G1Monomial)*(len(trustedSetup.G1Monomial[0])-2)/2)
+	for i, g1 := range &trustedSetup.G1Monomial {
+		copy(g1MonomialBytes[i*(len(g1)-2)/2:], hexutil.MustDecode(g1))
+	}
+
+	// Length of a G1 point, converted from hex to binary.
+	g1LagrangeBytes := make([]byte, len(trustedSetup.G1Lagrange)*(len(trustedSetup.G1Lagrange[0])-2)/2)
+	for i, g1 := range &trustedSetup.G1Lagrange {
+		copy(g1LagrangeBytes[i*(len(g1)-2)/2:], hexutil.MustDecode(g1))
+	}
+
+	// Length of a G2 point, converted from hex to binary.
+	g2MonomialBytes := make([]byte, len(trustedSetup.G2Monomial)*(len(trustedSetup.G2Monomial[0])-2)/2)
+	for i, g2 := range &trustedSetup.G2Monomial {
+		copy(g2MonomialBytes[i*(len(g2)-2)/2:], hexutil.MustDecode(g2))
+	}
+
+	if !kzgLoaded {
+		const precompute uint = 8
+
+		kzgLoaded = true
+
+		// Free the current trusted setup before running this method.
+		// CKZG panics if the same setup is run multiple times.
+		if err = CKZG.LoadTrustedSetup(g1MonomialBytes, g1LagrangeBytes, g2MonomialBytes, precompute); err != nil {
+			return errors.Wrap(err, "load trust setup")
+		}
+	}
+
 	return nil
 }

changelog/manu-kzg.md

@@ -0,0 +1,3 @@
+### Added
+
+- PeerDAS related KZG wrappers.

deps.bzl

@@ -755,11 +755,11 @@ def prysm_deps():
         build_directives = [
             "gazelle:resolve go github.com/supranational/blst/bindings/go @com_github_supranational_blst//:go_default_library",
         ],
-        importpath = "github.com/ethereum/c-kzg-4844",
+        importpath = "github.com/ethereum/c-kzg-4844/v2",
         patch_args = ["-p1"],
         patches = ["//third_party:com_github_ethereum_c_kzg_4844.patch"],
-        sum = "h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA=",
-        version = "v1.0.0",
+        sum = "h1:KhzBVjmURsfr1+S3k/VE35T02+AW2qU9t9gr4R6YpSo=",
+        version = "v2.1.1",
     )
     go_repository(
         name = "com_github_ethereum_go_ethereum",

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/dgraph-io/ristretto v0.0.4-0.20210318174700-74754f61e018
 	github.com/dustin/go-humanize v1.0.0
 	github.com/emicklei/dot v0.11.0
+	github.com/ethereum/c-kzg-4844/v2 v2.1.1
 	github.com/ethereum/go-ethereum v1.15.0
 	github.com/fsnotify/fsnotify v1.6.0
 	github.com/ghodss/yaml v1.0.0

go.sum

@@ -237,6 +237,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA=
 github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0=
+github.com/ethereum/c-kzg-4844/v2 v2.1.1 h1:KhzBVjmURsfr1+S3k/VE35T02+AW2qU9t9gr4R6YpSo=
+github.com/ethereum/c-kzg-4844/v2 v2.1.1/go.mod h1:TC48kOKjJKPbN7C++qIgt0TJzZ70QznYR7Ob+WXl57E=
 github.com/ethereum/go-ethereum v1.15.0 h1:LLb2jCPsbJZcB4INw+E/MgzUX5wlR6SdwXcv09/1ME4=
 github.com/ethereum/go-ethereum v1.15.0/go.mod h1:4q+4t48P2C03sjqGvTXix5lEOplf5dz4CTosbjt5tGs=
 github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8=

third_party/com_github_ethereum_c_kzg_4844.patch

@@ -7,7 +7,7 @@ index 7e49df7..1d476f7 100644
      name = "go",
      srcs = ["main.go"],
 +    cdeps = [
-+        "//bindings/go/blst_headers:hdrs",
++        "@com_github_supranational_blst//:blst",
 +        "//src:kzg",
 +    ],
      cgo = True,
@@ -15,9 +15,10 @@ index 7e49df7..1d476f7 100644
 -        "-Ibindings/go/src",
 -        "-Ibindings/go/bindings/go/blst_headers",
 +        "-Iexternal/com_github_ethereum_c_kzg_4844/src",
-+        "-Iexternal/com_github_ethereum_c_kzg_4844/bindings/go/blst_headers",
++        "-Iexternal/com_github_supranational_blst/bindings/blst_headers",
      ],
-     importpath = "github.com/ethereum/c-kzg-4844/bindings/go",
+     importpath = "github.com/ethereum/c-kzg-4844/v2/bindings/go",
+     importpath_aliases = ["github.com/ethereum/c-kzg-4844/bindings/go"],
      visibility = ["//visibility:public"],
 diff --git a/bindings/go/blst_headers/BUILD.bazel b/bindings/go/blst_headers/BUILD.bazel
 new file mode 100644
@@ -35,12 +36,9 @@ new file mode 100644
 index 0000000..b3f845d
 --- /dev/null
 +++ b/src/BUILD.bazel
-@@ -0,0 +1,8 @@
+@@ -0,0 +1,5 @@
 +cc_library(
 +    name = "kzg",
-+    hdrs = [
-+        "c_kzg_4844.c",
-+        "c_kzg_4844.h",
-+    ],
++    hdrs = glob(["ckzg.*", "*/*.c", "*/*.h"]),
 +    visibility = ["//visibility:public"],
 +)