chore!: Make Cell be a flat sequence of bytes (#14159)

* chore: move all ckzg related functionality into kzg package

* refactor code to match

* run: bazel run //:gazelle -- fix

* chore: add some docs and stop copying large objects when converting between types

* fixes

* manually add kzg.go dep to Build.Hazel

* move kzg methods to kzg.go

* chore: add RecoverCellsAndProofs method

* bazel run //:gazelle -- fix

* make Cells be flattened sequence of bytes

* chore: add test for flattening roundtrip

* chore: remove code that was doing the flattening outside of the kzg package

* fix merge

* fix

* remove now un-needed conversion

* use pointers for Cell parameters

* linter

* rename cell conversion methods (this only applies to old version of c-kzg)

beacon-chain/blockchain/kzg/BUILD.bazel

@@ -23,6 +23,7 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = [
+        "kzg_test.go",
         "trusted_setup_test.go",
         "validation_test.go",
     ],

beacon-chain/blockchain/kzg/kzg.go

@@ -28,19 +28,15 @@ const BytesPerCell = ckzg4844.FieldElementsPerCell * ckzg4844.BytesPerFieldEleme
 // BytesPerBlob is the number of bytes in a single blob.
 const BytesPerBlob = ckzg4844.BytesPerBlob
 
-// FieldElementsPerCell is the number of field elements in a single cell.
-// TODO: This should not be exposed.
-const FieldElementsPerCell = ckzg4844.FieldElementsPerCell
+// fieldElementsPerCell is the number of field elements in a single cell.
+const fieldElementsPerCell = ckzg4844.FieldElementsPerCell
 
 // CellsPerExtBlob is the number of cells that we generate for a single blob.
 // This is equivalent to the number of columns in the data matrix.
 const CellsPerExtBlob = ckzg4844.CellsPerExtBlob
 
 // Cell represents a chunk of an encoded Blob.
-// TODO: This is not correctly sized in c-kzg
-// TODO: It should be a vector of bytes
-// TODO: Note that callers of this package rely on `BytesPerCell`
-type Cell ckzg4844.Cell
+type Cell [BytesPerCell]byte
 
 // CellsAndProofs represents the Cells and Proofs corresponding to
 // a single blob.
@@ -75,7 +71,7 @@ func ComputeCellsAndKZGProofs(blob *Blob) ([ckzg4844.CellsPerExtBlob]Cell, [ckzg
 	// Convert Cells and Proofs to types defined in this package
 	var cells [ckzg4844.CellsPerExtBlob]Cell
 	for i := range _cells {
-		cells[i] = Cell(_cells[i])
+		cells[i] = ckzgCellToCell(&_cells[i])
 	}
 
 	var proofs [ckzg4844.CellsPerExtBlob]Proof
@@ -89,14 +85,14 @@ func ComputeCellsAndKZGProofs(blob *Blob) ([ckzg4844.CellsPerExtBlob]Cell, [ckzg
 // VerifyCellKZGProof is unused. TODO: We can check when the batch size for `VerifyCellKZGProofBatch` is 1
 // and call this, though I think its better if the cryptography library handles this.
 func VerifyCellKZGProof(commitmentBytes Bytes48, cellId uint64, cell *Cell, proofBytes Bytes48) (bool, error) {
-	return ckzg4844.VerifyCellKZGProof(commitmentBytes, cellId, ckzg4844.Cell(*cell), proofBytes)
+	return ckzg4844.VerifyCellKZGProof(commitmentBytes, cellId, cellToCKZGCell(cell), proofBytes)
 }
 
 func VerifyCellKZGProofBatch(commitmentsBytes []Bytes48, rowIndices, columnIndices []uint64, _cells []Cell, proofsBytes []Bytes48) (bool, error) {
 	// Convert `Cell` type to `ckzg4844.Cell`
 	ckzgCells := make([]ckzg4844.Cell, len(_cells))
 	for i := range _cells {
-		ckzgCells[i] = ckzg4844.Cell(_cells[i])
+		ckzgCells[i] = cellToCKZGCell(&_cells[i])
 	}
 
 	return ckzg4844.VerifyCellKZGProofBatch(commitmentsBytes, rowIndices, columnIndices, ckzgCells, proofsBytes)
@@ -106,7 +102,7 @@ func RecoverAllCells(cellIds []uint64, _cells []Cell) ([ckzg4844.CellsPerExtBlob
 	// Convert `Cell` type to `ckzg4844.Cell`
 	ckzgCells := make([]ckzg4844.Cell, len(_cells))
 	for i := range _cells {
-		ckzgCells[i] = ckzg4844.Cell(_cells[i])
+		ckzgCells[i] = cellToCKZGCell(&_cells[i])
 	}
 
 	recoveredCells, err := ckzg4844.RecoverAllCells(cellIds, ckzgCells)
@@ -122,7 +118,7 @@ func RecoverAllCells(cellIds []uint64, _cells []Cell) ([ckzg4844.CellsPerExtBlob
 	// Convert `ckzg4844.Cell` type to `Cell`
 	var ret [ckzg4844.CellsPerExtBlob]Cell
 	for i := range recoveredCells {
-		ret[i] = Cell(recoveredCells[i])
+		ret[i] = ckzgCellToCell(&recoveredCells[i])
 	}
 	return ret, nil
 }
@@ -151,7 +147,7 @@ func CellsToBlob(_cells *[ckzg4844.CellsPerExtBlob]Cell) (Blob, error) {
 	// Convert `Cell` type to `ckzg4844.Cell`
 	var ckzgCells [ckzg4844.CellsPerExtBlob]ckzg4844.Cell
 	for i := range _cells {
-		ckzgCells[i] = ckzg4844.Cell(_cells[i])
+		ckzgCells[i] = cellToCKZGCell(&_cells[i])
 	}
 
 	blob, err := ckzg4844.CellsToBlob(ckzgCells)
@@ -161,3 +157,21 @@ func CellsToBlob(_cells *[ckzg4844.CellsPerExtBlob]Cell) (Blob, error) {
 
 	return Blob(blob), nil
 }
+
+// The correct type for Cell is [BytesPerCell]byte
+// c-kzg currently uses [BytesPerFieldElement]Bytes32
+// so we have these helper methods to convert between the two.
+func cellToCKZGCell(flattened *Cell) ckzg4844.Cell {
+	var cell ckzg4844.Cell
+	for i := 0; i < fieldElementsPerCell; i++ {
+		copy(cell[i][:], flattened[i*32:(i+1)*32])
+	}
+	return cell
+}
+func ckzgCellToCell(cell *ckzg4844.Cell) Cell {
+	var flattened Cell
+	for i, fieldElement := range cell {
+		copy(flattened[i*32:(i+1)*32], fieldElement[:])
+	}
+	return flattened
+}

beacon-chain/blockchain/kzg/kzg_test.go

@@ -0,0 +1,21 @@
+package kzg
+
+import "testing"
+
+func TestCellFlattenedChunked(t *testing.T) {
+	cell := makeCell()
+	chunkedCell := cellToCKZGCell(&cell)
+	flattenedCell := ckzgCellToCell(&chunkedCell)
+	if cell != flattenedCell {
+		t.Errorf("cell != flattenedCell")
+	}
+}
+
+func makeCell() Cell {
+	var cell Cell
+	for i := 0; i < fieldElementsPerCell; i++ {
+		rand32 := deterministicRandomness(int64(i))
+		copy(cell[i*32:], rand32[:])
+	}
+	return cell
+}

beacon-chain/core/peerdas/helpers.go

@@ -168,15 +168,7 @@ func DataColumnSidecars(signedBlock interfaces.ReadOnlySignedBeaconBlock, blobs
 
 		columnBytes := make([][]byte, 0, blobsCount)
 		for i := range column {
-			cell := column[i]
-
-			cellBytes := make([]byte, 0, kzg.BytesPerCell)
-			for _, fieldElement := range cell {
-				copiedElem := fieldElement
-				cellBytes = append(cellBytes, copiedElem[:]...)
-			}
-
-			columnBytes = append(columnBytes, cellBytes)
+			columnBytes = append(columnBytes, column[i][:])
 		}
 
 		kzgProofOfColumnBytes := make([][]byte, 0, blobsCount)
@@ -234,15 +226,7 @@ func DataColumnSidecarsForReconstruct(
 
 		columnBytes := make([][]byte, 0, blobsCount)
 		for i := range column {
-			cell := column[i]
-
-			cellBytes := make([]byte, 0, kzg.BytesPerCell)
-			for _, fieldElement := range cell {
-				copiedElem := fieldElement
-				cellBytes = append(cellBytes, copiedElem[:]...)
-			}
-
-			columnBytes = append(columnBytes, cellBytes)
+			columnBytes = append(columnBytes, column[i][:])
 		}
 
 		kzgProofOfColumnBytes := make([][]byte, 0, blobsCount)
@@ -290,12 +274,8 @@ func VerifyDataColumnSidecarKZGProofs(sc *ethpb.DataColumnSidecar) (bool, error)
 		ckzgComms = append(ckzgComms, kzg.Bytes48(com))
 	}
 	var cells []kzg.Cell
-	for _, ce := range sc.DataColumn {
-		var newCell []kzg.Bytes32
-		for i := 0; i < len(ce); i += 32 {
-			newCell = append(newCell, kzg.Bytes32(ce[i:i+32]))
-		}
-		cells = append(cells, kzg.Cell(newCell))
+	for _, cell := range sc.DataColumn {
+		cells = append(cells, kzg.Cell(cell))
 	}
 	var proofs []kzg.Bytes48
 	for _, p := range sc.KzgProof {

beacon-chain/sync/data_columns_reconstruct.go

@@ -47,7 +47,7 @@ func recoverCellsAndProofs(
 		start := time.Now()
 
 		cellsId := make([]uint64, 0, columnsCount)
-		cKzgCells := make([]kzg.Cell, 0, columnsCount)
+		cells := make([]kzg.Cell, 0, columnsCount)
 
 		for _, sidecar := range dataColumnSideCars {
 			// Build the cell ids.
@@ -57,17 +57,11 @@ func recoverCellsAndProofs(
 			column := sidecar.DataColumn
 			cell := column[blobIndex]
 
-			// Transform the cell as a cKzg cell.
-			var ckzgCell kzg.Cell
-			for i := 0; i < kzg.FieldElementsPerCell; i++ {
-				copy(ckzgCell[i][:], cell[32*i:32*(i+1)])
-			}
-
-			cKzgCells = append(cKzgCells, ckzgCell)
+			cells = append(cells, kzg.Cell(cell))
 		}
 
 		// Recover the blob.
-		recoveredCells, err := kzg.RecoverAllCells(cellsId, cKzgCells)
+		recoveredCells, err := kzg.RecoverAllCells(cellsId, cells)
 		if err != nil {
 			return nil, errors.Wrapf(err, "recover all cells for blob %d", blobIndex)
 		}