From 8b3ac46a2dff4bddbefc4e31259ec7fc1b47eb19 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 31 Jan 2024 13:45:49 +0200 Subject: [PATCH 01/11] Add helper method: construct_vanishing_polynomial() --- .../polynomial-commitments-sampling.md | 66 ++++++++++++------- 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index d3a8cd23b..590a4efec 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -471,6 +471,45 @@ def verify_cell_proof_batch(row_commitments_bytes: Sequence[Bytes48], ## Reconstruction +### `construct_vanishing_polynomial` + +```python +def construct_vanishing_polynomial(cell_ids: Sequence[CellID], + cells: Sequence[Cell]) -> Tuple[ + Sequence[BLSFieldElement], + Sequence[BLSFieldElement]]: + missing_cell_ids = [cell_id for cell_id in range(CELLS_PER_BLOB) if cell_id not in cell_ids] + roots_of_unity_reduced = compute_roots_of_unity(CELLS_PER_BLOB) + short_zero_poly = vanishing_polynomialcoeff([ + roots_of_unity_reduced[reverse_bits(cell_id, CELLS_PER_BLOB)] + for cell_id in missing_cell_ids + ]) + + zero_poly_coeff = [] + for i in short_zero_poly: + zero_poly_coeff.append(i) + zero_poly_coeff.extend([0] * (FIELD_ELEMENTS_PER_CELL - 1)) + zero_poly_coeff = zero_poly_coeff + [0] * (2 * FIELD_ELEMENTS_PER_BLOB - len(zero_poly_coeff)) + + zero_poly_eval = fft_field(zero_poly_coeff, + compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB)) + zero_poly_eval_brp = bit_reversal_permutation(zero_poly_eval) + for cell_id in missing_cell_ids: + start = cell_id * FIELD_ELEMENTS_PER_CELL + end = (cell_id + 1) * FIELD_ELEMENTS_PER_CELL + assert zero_poly_eval_brp[start:end] == [0] * FIELD_ELEMENTS_PER_CELL + for cell_id in cell_ids: + start = cell_id * FIELD_ELEMENTS_PER_CELL + end = (cell_id + 1) * FIELD_ELEMENTS_PER_CELL + assert all(a != 0 for a in zero_poly_eval_brp[start:end]) + + return zero_poly_coeff, zero_poly_eval, zero_poly_eval_brp +``` + +### `recover_shifted_data` + +### `recover_original_data` + ### `recover_polynomial` ```python @@ -488,32 +527,9 @@ def recover_polynomial(cell_ids: Sequence[CellID], assert len(cell_ids) == len(cells_bytes) cells = [bytes_to_cell(cell_bytes) for cell_bytes in cells_bytes] - assert len(cells) >= CELLS_PER_BLOB // 2 - missing_cell_ids = [cell_id for cell_id in range(CELLS_PER_BLOB) if cell_id not in cell_ids] - roots_of_unity_reduced = compute_roots_of_unity(CELLS_PER_BLOB) - short_zero_poly = vanishing_polynomialcoeff([ - roots_of_unity_reduced[reverse_bits(cell_id, CELLS_PER_BLOB)] - for cell_id in missing_cell_ids - ]) - full_zero_poly = [] - for i in short_zero_poly: - full_zero_poly.append(i) - full_zero_poly.extend([0] * (FIELD_ELEMENTS_PER_CELL - 1)) - full_zero_poly = full_zero_poly + [0] * (2 * FIELD_ELEMENTS_PER_BLOB - len(full_zero_poly)) - - zero_poly_eval = fft_field(full_zero_poly, - compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB)) - zero_poly_eval_brp = bit_reversal_permutation(zero_poly_eval) - for cell_id in missing_cell_ids: - start = cell_id * FIELD_ELEMENTS_PER_CELL - end = (cell_id + 1) * FIELD_ELEMENTS_PER_CELL - assert zero_poly_eval_brp[start:end] == [0] * FIELD_ELEMENTS_PER_CELL - for cell_id in cell_ids: - start = cell_id * FIELD_ELEMENTS_PER_CELL - end = (cell_id + 1) * FIELD_ELEMENTS_PER_CELL - assert all(a != 0 for a in zero_poly_eval_brp[start:end]) + zero_poly_coeff, zero_poly_eval, zero_poly_eval_brp = construct_vanishing_polynomial(cell_ids, cells) extended_evaluation_rbo = [0] * (FIELD_ELEMENTS_PER_BLOB * 2) for cell_id, cell in zip(cell_ids, cells): @@ -533,7 +549,7 @@ def recover_polynomial(cell_ids: Sequence[CellID], shift_inv = div(BLSFieldElement(1), shift_factor) shifted_extended_evaluation = shift_polynomialcoeff(extended_evaluations_fft, shift_factor) - shifted_zero_poly = shift_polynomialcoeff(full_zero_poly, shift_factor) + shifted_zero_poly = shift_polynomialcoeff(zero_poly_coeff, shift_factor) eval_shifted_extended_evaluation = fft_field(shifted_extended_evaluation, roots_of_unity_extended) eval_shifted_zero_poly = fft_field(shifted_zero_poly, roots_of_unity_extended) From be5e5c5a75231cd6c0bd255470181bc30613d425 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 23 Jan 2024 18:22:05 +0200 Subject: [PATCH 02/11] Add helpers: recover_shifted_data() and recover_original_data() --- .../polynomial-commitments-sampling.md | 92 ++++++++++++------- 1 file changed, 61 insertions(+), 31 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index 590a4efec..be215ee04 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -42,6 +42,9 @@ - [`verify_cell_proof`](#verify_cell_proof) - [`verify_cell_proof_batch`](#verify_cell_proof_batch) - [Reconstruction](#reconstruction) + - [`construct_vanishing_polynomial`](#construct_vanishing_polynomial) + - [`recover_shifted_data`](#recover_shifted_data) + - [`recover_original_data`](#recover_original_data) - [`recover_polynomial`](#recover_polynomial) @@ -508,8 +511,60 @@ def construct_vanishing_polynomial(cell_ids: Sequence[CellID], ### `recover_shifted_data` +```python +def recover_shifted_data(cell_ids: Sequence[CellID], + cells: Sequence[Cell], + zero_poly_eval: Sequence[BLSFieldElement], + zero_poly_coeff: Sequence[BLSFieldElement], + roots_of_unity_extended: Sequence[BLSFieldElement]) -> Tuple[ + Sequence[BLSFieldElement], + Sequence[BLSFieldElement], + BLSFieldElement]: + extended_evaluation_rbo = [0] * (FIELD_ELEMENTS_PER_BLOB * 2) + for cell_id, cell in zip(cell_ids, cells): + start = cell_id * FIELD_ELEMENTS_PER_CELL + end = (cell_id + 1) * FIELD_ELEMENTS_PER_CELL + extended_evaluation_rbo[start:end] = cell + extended_evaluation = bit_reversal_permutation(extended_evaluation_rbo) + + extended_evaluation_times_zero = [BLSFieldElement(int(a) * int(b) % BLS_MODULUS) + for a, b in zip(zero_poly_eval, extended_evaluation)] + + extended_evaluations_fft = fft_field(extended_evaluation_times_zero, roots_of_unity_extended, inv=True) + + shift_factor = BLSFieldElement(PRIMITIVE_ROOT_OF_UNITY) + shift_inv = div(BLSFieldElement(1), shift_factor) + + shifted_extended_evaluation = shift_polynomialcoeff(extended_evaluations_fft, shift_factor) + shifted_zero_poly = shift_polynomialcoeff(zero_poly_coeff, shift_factor) + + eval_shifted_extended_evaluation = fft_field(shifted_extended_evaluation, roots_of_unity_extended) + eval_shifted_zero_poly = fft_field(shifted_zero_poly, roots_of_unity_extended) + + return eval_shifted_extended_evaluation, eval_shifted_zero_poly, shift_inv +``` + ### `recover_original_data` +```python +def recover_original_data(eval_shifted_extended_evaluation: Sequence[BLSFieldElement], + eval_shifted_zero_poly: Sequence[BLSFieldElement], + shift_inv: BLSFieldElement, + roots_of_unity_extended: Sequence[BLSFieldElement]) -> Sequence[BLSFieldElement]: + eval_shifted_reconstructed_poly = [ + div(a, b) + for a, b in zip(eval_shifted_extended_evaluation, eval_shifted_zero_poly) + ] + + shifted_reconstructed_poly = fft_field(eval_shifted_reconstructed_poly, roots_of_unity_extended, inv=True) + + reconstructed_poly = shift_polynomialcoeff(shifted_reconstructed_poly, shift_inv) + + reconstructed_data = bit_reversal_permutation(fft_field(reconstructed_poly, roots_of_unity_extended)) + + return reconstructed_data +``` + ### `recover_polynomial` ```python @@ -529,41 +584,16 @@ def recover_polynomial(cell_ids: Sequence[CellID], cells = [bytes_to_cell(cell_bytes) for cell_bytes in cells_bytes] assert len(cells) >= CELLS_PER_BLOB // 2 - zero_poly_coeff, zero_poly_eval, zero_poly_eval_brp = construct_vanishing_polynomial(cell_ids, cells) - - extended_evaluation_rbo = [0] * (FIELD_ELEMENTS_PER_BLOB * 2) - for cell_id, cell in zip(cell_ids, cells): - start = cell_id * FIELD_ELEMENTS_PER_CELL - end = (cell_id + 1) * FIELD_ELEMENTS_PER_CELL - extended_evaluation_rbo[start:end] = cell - extended_evaluation = bit_reversal_permutation(extended_evaluation_rbo) - - extended_evaluation_times_zero = [BLSFieldElement(int(a) * int(b) % BLS_MODULUS) - for a, b in zip(zero_poly_eval, extended_evaluation)] - roots_of_unity_extended = compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB) - extended_evaluations_fft = fft_field(extended_evaluation_times_zero, roots_of_unity_extended, inv=True) + zero_poly_coeff, zero_poly_eval, zero_poly_eval_brp = construct_vanishing_polynomial(cell_ids, cells) - shift_factor = BLSFieldElement(PRIMITIVE_ROOT_OF_UNITY) - shift_inv = div(BLSFieldElement(1), shift_factor) + eval_shifted_extended_evaluation, eval_shifted_zero_poly, shift_inv = \ + recover_shifted_data(cell_ids, cells, zero_poly_eval, zero_poly_coeff, roots_of_unity_extended) - shifted_extended_evaluation = shift_polynomialcoeff(extended_evaluations_fft, shift_factor) - shifted_zero_poly = shift_polynomialcoeff(zero_poly_coeff, shift_factor) - - eval_shifted_extended_evaluation = fft_field(shifted_extended_evaluation, roots_of_unity_extended) - eval_shifted_zero_poly = fft_field(shifted_zero_poly, roots_of_unity_extended) - - eval_shifted_reconstructed_poly = [ - div(a, b) - for a, b in zip(eval_shifted_extended_evaluation, eval_shifted_zero_poly) - ] - - shifted_reconstructed_poly = fft_field(eval_shifted_reconstructed_poly, roots_of_unity_extended, inv=True) - - reconstructed_poly = shift_polynomialcoeff(shifted_reconstructed_poly, shift_inv) - - reconstructed_data = bit_reversal_permutation(fft_field(reconstructed_poly, roots_of_unity_extended)) + reconstructed_data = \ + recover_original_data(eval_shifted_extended_evaluation, eval_shifted_zero_poly, shift_inv, + roots_of_unity_extended) for cell_id, cell in zip(cell_ids, cells): start = cell_id * FIELD_ELEMENTS_PER_CELL From 4d01d5043720ed9516186521fe1f8696ef809420 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 31 Jan 2024 13:45:04 +0200 Subject: [PATCH 03/11] Initial docs added to recover_shifted_data() and recover_original_data() --- .../polynomial-commitments-sampling.md | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index be215ee04..b9e31b3bc 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -520,6 +520,12 @@ def recover_shifted_data(cell_ids: Sequence[CellID], Sequence[BLSFieldElement], Sequence[BLSFieldElement], BLSFieldElement]: + """ + Given Z(x), return polynomial Q_1(x)=(E*Z)(k*x) and Q_2(x)=Z(k*x) and k^{-1} + """ + shift_factor = BLSFieldElement(PRIMITIVE_ROOT_OF_UNITY) + shift_inv = div(BLSFieldElement(1), shift_factor) + extended_evaluation_rbo = [0] * (FIELD_ELEMENTS_PER_BLOB * 2) for cell_id, cell in zip(cell_ids, cells): start = cell_id * FIELD_ELEMENTS_PER_CELL @@ -527,15 +533,15 @@ def recover_shifted_data(cell_ids: Sequence[CellID], extended_evaluation_rbo[start:end] = cell extended_evaluation = bit_reversal_permutation(extended_evaluation_rbo) + # Compute (E*Z)(x) extended_evaluation_times_zero = [BLSFieldElement(int(a) * int(b) % BLS_MODULUS) for a, b in zip(zero_poly_eval, extended_evaluation)] extended_evaluations_fft = fft_field(extended_evaluation_times_zero, roots_of_unity_extended, inv=True) - shift_factor = BLSFieldElement(PRIMITIVE_ROOT_OF_UNITY) - shift_inv = div(BLSFieldElement(1), shift_factor) - + # Compute (E*Z)(k*x) shifted_extended_evaluation = shift_polynomialcoeff(extended_evaluations_fft, shift_factor) + # Compute Z(k*x) shifted_zero_poly = shift_polynomialcoeff(zero_poly_coeff, shift_factor) eval_shifted_extended_evaluation = fft_field(shifted_extended_evaluation, roots_of_unity_extended) @@ -551,6 +557,10 @@ def recover_original_data(eval_shifted_extended_evaluation: Sequence[BLSFieldEle eval_shifted_zero_poly: Sequence[BLSFieldElement], shift_inv: BLSFieldElement, roots_of_unity_extended: Sequence[BLSFieldElement]) -> Sequence[BLSFieldElement]: + """ + Given Q_1, Q_2 and k^{-1}, compute P(x) + """ + # Compute Q_3 = Q_1(x)/Q_2(x) = P(k*x) eval_shifted_reconstructed_poly = [ div(a, b) for a, b in zip(eval_shifted_extended_evaluation, eval_shifted_zero_poly) @@ -558,6 +568,7 @@ def recover_original_data(eval_shifted_extended_evaluation: Sequence[BLSFieldEle shifted_reconstructed_poly = fft_field(eval_shifted_reconstructed_poly, roots_of_unity_extended, inv=True) + # Unshift P(k*x) by k^{-1} to get P(x) reconstructed_poly = shift_polynomialcoeff(shifted_reconstructed_poly, shift_inv) reconstructed_data = bit_reversal_permutation(fft_field(reconstructed_poly, roots_of_unity_extended)) @@ -571,10 +582,11 @@ def recover_original_data(eval_shifted_extended_evaluation: Sequence[BLSFieldEle def recover_polynomial(cell_ids: Sequence[CellID], cells_bytes: Sequence[Vector[Bytes32, FIELD_ELEMENTS_PER_CELL]]) -> Polynomial: """ - Recovers a polynomial from 2 * FIELD_ELEMENTS_PER_CELL evaluations, half of which can be missing. + Recover original polynomial from 2 * FIELD_ELEMENTS_PER_CELL evaluations, half of which can be missing. This + algorithm uses FFTs to recover cells faster than using Lagrange implementation, as can be seen here: + https://ethresear.ch/t/reed-solomon-erasure-code-recovery-in-n-log-2-n-time-with-ffts/3039 - This algorithm uses FFTs to recover cells faster than using Lagrange implementation. However, - a faster version thanks to Qi Zhou can be found here: + A faster version thanks to Qi Zhou can be found here: https://github.com/ethereum/research/blob/51b530a53bd4147d123ab3e390a9d08605c2cdb8/polynomial_reconstruction/polynomial_reconstruction_danksharding.py Public method. From 212c1fe624f8f84c77f30201ace8045f25ed0227 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Wed, 31 Jan 2024 13:32:23 +0200 Subject: [PATCH 04/11] Small improvements on construct_vanishing_polynomial() --- .../polynomial-commitments-sampling.md | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index b9e31b3bc..3b89f0799 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -477,34 +477,40 @@ def verify_cell_proof_batch(row_commitments_bytes: Sequence[Bytes48], ### `construct_vanishing_polynomial` ```python -def construct_vanishing_polynomial(cell_ids: Sequence[CellID], - cells: Sequence[Cell]) -> Tuple[ - Sequence[BLSFieldElement], - Sequence[BLSFieldElement]]: - missing_cell_ids = [cell_id for cell_id in range(CELLS_PER_BLOB) if cell_id not in cell_ids] +def construct_vanishing_polynomial(missing_cell_ids: Sequence[CellID]) -> Tuple[ + Sequence[BLSFieldElement], + Sequence[BLSFieldElement]]: + """ + Given the cells that are missing from the data, compute the polynomial that vanishes at every point that + corresponds to a missing field element + """ + # Get the small domain roots_of_unity_reduced = compute_roots_of_unity(CELLS_PER_BLOB) + + # Compute polynomial that vanishes at all the missing cells (over the small domain) short_zero_poly = vanishing_polynomialcoeff([ - roots_of_unity_reduced[reverse_bits(cell_id, CELLS_PER_BLOB)] - for cell_id in missing_cell_ids + roots_of_unity_reduced[reverse_bits(missing_cell_id, CELLS_PER_BLOB)] + for missing_cell_id in missing_cell_ids ]) - zero_poly_coeff = [] - for i in short_zero_poly: - zero_poly_coeff.append(i) - zero_poly_coeff.extend([0] * (FIELD_ELEMENTS_PER_CELL - 1)) - zero_poly_coeff = zero_poly_coeff + [0] * (2 * FIELD_ELEMENTS_PER_BLOB - len(zero_poly_coeff)) + # Extend vanishing polynomial to full domain using the closed form of the vanishing polynomial over a coset + zero_poly_coeff = [0] * (FIELD_ELEMENTS_PER_BLOB * 2) + for (i, coeff) in enumerate(short_zero_poly): + zero_poly_coeff[i * FIELD_ELEMENTS_PER_CELL] = coeff + # Compute evaluations of the extended vanishing polynomial zero_poly_eval = fft_field(zero_poly_coeff, compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB)) zero_poly_eval_brp = bit_reversal_permutation(zero_poly_eval) - for cell_id in missing_cell_ids: + + # Sanity check + for cell_id in range(CELLS_PER_BLOB): start = cell_id * FIELD_ELEMENTS_PER_CELL end = (cell_id + 1) * FIELD_ELEMENTS_PER_CELL - assert zero_poly_eval_brp[start:end] == [0] * FIELD_ELEMENTS_PER_CELL - for cell_id in cell_ids: - start = cell_id * FIELD_ELEMENTS_PER_CELL - end = (cell_id + 1) * FIELD_ELEMENTS_PER_CELL - assert all(a != 0 for a in zero_poly_eval_brp[start:end]) + if cell_id in missing_cell_ids: + assert zero_poly_eval_brp[start:end] == [0] * FIELD_ELEMENTS_PER_CELL + else: # cell_id in cell_ids + assert all(a != 0 for a in zero_poly_eval_brp[start:end]) return zero_poly_coeff, zero_poly_eval, zero_poly_eval_brp ``` @@ -593,12 +599,15 @@ def recover_polynomial(cell_ids: Sequence[CellID], """ assert len(cell_ids) == len(cells_bytes) + # Get the extended domain + roots_of_unity_extended = compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB) + + # Convert from bytes to cells cells = [bytes_to_cell(cell_bytes) for cell_bytes in cells_bytes] assert len(cells) >= CELLS_PER_BLOB // 2 - roots_of_unity_extended = compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB) - - zero_poly_coeff, zero_poly_eval, zero_poly_eval_brp = construct_vanishing_polynomial(cell_ids, cells) + missing_cell_ids = [cell_id for cell_id in range(CELLS_PER_BLOB) if cell_id not in cell_ids] + zero_poly_coeff, zero_poly_eval, zero_poly_eval_brp = construct_vanishing_polynomial(missing_cell_ids) eval_shifted_extended_evaluation, eval_shifted_zero_poly, shift_inv = \ recover_shifted_data(cell_ids, cells, zero_poly_eval, zero_poly_coeff, roots_of_unity_extended) From d60580bb5267a2d58d819f612bd246c6289db1c0 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 1 Feb 2024 18:09:37 +0800 Subject: [PATCH 05/11] Apply suggestions from code review Co-authored-by: Justin Traglia <95511699+jtraglia@users.noreply.github.com> --- .../eip7594/polynomial-commitments-sampling.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index 3b89f0799..34e7f7953 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -482,7 +482,7 @@ def construct_vanishing_polynomial(missing_cell_ids: Sequence[CellID]) -> Tuple[ Sequence[BLSFieldElement]]: """ Given the cells that are missing from the data, compute the polynomial that vanishes at every point that - corresponds to a missing field element + corresponds to a missing field element. """ # Get the small domain roots_of_unity_reduced = compute_roots_of_unity(CELLS_PER_BLOB) @@ -495,7 +495,7 @@ def construct_vanishing_polynomial(missing_cell_ids: Sequence[CellID]) -> Tuple[ # Extend vanishing polynomial to full domain using the closed form of the vanishing polynomial over a coset zero_poly_coeff = [0] * (FIELD_ELEMENTS_PER_BLOB * 2) - for (i, coeff) in enumerate(short_zero_poly): + for i, coeff in enumerate(short_zero_poly): zero_poly_coeff[i * FIELD_ELEMENTS_PER_CELL] = coeff # Compute evaluations of the extended vanishing polynomial @@ -508,7 +508,7 @@ def construct_vanishing_polynomial(missing_cell_ids: Sequence[CellID]) -> Tuple[ start = cell_id * FIELD_ELEMENTS_PER_CELL end = (cell_id + 1) * FIELD_ELEMENTS_PER_CELL if cell_id in missing_cell_ids: - assert zero_poly_eval_brp[start:end] == [0] * FIELD_ELEMENTS_PER_CELL + assert all(a == 0 for a in zero_poly_eval_brp[start:end]) else: # cell_id in cell_ids assert all(a != 0 for a in zero_poly_eval_brp[start:end]) @@ -527,7 +527,7 @@ def recover_shifted_data(cell_ids: Sequence[CellID], Sequence[BLSFieldElement], BLSFieldElement]: """ - Given Z(x), return polynomial Q_1(x)=(E*Z)(k*x) and Q_2(x)=Z(k*x) and k^{-1} + Given Z(x), return polynomial Q_1(x)=(E*Z)(k*x) and Q_2(x)=Z(k*x) and k^{-1}. """ shift_factor = BLSFieldElement(PRIMITIVE_ROOT_OF_UNITY) shift_inv = div(BLSFieldElement(1), shift_factor) @@ -564,7 +564,7 @@ def recover_original_data(eval_shifted_extended_evaluation: Sequence[BLSFieldEle shift_inv: BLSFieldElement, roots_of_unity_extended: Sequence[BLSFieldElement]) -> Sequence[BLSFieldElement]: """ - Given Q_1, Q_2 and k^{-1}, compute P(x) + Given Q_1, Q_2 and k^{-1}, compute P(x). """ # Compute Q_3 = Q_1(x)/Q_2(x) = P(k*x) eval_shifted_reconstructed_poly = [ From e3b83d5450d472e12ed7b902b07e9dc613a074e7 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 5 Feb 2024 16:53:28 +0200 Subject: [PATCH 06/11] Add defensive asserts in recover_polynomial() --- specs/_features/eip7594/polynomial-commitments-sampling.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index 34e7f7953..efef2779b 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -598,13 +598,16 @@ def recover_polynomial(cell_ids: Sequence[CellID], Public method. """ assert len(cell_ids) == len(cells_bytes) + # Check we have enough cells to be able to perform the reconstruction + assert CELLS_PER_BLOB / 2 <= len(cell_ids) <= CELLS_PER_BLOB + # Check for duplicates + assert len(cell_ids) == len(set(cell_ids)) # Get the extended domain roots_of_unity_extended = compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB) # Convert from bytes to cells cells = [bytes_to_cell(cell_bytes) for cell_bytes in cells_bytes] - assert len(cells) >= CELLS_PER_BLOB // 2 missing_cell_ids = [cell_id for cell_id in range(CELLS_PER_BLOB) if cell_id not in cell_ids] zero_poly_coeff, zero_poly_eval, zero_poly_eval_brp = construct_vanishing_polynomial(missing_cell_ids) From 1e41a6bc698d93131e96cc069931fe630902ccf5 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 5 Feb 2024 16:53:59 +0200 Subject: [PATCH 07/11] New FIELD_ELEMENTS_PER_EXT_BLOB: number of cells in an extended blob --- .../eip7594/polynomial-commitments-sampling.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index efef2779b..742602544 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -79,9 +79,10 @@ Cells are the smallest unit of blob data that can come with their own KZG proofs | Name | Value | Description | | - | - | - | +| `FIELD_ELEMENTS_PER_EXT_BLOB` | `2 * FIELD_ELEMENTS_PER_BLOB` | Number of field elements in a Reed-Solomon extended blob | | `FIELD_ELEMENTS_PER_CELL` | `uint64(64)` | Number of field elements in a cell | | `BYTES_PER_CELL` | `FIELD_ELEMENTS_PER_CELL * BYTES_PER_FIELD_ELEMENT` | The number of bytes in a cell | -| `CELLS_PER_BLOB` | `((2 * FIELD_ELEMENTS_PER_BLOB) // FIELD_ELEMENTS_PER_CELL)` | The number of cells in a blob | +| `CELLS_PER_BLOB` | `FIELD_ELEMENTS_PER_EXT_BLOB // FIELD_ELEMENTS_PER_CELL` | The number of cells in a blob | | `RANDOM_CHALLENGE_KZG_CELL_BATCH_DOMAIN` | `b'RCKZGCBATCH__V1_'` | ## Helper functions @@ -355,7 +356,7 @@ def coset_for_cell(cell_id: CellID) -> Cell: """ assert cell_id < CELLS_PER_BLOB roots_of_unity_brp = bit_reversal_permutation( - compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB) + compute_roots_of_unity(FIELD_ELEMENTS_PER_EXT_BLOB) ) return Cell(roots_of_unity_brp[FIELD_ELEMENTS_PER_CELL * cell_id:FIELD_ELEMENTS_PER_CELL * (cell_id + 1)]) ``` @@ -405,7 +406,7 @@ def compute_cells(blob: Blob) -> Vector[Cell, CELLS_PER_BLOB]: polynomial_coeff = polynomial_eval_to_coeff(polynomial) extended_data = fft_field(polynomial_coeff + [0] * FIELD_ELEMENTS_PER_BLOB, - compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB)) + compute_roots_of_unity(FIELD_ELEMENTS_PER_EXT_BLOB)) extended_data_rbo = bit_reversal_permutation(extended_data) return [extended_data_rbo[i * FIELD_ELEMENTS_PER_CELL:(i + 1) * FIELD_ELEMENTS_PER_CELL] for i in range(CELLS_PER_BLOB)] @@ -494,13 +495,13 @@ def construct_vanishing_polynomial(missing_cell_ids: Sequence[CellID]) -> Tuple[ ]) # Extend vanishing polynomial to full domain using the closed form of the vanishing polynomial over a coset - zero_poly_coeff = [0] * (FIELD_ELEMENTS_PER_BLOB * 2) + zero_poly_coeff = [0] * (FIELD_ELEMENTS_PER_EXT_BLOB) for i, coeff in enumerate(short_zero_poly): zero_poly_coeff[i * FIELD_ELEMENTS_PER_CELL] = coeff # Compute evaluations of the extended vanishing polynomial zero_poly_eval = fft_field(zero_poly_coeff, - compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB)) + compute_roots_of_unity(FIELD_ELEMENTS_PER_EXT_BLOB)) zero_poly_eval_brp = bit_reversal_permutation(zero_poly_eval) # Sanity check @@ -532,7 +533,7 @@ def recover_shifted_data(cell_ids: Sequence[CellID], shift_factor = BLSFieldElement(PRIMITIVE_ROOT_OF_UNITY) shift_inv = div(BLSFieldElement(1), shift_factor) - extended_evaluation_rbo = [0] * (FIELD_ELEMENTS_PER_BLOB * 2) + extended_evaluation_rbo = [0] * (FIELD_ELEMENTS_PER_EXT_BLOB) for cell_id, cell in zip(cell_ids, cells): start = cell_id * FIELD_ELEMENTS_PER_CELL end = (cell_id + 1) * FIELD_ELEMENTS_PER_CELL @@ -588,7 +589,7 @@ def recover_original_data(eval_shifted_extended_evaluation: Sequence[BLSFieldEle def recover_polynomial(cell_ids: Sequence[CellID], cells_bytes: Sequence[Vector[Bytes32, FIELD_ELEMENTS_PER_CELL]]) -> Polynomial: """ - Recover original polynomial from 2 * FIELD_ELEMENTS_PER_CELL evaluations, half of which can be missing. This + Recover original polynomial from FIELD_ELEMENTS_PER_EXT_BLOB evaluations, half of which can be missing. This algorithm uses FFTs to recover cells faster than using Lagrange implementation, as can be seen here: https://ethresear.ch/t/reed-solomon-erasure-code-recovery-in-n-log-2-n-time-with-ffts/3039 @@ -604,7 +605,7 @@ def recover_polynomial(cell_ids: Sequence[CellID], assert len(cell_ids) == len(set(cell_ids)) # Get the extended domain - roots_of_unity_extended = compute_roots_of_unity(2 * FIELD_ELEMENTS_PER_BLOB) + roots_of_unity_extended = compute_roots_of_unity(FIELD_ELEMENTS_PER_EXT_BLOB) # Convert from bytes to cells cells = [bytes_to_cell(cell_bytes) for cell_bytes in cells_bytes] From 4b1106b39c1ffe9604ab2b961a40409e93bad2ef Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Mon, 5 Feb 2024 16:59:08 +0200 Subject: [PATCH 08/11] Update specs/_features/eip7594/polynomial-commitments-sampling.md Co-authored-by: Hsiao-Wei Wang --- .../_features/eip7594/polynomial-commitments-sampling.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index 742602544..fb723fb02 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -616,9 +616,12 @@ def recover_polynomial(cell_ids: Sequence[CellID], eval_shifted_extended_evaluation, eval_shifted_zero_poly, shift_inv = \ recover_shifted_data(cell_ids, cells, zero_poly_eval, zero_poly_coeff, roots_of_unity_extended) - reconstructed_data = \ - recover_original_data(eval_shifted_extended_evaluation, eval_shifted_zero_poly, shift_inv, - roots_of_unity_extended) + reconstructed_data = recover_original_data( + eval_shifted_extended_evaluation, + eval_shifted_zero_poly, + shift_inv, + roots_of_unity_extended, + ) for cell_id, cell in zip(cell_ids, cells): start = cell_id * FIELD_ELEMENTS_PER_CELL From f1dd735784c9992fac4260b2c595d9adc8cb2ffc Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 6 Feb 2024 13:10:24 +0200 Subject: [PATCH 09/11] Update specs/_features/eip7594/polynomial-commitments-sampling.md Co-authored-by: Justin Traglia <95511699+jtraglia@users.noreply.github.com> --- specs/_features/eip7594/polynomial-commitments-sampling.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index fb723fb02..7abc6cb34 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -495,7 +495,7 @@ def construct_vanishing_polynomial(missing_cell_ids: Sequence[CellID]) -> Tuple[ ]) # Extend vanishing polynomial to full domain using the closed form of the vanishing polynomial over a coset - zero_poly_coeff = [0] * (FIELD_ELEMENTS_PER_EXT_BLOB) + zero_poly_coeff = [0] * FIELD_ELEMENTS_PER_EXT_BLOB for i, coeff in enumerate(short_zero_poly): zero_poly_coeff[i * FIELD_ELEMENTS_PER_CELL] = coeff From 385b0f35ac3e924e6d1a4fdd46d0e93c4eb1b305 Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Tue, 6 Feb 2024 13:10:31 +0200 Subject: [PATCH 10/11] Update specs/_features/eip7594/polynomial-commitments-sampling.md Co-authored-by: Justin Traglia <95511699+jtraglia@users.noreply.github.com> --- specs/_features/eip7594/polynomial-commitments-sampling.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index 7abc6cb34..2b96716ad 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -533,7 +533,7 @@ def recover_shifted_data(cell_ids: Sequence[CellID], shift_factor = BLSFieldElement(PRIMITIVE_ROOT_OF_UNITY) shift_inv = div(BLSFieldElement(1), shift_factor) - extended_evaluation_rbo = [0] * (FIELD_ELEMENTS_PER_EXT_BLOB) + extended_evaluation_rbo = [0] * FIELD_ELEMENTS_PER_EXT_BLOB for cell_id, cell in zip(cell_ids, cells): start = cell_id * FIELD_ELEMENTS_PER_CELL end = (cell_id + 1) * FIELD_ELEMENTS_PER_CELL From 90afb238eddf37852813784544a7ddc923c637e6 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 14 Feb 2024 20:51:50 +0800 Subject: [PATCH 11/11] Update specs/_features/eip7594/polynomial-commitments-sampling.md --- .../_features/eip7594/polynomial-commitments-sampling.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/specs/_features/eip7594/polynomial-commitments-sampling.md b/specs/_features/eip7594/polynomial-commitments-sampling.md index 2b96716ad..9cd862589 100644 --- a/specs/_features/eip7594/polynomial-commitments-sampling.md +++ b/specs/_features/eip7594/polynomial-commitments-sampling.md @@ -613,8 +613,13 @@ def recover_polynomial(cell_ids: Sequence[CellID], missing_cell_ids = [cell_id for cell_id in range(CELLS_PER_BLOB) if cell_id not in cell_ids] zero_poly_coeff, zero_poly_eval, zero_poly_eval_brp = construct_vanishing_polynomial(missing_cell_ids) - eval_shifted_extended_evaluation, eval_shifted_zero_poly, shift_inv = \ - recover_shifted_data(cell_ids, cells, zero_poly_eval, zero_poly_coeff, roots_of_unity_extended) + eval_shifted_extended_evaluation, eval_shifted_zero_poly, shift_inv = recover_shifted_data( + cell_ids, + cells, + zero_poly_eval, + zero_poly_coeff, + roots_of_unity_extended, + ) reconstructed_data = recover_original_data( eval_shifted_extended_evaluation,