Merge branch 'dev' into sf-ispostfork

Makefile

@@ -41,6 +41,8 @@ CURRENT_DIR = ${CURDIR}
 LINTER_CONFIG_FILE = $(CURRENT_DIR)/linter.ini
 GENERATOR_ERROR_LOG_FILE = $(CURRENT_DIR)/$(TEST_VECTOR_DIR)/testgen_error_log.txt
 
+SCRIPTS_DIR = ${CURRENT_DIR}/scripts
+
 export DAPP_SKIP_BUILD:=1
 export DAPP_SRC:=$(SOLIDITY_DEPOSIT_CONTRACT_DIR)
 export DAPP_LIB:=$(SOLIDITY_DEPOSIT_CONTRACT_DIR)/lib
@@ -195,6 +197,14 @@ $(TEST_VECTOR_DIR):
 $(TEST_VECTOR_DIR)/:
 	$(info ignoring duplicate tests dir)
 
+gen_kzg_setups:
+	cd $(SCRIPTS_DIR); \
+	if ! test -d venv; then python3 -m venv venv; fi; \
+	. venv/bin/activate; \
+	pip3 install -r requirements.txt; \
+	python3 ./gen_kzg_trusted_setups.py --secret=1337 --length=4 --output-dir ${CURRENT_DIR}/presets/minimal/trusted_setups; \
+	python3 ./gen_kzg_trusted_setups.py --secret=1337 --length=4096 --output-dir ${CURRENT_DIR}/presets/mainnet/trusted_setups
+
 # For any generator, build it using the run_generator function.
 # (creation of output dir is a dependency)
 gen_%: $(TEST_VECTOR_DIR)

presets/minimal/trusted_setups/testing_trusted_setups.json

@@ -0,0 +1 @@
+{"setup_G1": ["0x97f1d3a73197d7942695638c4fa9ac0fc3688c4f9774b905a14e3a3f171bac586c55e83ff97a1aeffb3af00adb22c6bb", "0x854262641262cb9e056a8512808ea6864d903dbcad713fd6da8dddfa5ce40d85612c912063ace060ed8c4bf005bab839", "0x86f708eee5ae0cf40be36993e760d9cb3b2371f22db3209947c5d21ea68e55186b30871c50bf11ef29e5248bf42d5678", "0x94f9c0bafb23cbbf34a93a64243e3e0f934b57593651f3464de7dc174468123d9698f1b9dfa22bb5b6eb96eae002f29f"], "setup_G2": ["0x93e02b6052719f607dacd3a088274f65596bd0d09920b61ab5da61bbdc7f5049334cf11213945d57e5ac7d055d042b7e024aa2b2f08f0a91260805272dc51051c6e47ad4fa403b02b4510b647ae3d1770bac0326a805bbefd48056c8c121bdb8", "0x99aca9fb2f7760cecb892bf7262c176b334824f5727f680bba701a33e322cb6667531410dfc7c8e4321a3f0ea8af48cb1436638a2093123f046f0f504cc2a864825542873edbbc5d7ed17af125a4f2cf6433c6f4f61b81173726981dd989761d", "0x88e2e982982bf8231e747e9dfcd14c05bd02623d1332734d2af26246c6869fb56ee6c994843f593178a040495ba61f4a083b0e18110b1d9f5224783d8f9a895e8ee744e87929430e9ba96bd29251cbf61240b256d1525600f3d562894d93d659", "0xa2d33775e3d9e6af0d1b27d389e6c021a578e617a3d6627686db6288d4b3dffd7a847a00f7ef01828b7f42885b660e4204923402aca18fbae74ccd4e9c50dd8c2281b38dc09c022342ed1ac695d53f7081cb21f05fdfc0a3508c04759196fcd3"], "setup_G1_lagrange": ["0x91131b2e3c1e5f0b51df8970e67080032f411571b66d301436c46f25bbfddf9ca16756430dc470bdb0d85b47fedcdbc1", "0x934d35b2a46e169915718b77127b0d4efbacdad7fdde4593af7d21d37ebcb77fe6c8dde6b8a9537854d70ef1f291a585", "0x9410ca1d0342fe7419f02194281df45e1c1ff42fd8b439de5644cc312815c21ddd2e3eeb63fb807cf837e68b76668bd5", "0xb163df7e9baeb60f69b6ee5faa538c3a564b62eb8cde6a3616083c8cb2171eedd583c9143e7e916df59bf27da5e024e8"], "roots_of_unity": [1, 3465144826073652318776269530687742778270252468765361963008, 52435875175126190479447740508185965837690552500527637822603658699938581184512, 52435875175126190475982595682112313518914282969839895044333406231173219221505]}

scripts/gen_kzg_trusted_setups.py

@@ -0,0 +1,36 @@
+import os
+from pathlib import Path
+
+from eth2spec.utils.kzg import (
+    dump_kzg_trusted_setup_files,
+)
+
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--secret",
+        dest="secret",
+        type=int,
+        required=True,
+        help='the secret of trusted setup',
+    )
+    parser.add_argument(
+        "--length",
+        dest="length",
+        type=int,
+        required=True,
+        help='the length of trusted setup',
+    )
+    parser.add_argument(
+        "-o",
+        "--output-dir",
+        dest="output_dir",
+        required=True,
+        help='the output directory',
+    )
+    args = parser.parse_args()
+
+    dump_kzg_trusted_setup_files(args.secret, args.length, args.output_dir)

scripts/requirements.txt

@@ -0,0 +1 @@
+../[generator]

setup.py

@@ -7,13 +7,14 @@ import os
 import re
 import string
 import textwrap
-from typing import Dict, NamedTuple, List, Sequence, Optional, TypeVar
+from typing import Dict, NamedTuple, List, Sequence, Optional, TypeVar, Tuple
 from abc import ABC, abstractmethod
 import ast
 import subprocess
 import sys
 import copy
 from collections import OrderedDict
+import json
 
 
 # NOTE: have to programmatically include third-party dependencies in `setup.py`.
@@ -121,7 +122,7 @@ def _get_self_type_from_source(source: str) -> Optional[str]:
     return args[0].annotation.id
 
 
-def _get_class_info_from_source(source: str) -> (str, Optional[str]):
+def _get_class_info_from_source(source: str) -> Tuple[str, Optional[str]]:
     class_def = ast.parse(source).body[0]
     base = class_def.bases[0]
     if isinstance(base, ast.Name):
@@ -140,6 +141,28 @@ def _is_constant_id(name: str) -> bool:
     return all(map(lambda c: c in string.ascii_uppercase + '_' + string.digits, name[1:]))
 
 
+def _load_kzg_trusted_setups(preset_name):
+    """
+    [TODO] it's not the final mainnet trusted setup.
+    We will update it after the KZG ceremony.
+    """
+    file_path = str(Path(__file__).parent) + '/presets/' + preset_name + '/trusted_setups/testing_trusted_setups.json'
+
+    with open(file_path, 'r') as f:
+        json_data = json.load(f)
+
+    trusted_setup_G1 = json_data['setup_G1']
+    trusted_setup_G2 = json_data['setup_G2']
+    trusted_setup_G1_lagrange = json_data['setup_G1_lagrange']
+    roots_of_unity = json_data['roots_of_unity']
+
+    return trusted_setup_G1, trusted_setup_G2, trusted_setup_G1_lagrange, roots_of_unity
+
+ALL_KZG_SETUPS = {
+    'minimal': _load_kzg_trusted_setups('minimal'),
+    'mainnet': _load_kzg_trusted_setups('mainnet')
+}
+
 ETH2_SPEC_COMMENT_PREFIX = "eth2spec:"
 
 
@@ -167,7 +190,16 @@ def _parse_value(name: str, typed_value: str, type_hint: Optional[str]=None) ->
     return VariableDefinition(type_name=type_name, value=typed_value[i+1:-1], comment=comment, type_hint=type_hint)
 
 
-def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str]) -> SpecObject:
+def _update_constant_vars_with_kzg_setups(constant_vars, preset_name):
+    comment = "noqa: E501"
+    kzg_setups = ALL_KZG_SETUPS[preset_name]
+    constant_vars['KZG_SETUP_G1'] = VariableDefinition(constant_vars['KZG_SETUP_G1'].value, str(kzg_setups[0]), comment, None)
+    constant_vars['KZG_SETUP_G2'] = VariableDefinition(constant_vars['KZG_SETUP_G2'].value, str(kzg_setups[1]), comment, None)
+    constant_vars['KZG_SETUP_LAGRANGE'] = VariableDefinition(constant_vars['KZG_SETUP_LAGRANGE'].value, str(kzg_setups[2]), comment, None)
+    constant_vars['ROOTS_OF_UNITY'] = VariableDefinition(constant_vars['ROOTS_OF_UNITY'].value, str(kzg_setups[3]), comment, None)
+
+
+def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], preset_name=str) -> SpecObject:
     functions: Dict[str, str] = {}
     protocols: Dict[str, ProtocolDefinition] = {}
     constant_vars: Dict[str, VariableDefinition] = {}
@@ -256,6 +288,10 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str]) ->
             if comment == "skip":
                 should_skip = True
 
+    # Load KZG trusted setup from files
+    if any('KZG_SETUP' in name for name in constant_vars):
+        _update_constant_vars_with_kzg_setups(constant_vars, preset_name)
+
     return SpecObject(
         functions=functions,
         protocols=protocols,
@@ -629,7 +665,6 @@ def retrieve_blobs_sidecar(slot: Slot, beacon_block_root: Root) -> Optional[Blob
         return {**super().hardcoded_custom_type_dep_constants(spec_object), **constants}
 
 
-
 spec_builders = {
     builder.fork: builder
     for builder in (Phase0SpecBuilder, AltairSpecBuilder, BellatrixSpecBuilder, CapellaSpecBuilder, EIP4844SpecBuilder)
@@ -880,7 +915,7 @@ def _build_spec(preset_name: str, fork: str,
                 source_files: Sequence[Path], preset_files: Sequence[Path], config_file: Path) -> str:
     preset = load_preset(preset_files)
     config = load_config(config_file)
-    all_specs = [get_spec(spec, preset, config) for spec in source_files]
+    all_specs = [get_spec(spec, preset, config, preset_name) for spec in source_files]
 
     spec_object = all_specs[0]
     for value in all_specs[1:]:

specs/eip4844/beacon-chain.md

@@ -12,7 +12,6 @@
 - [Custom types](#custom-types)
 - [Constants](#constants)
   - [Blob](#blob)
-  - [Domain types](#domain-types)
 - [Preset](#preset)
   - [Execution](#execution)
 - [Configuration](#configuration)
@@ -57,12 +56,6 @@ This upgrade adds blobs to the beacon chain as part of EIP-4844.
 | `BLOB_TX_TYPE` | `uint8(0x05)` |
 | `VERSIONED_HASH_VERSION_KZG` | `Bytes1(0x01)` | 
 
-### Domain types
-
-| Name | Value |
-| - | - |
-| `DOMAIN_BLOBS_SIDECAR` | `DomainType('0x0a000000')` |
-
 ## Preset
 
 ### Execution
@@ -203,10 +196,10 @@ See [the full details of `blob_versioned_hashes` offset calculation](https://gis
 def tx_peek_blob_versioned_hashes(opaque_tx: Transaction) -> Sequence[VersionedHash]:
     assert opaque_tx[0] == BLOB_TX_TYPE
     message_offset = 1 + uint32.decode_bytes(opaque_tx[1:5])
-    # field offset: 32 + 8 + 32 + 32 + 8 + 4 + 32 + 4 + 4 = 156
+    # field offset: 32 + 8 + 32 + 32 + 8 + 4 + 32 + 4 + 4 + 32 = 188
     blob_versioned_hashes_offset = (
         message_offset
-        + uint32.decode_bytes(opaque_tx[(message_offset + 156):(message_offset + 160)])
+        + uint32.decode_bytes(opaque_tx[(message_offset + 188):(message_offset + 192)])
     )
     return [
         VersionedHash(opaque_tx[x:(x + 32)])

specs/eip4844/polynomial-commitments.md

@@ -329,7 +329,7 @@ def compute_kzg_proof(polynomial: Polynomial, z: BLSFieldElement) -> KZGProof:
 
     # Make sure we won't divide by zero during division
     assert z not in ROOTS_OF_UNITY
-    denominator_poly = [(x - z) % BLS_MODULUS for x in bit_reversal_permutation(ROOTS_OF_UNITY)]
+    denominator_poly = [(int(x) - z) % BLS_MODULUS for x in bit_reversal_permutation(ROOTS_OF_UNITY)]
 
     # Calculate quotient polynomial by doing point-by-point division
     quotient_polynomial = [div(a, b) for a, b in zip(polynomial_shifted, denominator_poly)]

tests/core/pyspec/eth2spec/test/eip4844/unittests/test_offset.py

@@ -0,0 +1,23 @@
+
+from eth2spec.test.helpers.constants import (
+    EIP4844,
+    MINIMAL,
+)
+from eth2spec.test.helpers.sharding import (
+    get_sample_opaque_tx,
+)
+from eth2spec.test.context import (
+    with_phases,
+    spec_state_test,
+    with_presets,
+)
+
+
+@with_phases([EIP4844])
+@spec_state_test
+@with_presets([MINIMAL])
+def test_tx_peek_blob_versioned_hashes(spec, state):
+    otx, blobs, commitments = get_sample_opaque_tx(spec)
+    data_hashes = spec.tx_peek_blob_versioned_hashes(otx)
+    expected = [spec.kzg_commitment_to_versioned_hash(blob_commitment) for blob_commitment in commitments]
+    assert expected == data_hashes

tests/core/pyspec/eth2spec/test/helpers/sharding.py

@@ -34,13 +34,14 @@ class ECDSASignature(Container):
 class BlobTransaction(Container):
     chain_id: uint256
     nonce: uint64
-    priority_fee_per_gas: uint256
-    max_basefee_per_gas: uint256
+    max_priority_fee_per_gas: uint256
+    max_fee_per_gas: uint256
     gas: uint64
     to: Union[None, Bytes20]  # Address = Bytes20
     value: uint256
     data: ByteList[MAX_CALLDATA_SIZE]
     access_list: List[AccessTuple, MAX_ACCESS_LIST_SIZE]
+    max_fee_per_data_gas: uint256
     blob_versioned_hashes: List[Bytes32, MAX_VERSIONED_HASHES_LIST_SIZE]
 
 

tests/core/pyspec/eth2spec/utils/bls.py

@@ -5,6 +5,7 @@ from py_ecc.optimized_bls12_381 import (  # noqa: F401
     G2,
     Z1,
     Z2,
+    FQ,
     add,
     multiply,
     neg,

tests/core/pyspec/eth2spec/utils/kzg.py

@@ -1,6 +1,15 @@
 # Ref:
 # - https://github.com/ethereum/research/blob/8f084630528ba33d92b2bc05edf5338dd193c6f1/trusted_setup/trusted_setup.py
 # - https://github.com/asn-d6/kzgverify
+import json
+import os
+from typing import (
+    Tuple,
+    Sequence,
+)
+from pathlib import Path
+
+from eth_utils import encode_hex
 from py_ecc.optimized_bls12_381 import (  # noqa: F401
     G1,
     G2,
@@ -11,13 +20,16 @@ from py_ecc.optimized_bls12_381 import (  # noqa: F401
     multiply,
     neg,
 )
+from py_ecc.typing import (
+    Optimized_Point3D,
+)
 from eth2spec.utils import bls
 
 
 PRIMITIVE_ROOT_OF_UNITY = 7
 
 
-def generate_setup(generator, secret, length):
+def generate_setup(generator: Optimized_Point3D, secret: int, length: int) -> Tuple[Optimized_Point3D]:
     """
     Generate trusted setup of ``generator`` in ``length``.
     """
@@ -27,7 +39,7 @@ def generate_setup(generator, secret, length):
     return tuple(result)
 
 
-def fft(vals, modulus, domain):
+def fft(vals: Sequence[Optimized_Point3D], modulus: int, domain: int) -> Sequence[Optimized_Point3D]:
     """
     FFT for group elements
     """
@@ -43,7 +55,7 @@ def fft(vals, modulus, domain):
     return o
 
 
-def compute_root_of_unity(length) -> int:
+def compute_root_of_unity(length: int) -> int:
     """
     Generate a w such that ``w**length = 1``.
     """
@@ -51,11 +63,12 @@ def compute_root_of_unity(length) -> int:
     return pow(PRIMITIVE_ROOT_OF_UNITY, (BLS_MODULUS - 1) // length, BLS_MODULUS)
 
 
-def compute_roots_of_unity(field_elements_per_blob):
+def compute_roots_of_unity(field_elements_per_blob: int) -> Tuple[int]:
     """
     Compute a list of roots of unity for a given order.
     The order must divide the BLS multiplicative group order, i.e. BLS_MODULUS - 1
     """
+    field_elements_per_blob = int(field_elements_per_blob)  # to non-SSZ int
     assert (BLS_MODULUS - 1) % field_elements_per_blob == 0
     root_of_unity = compute_root_of_unity(length=field_elements_per_blob)
 
@@ -64,10 +77,10 @@ def compute_roots_of_unity(field_elements_per_blob):
     for _ in range(field_elements_per_blob):
         roots.append(current_root_of_unity)
         current_root_of_unity = current_root_of_unity * root_of_unity % BLS_MODULUS
-    return roots
+    return tuple(roots)
 
 
-def get_lagrange(setup):
+def get_lagrange(setup: Sequence[Optimized_Point3D]) -> Tuple[bytes]:
     """
     Convert a G1 or G2 portion of a setup into the Lagrange basis.
     """
@@ -77,4 +90,34 @@ def get_lagrange(setup):
     # TODO: introduce an IFFT function for simplicity
     fft_output = fft(setup, BLS_MODULUS, domain)
     inv_length = pow(len(setup), BLS_MODULUS - 2, BLS_MODULUS)
-    return [bls.G1_to_bytes48(multiply(fft_output[-i], inv_length)) for i in range(len(fft_output))]
+    return tuple(bls.G1_to_bytes48(multiply(fft_output[-i], inv_length)) for i in range(len(fft_output)))
+
+
+def dump_kzg_trusted_setup_files(secret: int, length: int, output_dir: str) -> None:
+    setup_g1 = generate_setup(bls.G1, secret, length)
+    setup_g2 = generate_setup(bls.G2, secret, length)
+    setup_g1_lagrange = get_lagrange(setup_g1)
+    roots_of_unity = compute_roots_of_unity(length)
+
+    serailized_setup_g1 = [encode_hex(bls.G1_to_bytes48(p)) for p in setup_g1]
+    serialized_setup_g2 = [encode_hex(bls.G2_to_bytes96(p)) for p in setup_g2]
+    serialized_setup_g1_lagrange = [encode_hex(x) for x in setup_g1_lagrange]
+
+    output_dir_path = Path(output_dir)
+
+    if not os.path.exists(output_dir_path):
+        os.makedirs(output_dir_path)
+        print("Created directory: ", output_dir_path)
+
+    file_path = output_dir_path / 'testing_trusted_setups.json'
+
+    with open(file_path, 'w+') as f:
+        json.dump(
+            {
+                "setup_G1": serailized_setup_g1,
+                "setup_G2": serialized_setup_g2,
+                "setup_G1_lagrange": serialized_setup_g1_lagrange,
+                "roots_of_unity": roots_of_unity,
+            }, f)
+
+    print(f'Generated trusted setup file: {file_path}\n')