From 819bb6d21d454974930281ccafb94f71478a9bfd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 24 Jul 2020 01:55:58 +0800 Subject: [PATCH 1/4] Use Milagro BLS in CI --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5ef25289e..f5cb1c702 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ find_test: pyspec citest: pyspec mkdir -p tests/core/pyspec/test-reports/eth2spec; . venv/bin/activate; cd $(PY_SPEC_DIR); \ - python -m pytest -n 4 --disable-bls --junitxml=eth2spec/test_results.xml eth2spec + python -m pytest -n 4 --bls-type=milagro --junitxml=eth2spec/test_results.xml eth2spec open_cov: ((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) & From e7f070d5251a36e13934bf7fd83751e7eb68f5b5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 22 Jul 2020 23:06:16 +0800 Subject: [PATCH 2/4] Use `hash_tree_root` to generate `ShardBlockHeader.body_root` Include the mix-in length to the proof. --- setup.py | 6 ---- specs/phase1/custody-game.md | 12 ++------ specs/phase1/validator.md | 2 +- .../pyspec/eth2spec/test/helpers/custody.py | 12 ++++---- .../test_process_chunk_challenge.py | 29 +++++++++++++++++++ 5 files changed, 40 insertions(+), 21 deletions(-) diff --git a/setup.py b/setup.py index bac58e16a..59740a576 100644 --- a/setup.py +++ b/setup.py @@ -225,12 +225,6 @@ get_attesting_indices = cache_this( PHASE1_SUNDRY_FUNCTIONS = ''' -def get_block_data_merkle_root(data: ByteList) -> Root: - # To get the Merkle root of the block data, we need the Merkle root without the length mix-in - # The below implements this in the Remerkleable framework - return Root(data.get_backing().get_left().merkle_root()) - - _get_start_shard = get_start_shard get_start_shard = cache_this( lambda state, slot: (state.validators.hash_tree_root(), slot), diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 3688442a0..8774dc337 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -25,7 +25,6 @@ - [`CustodyKeyReveal`](#custodykeyreveal) - [`EarlyDerivedSecretReveal`](#earlyderivedsecretreveal) - [Helpers](#helpers) - - [`get_block_data_merkle_root`](#get_block_data_merkle_root) - [`replace_empty_or_append`](#replace_empty_or_append) - [`legendre_bit`](#legendre_bit) - [`get_custody_atoms`](#get_custody_atoms) @@ -127,7 +126,7 @@ class CustodyChunkResponse(Container): challenge_index: uint64 chunk_index: uint64 chunk: ByteVector[BYTES_PER_CUSTODY_CHUNK] - branch: Vector[Root, CUSTODY_RESPONSE_DEPTH] + branch: Vector[Root, CUSTODY_RESPONSE_DEPTH + 1] ``` #### `CustodySlashing` @@ -180,13 +179,8 @@ class EarlyDerivedSecretReveal(Container): mask: Bytes32 ``` - ## Helpers -### `get_block_data_merkle_root` - -`get_block_data_merkle_root(data: ByteList) -> Root` is the function that returns the Merkle root of the block data without the length mix-in. - ### `replace_empty_or_append` ```python @@ -381,7 +375,7 @@ def process_chunk_challenge_response(state: BeaconState, assert is_valid_merkle_branch( leaf=hash_tree_root(response.chunk), branch=response.branch, - depth=CUSTODY_RESPONSE_DEPTH, + depth=CUSTODY_RESPONSE_DEPTH + 1, # Add 1 for the List length mix-in index=response.chunk_index, root=challenge.data_root, ) @@ -523,7 +517,7 @@ def process_custody_slashing(state: BeaconState, signed_custody_slashing: Signed assert hash_tree_root(shard_transition) == attestation.data.shard_transition_root # Verify that the provided data matches the shard-transition assert ( - get_block_data_merkle_root(custody_slashing.data) + hash_tree_root(custody_slashing.data) == shard_transition.shard_data_roots[custody_slashing.data_index] ) assert len(custody_slashing.data) == shard_transition.shard_block_lengths[custody_slashing.data_index] diff --git a/specs/phase1/validator.md b/specs/phase1/validator.md index ad2fa09e3..7e22c5405 100644 --- a/specs/phase1/validator.md +++ b/specs/phase1/validator.md @@ -294,7 +294,7 @@ def get_shard_transition_fields( for slot in offset_slots: if slot in shard_block_slots: shard_block = shard_blocks[shard_block_slots.index(slot)] - shard_data_roots.append(get_block_data_merkle_root(shard_block.message.body)) + shard_data_roots.append(hash_tree_root(shard_block.message.body)) else: shard_block = SignedShardBlock(message=ShardBlock(slot=slot, shard=shard)) shard_data_roots.append(Root()) diff --git a/tests/core/pyspec/eth2spec/test/helpers/custody.py b/tests/core/pyspec/eth2spec/test/helpers/custody.py index 4cb9947fa..c1e493931 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/custody.py +++ b/tests/core/pyspec/eth2spec/test/helpers/custody.py @@ -113,7 +113,7 @@ def get_valid_chunk_challenge(spec, state, attestation, shard_transition, data_i def custody_chunkify(spec, x): chunks = [bytes(x[i:i + spec.BYTES_PER_CUSTODY_CHUNK]) for i in range(0, len(x), spec.BYTES_PER_CUSTODY_CHUNK)] chunks[-1] = chunks[-1].ljust(spec.BYTES_PER_CUSTODY_CHUNK, b"\0") - return chunks + return [ByteVector[spec.BYTES_PER_CUSTODY_CHUNK](c) for c in chunks] def build_proof(anchor, leaf_index): @@ -149,12 +149,14 @@ def get_valid_custody_chunk_response(spec, state, chunk_challenge, challenge_ind chunk_index = chunk_challenge.chunk_index - data_branch = build_proof(custody_data_block.get_backing().get_left(), chunk_index + 2**spec.CUSTODY_RESPONSE_DEPTH) + leaf_index = chunk_index + 2**spec.CUSTODY_RESPONSE_DEPTH + serialized_length = (len(custody_data_block)).to_bytes(32, 'little') + data_branch = build_proof(custody_data_block.get_backing().get_left(), leaf_index) + [serialized_length] return spec.CustodyChunkResponse( challenge_index=challenge_index, chunk_index=chunk_index, - chunk=ByteVector[spec.BYTES_PER_CUSTODY_CHUNK](chunks[chunk_index]), + chunk=chunks[chunk_index], branch=data_branch, ) @@ -165,7 +167,7 @@ def get_custody_test_vector(bytelength, offset=0): def get_sample_shard_transition(spec, start_slot, block_lengths): - b = [spec.get_block_data_merkle_root(ByteList[spec.MAX_SHARD_BLOCK_SIZE](get_custody_test_vector(x))) + b = [spec.hash_tree_root(ByteList[spec.MAX_SHARD_BLOCK_SIZE](get_custody_test_vector(x))) for x in block_lengths] shard_transition = spec.ShardTransition( start_slot=start_slot, @@ -200,5 +202,5 @@ def get_custody_slashable_shard_transition(spec, start_slot, block_lengths, cust slashable_test_vector = get_custody_slashable_test_vector(spec, custody_secret, block_lengths[0], slashable=slashable) block_data = ByteList[spec.MAX_SHARD_BLOCK_SIZE](slashable_test_vector) - shard_transition.shard_data_roots[0] = spec.get_block_data_merkle_root(block_data) + shard_transition.shard_data_roots[0] = spec.hash_tree_root(block_data) return shard_transition, slashable_test_vector diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py index 4ef1d667a..322aaf5e2 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_chunk_challenge.py @@ -248,6 +248,35 @@ def test_custody_response(spec, state): yield from run_custody_chunk_response_processing(spec, state, custody_response) +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_custody_response_chunk_index_2(spec, state): + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH) + + shard = 0 + offset_slots = spec.get_offset_slots(state, shard) + shard_transition = get_sample_shard_transition(spec, state.slot, [2**15 // 3] * len(offset_slots)) + attestation = get_valid_on_time_attestation(spec, state, index=shard, signed=True, + shard_transition=shard_transition) + + transition_to(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + _, _, _ = run_attestation_processing(spec, state, attestation) + + transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH * (spec.EPOCHS_PER_CUSTODY_PERIOD - 1)) + + challenge = get_valid_chunk_challenge(spec, state, attestation, shard_transition, chunk_index=2) + + _, _, _ = run_chunk_challenge_processing(spec, state, challenge) + + chunk_challenge_index = state.custody_chunk_challenge_index - 1 + + custody_response = get_valid_custody_chunk_response( + spec, state, challenge, chunk_challenge_index, block_length_or_custody_data=2**15 // 3) + + yield from run_custody_chunk_response_processing(spec, state, custody_response) + + @with_all_phases_except([PHASE0]) @spec_state_test def test_custody_response_multiple_epochs(spec, state): From a8370f7448786a3d1932f8f2f71b746d5fc30bc8 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 28 Jul 2020 21:51:00 +0800 Subject: [PATCH 3/4] Apply refactoring suggestions from Danny --- specs/phase1/custody-game.md | 6 +----- tests/core/pyspec/eth2spec/test/helpers/custody.py | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 8774dc337..8d9c89c56 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -516,12 +516,8 @@ def process_custody_slashing(state: BeaconState, signed_custody_slashing: Signed shard_transition = custody_slashing.shard_transition assert hash_tree_root(shard_transition) == attestation.data.shard_transition_root # Verify that the provided data matches the shard-transition - assert ( - hash_tree_root(custody_slashing.data) - == shard_transition.shard_data_roots[custody_slashing.data_index] - ) assert len(custody_slashing.data) == shard_transition.shard_block_lengths[custody_slashing.data_index] - + assert hash_tree_root(custody_slashing.data) == shard_transition.shard_data_roots[custody_slashing.data_index] # Verify existence and participation of claimed malefactor attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) assert custody_slashing.malefactor_index in attesters diff --git a/tests/core/pyspec/eth2spec/test/helpers/custody.py b/tests/core/pyspec/eth2spec/test/helpers/custody.py index c1e493931..162ca3263 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/custody.py +++ b/tests/core/pyspec/eth2spec/test/helpers/custody.py @@ -150,7 +150,7 @@ def get_valid_custody_chunk_response(spec, state, chunk_challenge, challenge_ind chunk_index = chunk_challenge.chunk_index leaf_index = chunk_index + 2**spec.CUSTODY_RESPONSE_DEPTH - serialized_length = (len(custody_data_block)).to_bytes(32, 'little') + serialized_length = len(custody_data_block).to_bytes(32, 'little') data_branch = build_proof(custody_data_block.get_backing().get_left(), leaf_index) + [serialized_length] return spec.CustodyChunkResponse( From b58633ce5dcca48aa51b9f04bf9daa230da0da65 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 29 Jul 2020 02:06:25 +0800 Subject: [PATCH 4/4] Quick fix the phase 1 configurations and tests (#1990) * Quick fix the configurations * Remove the unused `CUSTODY_RESPONSE_DEADLINE` --- configs/mainnet/phase1.yaml | 2 -- configs/minimal/phase1.yaml | 8 +++----- specs/phase1/custody-game.md | 1 - .../block_processing/test_process_custody_slashing.py | 2 +- 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/configs/mainnet/phase1.yaml b/configs/mainnet/phase1.yaml index 8c0800957..78630357f 100644 --- a/configs/mainnet/phase1.yaml +++ b/configs/mainnet/phase1.yaml @@ -69,8 +69,6 @@ EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 32768 EPOCHS_PER_CUSTODY_PERIOD: 16384 # 2**11 (= 2,048) epochs, ~9 days CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 -# 2**14 (= 16,384) epochs -CUSTODY_RESPONSE_DEADLINE: 16384 # 2**15 (= 32,768) epochs, ~146 days MAX_CHUNK_CHALLENGE_DELAY: 32768 diff --git a/configs/minimal/phase1.yaml b/configs/minimal/phase1.yaml index 7fbc7e5a3..b4b738660 100644 --- a/configs/minimal/phase1.yaml +++ b/configs/minimal/phase1.yaml @@ -66,15 +66,13 @@ DOMAIN_LIGHT_AGGREGATE_AND_PROOF: 0x85000000 # 2**1 (= 2) epochs RANDAO_PENALTY_EPOCHS: 2 # [customized] quicker for testing -EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 128 +EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 64 # [customized] quicker for testing -EPOCHS_PER_CUSTODY_PERIOD: 64 +EPOCHS_PER_CUSTODY_PERIOD: 32 # [customized] quicker for testing CUSTODY_PERIOD_TO_RANDAO_PADDING: 8 -# [customized] quicker for testing -CUSTODY_RESPONSE_DEADLINE: 128 # [customize for faster testing] -MAX_CHUNK_CHALLENGE_DELAY: 128 +MAX_CHUNK_CHALLENGE_DELAY: 64 # Misc parameters diff --git a/specs/phase1/custody-game.md b/specs/phase1/custody-game.md index 8d9c89c56..6746c26e2 100644 --- a/specs/phase1/custody-game.md +++ b/specs/phase1/custody-game.md @@ -72,7 +72,6 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | `EPOCHS_PER_CUSTODY_PERIOD` | `2**14` (= 16,384) | epochs | ~73 days | | `CUSTODY_PERIOD_TO_RANDAO_PADDING` | `2**11` (= 2,048) | epochs | ~9 days | | `MAX_CHUNK_CHALLENGE_DELAY` | `2**15` (= 32,768) | epochs | ~146 days | -| `CHUNK_RESPONSE_DEADLINE` | `2**14` (= 16,384) | epochs | ~73 days | ### Max operations per block diff --git a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py index 07cd76996..861922884 100644 --- a/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py +++ b/tests/core/pyspec/eth2spec/test/phase1/block_processing/test_process_custody_slashing.py @@ -124,7 +124,7 @@ def test_multiple_epochs_custody(spec, state): @with_all_phases_except([PHASE0]) @spec_state_test def test_many_epochs_custody(spec, state): - yield from run_standard_custody_slashing_test(spec, state, shard_lateness=spec.SLOTS_PER_EPOCH * 10) + yield from run_standard_custody_slashing_test(spec, state, shard_lateness=spec.SLOTS_PER_EPOCH * 5) @with_all_phases_except([PHASE0])