From 6422acdcdd80afc3ffbf88fb6cdafecbb7c1eea7 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Wed, 3 Jul 2019 13:31:03 +0100 Subject: [PATCH 01/26] Cosmetic change: Define Bitlist/Bitvector serialization using bytes, not bigints --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index d5855a755..a4abef966 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -41,11 +41,16 @@ def serialize(obj: SSZValue): if isinstance(obj, BasicValue): return serialize_basic(obj) elif isinstance(obj, Bitvector): - as_integer = sum([obj[i] << i for i in range(len(obj))]) - return as_integer.to_bytes((len(obj) + 7) // 8, "little") + as_bytearray = [0] * ((len(obj) + 7) // 8) + for i in range(len(obj)): + as_bytearray[i // 8] |= obj[i] << (i % 8) + return bytes(as_bytearray) elif isinstance(obj, Bitlist): - as_integer = (1 << len(obj)) + sum([obj[i] << i for i in range(len(obj))]) - return as_integer.to_bytes((as_integer.bit_length() + 7) // 8, "little") + as_bytearray = [0] * (len(obj) // 8 + 1) + for i in range(len(obj)): + as_bytearray[i // 8] |= obj[i] << (i % 8) + as_bytearray[len(obj) // 8] |= 1 << (len(obj) % 8) + return bytes(as_bytearray) elif isinstance(obj, Series): return encode_series(obj) else: @@ -92,12 +97,11 @@ def encode_series(values: Series): def pack(values: Series): if isinstance(values, bytes): # Bytes and BytesN are already packed return values - elif isinstance(values, Bitvector): - as_integer = sum([values[i] << i for i in range(len(values))]) - return as_integer.to_bytes((values.length + 7) // 8, "little") - elif isinstance(values, Bitlist): - as_integer = sum([values[i] << i for i in range(len(values))]) - return as_integer.to_bytes((values.length + 7) // 8, "little") + elif isinstance(values, Bitvector) or isinstance(values, Bitlist): + as_bytearray = [0] * ((len(values) + 7) // 8) + for i in range(len(values)): + as_bytearray[i // 8] |= values[i] << (i % 8) + return bytes(as_bytearray) return b''.join([serialize_basic(value) for value in values]) From 619b2a3573b3a0c83cc50e13adb7d97a3816539a Mon Sep 17 00:00:00 2001 From: dankrad Date: Wed, 3 Jul 2019 15:10:37 +0100 Subject: [PATCH 02/26] Update test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py Co-Authored-By: Diederik Loerakker --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index a4abef966..1e0c806d9 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -97,7 +97,7 @@ def encode_series(values: Series): def pack(values: Series): if isinstance(values, bytes): # Bytes and BytesN are already packed return values - elif isinstance(values, Bitvector) or isinstance(values, Bitlist): + elif isinstance(values, (Bitvector, Bitlist)): as_bytearray = [0] * ((len(values) + 7) // 8) for i in range(len(values)): as_bytearray[i // 8] |= values[i] << (i % 8) From b3d65368a1e5747155fe6cdcd557ba3efc023737 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jul 2019 20:38:18 +0800 Subject: [PATCH 03/26] `PERSISTENT_COMMITTEE_PERIOD` has been defined in phase 0 --- specs/core/1_shard-data-chains.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 613b4c4c2..330840598 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -13,7 +13,7 @@ - [Misc](#misc) - [Initial values](#initial-values) - [Time parameters](#time-parameters) - - [Signature domains](#signature-domains) + - [Signature domain types](#signature-domain-types) - [TODO PLACEHOLDER](#todo-placeholder) - [Data structures](#data-structures) - [`ShardBlockBody`](#shardblockbody) @@ -61,7 +61,6 @@ This document describes the shard data layer and the shard fork choice rule in P | Name | Value | Unit | Duration | | - | - | :-: | :-: | | `CROSSLINK_LOOKBACK` | `2**0` (= 1) | epochs | 6.2 minutes | -| `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | ~9 days | ### Signature domain types From bc39f39d5dd502ff8e317c85d9e8fbcfabf5b14b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jul 2019 20:41:00 +0800 Subject: [PATCH 04/26] Move `SECONDS_PER_SLOT` back to 0_beacon_chain spec so that the "duration" notes make more sense --- specs/core/0_beacon-chain.md | 1 + specs/core/0_fork-choice.md | 10 ---------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 975874d51..c0d52463a 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -207,6 +207,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | Unit | Duration | | - | - | :-: | :-: | +| `SECONDS_PER_SLOT` | `6` | seconds | 6 seconds | | `MIN_ATTESTATION_INCLUSION_DELAY` | `2**0` (= 1) | slots | 6 seconds | | `SLOTS_PER_EPOCH` | `2**6` (= 64) | slots | 6.4 minutes | | `MIN_SEED_LOOKAHEAD` | `2**0` (= 1) | epochs | 6.4 minutes | diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 9fd8ab53e..fed5457d7 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -8,8 +8,6 @@ - [Ethereum 2.0 Phase 0 -- Beacon Chain Fork Choice](#ethereum-20-phase-0----beacon-chain-fork-choice) - [Table of contents](#table-of-contents) - [Introduction](#introduction) - - [Configuration](#configuration) - - [Time parameters](#time-parameters) - [Fork choice](#fork-choice) - [Helpers](#helpers) - [`LatestMessage`](#latestmessage) @@ -29,14 +27,6 @@ This document is the beacon chain fork choice spec, part of Ethereum 2.0 Phase 0. It assumes the [beacon chain state transition function spec](./0_beacon-chain.md). -## Configuration - -### Time parameters - -| Name | Value | Unit | Duration | -| - | - | :-: | :-: | -| `SECONDS_PER_SLOT` | `6` | seconds | 6 seconds | - ## Fork choice The head block root associated with a `store` is defined as `get_head(store)`. At genesis, let `store = get_genesis_store(genesis_state)` and update `store` by running: From ff96eea3ac7042a36bbc91ae73799f859b77e9c9 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jul 2019 20:46:47 +0800 Subject: [PATCH 05/26] Add phase 1 domain to constant_presets files --- configs/constant_presets/mainnet.yaml | 3 +++ configs/constant_presets/minimal.yaml | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index 10ab26a00..d4e69dab5 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -128,3 +128,6 @@ DOMAIN_ATTESTATION: 0x02000000 DOMAIN_DEPOSIT: 0x03000000 DOMAIN_VOLUNTARY_EXIT: 0x04000000 DOMAIN_TRANSFER: 0x05000000 +DOMAIN_CUSTODY_BIT_CHALLENGE: 0x06000000 +DOMAIN_SHARD_PROPOSER: 0x80000000 +DOMAIN_SHARD_ATTESTER: 0x81000000 diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index b030333ff..34419a223 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -125,4 +125,7 @@ DOMAIN_RANDAO: 0x01000000 DOMAIN_ATTESTATION: 0x02000000 DOMAIN_DEPOSIT: 0x03000000 DOMAIN_VOLUNTARY_EXIT: 0x04000000 -DOMAIN_TRANSFER: 0x05000000 \ No newline at end of file +DOMAIN_TRANSFER: 0x05000000 +DOMAIN_CUSTODY_BIT_CHALLENGE: 0x06000000 +DOMAIN_SHARD_PROPOSER: 0x80000000 +DOMAIN_SHARD_ATTESTER: 0x81000000 From 733653f1695a01e23c6266205c8d05dcc698bb9a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jul 2019 20:52:58 +0800 Subject: [PATCH 06/26] Update some missing type hinting of phase 1 --- specs/core/1_custody-game.md | 16 ++++++++-------- specs/core/1_shard-data-chains.md | 20 ++++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 48100e7ba..f79977442 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -15,7 +15,7 @@ - [Time parameters](#time-parameters) - [Max operations per block](#max-operations-per-block) - [Reward and penalty quotients](#reward-and-penalty-quotients) - - [Signature domains](#signature-domains) + - [Signature domain types](#signature-domain-types) - [TODO PLACEHOLDER](#todo-placeholder) - [Data structures](#data-structures) - [Custody objects](#custody-objects) @@ -156,7 +156,7 @@ class CustodyChunkChallengeRecord(Container): challenger_index: ValidatorIndex responder_index: ValidatorIndex inclusion_epoch: Epoch - data_root: Bytes32 + data_root: Hash depth: uint64 chunk_index: uint64 ``` @@ -169,9 +169,9 @@ class CustodyBitChallengeRecord(Container): challenger_index: ValidatorIndex responder_index: ValidatorIndex inclusion_epoch: Epoch - data_root: Bytes32 + data_root: Hash chunk_count: uint64 - chunk_bits_merkle_root: Bytes32 + chunk_bits_merkle_root: Hash responder_key: BLSSignature ``` @@ -182,9 +182,9 @@ class CustodyResponse(Container): challenge_index: uint64 chunk_index: uint64 chunk: Vector[Bytes[PLACEHOLDER], BYTES_PER_CUSTODY_CHUNK] - data_branch: List[Bytes32, PLACEHOLDER] - chunk_bits_branch: List[Bytes32, PLACEHOLDER] - chunk_bits_leaf: Bytes32 + data_branch: List[Hash, PLACEHOLDER] + chunk_bits_branch: List[Hash, PLACEHOLDER] + chunk_bits_leaf: Hash ``` ### New beacon operations @@ -296,7 +296,7 @@ def get_custody_chunk_bit(key: BLSSignature, chunk: bytes) -> bool: ### `get_chunk_bits_root` ```python -def get_chunk_bits_root(chunk_bits: bytes) -> Bytes32: +def get_chunk_bits_root(chunk_bits: bytes) -> Hash: aggregated_bits = bytearray([0] * 32) for i in range(0, len(chunk_bits), 32): for j in range(32): diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 330840598..fb4f353ef 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -93,7 +93,7 @@ class ShardAttestation(Container): class data(Container): slot: Slot shard: Shard - shard_block_root: Bytes32 + shard_block_root: Hash aggregation_bits: Bitlist[PLACEHOLDER] aggregate_signature: BLSSignature ``` @@ -104,10 +104,10 @@ class ShardAttestation(Container): class ShardBlock(Container): slot: Slot shard: Shard - beacon_chain_root: Bytes32 - parent_root: Bytes32 + beacon_chain_root: Hash + parent_root: Hash data: ShardBlockBody - state_root: Bytes32 + state_root: Hash attestations: List[ShardAttestation, PLACEHOLDER] signature: BLSSignature ``` @@ -118,10 +118,10 @@ class ShardBlock(Container): class ShardBlockHeader(Container): slot: Slot shard: Shard - beacon_chain_root: Bytes32 - parent_root: Bytes32 - body_root: Bytes32 - state_root: Bytes32 + beacon_chain_root: Hash + parent_root: Hash + body_root: Hash + state_root: Hash attestations: List[ShardAttestation, PLACEHOLDER] signature: BLSSignature ``` @@ -249,7 +249,7 @@ def verify_shard_attestation_signature(state: BeaconState, ### `compute_crosslink_data_root` ```python -def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Bytes32: +def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Hash: def is_power_of_two(value: uint64) -> bool: return (value > 0) and (value & (value - 1) == 0) @@ -258,7 +258,7 @@ def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Bytes32: values.append(b'\x00' * BYTES_PER_SHARD_BLOCK_BODY) return values - def hash_tree_root_of_bytes(data: bytes) -> bytes: + def hash_tree_root_of_bytes(data: bytes) -> Hash: return hash_tree_root([data[i:i + 32] for i in range(0, len(data), 32)]) def zpad(data: bytes, length: uint64) -> bytes: From c8c810c0e107a20a6ed086aa3187d6e1ddeeb330 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Fri, 5 Jul 2019 15:03:37 +0100 Subject: [PATCH 07/26] Minor fixes --- specs/core/0_beacon-chain.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 975874d51..75fa127f7 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -540,8 +540,6 @@ class BeaconState(Container): ### Math -#### `int_to_bytes` - #### `integer_squareroot` ```python @@ -560,13 +558,15 @@ def integer_squareroot(n: uint64) -> uint64: #### `xor` ```python -def xor(bytes1: Bytes32, bytes2: Bytes32) -> Bytes32: +def xor(bytes_1: Bytes32, bytes_2: Bytes32) -> Bytes32: """ Return the exclusive-or of two 32-byte strings. """ - return Bytes32(a ^ b for a, b in zip(bytes1, bytes2)) + return Bytes32(a ^ b for a, b in zip(bytes_1, bytes_2)) ``` +#### `int_to_bytes` + ```python def int_to_bytes(n: uint64, length: uint64) -> bytes: """ @@ -653,7 +653,7 @@ def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationDa ```python def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: """ - Verify validity of ``indexed_attestation``. + Check indices and signature of ``indexed_attestation``. """ bit_0_indices = indexed_attestation.custody_bit_0_indices bit_1_indices = indexed_attestation.custody_bit_1_indices @@ -989,7 +989,7 @@ def get_total_balance(state: BeaconState, indices: Set[ValidatorIndex]) -> Gwei: """ Return the combined effective balance of the ``indices``. (1 Gwei minimum to avoid divisions by zero.) """ - return Gwei(max(sum([state.validators[index].effective_balance for index in indices]), 1)) + return Gwei(max(1, sum([state.validators[index].effective_balance for index in indices]))) ``` #### `get_total_active_balance` From 591f9658d349c8a970865d53e716ea7aed9e24d8 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Fri, 5 Jul 2019 15:04:57 +0100 Subject: [PATCH 08/26] Copy edit --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 75fa127f7..441d6f72b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -653,7 +653,7 @@ def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationDa ```python def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: """ - Check indices and signature of ``indexed_attestation``. + Check if ``indexed_attestation`` has valid indices and signature. """ bit_0_indices = indexed_attestation.custody_bit_0_indices bit_1_indices = indexed_attestation.custody_bit_1_indices From 0eadf61631410281514ee0dce0bedeee9ccf49bd Mon Sep 17 00:00:00 2001 From: Dean Eigenmann Date: Wed, 10 Jul 2019 13:11:34 -0400 Subject: [PATCH 09/26] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 975874d51..357ad44fc 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -49,9 +49,9 @@ - [`BeaconState`](#beaconstate) - [Helper functions](#helper-functions) - [Math](#math) - - [`int_to_bytes`](#int_to_bytes) - [`integer_squareroot`](#integer_squareroot) - [`xor`](#xor) + - [`int_to_bytes`](#int_to_bytes) - [`bytes_to_int`](#bytes_to_int) - [Crypto](#crypto) - [`hash`](#hash) @@ -540,8 +540,6 @@ class BeaconState(Container): ### Math -#### `int_to_bytes` - #### `integer_squareroot` ```python @@ -567,6 +565,8 @@ def xor(bytes1: Bytes32, bytes2: Bytes32) -> Bytes32: return Bytes32(a ^ b for a, b in zip(bytes1, bytes2)) ``` +#### `int_to_bytes` + ```python def int_to_bytes(n: uint64, length: uint64) -> bytes: """ From b80d6e049599355f3ac214d0aba5a645baa4a570 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Wed, 10 Jul 2019 17:00:11 -0700 Subject: [PATCH 10/26] Avoid unnecessary materialization of list There is a realization of a `list` in the `get_unslashed_attesting_indices` helper that is unnecessary. The functionality in this PR is the same so this change should only really be cosmetic wrt the spec. --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index c0d52463a..e2d734e13 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1271,7 +1271,7 @@ def get_unslashed_attesting_indices(state: BeaconState, output = set() # type: Set[ValidatorIndex] for a in attestations: output = output.union(get_attesting_indices(state, a.data, a.aggregation_bits)) - return set(filter(lambda index: not state.validators[index].slashed, list(output))) + return set(filter(lambda index: not state.validators[index].slashed, output)) ``` ```python From 4def681a4ee05f4852828bcafc39daa832d4036f Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Wed, 10 Jul 2019 17:05:49 -0700 Subject: [PATCH 11/26] Remove another unnecessary list materialization --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index e2d734e13..569d2a80c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1284,10 +1284,10 @@ def get_winning_crosslink_and_attesting_indices(state: BeaconState, epoch: Epoch, shard: Shard) -> Tuple[Crosslink, Set[ValidatorIndex]]: attestations = [a for a in get_matching_source_attestations(state, epoch) if a.data.crosslink.shard == shard] - crosslinks = list(filter( + crosslinks = filter( lambda c: hash_tree_root(state.current_crosslinks[shard]) in (c.parent_root, hash_tree_root(c)), [a.data.crosslink for a in attestations] - )) + ) # Winning crosslink has the crosslink data root with the most balance voting for it (ties broken lexicographically) winning_crosslink = max(crosslinks, key=lambda c: ( get_attesting_balance(state, [a for a in attestations if a.data.crosslink == c]), c.data_root From 6dc306700b0b343bb788c1bfca083253da3cffdb Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 11 Jul 2019 02:53:51 -0600 Subject: [PATCH 12/26] avoid overflow in slashing penalty calculation (#1286) Change presentation to avoid uint64 overflow in slashing penalty calculation. (Factor out `EFFECTIVE_BALANCE_INCREMENT` from `validator.effective_balance`.) --- specs/core/0_beacon-chain.md | 4 +++- .../epoch_processing/test_process_slashings.py | 18 ++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b0b4fdce5..7f9d3964a 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1482,7 +1482,9 @@ def process_slashings(state: BeaconState) -> None: total_balance = get_total_active_balance(state) for index, validator in enumerate(state.validators): if validator.slashed and epoch + EPOCHS_PER_SLASHINGS_VECTOR // 2 == validator.withdrawable_epoch: - penalty = validator.effective_balance * min(sum(state.slashings) * 3, total_balance) // total_balance + increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from penalty numerator to avoid uint64 overflow + penalty_numerator = validator.effective_balance // increment * min(sum(state.slashings) * 3, total_balance) + penalty = penalty_numerator // total_balance * increment decrease_balance(state, ValidatorIndex(index), penalty) ``` diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py index 7be23a04d..c58da5a4a 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_slashings.py @@ -66,8 +66,13 @@ def test_small_penalty(spec, state): spec.process_slashings(state) yield 'post', state - assert state.balances[0] == pre_slash_balances[0] - (state.validators[0].effective_balance - * 3 * total_penalties // total_balance) + expected_penalty = ( + state.validators[0].effective_balance // spec.EFFECTIVE_BALANCE_INCREMENT + * (3 * total_penalties) + // total_balance + * spec.EFFECTIVE_BALANCE_INCREMENT + ) + assert state.balances[0] == pre_slash_balances[0] - expected_penalty @with_all_phases @@ -121,5 +126,10 @@ def test_scaled_penalties(spec, state): for i in slashed_indices: v = state.validators[i] - penalty = v.effective_balance * total_penalties * 3 // total_balance - assert state.balances[i] == pre_slash_balances[i] - penalty + expected_penalty = ( + v.effective_balance // spec.EFFECTIVE_BALANCE_INCREMENT + * (3 * total_penalties) + // (total_balance) + * spec.EFFECTIVE_BALANCE_INCREMENT + ) + assert state.balances[i] == pre_slash_balances[i] - expected_penalty From f3460809015d1bea884988b339a577fb245f6395 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 11 Jul 2019 11:22:08 -0600 Subject: [PATCH 13/26] phase 2 wiki to readme --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 2f32df03f..f1d7099dd 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,12 @@ Core specifications for Eth 2.0 client validation can be found in [specs/core](s * [Custody Game](specs/core/1_custody-game.md) * [Shard Data Chains](specs/core/1_shard-data-chains.md) +### Phase 2 + +Phase 2 is still heavily in R&D and does not yet have any formal specifications. + +See the [Eth 2.0 Phase 2 Wiki](https://hackmd.io/UzysWse1Th240HELswKqVA?view) for current progress, discussions, and definitions regarding this work. + ### Accompanying documents can be found in [specs](specs) and include: * [SimpleSerialize (SSZ) spec](specs/simple-serialize.md) From 0b999e39bd312dd249589e7f2ca361f59b21eb6c Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 12 Jul 2019 06:38:00 -0600 Subject: [PATCH 14/26] Update README.md Co-Authored-By: Hsiao-Wei Wang --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f1d7099dd..0ae6156e7 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Core specifications for Eth 2.0 client validation can be found in [specs/core](s ### Phase 2 -Phase 2 is still heavily in R&D and does not yet have any formal specifications. +Phase 2 is still actively in R&D and does not yet have any formal specifications. See the [Eth 2.0 Phase 2 Wiki](https://hackmd.io/UzysWse1Th240HELswKqVA?view) for current progress, discussions, and definitions regarding this work. From b2c85706062d6ce3134ca6a3f02b7d5d9ebb00b9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 12 Jul 2019 19:09:33 +0200 Subject: [PATCH 15/26] fix merkleization with 0-limit case, and enforce padding limit --- .../pyspec/eth2spec/utils/merkle_minimal.py | 10 ++- .../eth2spec/utils/test_merkle_minimal.py | 81 +++++++++++-------- 2 files changed, 53 insertions(+), 38 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index e9416ea05..972b32d40 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -1,4 +1,4 @@ -from .hash_function import hash +from eth2spec.utils.hash_function import hash from math import log2 @@ -21,6 +21,8 @@ def calc_merkle_tree_from_leaves(values, layer_count=32): def get_merkle_root(values, pad_to=1): + if pad_to == 0: + return zerohashes[0] layer_count = int(log2(pad_to)) if len(values) == 0: return zerohashes[layer_count] @@ -36,9 +38,11 @@ def get_merkle_proof(tree, item_index): def merkleize_chunks(chunks, pad_to: int=1): - count = len(chunks) + if pad_to == 0: + return zerohashes[0] + count = min(len(chunks), pad_to) depth = max(count - 1, 0).bit_length() - max_depth = max(depth, (pad_to - 1).bit_length()) + max_depth = (pad_to - 1).bit_length() tmp = [None for _ in range(max_depth + 1)] def merge(h, i): diff --git a/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py index f1ed768e6..52e50d57a 100644 --- a/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py @@ -8,7 +8,8 @@ def h(a: bytes, b: bytes) -> bytes: def e(v: int) -> bytes: - return v.to_bytes(length=32, byteorder='little') + # prefix with 0xfff... to make it non-zero + return b'\xff' * 28 + v.to_bytes(length=4, byteorder='little') def z(i: int) -> bytes: @@ -16,44 +17,54 @@ def z(i: int) -> bytes: cases = [ - (0, 0, 1, z(0)), - (0, 1, 1, e(0)), - (1, 0, 2, h(z(0), z(0))), - (1, 1, 2, h(e(0), z(0))), - (1, 2, 2, h(e(0), e(1))), - (2, 0, 4, h(h(z(0), z(0)), z(1))), - (2, 1, 4, h(h(e(0), z(0)), z(1))), - (2, 2, 4, h(h(e(0), e(1)), z(1))), - (2, 3, 4, h(h(e(0), e(1)), h(e(2), z(0)))), - (2, 4, 4, h(h(e(0), e(1)), h(e(2), e(3)))), - (3, 0, 8, h(h(h(z(0), z(0)), z(1)), z(2))), - (3, 1, 8, h(h(h(e(0), z(0)), z(1)), z(2))), - (3, 2, 8, h(h(h(e(0), e(1)), z(1)), z(2))), - (3, 3, 8, h(h(h(e(0), e(1)), h(e(2), z(0))), z(2))), - (3, 4, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), z(2))), - (3, 5, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), z(0)), z(1)))), - (3, 6, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(z(0), z(0))))), - (3, 7, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), z(0))))), - (3, 8, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), e(7))))), - (4, 0, 16, h(h(h(h(z(0), z(0)), z(1)), z(2)), z(3))), - (4, 1, 16, h(h(h(h(e(0), z(0)), z(1)), z(2)), z(3))), - (4, 2, 16, h(h(h(h(e(0), e(1)), z(1)), z(2)), z(3))), - (4, 3, 16, h(h(h(h(e(0), e(1)), h(e(2), z(0))), z(2)), z(3))), - (4, 4, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), z(2)), z(3))), - (4, 5, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), z(0)), z(1))), z(3))), - (4, 6, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(z(0), z(0)))), z(3))), - (4, 7, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), z(0)))), z(3))), - (4, 8, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), e(7)))), z(3))), - (4, 9, 16, - h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), e(7)))), h(h(h(e(8), z(0)), z(1)), z(2)))), + # limit 0: always zero hash + (0, 0, z(0)), + (1, 0, z(0)), # cut-off due to limit + (2, 0, z(0)), # cut-off due to limit + # limit 1: padded to 1 element if not already. Returned (like identity func) + (0, 1, z(0)), + (1, 1, e(0)), + (2, 1, e(0)), # cut-off due to limit + (1, 1, e(0)), + (0, 2, h(z(0), z(0))), + (1, 2, h(e(0), z(0))), + (2, 2, h(e(0), e(1))), + (3, 2, h(e(0), e(1))), # cut-off due to limit + (16, 2, h(e(0), e(1))), # bigger cut-off due to limit + (0, 4, h(h(z(0), z(0)), z(1))), + (1, 4, h(h(e(0), z(0)), z(1))), + (2, 4, h(h(e(0), e(1)), z(1))), + (3, 4, h(h(e(0), e(1)), h(e(2), z(0)))), + (4, 4, h(h(e(0), e(1)), h(e(2), e(3)))), + (5, 4, h(h(e(0), e(1)), h(e(2), e(3)))), # cut-off due to limit + (0, 8, h(h(h(z(0), z(0)), z(1)), z(2))), + (1, 8, h(h(h(e(0), z(0)), z(1)), z(2))), + (2, 8, h(h(h(e(0), e(1)), z(1)), z(2))), + (3, 8, h(h(h(e(0), e(1)), h(e(2), z(0))), z(2))), + (4, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), z(2))), + (5, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), z(0)), z(1)))), + (6, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(z(0), z(0))))), + (7, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), z(0))))), + (8, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), e(7))))), + (9, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), e(7))))), # cut-off due to limit + (0, 16, h(h(h(h(z(0), z(0)), z(1)), z(2)), z(3))), + (1, 16, h(h(h(h(e(0), z(0)), z(1)), z(2)), z(3))), + (2, 16, h(h(h(h(e(0), e(1)), z(1)), z(2)), z(3))), + (3, 16, h(h(h(h(e(0), e(1)), h(e(2), z(0))), z(2)), z(3))), + (4, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), z(2)), z(3))), + (5, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), z(0)), z(1))), z(3))), + (6, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(z(0), z(0)))), z(3))), + (7, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), z(0)))), z(3))), + (8, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), e(7)))), z(3))), + (9, 16, h(h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), e(7)))), h(h(h(e(8), z(0)), z(1)), z(2)))), ] @pytest.mark.parametrize( - 'depth,count,pow2,value', + 'count,pad_to,value', cases, ) -def test_merkleize_chunks_and_get_merkle_root(depth, count, pow2, value): +def test_merkleize_chunks_and_get_merkle_root(count, pad_to, value): chunks = [e(i) for i in range(count)] - assert merkleize_chunks(chunks, pad_to=pow2) == value - assert get_merkle_root(chunks, pad_to=pow2) == value + assert merkleize_chunks(chunks, pad_to=pad_to) == value + assert get_merkle_root(chunks, pad_to=pad_to) == value From 65b031158297e7ed938af1f73f3919100c3189d6 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 12 Jul 2019 20:39:55 +0200 Subject: [PATCH 16/26] more explicit about merkleization limit/pad --- test_libs/pyspec/eth2spec/utils/merkle_minimal.py | 13 +++++++++---- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 3 ++- .../pyspec/eth2spec/utils/test_merkle_minimal.py | 8 ++++---- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index 972b32d40..503926517 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -37,12 +37,17 @@ def get_merkle_proof(tree, item_index): return proof -def merkleize_chunks(chunks, pad_to: int=1): - if pad_to == 0: +def merkleize_chunks(chunks, limit=None): + # If no limit is defined, we are just merkleizing chunks (e.g. SSZ container). + if limit is None: + limit = len(chunks) + if limit == 0: return zerohashes[0] - count = min(len(chunks), pad_to) + # Limit strictly. Makes no sense to merkleize objects above the intended padding. + # And illegal to exceed list limits, just as with serialization. + count = min(len(chunks), limit) depth = max(count - 1, 0).bit_length() - max_depth = (pad_to - 1).bit_length() + max_depth = (limit - 1).bit_length() tmp = [None for _ in range(max_depth + 1)] def merge(h, i): diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index d5855a755..5b37a2bb7 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -126,6 +126,7 @@ def item_length(typ: SSZType) -> int: def chunk_count(typ: SSZType) -> int: + # note that for lists, .length *on the type* describes the list limit. if isinstance(typ, BasicType): return 1 elif issubclass(typ, Bits): @@ -150,7 +151,7 @@ def hash_tree_root(obj: SSZValue): raise Exception(f"Type not supported: {type(obj)}") if isinstance(obj, (List, Bytes, Bitlist)): - return mix_in_length(merkleize_chunks(leaves, pad_to=chunk_count(obj.type())), len(obj)) + return mix_in_length(merkleize_chunks(leaves, limit=chunk_count(obj.type())), len(obj)) else: return merkleize_chunks(leaves) diff --git a/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py index 52e50d57a..a40ec05cf 100644 --- a/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py @@ -61,10 +61,10 @@ cases = [ @pytest.mark.parametrize( - 'count,pad_to,value', + 'count,limit,value', cases, ) -def test_merkleize_chunks_and_get_merkle_root(count, pad_to, value): +def test_merkleize_chunks_and_get_merkle_root(count, limit, value): chunks = [e(i) for i in range(count)] - assert merkleize_chunks(chunks, pad_to=pad_to) == value - assert get_merkle_root(chunks, pad_to=pad_to) == value + assert merkleize_chunks(chunks, limit=limit) == value + assert get_merkle_root(chunks, pad_to=limit) == value From a8dc9157b8c288a8f2491370762cde89a05dedac Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 12 Jul 2019 21:15:28 +0200 Subject: [PATCH 17/26] clean up merkleization text in SSZ spec --- specs/simple-serialize.md | 40 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 8efd08c0a..6c6377843 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -25,8 +25,6 @@ - [Vectors, containers, lists, unions](#vectors-containers-lists-unions) - [Deserialization](#deserialization) - [Merkleization](#merkleization) - - [`Bitvector[N]`](#bitvectorn-1) - - [`Bitlist[N]`](#bitlistn-1) - [Self-signed containers](#self-signed-containers) - [Implementations](#implementations) @@ -177,38 +175,36 @@ Note that deserialization requires hardening against invalid inputs. A non-exhau We first define helper functions: +* `chunk_count(type)`: calculate the amount of leafs for merkleization of the type. + * all basic types: `1` + * bitlists and bitvectors: `(N + 255) // 256` (dividing by chunk size, rounding up) + * lists and vectors of basic types: `N * item_length(elem_type) + 31) // 32` (dividing by chunk size, rounding up) + * lists and vectors of composite types: `N` + * containers: `len(fields)` +* `bitfield_bytes(bits)`: return the bits of the bitlist or bitvector, packed in bytes, aligned to the start. Exclusive length-delimiting bit for bitlists. * `pack`: Given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. * `next_pow_of_two(i)`: get the next power of 2 of `i`, if not already a power of 2, with 0 mapping to 1. Examples: `0->1, 1->1, 2->2, 3->4, 4->4, 6->8, 9->16` -* `merkleize(data, pad_for=1)`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. - * The merkleization depends on the effective input, which can be padded: if `pad_for=L`, then pad the `data` with zeroed chunks to `next_pow_of_two(L)` (virtually for memory efficiency). +* `merkleize(chunks, limit=None)`: Given ordered `BYTES_PER_CHUNK`-byte chunks, merkleize the chunks, and return the root: + * The merkleization depends on the effective input, which can be padded/limited: + - if no limit: pad the `chunks` with zeroed chunks to `next_pow_of_two(len(chunks))` (virtually for memory efficiency). + - if `limit > len(chunks)`, pad the `chunks` with zeroed chunks to `next_pow_of_two(limit)` (virtually for memory efficiency). + - if `limit < len(chunks)`: do not merkleize, input exceeds limit. Raise an error instead. * Then, merkleize the chunks (empty input is padded to 1 zero chunk): - - If `1` chunk: A single chunk is simply that chunk, i.e. the identity when the number of chunks is one. - - If `> 1` chunks: pad to `next_pow_of_two(len(chunks))`, merkleize as binary tree. + - If `1` chunk: the root is the chunk itself. + - If `> 1` chunks: merkleize as binary tree. * `mix_in_length`: Given a Merkle root `root` and a length `length` (`"uint256"` little-endian serialization) return `hash(root + length)`. * `mix_in_type`: Given a Merkle root `root` and a type_index `type_index` (`"uint256"` little-endian serialization) return `hash(root + type_index)`. We now define Merkleization `hash_tree_root(value)` of an object `value` recursively: * `merkleize(pack(value))` if `value` is a basic object or a vector of basic objects. -* `mix_in_length(merkleize(pack(value), pad_for=(N * elem_size / BYTES_PER_CHUNK)), len(value))` if `value` is a list of basic objects. +* `merkleize(bitfield_bytes(value), limit=chunk_count(type))` if `value` is a bitvector. +* `mix_in_length(merkleize(pack(value), limit=chunk_count(type)), len(value))` if `value` is a list of basic objects. +* `mix_in_length(merkleize(bitfield_bytes(value), limit=chunk_count(type)), len(value))` if `value` is a bitlist. * `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container. -* `mix_in_length(merkleize([hash_tree_root(element) for element in value], pad_for=N), len(value))` if `value` is a list of composite objects. +* `mix_in_length(merkleize([hash_tree_root(element) for element in value], limit=chunk_count(type)), len(value))` if `value` is a list of composite objects. * `mix_in_type(merkleize(value.value), value.type_index)` if `value` is of union type. -### `Bitvector[N]` - -```python -as_integer = sum([value[i] << i for i in range(len(value))]) -return merkleize(pack(as_integer.to_bytes((N + 7) // 8, "little"))) -``` - -### `Bitlist[N]` - -```python -as_integer = sum([value[i] << i for i in range(len(value))]) -return mix_in_length(merkleize(pack(as_integer.to_bytes((N + 7) // 8, "little"))), len(value)) -``` - ## Self-signed containers Let `value` be a self-signed container object. The convention is that the signature (e.g. a `"bytes96"` BLS12-381 signature) be the last field of `value`. Further, the signed message for `value` is `signing_root(value) = hash_tree_root(truncate_last(value))` where `truncate_last` truncates the last element of `value`. From 5a13684c7f2a9b274d1772ce6aa2db03cb0112fb Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 12 Jul 2019 21:23:45 +0200 Subject: [PATCH 18/26] make exceeding limit raise an error --- .../pyspec/eth2spec/utils/merkle_minimal.py | 10 +++++-- .../eth2spec/utils/test_merkle_minimal.py | 28 +++++++++++++------ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index 503926517..9d7138d7d 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -41,11 +41,15 @@ def merkleize_chunks(chunks, limit=None): # If no limit is defined, we are just merkleizing chunks (e.g. SSZ container). if limit is None: limit = len(chunks) + + count = len(chunks) + # See if the input is within expected size. + # If not, a list-limit is set incorrectly, or a value is unexpectedly large. + assert count <= limit + if limit == 0: return zerohashes[0] - # Limit strictly. Makes no sense to merkleize objects above the intended padding. - # And illegal to exceed list limits, just as with serialization. - count = min(len(chunks), limit) + depth = max(count - 1, 0).bit_length() max_depth = (limit - 1).bit_length() tmp = [None for _ in range(max_depth + 1)] diff --git a/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py index a40ec05cf..3746ea6ca 100644 --- a/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/test_merkle_minimal.py @@ -19,24 +19,24 @@ def z(i: int) -> bytes: cases = [ # limit 0: always zero hash (0, 0, z(0)), - (1, 0, z(0)), # cut-off due to limit - (2, 0, z(0)), # cut-off due to limit + (1, 0, None), # cut-off due to limit + (2, 0, None), # cut-off due to limit # limit 1: padded to 1 element if not already. Returned (like identity func) (0, 1, z(0)), (1, 1, e(0)), - (2, 1, e(0)), # cut-off due to limit + (2, 1, None), # cut-off due to limit (1, 1, e(0)), (0, 2, h(z(0), z(0))), (1, 2, h(e(0), z(0))), (2, 2, h(e(0), e(1))), - (3, 2, h(e(0), e(1))), # cut-off due to limit - (16, 2, h(e(0), e(1))), # bigger cut-off due to limit + (3, 2, None), # cut-off due to limit + (16, 2, None), # bigger cut-off due to limit (0, 4, h(h(z(0), z(0)), z(1))), (1, 4, h(h(e(0), z(0)), z(1))), (2, 4, h(h(e(0), e(1)), z(1))), (3, 4, h(h(e(0), e(1)), h(e(2), z(0)))), (4, 4, h(h(e(0), e(1)), h(e(2), e(3)))), - (5, 4, h(h(e(0), e(1)), h(e(2), e(3)))), # cut-off due to limit + (5, 4, None), # cut-off due to limit (0, 8, h(h(h(z(0), z(0)), z(1)), z(2))), (1, 8, h(h(h(e(0), z(0)), z(1)), z(2))), (2, 8, h(h(h(e(0), e(1)), z(1)), z(2))), @@ -46,7 +46,7 @@ cases = [ (6, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(z(0), z(0))))), (7, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), z(0))))), (8, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), e(7))))), - (9, 8, h(h(h(e(0), e(1)), h(e(2), e(3))), h(h(e(4), e(5)), h(e(6), e(7))))), # cut-off due to limit + (9, 8, None), # cut-off due to limit (0, 16, h(h(h(h(z(0), z(0)), z(1)), z(2)), z(3))), (1, 16, h(h(h(h(e(0), z(0)), z(1)), z(2)), z(3))), (2, 16, h(h(h(h(e(0), e(1)), z(1)), z(2)), z(3))), @@ -66,5 +66,15 @@ cases = [ ) def test_merkleize_chunks_and_get_merkle_root(count, limit, value): chunks = [e(i) for i in range(count)] - assert merkleize_chunks(chunks, limit=limit) == value - assert get_merkle_root(chunks, pad_to=limit) == value + if value is None: + bad = False + try: + merkleize_chunks(chunks, limit=limit) + bad = True + except AssertionError: + pass + if bad: + assert False, "expected merkleization to be invalid" + else: + assert merkleize_chunks(chunks, limit=limit) == value + assert get_merkle_root(chunks, pad_to=limit) == value From b98679957b5e056e852c4a5f57b1eab4d29da118 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 12 Jul 2019 22:11:33 +0200 Subject: [PATCH 19/26] use as_bytes function to reduce code duplication, and for later usage --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 23 ++++++++----------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 7 +++++- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 1e0c806d9..2a7d92314 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -41,15 +41,13 @@ def serialize(obj: SSZValue): if isinstance(obj, BasicValue): return serialize_basic(obj) elif isinstance(obj, Bitvector): - as_bytearray = [0] * ((len(obj) + 7) // 8) - for i in range(len(obj)): - as_bytearray[i // 8] |= obj[i] << (i % 8) - return bytes(as_bytearray) + return obj.as_bytes() elif isinstance(obj, Bitlist): - as_bytearray = [0] * (len(obj) // 8 + 1) - for i in range(len(obj)): - as_bytearray[i // 8] |= obj[i] << (i % 8) - as_bytearray[len(obj) // 8] |= 1 << (len(obj) % 8) + as_bytearray = list(obj.as_bytes()) + if len(obj) % 8 == 0: + as_bytearray.append(1) + else: + as_bytearray[len(obj) // 8] |= 1 << (len(obj) % 8) return bytes(as_bytearray) elif isinstance(obj, Series): return encode_series(obj) @@ -97,11 +95,10 @@ def encode_series(values: Series): def pack(values: Series): if isinstance(values, bytes): # Bytes and BytesN are already packed return values - elif isinstance(values, (Bitvector, Bitlist)): - as_bytearray = [0] * ((len(values) + 7) // 8) - for i in range(len(values)): - as_bytearray[i // 8] |= values[i] << (i % 8) - return bytes(as_bytearray) + elif isinstance(values, Bits): + # packs the bits in bytes, left-aligned. + # Exclusive length delimiting bits for bitlists. + return values.as_bytes() return b''.join([serialize_basic(value) for value in values]) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 2ec4b5ce2..1f199e6e1 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -354,7 +354,12 @@ class BitElementsType(ElementsType): class Bits(BaseList, metaclass=BitElementsType): - pass + + def as_bytes(self): + as_bytearray = [0] * ((len(self) + 7) // 8) + for i in range(len(self)): + as_bytearray[i // 8] |= int(self[i]) << (i % 8) + return bytes(as_bytearray) class Bitlist(Bits): From ac6d019870b8a79ad57a9043f30ec5cc4eafe82e Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 12 Jul 2019 22:20:07 +0200 Subject: [PATCH 20/26] bits serialization clear now, directly to bytes --- specs/simple-serialize.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 8d9c33103..915cb772a 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -120,8 +120,10 @@ return b"" ### `Bitvector[N]` ```python -as_integer = sum([value[i] << i for i in range(len(value))]) -return as_integer.to_bytes((N + 7) // 8, "little") +array = [0] * ((N + 7) // 8) +for i in range(N): + array[i // 8] |= value[i] << (i % 8) +return bytes(array) ``` ### `Bitlist[N]` @@ -129,8 +131,11 @@ return as_integer.to_bytes((N + 7) // 8, "little") Note that from the offset coding, the length (in bytes) of the bitlist is known. An additional leading `1` bit is added so that the length in bits will also be known. ```python -as_integer = (1 << len(value)) + sum([value[i] << i for i in range(len(value))]) -return as_integer.to_bytes((as_integer.bit_length() + 7) // 8, "little") +array = [0] * ((len(value) // 8) + 1) +for i in range(len(value)): + array[i // 8] |= value[i] << (i % 8) +array[len(value) // 8] |= 1 << (len(value) % 8) +return bytes(array) ``` ### Vectors, containers, lists, unions From 8970b71ca405d3971ca973d5a5f3b6afba7d6964 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 14 Jul 2019 16:05:51 -0600 Subject: [PATCH 21/26] ensure min_seed_lookahead functions properly --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ebfe41a6a..49c64e3ed 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -865,7 +865,7 @@ def get_seed(state: BeaconState, epoch: Epoch) -> Hash: """ Return the seed at ``epoch``. """ - mix = get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD)) # Avoid underflow + mix = get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD - 1)) # Avoid underflow active_index_root = state.active_index_roots[epoch % EPOCHS_PER_HISTORICAL_VECTOR] return hash(mix + active_index_root + int_to_bytes(epoch, length=32)) ``` From d9fd1d3a2a4c73148757adf92787f5600807a21c Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 15 Jul 2019 00:12:12 +0200 Subject: [PATCH 22/26] improve type wording based on PR 1292 feedback --- specs/simple-serialize.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 6c6377843..01440c1cd 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -175,11 +175,12 @@ Note that deserialization requires hardening against invalid inputs. A non-exhau We first define helper functions: +* `size_of(B)`, where `B` is a basic type: the length, in bytes, of the serialized form of the basic type. * `chunk_count(type)`: calculate the amount of leafs for merkleization of the type. * all basic types: `1` - * bitlists and bitvectors: `(N + 255) // 256` (dividing by chunk size, rounding up) - * lists and vectors of basic types: `N * item_length(elem_type) + 31) // 32` (dividing by chunk size, rounding up) - * lists and vectors of composite types: `N` + * `Bitlist[N]` and `Bitvector[N]`: `(N + 255) // 256` (dividing by chunk size, rounding up) + * `List[B, N]` and `Vector[B, N]`, where `B` is a basic type: `(N * size_of(B) + 31) // 32` (dividing by chunk size, rounding up) + * `List[C, N]` and `Vector[C, N]`, where `C` is a composite type: `N` * containers: `len(fields)` * `bitfield_bytes(bits)`: return the bits of the bitlist or bitvector, packed in bytes, aligned to the start. Exclusive length-delimiting bit for bitlists. * `pack`: Given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. From ef659144b48ca45afd3b67cd5162ba25144a4e21 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 15 Jul 2019 02:05:04 +0200 Subject: [PATCH 23/26] make zero hash representation clear, fixes #1282 --- specs/core/0_beacon-chain.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 49c64e3ed..2682807b2 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1216,7 +1216,7 @@ def process_slot(state: BeaconState) -> None: previous_state_root = hash_tree_root(state) state.state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_state_root # Cache latest block header state root - if state.latest_block_header.state_root == Hash(): + if state.latest_block_header.state_root == Bytes32(): state.latest_block_header.state_root = previous_state_root # Cache block root previous_block_root = signing_root(state.latest_block_header) @@ -1548,8 +1548,9 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: state.latest_block_header = BeaconBlockHeader( slot=block.slot, parent_root=block.parent_root, - state_root=Hash(), # Overwritten in the next `process_slot` call + # state_root: zeroed, overwritten in the next `process_slot` call body_root=hash_tree_root(block.body), + # signature is always zeroed ) # Verify proposer is not slashed proposer = state.validators[get_beacon_proposer_index(state)] @@ -1672,7 +1673,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.crosslink.parent_root == hash_tree_root(parent_crosslink) assert data.crosslink.start_epoch == parent_crosslink.end_epoch assert data.crosslink.end_epoch == min(data.target.epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) - assert data.crosslink.data_root == Hash() # [to be removed in phase 1] + assert data.crosslink.data_root == Bytes32() # [to be removed in phase 1] # Check signature assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) From bfd5010f26acc6de8f895984c0335fe792f97b5c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 Jul 2019 14:27:34 +0800 Subject: [PATCH 24/26] Bump `py_ecc==1.7.1` --- test_generators/bls/main.py | 48 +++++++++++++------------- test_generators/bls/requirements.txt | 2 +- test_libs/pyspec/eth2spec/utils/bls.py | 6 ++-- test_libs/pyspec/requirements.txt | 2 +- test_libs/pyspec/setup.py | 2 +- 5 files changed, 30 insertions(+), 30 deletions(-) diff --git a/test_generators/bls/main.py b/test_generators/bls/main.py index 284cf68b0..2e328d1dd 100644 --- a/test_generators/bls/main.py +++ b/test_generators/bls/main.py @@ -5,7 +5,9 @@ BLS test vectors generator from typing import Tuple from eth_utils import ( - to_tuple, int_to_big_endian + encode_hex, + int_to_big_endian, + to_tuple, ) from gen_base import gen_runner, gen_suite, gen_typing @@ -20,7 +22,7 @@ def int_to_hex(n: int, byte_length: int=None) -> str: byte_value = int_to_big_endian(n) if byte_length: byte_value = byte_value.rjust(byte_length, b'\x00') - return '0x' + byte_value.hex() + return encode_hex(byte_value) def hex_to_int(x: str) -> int: @@ -28,11 +30,9 @@ def hex_to_int(x: str) -> int: DOMAINS = [ - 0, - 1, - 1234, - 2**32-1, - 2**64-1 + b'\x00\x00\x00\x00\x00\x00\x00\x00', + b'\x00\x00\x00\x00\x00\x00\x00\x01', + b'\xff\xff\xff\xff\xff\xff\xff\xff' ] MESSAGES = [ @@ -51,12 +51,12 @@ PRIVKEYS = [ def hash_message(msg: bytes, - domain: int) ->Tuple[Tuple[str, str], Tuple[str, str], Tuple[str, str]]: + domain: bytes) ->Tuple[Tuple[str, str], Tuple[str, str], Tuple[str, str]]: """ Hash message Input: - - Message as bytes - - domain as uint64 + - Message as bytes32 + - domain as bytes8 Output: - Message hash as a G2 point """ @@ -69,12 +69,12 @@ def hash_message(msg: bytes, ] -def hash_message_compressed(msg: bytes, domain: int) -> Tuple[str, str]: +def hash_message_compressed(msg: bytes, domain: bytes) -> Tuple[str, str]: """ Hash message Input: - - Message as bytes - - domain as uint64 + - Message as bytes32 + - domain as bytes8 Output: - Message hash as a compressed G2 point """ @@ -88,8 +88,8 @@ def case01_message_hash_G2_uncompressed(): for domain in DOMAINS: yield { 'input': { - 'message': '0x' + msg.hex(), - 'domain': int_to_hex(domain, byte_length=8) + 'message': encode_hex(msg), + 'domain': encode_hex(domain), }, 'output': hash_message(msg, domain) } @@ -100,8 +100,8 @@ def case02_message_hash_G2_compressed(): for domain in DOMAINS: yield { 'input': { - 'message': '0x' + msg.hex(), - 'domain': int_to_hex(domain, byte_length=8) + 'message': encode_hex(msg), + 'domain': encode_hex(domain), }, 'output': hash_message_compressed(msg, domain) } @@ -125,10 +125,10 @@ def case04_sign_messages(): yield { 'input': { 'privkey': int_to_hex(privkey), - 'message': '0x' + message.hex(), - 'domain': int_to_hex(domain, byte_length=8) + 'message': encode_hex(message), + 'domain': encode_hex(domain), }, - 'output': '0x' + sig.hex() + 'output': encode_hex(sig) } # TODO: case05_verify_messages: Verify messages signed in case04 @@ -141,17 +141,17 @@ def case06_aggregate_sigs(): for message in MESSAGES: sigs = [bls.sign(message, privkey, domain) for privkey in PRIVKEYS] yield { - 'input': ['0x' + sig.hex() for sig in sigs], - 'output': '0x' + bls.aggregate_signatures(sigs).hex(), + 'input': [encode_hex(sig) for sig in sigs], + 'output': encode_hex(bls.aggregate_signatures(sigs)), } @to_tuple def case07_aggregate_pubkeys(): pubkeys = [bls.privtopub(privkey) for privkey in PRIVKEYS] - pubkeys_serial = ['0x' + pubkey.hex() for pubkey in pubkeys] + pubkeys_serial = [encode_hex(pubkey) for pubkey in pubkeys] yield { 'input': pubkeys_serial, - 'output': '0x' + bls.aggregate_pubkeys(pubkeys).hex(), + 'output': encode_hex(bls.aggregate_pubkeys(pubkeys)), } diff --git a/test_generators/bls/requirements.txt b/test_generators/bls/requirements.txt index 6d83bdfb5..84a28d357 100644 --- a/test_generators/bls/requirements.txt +++ b/test_generators/bls/requirements.txt @@ -1,3 +1,3 @@ -py-ecc==1.7.0 +py_ecc==1.7.1 eth-utils==1.6.0 ../../test_libs/gen_helpers diff --git a/test_libs/pyspec/eth2spec/utils/bls.py b/test_libs/pyspec/eth2spec/utils/bls.py index ab2327f43..d8a9ab5be 100644 --- a/test_libs/pyspec/eth2spec/utils/bls.py +++ b/test_libs/pyspec/eth2spec/utils/bls.py @@ -24,13 +24,13 @@ def only_with_bls(alt_return=None): @only_with_bls(alt_return=True) def bls_verify(pubkey, message_hash, signature, domain): return bls.verify(message_hash=message_hash, pubkey=pubkey, - signature=signature, domain=int.from_bytes(domain, byteorder='little')) + signature=signature, domain=domain) @only_with_bls(alt_return=True) def bls_verify_multiple(pubkeys, message_hashes, signature, domain): return bls.verify_multiple(pubkeys=pubkeys, message_hashes=message_hashes, - signature=signature, domain=int.from_bytes(domain, byteorder='little')) + signature=signature, domain=domain) @only_with_bls(alt_return=STUB_PUBKEY) @@ -46,4 +46,4 @@ def bls_aggregate_signatures(signatures): @only_with_bls(alt_return=STUB_SIGNATURE) def bls_sign(message_hash, privkey, domain): return bls.sign(message_hash=message_hash, privkey=privkey, - domain=int.from_bytes(domain, byteorder='little')) + domain=domain) diff --git a/test_libs/pyspec/requirements.txt b/test_libs/pyspec/requirements.txt index 83197af9c..713b4331a 100644 --- a/test_libs/pyspec/requirements.txt +++ b/test_libs/pyspec/requirements.txt @@ -1,6 +1,6 @@ eth-utils>=1.3.0,<2 eth-typing>=2.1.0,<3.0.0 pycryptodome==3.7.3 -py_ecc>=1.6.0 +py_ecc==1.7.1 dataclasses==0.6 ssz==0.1.0a10 diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index d8d54eab7..07e538e80 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -8,7 +8,7 @@ setup( "eth-utils>=1.3.0,<2", "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.7.3", - "py_ecc>=1.6.0", + "py_ecc==1.7.1", "ssz==0.1.0a10", "dataclasses==0.6", ] From 01af3044032735fa9266d08656090f1bc1e09351 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 20 Jul 2019 02:13:31 +0200 Subject: [PATCH 25/26] =?UTF-8?q?Found=20by=20Cem=20=C3=96zer:=20Ignore=20?= =?UTF-8?q?older=20latest=20messages=20in=20attesting=20balance=20sum,=20i?= =?UTF-8?q?nstead=20of=20assertion=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- specs/core/0_fork-choice.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 9fd8ab53e..00374fee9 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -101,8 +101,12 @@ def get_genesis_store(genesis_state: BeaconState) -> Store: ```python def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash: block = store.blocks[root] - assert block.slot >= slot - return root if block.slot == slot else get_ancestor(store, block.parent_root, slot) + if block.slot > slot: + return get_ancestor(store, block.parent_root, slot) + elif block.slot == slot: + return root + else: + return Bytes32() # root is older than queried slot: no results. ``` #### `get_latest_attesting_balance` From a90d273fbd3bc952892459a03284242de65022c8 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 22 Jul 2019 07:19:42 -0600 Subject: [PATCH 26/26] fix minor var typo --- specs/validator/0_beacon-chain-validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 58d87d450..188a6a291 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -322,13 +322,13 @@ Set `attestation.data = attestation_data` where `attestation_data` is the `Attes ##### Aggregation bits -- Let `attestation.aggregation_bits` be a `Bitlist[MAX_INDICES_PER_ATTESTATION]` where the bits at the index in the aggregated validator's `committee` is set to `0b1`. +- Let `attestation.aggregation_bits` be a `Bitlist[MAX_VALIDATORS_PER_COMMITTEE]` where the bits at the index in the aggregated validator's `committee` is set to `0b1`. *Note*: Calling `get_attesting_indices(state, attestation.data, attestation.aggregation_bits)` should return a list of length equal to 1, containing `validator_index`. ##### Custody bits -- Let `attestation.custody_bits` be a `Bitlist[MAX_INDICES_PER_ATTESTATION]` filled with zeros of length `len(committee)`. +- Let `attestation.custody_bits` be a `Bitlist[MAX_VALIDATORS_PER_COMMITTEE]` filled with zeros of length `len(committee)`. *Note*: This is a stub for Phase 0.