From f7ef9a1ba5e7b863538ab48097060f98ec536412 Mon Sep 17 00:00:00 2001 From: Matthew Slipper Date: Sat, 20 Apr 2019 17:26:07 -0700 Subject: [PATCH 01/11] Don't use SSZ in RPC request/response wrappers --- specs/networking/rpc-interface.md | 42 +++++++++++++++++-------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/specs/networking/rpc-interface.md b/specs/networking/rpc-interface.md index 5d408b5a0..ca6008a40 100644 --- a/specs/networking/rpc-interface.md +++ b/specs/networking/rpc-interface.md @@ -39,32 +39,36 @@ To facilitate RPC-over-`libp2p`, a single protocol name is used: `/eth/serenity/ Remote method calls are wrapped in a "request" structure: ``` -( - id: uint64 - method_id: uint16 - body: Request -) + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ id (uint64) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| method_id (uint16) | body_len (uint32) | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | body (body_len bytes) | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ``` and their corresponding responses are wrapped in a "response" structure: ``` -( - id: uint64 - response_code: uint16 - result: bytes -) + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ id (uint64) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| response_code (uint16) | result_len (uint32) | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | result (result_len bytes) | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ``` -If an error occurs, a variant of the response structure is returned: - -``` -( - id: uint64 - response_code: uint16 - result: bytes -) -``` +Note that the above structures are NOT encoded as SSZ but rather as sequences of bytes according to the packet diagrams above. This is because SSZ does not support structures without an explicit schema. Since the `body` and `result` fields depend on the value of `method_id` and `response_code`, a schema for the above structure cannot be known beforehand. The details of the RPC-Over-`libp2p` protocol are similar to [JSON-RPC 2.0](https://www.jsonrpc.org/specification). Specifically: From c2942c00c6555cd03aa0595b9ee947e5eb06e6f1 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 13 May 2019 17:02:20 -0400 Subject: [PATCH 02/11] lint requires install_test in ci --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fd7708f8d..2bfda20ff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -102,10 +102,12 @@ workflows: test_spec: jobs: - checkout_specs - - lint - install_test: requires: - checkout_specs - test: requires: - install_test + - lint: + requires: + - install_test From f830f83a1d6ae1b6728bfd357da6cbef0d4a5107 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 14 May 2019 10:32:20 +0800 Subject: [PATCH 03/11] Update config.yml --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2bfda20ff..439afc9a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -102,12 +102,12 @@ workflows: test_spec: jobs: - checkout_specs + - lint: + requires: + - checkout_specs - install_test: requires: - checkout_specs - test: requires: - install_test - - lint: - requires: - - install_test From 2885f853c28d2442b4507b80d24471292a41ddb2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 16:34:07 +0800 Subject: [PATCH 04/11] clean up lint --- .circleci/config.yml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 439afc9a9..3fe6643b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,7 +96,7 @@ jobs: reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run linter - command: make install_lint && make pyspec && make lint + command: make install_lint && make lint workflows: version: 2.1 test_spec: diff --git a/Makefile b/Makefile index 8cc889f21..a6b379b71 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ citest: $(PY_SPEC_ALL_TARGETS) install_lint: cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install flake8==3.5.0 -lint: +lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ flake8 --max-line-length=120 ./eth2spec; From b0747703a77c36d821b6bf049aa8abc8e1603603 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 17:23:26 +0800 Subject: [PATCH 05/11] Make CI job `lint` require `test` --- .circleci/config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3fe6643b7..d2b284e25 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,7 +52,7 @@ jobs: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} paths: - ~/specs-repo - install_test: + install_env: docker: - image: circleci/python:3.6 working_directory: ~/specs-repo @@ -64,7 +64,7 @@ jobs: reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Install pyspec requirements - command: make install_test + command: make install_test && make install_lint - save_cached_venv: venv_name: v1-pyspec reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' @@ -96,18 +96,18 @@ jobs: reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run linter - command: make install_lint && make lint + command: make lint workflows: version: 2.1 test_spec: jobs: - checkout_specs - - lint: - requires: - - checkout_specs - - install_test: + - install_env: requires: - checkout_specs - test: requires: - - install_test + - install_env + - lint: + requires: + - test From 1b3dfa67813e435761a71a3cc1c5e552efc043dd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 17:34:49 +0800 Subject: [PATCH 06/11] kick the cache --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d2b284e25..b9f09d9cc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,13 +60,13 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec + venv_name: v1-pyspec-02 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Install pyspec requirements command: make install_test && make install_lint - save_cached_venv: - venv_name: v1-pyspec + venv_name: v1-pyspec-02 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' venv_path: ./test_libs/pyspec/venv test: @@ -77,7 +77,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec + venv_name: v1-pyspec-02 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run py-tests @@ -92,7 +92,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec + venv_name: v1-pyspec-02 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run linter From 32f76641e34afb703355ab8b8ac93fafa0d59478 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 17:41:48 +0800 Subject: [PATCH 07/11] Add build_pyspec job --- .circleci/config.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b9f09d9cc..a0700a004 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,6 +69,19 @@ jobs: venv_name: v1-pyspec-02 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' venv_path: ./test_libs/pyspec/venv + build_pyspec: + docker: + - image: circleci/python:3.6 + working_directory: ~/specs-repo + steps: + - restore_cache: + key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} + - restore_cached_venv: + venv_name: v1-pyspec-02 + reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' + - run: + name: Build pyspec + command: make pyspec test: docker: - image: circleci/python:3.6 @@ -105,9 +118,13 @@ workflows: - install_env: requires: - checkout_specs + - build_pyspec: + requires: + - checkout_specs - test: requires: - install_env + - build_pyspec - lint: requires: - - test + - build_pyspec From ee4fdf5d06c909b4d7bc6f1dff3bce5a253b54e7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 17:45:09 +0800 Subject: [PATCH 08/11] Kick the cache again --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a0700a004..972f0e351 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,13 +60,13 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-02 + venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Install pyspec requirements command: make install_test && make install_lint - save_cached_venv: - venv_name: v1-pyspec-02 + venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' venv_path: ./test_libs/pyspec/venv build_pyspec: @@ -77,7 +77,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-02 + venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Build pyspec @@ -90,7 +90,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-02 + venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run py-tests @@ -105,7 +105,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-02 + venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run linter From be4c792fc301b7ee254b61466720e40e155b1601 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 18:20:20 +0800 Subject: [PATCH 09/11] Change it back to serial workflow --- .circleci/config.yml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 972f0e351..f3c5f6a81 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,19 +69,6 @@ jobs: venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' venv_path: ./test_libs/pyspec/venv - build_pyspec: - docker: - - image: circleci/python:3.6 - working_directory: ~/specs-repo - steps: - - restore_cache: - key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - - restore_cached_venv: - venv_name: v1-pyspec-03 - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - - run: - name: Build pyspec - command: make pyspec test: docker: - image: circleci/python:3.6 @@ -118,13 +105,9 @@ workflows: - install_env: requires: - checkout_specs - - build_pyspec: - requires: - - checkout_specs - test: requires: - install_env - - build_pyspec - lint: requires: - - build_pyspec + - test From 7f6896cca3931c0e4a6b352bfd0ba3fa3dca3212 Mon Sep 17 00:00:00 2001 From: Matthew Slipper Date: Thu, 16 May 2019 16:43:10 -0700 Subject: [PATCH 10/11] Update to use union type --- specs/networking/rpc-interface.md | 34 ++++++++++--------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/specs/networking/rpc-interface.md b/specs/networking/rpc-interface.md index ca6008a40..48ee9333b 100644 --- a/specs/networking/rpc-interface.md +++ b/specs/networking/rpc-interface.md @@ -39,36 +39,24 @@ To facilitate RPC-over-`libp2p`, a single protocol name is used: `/eth/serenity/ Remote method calls are wrapped in a "request" structure: ``` - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | -+ id (uint64) + -| | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| method_id (uint16) | body_len (uint32) | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | body (body_len bytes) | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +( + id: uint64 + method_id: uint16 + body: (message_body...) +) ``` and their corresponding responses are wrapped in a "response" structure: ``` - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | -+ id (uint64) + -| | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| response_code (uint16) | result_len (uint32) | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | result (result_len bytes) | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +( + id: uint64 + response_code: uint16 + result: bytes +) ``` -Note that the above structures are NOT encoded as SSZ but rather as sequences of bytes according to the packet diagrams above. This is because SSZ does not support structures without an explicit schema. Since the `body` and `result` fields depend on the value of `method_id` and `response_code`, a schema for the above structure cannot be known beforehand. +A union type is used to determine the contents of the `body` field in the request structure. Each "body" entry in the RPC calls below corresponds to one subtype in the `body` type union. The details of the RPC-Over-`libp2p` protocol are similar to [JSON-RPC 2.0](https://www.jsonrpc.org/specification). Specifically: From e1d973d5467e25fe496cc67ba3800aab096d1c54 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 17 May 2019 11:01:18 -0400 Subject: [PATCH 11/11] Update specs/validator/0_beacon-chain-validator.md Co-Authored-By: Hsiao-Wei Wang --- specs/validator/0_beacon-chain-validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 302deeae1..9e4fa7960 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -148,7 +148,7 @@ def get_committee_assignment( committees_per_slot = get_epoch_committee_count(state, epoch) // SLOTS_PER_EPOCH epoch_start_slot = get_epoch_start_slot(epoch) - for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH) + for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH): slot_start_shard = get_epoch_start_shard(state, epoch) + committees_per_slot * (slot % SLOTS_PER_EPOCH) for i in range(committees_per_slot): shard = (slot_start_shard + i) % SHARD_COUNT