From 97906a633910b8989576dc7371f4ca438d435cdf Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 22 Apr 2019 18:07:55 +0800 Subject: [PATCH 1/5] Only use `setup.py` --- .circleci/config.yml | 6 ++--- Makefile | 2 +- test_generators/README.md | 2 +- test_libs/config_helpers/requirements.txt | 1 - test_libs/config_helpers/setup.py | 15 +++++++++--- test_libs/gen_helpers/requirements.txt | 2 -- test_libs/gen_helpers/setup.py | 18 ++++++++++---- test_libs/pyspec/README.md | 2 +- test_libs/pyspec/requirements-testing.txt | 3 --- test_libs/pyspec/requirements.txt | 4 ---- test_libs/pyspec/setup.py | 29 ++++++++++++++++++----- 11 files changed, 55 insertions(+), 29 deletions(-) delete mode 100644 test_libs/config_helpers/requirements.txt delete mode 100644 test_libs/gen_helpers/requirements.txt delete mode 100644 test_libs/pyspec/requirements-testing.txt delete mode 100644 test_libs/pyspec/requirements.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index 4f806b00f..28d949de4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -61,13 +61,13 @@ jobs: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: venv_name: v1-pyspec - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' + reqs_checksum: '{{ checksum "test_libs/pyspec/setup.py" }}' - run: name: Install pyspec requirements command: make install_test - save_cached_venv: venv_name: v1-pyspec - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' + reqs_checksum: '{{ checksum "test_libs/pyspec/setup.py" }}' venv_path: ./test_libs/pyspec/venv test: docker: @@ -78,7 +78,7 @@ jobs: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: venv_name: v1-pyspec - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' + reqs_checksum: '{{ checksum "test_libs/pyspec/setup.py" }}' - run: name: Run py-tests command: make citest diff --git a/Makefile b/Makefile index 73d8adea8..10550750a 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ gen_yaml_tests: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_TARGETS) # installs the packages to run pyspec tests install_test: - cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt; + cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -e .[dev]; test: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; python -m pytest . diff --git a/test_generators/README.md b/test_generators/README.md index 66534e5a8..94db105c0 100644 --- a/test_generators/README.md +++ b/test_generators/README.md @@ -72,7 +72,7 @@ Note: make sure to run `make pyspec` from the root of the specs repository, to b Install all the necessary requirements (re-run when you add more): ```bash -pip3 install -r requirements.txt +pip3 install -e .[pyspec] ``` And write your initial test generator, extending the base generator: diff --git a/test_libs/config_helpers/requirements.txt b/test_libs/config_helpers/requirements.txt deleted file mode 100644 index e441a474b..000000000 --- a/test_libs/config_helpers/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -ruamel.yaml==0.15.87 diff --git a/test_libs/config_helpers/setup.py b/test_libs/config_helpers/setup.py index 90ad94ee4..b69d19cdb 100644 --- a/test_libs/config_helpers/setup.py +++ b/test_libs/config_helpers/setup.py @@ -1,9 +1,18 @@ from distutils.core import setup + +deps = { + 'config_helpers': [ + "ruamel.yaml==0.15.87", + ], +} + +deps['dev'] = ( + deps['config_helpers'] +) + setup( name='config_helpers', packages=['preset_loader'], - install_requires=[ - "ruamel.yaml==0.15.87" - ] + install_requires=deps['config_helpers'] ) diff --git a/test_libs/gen_helpers/requirements.txt b/test_libs/gen_helpers/requirements.txt deleted file mode 100644 index 3d6a39458..000000000 --- a/test_libs/gen_helpers/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -ruamel.yaml==0.15.87 -eth-utils==1.4.1 diff --git a/test_libs/gen_helpers/setup.py b/test_libs/gen_helpers/setup.py index 5de27a6db..d4d331c06 100644 --- a/test_libs/gen_helpers/setup.py +++ b/test_libs/gen_helpers/setup.py @@ -1,10 +1,20 @@ from distutils.core import setup + +deps = { + 'gen_helpers': [ + "ruamel.yaml==0.15.87", + "eth-utils==1.4.1", + ], +} + +deps['dev'] = ( + deps['gen_helpers'] +) + + setup( name='gen_helpers', packages=['gen_base'], - install_requires=[ - "ruamel.yaml==0.15.87", - "eth-utils==1.4.1" - ] + install_requires=deps['gen_helpers'], ) diff --git a/test_libs/pyspec/README.md b/test_libs/pyspec/README.md index df1834210..ab2967024 100644 --- a/test_libs/pyspec/README.md +++ b/test_libs/pyspec/README.md @@ -38,7 +38,7 @@ Install dependencies: ```bash python3 -m venv venv . venv/bin/activate -pip3 install -r requirements-testing.txt +pip3 install -e .[dev] ``` Note: make sure to run `make -B pyspec` from the root of the specs repository, to build the parts of the pyspec module derived from the markdown specs. diff --git a/test_libs/pyspec/requirements-testing.txt b/test_libs/pyspec/requirements-testing.txt deleted file mode 100644 index 388a878a9..000000000 --- a/test_libs/pyspec/requirements-testing.txt +++ /dev/null @@ -1,3 +0,0 @@ --r requirements.txt -pytest>=3.6,<3.7 -../config_helpers diff --git a/test_libs/pyspec/requirements.txt b/test_libs/pyspec/requirements.txt deleted file mode 100644 index 78d41708d..000000000 --- a/test_libs/pyspec/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -eth-utils>=1.3.0,<2 -eth-typing>=2.1.0,<3.0.0 -pycryptodome==3.7.3 -py_ecc>=1.6.0 diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index 1a131a417..13ea4d6e5 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -1,13 +1,30 @@ from setuptools import setup, find_packages -setup( - name='pyspec', - packages=find_packages(), - tests_require=["pytest"], - install_requires=[ + + + +deps = { + 'pyspec': [ "eth-utils>=1.3.0,<2", "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.7.3", "py_ecc>=1.6.0", - ] + ], + 'test': [ + "pytest>=3.6,<3.7", + ], +} + +deps['dev'] = ( + deps['pyspec'] + + deps['test'] +) + +install_requires = deps['pyspec'] + +setup( + name='pyspec', + packages=find_packages(), + install_requires=install_requires, + extras_require=deps, ) From 0fc31e3e7d2404567fd1d6e617ebcfa8e89e6dec Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 23 Apr 2019 09:36:18 +0800 Subject: [PATCH 2/5] Update .gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 3dd86fc80..0419f51b4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,10 @@ venv .venvs .venv /.pytest_cache +*.egg +*.egg-info +eggs +.eggs build/ output/ From ba99f8a284709a7bb648dbde59c22d4a9eb5d6e5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 23 Apr 2019 10:12:13 +0800 Subject: [PATCH 3/5] Update `install_requires` --- test_libs/config_helpers/setup.py | 8 +++++--- test_libs/gen_helpers/setup.py | 7 ++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/test_libs/config_helpers/setup.py b/test_libs/config_helpers/setup.py index b69d19cdb..836993d36 100644 --- a/test_libs/config_helpers/setup.py +++ b/test_libs/config_helpers/setup.py @@ -2,17 +2,19 @@ from distutils.core import setup deps = { - 'config_helpers': [ + 'preset_loader': [ "ruamel.yaml==0.15.87", ], } deps['dev'] = ( - deps['config_helpers'] + deps['preset_loader'] ) +install_requires = deps['preset_loader'] + setup( name='config_helpers', packages=['preset_loader'], - install_requires=deps['config_helpers'] + install_requires=install_requires, ) diff --git a/test_libs/gen_helpers/setup.py b/test_libs/gen_helpers/setup.py index d4d331c06..6d4003573 100644 --- a/test_libs/gen_helpers/setup.py +++ b/test_libs/gen_helpers/setup.py @@ -2,19 +2,20 @@ from distutils.core import setup deps = { - 'gen_helpers': [ + 'gen_base': [ "ruamel.yaml==0.15.87", "eth-utils==1.4.1", ], } deps['dev'] = ( - deps['gen_helpers'] + deps['gen_base'] ) +install_requires = deps['gen_base'] setup( name='gen_helpers', packages=['gen_base'], - install_requires=deps['gen_helpers'], + install_requires=install_requires, ) From 5437273e23c4380b33cf94aaae065fbb5686d2a7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 23 Apr 2019 10:15:01 +0800 Subject: [PATCH 4/5] Update `packages` --- .gitignore | 3 +++ test_libs/config_helpers/setup.py | 4 ++-- test_libs/gen_helpers/setup.py | 4 ++-- test_libs/pyspec/setup.py | 4 +--- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 0419f51b4..84938d298 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ eth2.0-spec-tests/ # Dynamically built from Markdown spec test_libs/pyspec/eth2spec/phase0/spec.py + +# vscode +.vscode/** diff --git a/test_libs/config_helpers/setup.py b/test_libs/config_helpers/setup.py index 836993d36..88669c092 100644 --- a/test_libs/config_helpers/setup.py +++ b/test_libs/config_helpers/setup.py @@ -1,4 +1,4 @@ -from distutils.core import setup +from setuptools import setup, find_packages deps = { @@ -15,6 +15,6 @@ install_requires = deps['preset_loader'] setup( name='config_helpers', - packages=['preset_loader'], + packages=find_packages(exclude=["tests", "tests.*"]), install_requires=install_requires, ) diff --git a/test_libs/gen_helpers/setup.py b/test_libs/gen_helpers/setup.py index 6d4003573..0c84ff2e2 100644 --- a/test_libs/gen_helpers/setup.py +++ b/test_libs/gen_helpers/setup.py @@ -1,4 +1,4 @@ -from distutils.core import setup +from setuptools import setup, find_packages deps = { @@ -16,6 +16,6 @@ install_requires = deps['gen_base'] setup( name='gen_helpers', - packages=['gen_base'], + packages=find_packages(exclude=["tests", "tests.*"]), install_requires=install_requires, ) diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index 13ea4d6e5..3fd9d4c0f 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -1,8 +1,6 @@ from setuptools import setup, find_packages - - deps = { 'pyspec': [ "eth-utils>=1.3.0,<2", @@ -24,7 +22,7 @@ install_requires = deps['pyspec'] setup( name='pyspec', - packages=find_packages(), + packages=find_packages(exclude=["tests", "tests.*"]), install_requires=install_requires, extras_require=deps, ) From b1874dc18b6cd7cc65dd52ce4f9b9c8e9ad14f3c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 23 Apr 2019 10:39:52 +0800 Subject: [PATCH 5/5] Update Makefile and CI setting 1. Move .venv to TEST_LIBS_DIR/ 2. Install `config_helpers` separately --- .circleci/config.yml | 8 ++++---- Makefile | 14 ++++++++++---- test_libs/setup.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 8 deletions(-) create mode 100644 test_libs/setup.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 28d949de4..41e809207 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,15 +60,15 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec + venv_name: v1-test_libs reqs_checksum: '{{ checksum "test_libs/pyspec/setup.py" }}' - run: name: Install pyspec requirements command: make install_test - save_cached_venv: - venv_name: v1-pyspec + venv_name: v1-test_libs reqs_checksum: '{{ checksum "test_libs/pyspec/setup.py" }}' - venv_path: ./test_libs/pyspec/venv + venv_path: ./test_libs/venv test: docker: - image: circleci/python:3.6 @@ -77,7 +77,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec + venv_name: v1-test_libs reqs_checksum: '{{ checksum "test_libs/pyspec/setup.py" }}' - run: name: Run py-tests diff --git a/Makefile b/Makefile index 10550750a..135124898 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ SPEC_DIR = ./specs SCRIPT_DIR = ./scripts TEST_LIBS_DIR = ./test_libs PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec +CONFIG_HELPERS_DIR = $(TEST_LIBS_DIR)/config_helpers YAML_TEST_DIR = ./eth2.0-spec-tests/tests GENERATOR_DIR = ./test_generators CONFIGS_DIR = ./configs @@ -23,7 +24,8 @@ all: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS) clean: rm -rf $(YAML_TEST_DIR) rm -rf $(GENERATOR_VENVS) - rm -rf $(PY_SPEC_DIR)/venv $(PY_SPEC_DIR)/.pytest_cache + rm -rf $(TEST_LIBS_DIR)/venv + rm -rf $(PY_SPEC_DIR)/.pytest_cache rm -rf $(PY_SPEC_ALL_TARGETS) # "make gen_yaml_tests" to run generators @@ -31,13 +33,17 @@ gen_yaml_tests: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_TARGETS) # installs the packages to run pyspec tests install_test: - cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -e .[dev]; + cd $(TEST_LIBS_DIR); python3 -m venv venv; . venv/bin/activate; \ + cd ..; cd $(CONFIG_HELPERS_DIR); pip3 install -e .; \ + cd ../..; cd $(PY_SPEC_DIR); pip3 install -e .[dev]; test: $(PY_SPEC_ALL_TARGETS) - cd $(PY_SPEC_DIR); . venv/bin/activate; python -m pytest . + cd $(TEST_LIBS_DIR); . venv/bin/activate; \ + cd ..; cd $(PY_SPEC_DIR); python -m pytest .; citest: $(PY_SPEC_ALL_TARGETS) - cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; python -m pytest --junitxml=test-reports/eth2spec/test_results.xml . + cd $(TEST_LIBS_DIR); . venv/bin/activate; \ + cd ..; cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; python -m pytest --junitxml=test-reports/eth2spec/test_results.xml . # "make pyspec" to create the pyspec for all phases. pyspec: $(PY_SPEC_ALL_TARGETS) diff --git a/test_libs/setup.py b/test_libs/setup.py new file mode 100644 index 000000000..b82fc369c --- /dev/null +++ b/test_libs/setup.py @@ -0,0 +1,29 @@ +from setuptools import setup, find_packages + + +deps = { + 'pyspec': [ + "eth-utils>=1.3.0,<2", + "eth-typing>=2.1.0,<3.0.0", + "pycryptodome==3.7.3", + "py_ecc>=1.6.0", + ], + 'test': [ + "pytest>=3.6,<3.7", + ], +} + +deps['dev'] = ( + deps['pyspec'] + + deps['test'] +) + +install_requires = deps['pyspec'] + + +setup( + name='pyspec', + packages=find_packages(exclude=["tests", "tests.*"]), + install_requires=install_requires, + extras_require=deps, +)