From c9de95541feba6ffd1f2307f31be13e39f184d7a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 Mar 2021 23:56:34 +0800 Subject: [PATCH 1/4] Use a dummy `INCOMPLETE` file to indicate that the test generation is incomplete --- .../gen_helpers/gen_base/gen_runner.py | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index f14f7e323..ee2410894 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -1,3 +1,5 @@ +import os +import shutil import argparse from pathlib import Path import sys @@ -102,8 +104,11 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): yaml = YAML(pure=True) yaml.default_flow_style = None + log_file = Path(output_dir) / 'testgen_error_log.txt' + print(f"Generating tests into {output_dir}") print(f"Reading configs from {args.configs_path}") + print(f'Error log file: {log_file}') configs = args.config_list if configs is None: @@ -126,14 +131,25 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): / Path(test_case.runner_name) / Path(test_case.handler_name) / Path(test_case.suite_name) / Path(test_case.case_name) ) + incomplete_tag_file = case_dir / "INCOMPLETE" + if case_dir.exists(): - if not args.force: + if not args.force and not incomplete_tag_file.exists(): print(f'Skipping already existing test: {case_dir}') continue - print(f'Warning, output directory {case_dir} already exist,' - f' old files are not deleted but will be overwritten when a new version is produced') + else: + print(f'Warning, output directory {case_dir} already exist,' + f' old files will be deleted and it will generate test vector files with the latest version') + # Clear the existing case_dir folder + shutil.rmtree(case_dir) print(f'Generating test: {case_dir}') + + # Add `INCOMPLETE` tag file to indicate that the test generation has not completed. + case_dir.mkdir(parents=True, exist_ok=True) + with incomplete_tag_file.open("w") as f: + f.write("\n") + try: def output_part(out_kind: str, name: str, fn: Callable[[Path, ], None]): # make sure the test case directory is created before any test part is written. @@ -170,6 +186,19 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): except Exception as e: print(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}") traceback.print_exc() + # Write to log file + with log_file.open("a+") as f: + f.write(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}") + traceback.print_exc(file=f) + f.write('\n') + else: + # If no written_part, the only file was incomplete_tag_file. Clear the existing case_dir folder. + if not written_part: + shutil.rmtree(case_dir) + else: + # Only remove `INCOMPLETE` tag file + os.remove(incomplete_tag_file) + print(f"completed {generator_name}") From d3c9eacd1438655b31fa18a79889e35d1165720f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 Mar 2021 23:59:20 +0800 Subject: [PATCH 2/4] Disable MAINNET test_valid_signature_future_committee --- .../test/altair/block_processing/test_process_sync_committee.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py b/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py index 58cf48314..d24294272 100644 --- a/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py +++ b/tests/core/pyspec/eth2spec/test/altair/block_processing/test_process_sync_committee.py @@ -284,6 +284,7 @@ def test_invalid_signature_previous_committee(spec, state): @with_all_phases_except([PHASE0, PHASE1]) @spec_state_test @always_bls +@with_configs([MINIMAL], reason="too slow") def test_valid_signature_future_committee(spec, state): # NOTE: the `state` provided is at genesis and the process to select # sync committees currently returns the same committee for the first and second From 2ee559eca1c5f94e9a9a2b3d05619a121e6ebeae Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 17 Mar 2021 02:40:59 +0800 Subject: [PATCH 3/4] Fix `SkippedTest` cases. Should remove the folder --- .../core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index ee2410894..e4f1318ae 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -145,6 +145,8 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): print(f'Generating test: {case_dir}') + written_part = False + # Add `INCOMPLETE` tag file to indicate that the test generation has not completed. case_dir.mkdir(parents=True, exist_ok=True) with incomplete_tag_file.open("w") as f: @@ -159,7 +161,6 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): except IOError as e: sys.exit(f'Error when dumping test "{case_dir}", part "{name}", kind "{out_kind}": {e}') - written_part = False meta = dict() try: @@ -173,6 +174,7 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): output_part("ssz", name, dump_ssz_fn(data, name, file_mode)) except SkippedTest as e: print(e) + shutil.rmtree(case_dir) continue # Once all meta data is collected (if any), write it to a meta data file. @@ -182,7 +184,6 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): if not written_part: print(f"test case {case_dir} did not produce any test case parts") - except Exception as e: print(f"ERROR: failed to generate vector(s) for test {case_dir}: {e}") traceback.print_exc() From f360a9823f91d4cbc0714eb90b62646f515dd490 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 17 Mar 2021 02:41:44 +0800 Subject: [PATCH 4/4] Add Makefile commands `detect_generator_error_log` and `detect_generator_incomplete` --- Makefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 951237808..d1ebe0fec 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html CURRENT_DIR = ${CURDIR} LINTER_CONFIG_FILE = $(CURRENT_DIR)/linter.ini +GENERATOR_ERROR_LOG_FILE = $(CURRENT_DIR)/$(TEST_VECTOR_DIR)/testgen_error_log.txt export DAPP_SKIP_BUILD:=1 export DAPP_SRC:=$(SOLIDITY_DEPOSIT_CONTRACT_DIR) @@ -35,7 +36,8 @@ export DAPP_JSON:=build/combined.json .PHONY: clean partial_clean all test citest lint generate_tests pyspec install_test open_cov \ install_deposit_contract_tester test_deposit_contract install_deposit_contract_compiler \ - compile_deposit_contract test_compile_deposit_contract check_toc + compile_deposit_contract test_compile_deposit_contract check_toc \ + detect_generator_incomplete detect_generator_error_log all: $(PY_SPEC_ALL_TARGETS) @@ -171,3 +173,9 @@ $(TEST_VECTOR_DIR)/: # (creation of output dir is a dependency) gen_%: $(TEST_VECTOR_DIR) $(call run_generator,$*) + +detect_generator_incomplete: $(TEST_VECTOR_DIR) + find $(TEST_VECTOR_DIR) -name "INCOMPLETE" + +detect_generator_error_log: $(TEST_VECTOR_DIR) + [ -f $(GENERATOR_ERROR_LOG_FILE) ] && echo "[ERROR] $(GENERATOR_ERROR_LOG_FILE) file exists" || echo "[PASSED] error log file does not exist"