From d244bcace9e35f1cb19504c6f62bd2765352fc05 Mon Sep 17 00:00:00 2001 From: Umut Date: Fri, 12 Nov 2021 16:46:58 +0300 Subject: [PATCH] test(scripts): create measurement script tests --- .github/workflows/continuous-integration.yaml | 4 + Makefile | 9 +- script/progress_tracker_utils/measure.py | 4 +- .../test_progress_tracker.py | 137 ++++++++++++++++++ .../alert_invalid_comparison_constant.py | 7 + .../alert_invalid_comparison_operator.py | 7 + .../test_scripts/alert_on_undefined_metric.py | 7 + .../test_scripts/measure_end_before_start.py | 7 + .../measure_invalid_indentation.py | 6 + .../test_scripts/measure_nested.py | 13 ++ .../test_scripts/measure_unfinished.py | 5 + .../two_targets_with_the_same_name.1.py | 5 + .../two_targets_with_the_same_name.2.py | 5 + 13 files changed, 213 insertions(+), 3 deletions(-) create mode 100644 script/progress_tracker_utils/test_progress_tracker.py create mode 100644 script/progress_tracker_utils/test_scripts/alert_invalid_comparison_constant.py create mode 100644 script/progress_tracker_utils/test_scripts/alert_invalid_comparison_operator.py create mode 100644 script/progress_tracker_utils/test_scripts/alert_on_undefined_metric.py create mode 100644 script/progress_tracker_utils/test_scripts/measure_end_before_start.py create mode 100644 script/progress_tracker_utils/test_scripts/measure_invalid_indentation.py create mode 100644 script/progress_tracker_utils/test_scripts/measure_nested.py create mode 100644 script/progress_tracker_utils/test_scripts/measure_unfinished.py create mode 100644 script/progress_tracker_utils/test_scripts/two_targets_with_the_same_name.1.py create mode 100644 script/progress_tracker_utils/test_scripts/two_targets_with_the_same_name.2.py diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index 1b9231ac2..9e3bcb7d9 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -285,6 +285,10 @@ jobs: if: ${{ github.event_name == 'schedule' && steps.conformance.outcome == 'success' && !cancelled() }} run: | make pytest_nb + - name: PyTest Progress Tracker + if: ${{ steps.conformance.outcome == 'success' && !cancelled() }} + run: | + make pytest_progress_tracker - name: Test coverage id: coverage if: ${{ always() && steps.pytest.outcome != 'skipped' && !cancelled() }} diff --git a/Makefile b/Makefile index 5973cfa77..130c0c76d 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,9 @@ pylint_benchmarks: .PHONY: pylint_benchmarks pylint_script: - find ./script/ -type f -name "*.py" | xargs poetry run pylint --rcfile=pylintrc + @# disable linting python files under `progress_tracker_utils/test_scripts` folder + @# because they are intentionally ill-formed so that progress tracker can be tested + find ./script/ -type f -name "*.py" -not -path "./script/progress_tracker_utils/test_scripts/*" | xargs poetry run pylint --rcfile=pylintrc .PHONY: pylint_script flake8: @@ -93,6 +95,11 @@ pytest: --cov-report=term-missing:skip-covered tests/ .PHONY: pytest +pytest_progress_tracker: + poetry run python script/progress_tracker_utils/extract_machine_info.py + poetry run pytest -svv script/progress_tracker_utils/test_progress_tracker.py +.PHONY: pytest_progress_tracker + # Not a huge fan of ignoring missing imports, but some packages do not have typing stubs mypy: poetry run mypy -p $(SRC_DIR) --ignore-missing-imports diff --git a/script/progress_tracker_utils/measure.py b/script/progress_tracker_utils/measure.py index dc4024d72..2609d04b5 100644 --- a/script/progress_tracker_utils/measure.py +++ b/script/progress_tracker_utils/measure.py @@ -39,7 +39,7 @@ def register_alert(script, index, line, metrics, alerts): # Parse the alert and append it to list of alerts supported_operators = ["==", "!=", "<=", ">=", "<", ">"] for operator in supported_operators: - alert_details = alert_line.split(operator) + alert_details = alert_line.split(f" {operator} ") # An alert should be of form `{metric} {operator} {constant}` if len(alert_details) == 2: @@ -59,7 +59,7 @@ def register_alert(script, index, line, metrics, alerts): except ValueError as error: raise SyntaxError( f"An alert is not using a constant floating point for comparison " - f"(at line {index + 1} of {script})", + f"(it uses `{value_str}` at line {index + 1} of {script})", ) from error break diff --git a/script/progress_tracker_utils/test_progress_tracker.py b/script/progress_tracker_utils/test_progress_tracker.py new file mode 100644 index 000000000..a6ca789e4 --- /dev/null +++ b/script/progress_tracker_utils/test_progress_tracker.py @@ -0,0 +1,137 @@ +"""Test file for progress tracker""" + +from typing import List + +import measure +import pytest + + +class Args: + """Class to mimic the command line arguments that can be passed to measurement script.""" + + base: str + files_to_benchmark: List[str] + samples: int + keep: bool + check: bool + + def __init__(self, files_to_benchmark: List[str]): + self.base = "script/progress_tracker_utils/test_scripts" + self.files_to_benchmark = files_to_benchmark + self.samples = 30 + self.keep = False + self.check = True + + +def test_alert_on_undefined_metric(): + """Test function for alert directive on unefined metric""" + + file = "script/progress_tracker_utils/test_scripts/alert_on_undefined_metric.py" + args = Args([file]) + + with pytest.raises(SyntaxError) as excinfo: + measure.main(args) + + assert str(excinfo.value) == ( + f"An alert is using an undefined metric `Accuracy (%)` (at line 7 of {file})" + ) + + +def test_alert_invalid_comparison_constant(): + """Test function for alert directive with invalid constant""" + + file = "script/progress_tracker_utils/test_scripts/alert_invalid_comparison_constant.py" + args = Args([file]) + + with pytest.raises(SyntaxError) as excinfo: + measure.main(args) + + assert str(excinfo.value) == ( + f"An alert is not using a constant floating point for comparison " + f'(it uses `"abc"` at line 7 of {file})' + ) + + +def test_alert_invalid_comparison_operator(): + """Test function for alert directive that use invalid comparison operator""" + + file = "script/progress_tracker_utils/test_scripts/alert_invalid_comparison_operator.py" + args = Args([file]) + + with pytest.raises(SyntaxError) as excinfo: + measure.main(args) + + assert str(excinfo.value) == ( + f"An alert is not using any of the supported comparisons ==, !=, <=, >=, <, > " + f"(at line 7 of {file})" + ) + + +def test_measure_end_before_start(): + """Test function for measure end directive before measure directive""" + + file = "script/progress_tracker_utils/test_scripts/measure_end_before_start.py" + args = Args([file]) + + with pytest.raises(SyntaxError) as excinfo: + measure.main(args) + + assert str(excinfo.value) == ( + f"Measurements cannot end before they are defined (at line 3 of {file})" + ) + + +def test_measure_invalid_indentation(): + """Test function for invalid indentation of measure directives""" + + file = "script/progress_tracker_utils/test_scripts/measure_invalid_indentation.py" + args = Args([file]) + + with pytest.raises(SyntaxError) as excinfo: + measure.main(args) + + assert str(excinfo.value) == ( + f"Measurements should finish with the same indentation as they are defined " + f"(at lines 4 and 6 of {file})" + ) + + +def test_measure_nested(): + """Test function for nested measure directives""" + + file = "script/progress_tracker_utils/test_scripts/measure_nested.py" + args = Args([file]) + + with pytest.raises(SyntaxError) as excinfo: + measure.main(args) + + assert str(excinfo.value) == ( + f"Nested measurements are not supported (at lines 3 and 7 of {file})" + ) + + +def test_measure_unfinished(): + """Test function for measure directives without a measure end directive""" + + file = "script/progress_tracker_utils/test_scripts/measure_unfinished.py" + args = Args([file]) + + with pytest.raises(SyntaxError) as excinfo: + measure.main(args) + + assert str(excinfo.value) == ( + f"Unfinished measurements are not supported (at line 3 of {file})" + ) + + +def test_two_targets_with_the_same_name(): + """Test function for target name collisions""" + + file1 = "script/progress_tracker_utils/test_scripts/two_targets_with_the_same_name.1.py" + file2 = "script/progress_tracker_utils/test_scripts/two_targets_with_the_same_name.2.py" + args = Args([file1, file2]) + + with pytest.raises(RuntimeError) as excinfo: + measure.main(args) + + assert str(excinfo.value) == "Target `X` is already registered" diff --git a/script/progress_tracker_utils/test_scripts/alert_invalid_comparison_constant.py b/script/progress_tracker_utils/test_scripts/alert_invalid_comparison_constant.py new file mode 100644 index 000000000..f8c01ccb1 --- /dev/null +++ b/script/progress_tracker_utils/test_scripts/alert_invalid_comparison_constant.py @@ -0,0 +1,7 @@ +# bench: Unit Target: X + +# bench: Measure: Printing Time (ms) +print(42) +# bench: Measure: End + +# bench: Alert: Printing Time (ms) != "abc" diff --git a/script/progress_tracker_utils/test_scripts/alert_invalid_comparison_operator.py b/script/progress_tracker_utils/test_scripts/alert_invalid_comparison_operator.py new file mode 100644 index 000000000..ea7e776d1 --- /dev/null +++ b/script/progress_tracker_utils/test_scripts/alert_invalid_comparison_operator.py @@ -0,0 +1,7 @@ +# bench: Unit Target: X + +# bench: Measure: Printing Time (ms) +print(42) +# bench: Measure: End + +# bench: Alert: Printing Time (ms) === 10 diff --git a/script/progress_tracker_utils/test_scripts/alert_on_undefined_metric.py b/script/progress_tracker_utils/test_scripts/alert_on_undefined_metric.py new file mode 100644 index 000000000..2f7acc9c1 --- /dev/null +++ b/script/progress_tracker_utils/test_scripts/alert_on_undefined_metric.py @@ -0,0 +1,7 @@ +# bench: Unit Target: X + +# bench: Measure: Printing Time (ms) +print(42) +# bench: Measure: End + +# bench: Alert: Accuracy (%) != 100 diff --git a/script/progress_tracker_utils/test_scripts/measure_end_before_start.py b/script/progress_tracker_utils/test_scripts/measure_end_before_start.py new file mode 100644 index 000000000..228f8a034 --- /dev/null +++ b/script/progress_tracker_utils/test_scripts/measure_end_before_start.py @@ -0,0 +1,7 @@ +# bench: Unit Target: X + +# bench: Measure: End + +# bench: Measure: Printing Time (ms) +print(42) +# bench: Measure: End diff --git a/script/progress_tracker_utils/test_scripts/measure_invalid_indentation.py b/script/progress_tracker_utils/test_scripts/measure_invalid_indentation.py new file mode 100644 index 000000000..6c61ce9b5 --- /dev/null +++ b/script/progress_tracker_utils/test_scripts/measure_invalid_indentation.py @@ -0,0 +1,6 @@ +# bench: Unit Target: X + +if True: + # bench: Measure: Printing Time (ms) + print(42) +# bench: Measure: End diff --git a/script/progress_tracker_utils/test_scripts/measure_nested.py b/script/progress_tracker_utils/test_scripts/measure_nested.py new file mode 100644 index 000000000..fceb1e778 --- /dev/null +++ b/script/progress_tracker_utils/test_scripts/measure_nested.py @@ -0,0 +1,13 @@ +# bench: Unit Target: X + +# bench: Measure: Printing Time Of Three (ms) + +print(42) + +# bench: Measure: Printing Time Of One (ms) +print(42) +# bench: Measure: End + +print(42) + +# bench: Measure: End diff --git a/script/progress_tracker_utils/test_scripts/measure_unfinished.py b/script/progress_tracker_utils/test_scripts/measure_unfinished.py new file mode 100644 index 000000000..9ba5e0c2d --- /dev/null +++ b/script/progress_tracker_utils/test_scripts/measure_unfinished.py @@ -0,0 +1,5 @@ +# bench: Unit Target: X + +# bench: Measure: Printing Time Of Three (ms) + +print(42) diff --git a/script/progress_tracker_utils/test_scripts/two_targets_with_the_same_name.1.py b/script/progress_tracker_utils/test_scripts/two_targets_with_the_same_name.1.py new file mode 100644 index 000000000..1803aca7d --- /dev/null +++ b/script/progress_tracker_utils/test_scripts/two_targets_with_the_same_name.1.py @@ -0,0 +1,5 @@ +# bench: Full Target: X + +# bench: Measure: Printing Time (ms) +print(420) +# bench: Measure: End diff --git a/script/progress_tracker_utils/test_scripts/two_targets_with_the_same_name.2.py b/script/progress_tracker_utils/test_scripts/two_targets_with_the_same_name.2.py new file mode 100644 index 000000000..a490d7e45 --- /dev/null +++ b/script/progress_tracker_utils/test_scripts/two_targets_with_the_same_name.2.py @@ -0,0 +1,5 @@ +# bench: Unit Target: X + +# bench: Measure: Printing Time (ms) +print(42) +# bench: Measure: End