diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1e0ed2ea6c..0dbb8ea8de 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -253,7 +253,7 @@ jobs: run: PYTHONPATH=. THREEFRY=1 GPU=1 python3 -m pytest test/test_randomness.py test/test_jit.py --durations=20 - name: Run process replay tests if: env.RUN_PROCESS_REPLAY == '1' - run: cp test/external/replay_codegen.py ./replay_codegen.py && git fetch origin master && git checkout origin/master && PYTHONPATH=. python3 replay_codegen.py + run: cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git checkout origin/master && PYTHONPATH=. python3 process_replay.py #testwebgpu: # name: WebGPU Tests @@ -348,7 +348,7 @@ jobs: run: FUZZ_SCHEDULE=1 FUZZ_SCHEDULE_MAX_PATHS=5 python -m pytest test/models/test_train.py test/models/test_end2end.py - name: Run process replay tests if: env.RUN_PROCESS_REPLAY == '1' - run: cp test/external/replay_codegen.py ./replay_codegen.py && git fetch origin master && git checkout origin/master && PYTHONPATH=. python3 replay_codegen.py + run: cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git checkout origin/master && PYTHONPATH=. python3 process_replay.py # testwebgl: # name: WebGL Tests @@ -538,7 +538,7 @@ jobs: cat test/models/efficientnet/Chicken.jpg | ./recognize | grep cock - name: Run process replay tests if: env.RUN_PROCESS_REPLAY == '1' - run: cp test/external/replay_codegen.py ./replay_codegen.py && git fetch origin master && git checkout origin/master && PYTHONPATH=. python3 replay_codegen.py + run: cp test/external/process_replay/process_replay.py ./process_replay.py && git fetch origin master && git checkout origin/master && PYTHONPATH=. python3 process_replay.py #testunicorn: # name: ARM64 unicorn Test diff --git a/README.md b/README.md index b5788bcf29..9d8455a4ac 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,6 @@ python3 -m pytest test/ # whole test suite #### Process replay tests -[Process replay](https://github.com/tinygrad/tinygrad/blob/master/test/external/replay_codegen.py) detects changes in the generated kernels of CI tests by comparing them against tinygrad master. If your PR is a refactor or speedup without any expected behavior change, it should include a green process replay pass to get merged. +[Process replay](https://github.com/tinygrad/tinygrad/blob/master/test/external/process_replay/process_replay.py) detects changes in the generated kernels of CI tests by comparing them against tinygrad master. If your PR is a refactor or speedup without any expected behavior change, it should include a green process replay pass to get merged. You can enable process replay by adding [run_process_replay] to your PR title. [example](https://github.com/tinygrad/tinygrad/pull/4995). Note that you should keep your branch up-to-date with master. diff --git a/test/external/process_replay/README.md b/test/external/process_replay/README.md new file mode 100644 index 0000000000..d6557adac6 --- /dev/null +++ b/test/external/process_replay/README.md @@ -0,0 +1,17 @@ +# Process replay tests + +Process replay is a tool for creating a diff of generated kernels between two commits. + +Refactor and speedup PRs need a green process replay check. + +Behavior change PRs can use process replay with `ASSERT_PROCESS_REPLAY=0` to check the diff is what was expected. It's also an indirect test coverage checker. + +## Running locally + +To run process replay locally: + +(optional: clear previous process replay runs with `test/external/process_replay/reset.py`) + +1. Run tests with `RUN_PROCESS_REPLAY=1` in your branch +2. Checkout master +3. Run `test/external/process_replay/process_replay.py` diff --git a/test/external/replay_codegen.py b/test/external/process_replay/process_replay.py old mode 100644 new mode 100755 similarity index 98% rename from test/external/replay_codegen.py rename to test/external/process_replay/process_replay.py index 8c04dfbbbc..0c91904f24 --- a/test/external/replay_codegen.py +++ b/test/external/process_replay/process_replay.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # compare kernels created by HEAD against master import difflib, pickle from tinygrad.codegen.linearizer import Linearizer diff --git a/test/external/process_replay/reset.py b/test/external/process_replay/reset.py new file mode 100755 index 0000000000..c40b6d0961 --- /dev/null +++ b/test/external/process_replay/reset.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python3 +from tinygrad.helpers import db_connection, VERSION +cur = db_connection() +cur.execute(f"drop table if exists process_replay_{VERSION}")