From fdc8fabae51dc0acc992f0319a3e306e28b711a1 Mon Sep 17 00:00:00 2001 From: chenyu Date: Tue, 30 Apr 2024 10:41:37 -0400 Subject: [PATCH] disable flaky mac gpt2 beam benchmark and add back cifar mac with JIT=2 (#4358) * debug flaky mac gpt2 beam run * disable for now --- .github/workflows/benchmark.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 01e365207b..e774f39270 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -27,9 +27,8 @@ jobs: ln -s ~/tinygrad/weights/bpe_simple_vocab_16e6.txt.gz weights/bpe_simple_vocab_16e6.txt.gz ln -s ~/tinygrad/weights/LLaMA weights/LLaMA ln -s ~/tinygrad/extra/datasets/cifar-10-python.tar.gz extra/datasets/cifar-10-python.tar.gz - # TODO: why is this test not reliable? - #- name: Run Stable Diffusion - # run: python3 examples/stable_diffusion.py --seed 0 --noshow --timing | tee sd.txt + - name: Run Stable Diffusion + run: JIT=2 python3 examples/stable_diffusion.py --seed 0 --noshow --timing | tee sd.txt - name: Run model inference benchmark run: METAL=1 python3 test/external/external_model_benchmark.py - name: Test speed vs torch @@ -56,8 +55,9 @@ jobs: JIT=1 python3 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_jitted.txt - name: Run GPT2 w HALF run: JIT=1 HALF=1 python3 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half.txt - - name: Run GPT2 w HALF/BEAM - run: JIT=1 HALF=1 BEAM=2 CACHELEVEL=0 python3 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half_beam.txt + # TODO: this is flaky + # - name: Run GPT2 w HALF/BEAM + # run: JIT=0 HALF=1 BEAM=2 CACHELEVEL=0 python3 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half_beam.txt - name: Train MNIST run: time PYTHONPATH=. TARGET_EVAL_ACC_PCT=97.3 python3 examples/beautiful_mnist.py | tee beautiful_mnist.txt - name: Run 10 CIFAR training steps @@ -66,9 +66,8 @@ jobs: run: JIT=2 STEPS=10 DEFAULT_FLOAT=HALF python3 examples/hlb_cifar10.py | tee train_cifar_half.txt #- name: Run 10 CIFAR training steps w BF16 # run: STEPS=10 DEFAULT_FLOAT=BFLOAT16 python3 examples/hlb_cifar10.py | tee train_cifar_bf16.txt - # TODO: this is flaky too - # - name: Run 10 CIFAR training steps w winograd - # run: WINO=1 STEPS=10 python3 examples/hlb_cifar10.py | tee train_cifar_wino.txt + - name: Run 10 CIFAR training steps w winograd + run: JIT=2 WINO=1 STEPS=10 python3 examples/hlb_cifar10.py | tee train_cifar_wino.txt - uses: actions/upload-artifact@v4 with: name: Speed (Mac)