mirror of
https://github.com/tinygrad/tinygrad.git
synced 2026-01-09 15:08:02 -05:00
e3c6c0c6db
* add gpt2 to examples * some cleanup * fixes * argparse + scaled_dot_product_attention * add timing * add to benchmark Co-authored-by: YassineYousfi <yassine.y10@gmail.com>
95 lines
3.6 KiB
YAML
95 lines
3.6 KiB
YAML
name: Benchmarks
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- master
|
|
- update_benchmark
|
|
|
|
jobs:
|
|
testmacbenchmark:
|
|
name: Mac Benchmark
|
|
runs-on: [self-hosted, macOS]
|
|
if: github.repository_owner == 'tinygrad'
|
|
env:
|
|
PYTHONPATH: .
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v3
|
|
- name: Run model inference benchmark
|
|
run: python3 test/external/external_model_benchmark.py
|
|
- name: Test speed vs torch
|
|
run: BIG=2 MPS=1 python3 test/test_speed_v_torch.py | tee torch_speed.txt
|
|
- name: Run Stable Diffusion
|
|
run: |
|
|
ln -s ~/tinygrad/weights/sd-v1-4.ckpt weights/sd-v1-4.ckpt
|
|
ln -s ~/tinygrad/weights/bpe_simple_vocab_16e6.txt.gz weights/bpe_simple_vocab_16e6.txt.gz
|
|
time python3 examples/stable_diffusion.py --noshow
|
|
- name: Run LLaMA
|
|
run: |
|
|
ln -s ~/tinygrad/weights/LLaMA weights/LLaMA
|
|
python3 examples/llama.py --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_unjitted.txt
|
|
JIT=1 python3 examples/llama.py --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_jitted.txt
|
|
- name: Run GPT2
|
|
run: |
|
|
python3 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_unjitted.txt
|
|
JIT=1 python3 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_jitted.txt
|
|
- name: Run 10 CIFAR training steps
|
|
run: |
|
|
ln -s ~/tinygrad/extra/datasets/cifar-10-python.tar.gz extra/datasets/cifar-10-python.tar.gz
|
|
STEPS=10 python3 examples/hlb_cifar10.py | tee train_cifar.txt
|
|
- uses: actions/upload-artifact@v3
|
|
with:
|
|
name: Speed (Mac)
|
|
path: |
|
|
onnx_inference_speed.csv
|
|
torch_speed.txt
|
|
train_cifar.txt
|
|
llama_unjitted.txt
|
|
llama_jitted.txt
|
|
gpt2_unjitted.txt
|
|
gpt2_jitted.txt
|
|
|
|
testamdbenchmark:
|
|
name: AMD Benchmark
|
|
runs-on: [self-hosted, Linux]
|
|
if: github.repository_owner == 'tinygrad'
|
|
env:
|
|
PYTHONPATH: .
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@v3
|
|
- name: Run model inference benchmark
|
|
run: python3 test/external/external_model_benchmark.py
|
|
- name: Test speed vs torch
|
|
run: BIG=2 TORCHCUDA=1 python3 test/test_speed_v_torch.py | tee torch_speed.txt
|
|
- name: Run Stable Diffusion
|
|
run: |
|
|
ln -s ~/tinygrad/weights/sd-v1-4.ckpt weights/sd-v1-4.ckpt
|
|
ln -s ~/tinygrad/weights/bpe_simple_vocab_16e6.txt.gz weights/bpe_simple_vocab_16e6.txt.gz
|
|
time DEBUG=1 python3 examples/stable_diffusion.py --noshow
|
|
- name: Run LLaMA
|
|
run: |
|
|
ln -s ~/tinygrad/weights/LLaMA weights/LLaMA
|
|
python3 examples/llama.py --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_unjitted.txt
|
|
JIT=1 python3 examples/llama.py --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_jitted.txt
|
|
- name: Run GPT2
|
|
run: |
|
|
python3 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_unjitted.txt
|
|
JIT=1 python3 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_jitted.txt
|
|
- name: Run 10 CIFAR training steps
|
|
run: |
|
|
ln -s ~/tinygrad/extra/datasets/cifar-10-python.tar.gz extra/datasets/cifar-10-python.tar.gz
|
|
STEPS=10 python3 examples/hlb_cifar10.py | tee train_cifar.txt
|
|
- uses: actions/upload-artifact@v3
|
|
with:
|
|
name: Speed (AMD)
|
|
path: |
|
|
onnx_inference_speed.csv
|
|
torch_speed.txt
|
|
train_cifar.txt
|
|
llama_unjitted.txt
|
|
llama_jitted.txt
|
|
gpt2_unjitted.txt
|
|
gpt2_jitted.txt
|