github/tinygrad
1
1
Fork 0
mirror of https://github.com/tinygrad/tinygrad.git synced 2026-01-08 22:48:25 -05:00
Code Issues Packages Projects Releases Wiki Activity

new generate_dataset.sh (#6423)

Browse Source 
* new generate_dataset.sh

* keep those there

* test: rm expected failures

* rename to extract
This commit is contained in:
qazal
2024-09-09 15:13:07 +08:00
committed by GitHub
parent 1941e66cc9
commit c5bae55ec8
5 changed files with 33 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
extra/optimization/extract_dataset.py Executable file
View File

@@ -0,0 +1,22 @@
#!/usr/bin/env python3
# extract asts from process replay artifacts
import os, pickle
from tinygrad.helpers import db_connection, getenv, VERSION
from test.external.process_replay.process_replay import _run_differ
PAGE_SIZE = 100
RUN_ID = os.getenv("GITHUB_RUN_ID", "HEAD")
TABLE_NAME = f"process_replay_{RUN_ID}_{getenv('GITHUB_RUN_ATTEMPT')}_{VERSION}"
LOGOPS = os.getenv("LOGOPS", "/tmp/sops")
def extract_ast(offset:int):
logops = open(LOGOPS, "a")
conn = db_connection()
for row in conn.execute(f"SELECT val FROM '{TABLE_NAME}' LIMIT ? OFFSET ?", (PAGE_SIZE, offset)).fetchall():
logops.write(str(pickle.loads(row[0])[0]).replace("\n", "").replace(" ", "")+"\n")
return False
if __name__ == "__main__":
conn = db_connection()
row_count = conn.execute(f"SELECT COUNT(*) FROM '{TABLE_NAME}'").fetchone()[0]
_run_differ(row_count, extract_ast)

17
extra/optimization/generate_dataset.sh
View File

@@ -1,12 +1,10 @@
#!/bin/bash
export LOGOPS=/tmp/ops
export LOGOPS=/tmp/sops
export RUN_PROCESS_REPLAY=1
rm $LOGOPS
test/external/process_replay/reset.py
# generate many kernels
PYTHONPATH="." OPT=2 GPU=1 python3 test/external/external_test_opt.py
PYTHONPATH="." OPT=3 GPU=1 python3 test/external/external_test_opt.py
GPU=1 IMAGE=1 python3 test/test_ops.py
FORWARD_ONLY=1 GPU=1 IMAGE=2 python test/test_ops.py
python3 -m pytest -n=auto test/ --ignore=test/unit --durations=20
STEPS=3 python3 examples/hlb_cifar10.py
WINO=1 STEPS=3 python3 examples/hlb_cifar10.py
python3 examples/stable_diffusion.py --noshow
@@ -23,6 +21,7 @@ JIT=2 BIG=1 MPS=1 python -m pytest test/test_gc.py
JIT=2 BIG=1 MPS=1 python -m pytest test/test_jit.py
JIT=2 BIG=1 MPS=1 python -m pytest test/test_speed_v_torch.py
# sort and uniq
sort -u /tmp/ops > /tmp/sops
ls -lh /tmp/ops /tmp/sops
# extract, sort and uniq
extra/optimization/extract_dataset.py
sort -u /tmp/ops > /tmp/sops
ls -lh /tmp/ops /tmp/sops