move autogen to different CI runner [pr] (#7346)

* move autogen to different CI runner [pr] * balance a bit * readme back there * compile enet in autogen
2026-04-29 03:00:14 -04:00 · 2024-10-29 12:35:22 +07:00
parent 4cb236a495
commit a5e0f59e41
1 changed files with 115 additions and 73 deletions
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -14,6 +14,119 @@ on:
  workflow_dispatch:

 jobs:
+  autogen:
+    name: Autogen+Docs
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v4
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.12
+    - name: Install docs dependencies (no cache)
+      run: pip install -e '.[docs]'
+    - name: Use as an external package
+      run: |
+        mkdir $HOME/test_external_dir
+        cd $HOME/test_external_dir
+        python -m venv venv
+        source venv/bin/activate
+        pip install $GITHUB_WORKSPACE
+        python -c "from tinygrad.tensor import Tensor; print(Tensor([1,2,3,4,5]))"
+        pip install mypy
+        mypy -c "from tinygrad.tensor import Tensor; print(Tensor([1,2,3,4,5]))"
+    - name: Run beautiful_mnist without numpy
+      run: |
+        mkdir $HOME/test_no_numpy_dir
+        cd $HOME/test_no_numpy_dir
+        python -m venv venv
+        source venv/bin/activate
+        pip install $GITHUB_WORKSPACE
+        cp $GITHUB_WORKSPACE/examples/beautiful_mnist.py .
+        PYTHONPATH=$GITHUB_WORKSPACE BS=2 STEPS=10 python beautiful_mnist.py
+    - name: Test Docs Build
+      run: python -m mkdocs build --strict
+    - name: Test Docs
+      run: |
+        python docs/abstractions2.py
+        python docs/abstractions3.py
+    - name: Test Quickstart
+      run: awk '/```python/{flag=1;next}/```/{flag=0}flag' docs/quickstart.md > quickstart.py &&  PYTHONPATH=. python quickstart.py
+    - name: Test DEBUG
+      run: DEBUG=100 python3 -c "from tinygrad import Tensor; N = 1024; a, b = Tensor.rand(N, N), Tensor.rand(N, N); c = (a.reshape(N, 1, N) * b.T.reshape(1, N, N)).sum(axis=2); print((c.numpy() - (a.numpy() @ b.numpy())).mean())"
+    - name: Install OpenCL
+      run: |
+        echo 'Acquire::http::Pipeline-Depth "5";' | sudo tee -a /etc/apt/apt.conf.d/99parallel
+        echo "deb [ allow-insecure=yes ] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
+        sudo apt update || true
+        sudo apt install --allow-unauthenticated -y --no-install-recommends opencl-headers \
+          intel-oneapi-runtime-openmp=2023.2.1-16 intel-oneapi-runtime-compilers-common=2023.2.1-16 intel-oneapi-runtime-compilers=2023.2.1-16 \
+          intel-oneapi-runtime-dpcpp-sycl-opencl-cpu=2023.2.1-16 intel-oneapi-runtime-tbb-common=2021.10.0-49541 \
+          intel-oneapi-runtime-tbb=2021.10.0-49541 intel-oneapi-runtime-opencl=2023.2.1-16
+    - name: Install packages (cuda)
+      run: |
+        echo 'Acquire::http::Pipeline-Depth "5";' | sudo tee -a /etc/apt/apt.conf.d/99parallel
+        sudo apt update -y || true
+        sudo apt install -y --no-install-recommends git g++ cmake ninja-build llvm-15-dev zlib1g-dev libglew-dev \
+          flex bison libfl-dev libboost-thread-dev libboost-filesystem-dev nvidia-cuda-toolkit-gcc libzstd-dev
+    - name: Install packages (amd)
+      run: |
+        echo 'Acquire::http::Pipeline-Depth "5";' | sudo tee -a /etc/apt/apt.conf.d/99parallel
+        wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
+        sudo tee /etc/apt/sources.list.d/rocm.list <<'EOF'
+        deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/6.1.2 jammy main
+        EOF
+        echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600
+        sudo apt update || true
+        sudo apt install --no-install-recommends --allow-unauthenticated -y hsa-rocr comgr hsa-rocr-dev liburing-dev libc6-dev
+        curl -s https://api.github.com/repos/Qazalin/remu/releases/latest | \
+        jq -r '.assets[] | select(.name == "libremu.so").browser_download_url' | \
+        sudo xargs curl -L -o /usr/local/lib/libremu.so
+        sudo tee --append /etc/ld.so.conf.d/rocm.conf <<'EOF'
+          /opt/rocm/lib
+          /opt/rocm/lib64
+        EOF
+        sudo ldconfig
+    - name: Compile EfficientNet to C and test it
+      run: |
+        CLANG=1 PYTHONPATH="." python examples/compile_efficientnet.py > recognize.c
+        clang -O2 recognize.c -lm -o recognize
+        cat test/models/efficientnet/Chicken.jpg | ./recognize | grep cock
+    - name: Verify OpenCL autogen
+      run: |
+        cp tinygrad/runtime/autogen/opencl.py /tmp/opencl.py.bak
+        ./autogen_stubs.sh opencl
+        diff /tmp/opencl.py.bak tinygrad/runtime/autogen/opencl.py
+    - name: Verify CUDA autogen
+      run: |
+        cp tinygrad/runtime/autogen/cuda.py /tmp/cuda.py.bak
+        cp tinygrad/runtime/autogen/nv_gpu.py /tmp/nv_gpu.py.bak
+        ./autogen_stubs.sh cuda
+        ./autogen_stubs.sh nv
+        diff /tmp/cuda.py.bak tinygrad/runtime/autogen/cuda.py
+        diff /tmp/nv_gpu.py.bak tinygrad/runtime/autogen/nv_gpu.py
+    - name: Verify AMD autogen
+      run: |
+        cp tinygrad/runtime/autogen/hsa.py /tmp/hsa.py.bak
+        cp tinygrad/runtime/autogen/comgr.py /tmp/comgr.py.bak
+        cp tinygrad/runtime/autogen/amd_gpu.py /tmp/amd_gpu.py.bak
+        ./autogen_stubs.sh hsa
+        ./autogen_stubs.sh comgr
+        ./autogen_stubs.sh amd
+        diff /tmp/hsa.py.bak tinygrad/runtime/autogen/hsa.py
+        diff /tmp/comgr.py.bak tinygrad/runtime/autogen/comgr.py
+        diff /tmp/amd_gpu.py.bak tinygrad/runtime/autogen/amd_gpu.py
+    - name: Verify Linux autogen
+      run: |
+        cp tinygrad/runtime/autogen/libc.py /tmp/libc.py.bak
+        cp tinygrad/runtime/autogen/io_uring.py /tmp/io_uring.py.bak
+        ./autogen_stubs.sh libc
+        ./autogen_stubs.sh io_uring
+        diff /tmp/libc.py.bak tinygrad/runtime/autogen/libc.py
+        diff /tmp/io_uring.py.bak tinygrad/runtime/autogen/io_uring.py
+
  uops:
    name: uops tests
    runs-on: ubuntu-latest
@@ -83,7 +196,7 @@ jobs:
      run: PYTHONPATH=. PYTHON=1 python3 -m pytest -rA test/test_linearizer_failures.py::TestLinearizerFailures::test_failure_1

  linter:
-    name: Linters
+    name: Linters+fuzz+unit Tests
    runs-on: ubuntu-latest
    timeout-minutes: 20

@@ -101,7 +214,7 @@ jobs:
        path: ${{ env.Python3_ROOT_DIR }}/lib/python3.8/site-packages
        key: linting-packages-${{ hashFiles('**/setup.py') }}-3.8
    - name: Install dependencies
-      run: pip install -e '.[linting,testing,docs]' --extra-index-url https://download.pytorch.org/whl/cpu
+      run: pip install -e '.[linting,testing]' --extra-index-url https://download.pytorch.org/whl/cpu
    - name: Lint bad-indentation and trailing-whitespace with pylint
      run: python -m pylint --disable=all -e W0311 -e C0303 --jobs=0 --indent-string='  ' --recursive=y .
    - name: Lint with ruff
@@ -112,14 +225,6 @@ jobs:
      run: python -m pylint tinygrad/
    - name: Run mypy
      run: python -m mypy --strict-equality
-    - name: Test Docs
-      run: |
-        python docs/abstractions2.py
-        python docs/abstractions3.py
-    - name: Test Docs Build
-      run: python -m mkdocs build --strict
-    - name: Test Quickstart
-      run: awk '/```python/{flag=1;next}/```/{flag=0}flag' docs/quickstart.md > quickstart.py &&  PYTHONPATH=. python quickstart.py
    - name: Test README
      run: awk '/```python/{flag=1;next}/```/{flag=0}flag' README.md > README.py &&  PYTHONPATH=. python README.py
    - name: Run unit tests
@@ -130,27 +235,6 @@ jobs:
      run: |
        PYTHONPATH="." python test/external/fuzz_shapetracker.py
        PYTHONPATH="." python test/external/fuzz_shapetracker_math.py
-    - name: Use as an external package
-      run: |
-        mkdir $HOME/test_external_dir
-        cd $HOME/test_external_dir
-        python -m venv venv
-        source venv/bin/activate
-        pip install $GITHUB_WORKSPACE
-        python -c "from tinygrad.tensor import Tensor; print(Tensor([1,2,3,4,5]))"
-        pip install mypy
-        mypy -c "from tinygrad.tensor import Tensor; print(Tensor([1,2,3,4,5]))"
-    - name: Run beautiful_mnist without numpy
-      run: |
-        mkdir $HOME/test_no_numpy_dir
-        cd $HOME/test_no_numpy_dir
-        python -m venv venv
-        source venv/bin/activate
-        pip install $GITHUB_WORKSPACE
-        cp $GITHUB_WORKSPACE/examples/beautiful_mnist.py .
-        PYTHONPATH=$GITHUB_WORKSPACE BS=2 STEPS=10 python beautiful_mnist.py
-    - name: Test DEBUG
-      run: DEBUG=100 python3 -c "from tinygrad import Tensor; N = 1024; a, b = Tensor.rand(N, N), Tensor.rand(N, N); c = (a.reshape(N, 1, N) * b.T.reshape(1, N, N)).sum(axis=2); print((c.numpy() - (a.numpy() @ b.numpy())).mean())"
    - name: Repo line count <= 9999 lines
      run: MAX_LINE_COUNT=9999 python sz.py

@@ -217,12 +301,6 @@ jobs:
      - if: ${{ matrix.task == 'optimage' }}
        name: Test openpilot fastvits model correctness (float32)
        run: PYTHONPATH="." FLOAT16=0 DEBUGCL=1 GPU=1 IMAGE=2 python examples/openpilot/compile2.py https://github.com/commaai/openpilot/raw/9118973ed03c1ae1d40cf69a29507ec2cc78efd7/selfdrive/modeld/models/supercombo.onnx
-      - if: ${{ matrix.task == 'optimage' }}
-        name: Compile EfficientNet to C and test it
-        run: |
-          CLANG=1 PYTHONPATH="." python examples/compile_efficientnet.py > recognize.c
-          clang -O2 recognize.c -lm -o recognize
-          cat test/models/efficientnet/Chicken.jpg | ./recognize | grep cock
      - if: ${{ matrix.task == 'onnx' }}
        name: Test ONNX (GPU)
        run: GPU=1 python -m pytest -n=auto test/external/external_test_onnx_backend.py --durations=20
@@ -486,42 +564,6 @@ jobs:
        run: |
          PYTHONPATH=${{ github.workspace }} python3 -c "from tinygrad import Device; assert Device.DEFAULT in ['LLVM','CLANG','CUDA','GPU','AMD','NV'], Device.DEFAULT"
          DEBUG=5 PYTHONPATH=${{ github.workspace }} FORWARD_ONLY=1 python3 test/test_ops.py TestOps.test_add
-      - name: Verify OpenCL autogen
-        if: matrix.backend == 'gpu'
-        run: |
-          cp tinygrad/runtime/autogen/opencl.py /tmp/opencl.py.bak
-          ./autogen_stubs.sh opencl
-          diff /tmp/opencl.py.bak tinygrad/runtime/autogen/opencl.py
-      - name: Verify CUDA autogen
-        if: matrix.backend == 'nv'
-        run: |
-          cp tinygrad/runtime/autogen/cuda.py /tmp/cuda.py.bak
-          cp tinygrad/runtime/autogen/nv_gpu.py /tmp/nv_gpu.py.bak
-          ./autogen_stubs.sh cuda
-          ./autogen_stubs.sh nv
-          diff /tmp/cuda.py.bak tinygrad/runtime/autogen/cuda.py
-          diff /tmp/nv_gpu.py.bak tinygrad/runtime/autogen/nv_gpu.py
-      - name: Verify AMD autogen
-        if: matrix.backend == 'amd'
-        run: |
-          cp tinygrad/runtime/autogen/hsa.py /tmp/hsa.py.bak
-          cp tinygrad/runtime/autogen/comgr.py /tmp/comgr.py.bak
-          cp tinygrad/runtime/autogen/amd_gpu.py /tmp/amd_gpu.py.bak
-          ./autogen_stubs.sh hsa
-          ./autogen_stubs.sh comgr
-          ./autogen_stubs.sh amd
-          diff /tmp/hsa.py.bak tinygrad/runtime/autogen/hsa.py
-          diff /tmp/comgr.py.bak tinygrad/runtime/autogen/comgr.py
-          diff /tmp/amd_gpu.py.bak tinygrad/runtime/autogen/amd_gpu.py
-      - name: Verify Linux autogen
-        if: matrix.backend == 'amd'
-        run: |
-          cp tinygrad/runtime/autogen/libc.py /tmp/libc.py.bak
-          cp tinygrad/runtime/autogen/io_uring.py /tmp/io_uring.py.bak
-          ./autogen_stubs.sh libc
-          ./autogen_stubs.sh io_uring
-          diff /tmp/libc.py.bak tinygrad/runtime/autogen/libc.py
-          diff /tmp/io_uring.py.bak tinygrad/runtime/autogen/io_uring.py
      - name: Run pytest (not cuda or amd)
        if: matrix.backend!='ptx' && matrix.backend!='triton' && matrix.backend != 'amd' && matrix.backend != 'nv'
        run: python -m pytest -n=auto test/ --ignore=test/unit --durations=20