github/tinygrad
1
1
Fork 0
mirror of https://github.com/tinygrad/tinygrad.git synced 2026-01-10 07:28:15 -05:00
Code Issues Packages Projects Releases Wiki Activity

these tests fail in CI due to segfaults and CPU crashes

Browse Source 
To confirm they're green locally, you can run the following commands:

1. For the tests skipped in test_ops.py (note: CLANG is very slow)

`for var in GPU CUDA CLANG; do export $var=1; for test in test/test_ops.py::TestOps::test_slice_fancy_indexing_no_dim_collapse test/test_ops.py::TestOps::test_slice_fancy_indexing_dim_collapse_int test/test_ops.py::TestOps::test_slice_fancy_indexing_dim_inject_none test/test_ops.py::TestOps::test_slice_fancy_indexing_dim_inject_and_collapse; do python3 -m pytest $test; done; unset $var; done`

2. For the ONNX tests skipped in CLANG:

```
CLANG=1 python3 -m pytest test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_ai_onnx_ml_array_feature_extractor_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_gather_elements_0_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_nllloss_NCd1_expanded_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_sce_mean_weight_ii_3d_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_gather_elements_1_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_sce_NCd1_mean_weight_negative_ii_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_nllloss_NCd1_weight_expanded_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_nllloss_NCd1d2d3_none_no_weight_negative_ii_expanded_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_nllloss_NCd1_ii_expanded_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_sce_mean_weight_ii_4d_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_sce_mean_weight_ii_3d_log_prob_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_gather_elements_negative_indices_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_sce_NCd1d2d3d4d5_mean_weight_log_prob_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_sce_NCd1_mean_weight_negative_ii_log_prob_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_nllloss_NCd1d2_no_weight_reduction_mean_ii_expanded_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_sce_NCd1d2d3d4d5_mean_weight_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_nllloss_NCd1d2d3d4d5_mean_weight_expanded_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_nllloss_NCd1_mean_weight_negative_ii_expanded_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_sce_mean_weight_ii_4d_log_prob_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_nllloss_NCd1d2_with_weight_reduction_mean_expanded_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_nllloss_NCd1_weight_ii_expanded_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_nllloss_NCd1d2_with_weight_reduction_sum_ii_expanded_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_nllloss_NCd1d2_with_weight_reduction_sum_expanded_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_nllloss_NCd1d2_expanded_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_nllloss_NCd1d2_reduction_sum_expanded_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_nllloss_NCd1d2d3d4d5_none_no_weight_expanded_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_nllloss_NCd1d2d3_sum_weight_high_ii_expanded_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_nllloss_NCd1d2_reduction_mean_expanded_cpu \
test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_nllloss_NCd1d2_with_weight_expanded_cpu
```

3. The LLVM test I skipped here is already [skipped in master for all backends](https://github.com/tinygrad/tinygrad/blob/master/test/external/external_test_onnx_backend.py#L186), I just made it more specific

`LLVM=1 python3 -m pytest test/external/external_test_onnx_backend.py::OnnxBackendNodeModelTest::test_dequantizelinear_e4m3fn_float16_cpu`
This commit is contained in:
qazal
2023-11-25 20:50:49 +02:00
parent 4d3fabaf3a
commit 15db570143
2 changed files with 40 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
test/external/external_test_onnx_backend.py vendored
View File

@@ -188,6 +188,11 @@ if getenv('GPU') and CI:
backend_test.exclude('test_max_float16_cpu')
backend_test.exclude('test_min_float16_cpu')
# Segfaults in CI
if getenv('LLVM') and CI:
backend_test.exclude('test_dequantizelinear_e4m3fn_float16_cpu')
# TODO: this somehow passes in CI but does not pass if run locally
if Device.DEFAULT in ['GPU', 'METAL', 'LLVM', 'CLANG']:
backend_test.exclude('test_MaxPool3d_stride_padding_cpu')
@@ -209,6 +214,37 @@ else:
backend_test.include('test_zfnet512')
backend_test.include('test_vgg19')
if getenv('CLANG') and CI:
backend_test.exclude('test_ai_onnx_ml_array_feature_extractor_cpu')
backend_test.exclude('test_gather_elements_0_cpu')
backend_test.exclude('test_nllloss_NCd1_expanded_cpu')
backend_test.exclude('test_sce_mean_weight_ii_3d_cpu')
backend_test.exclude('test_gather_elements_1_cpu')
backend_test.exclude('test_sce_NCd1_mean_weight_negative_ii_cpu')
backend_test.exclude('test_nllloss_NCd1_weight_expanded_cpu')
backend_test.exclude('test_nllloss_NCd1d2d3_none_no_weight_negative_ii_expanded_cpu')
backend_test.exclude('test_nllloss_NCd1_ii_expanded_cpu')
backend_test.exclude('test_sce_mean_weight_ii_4d_cpu')
backend_test.exclude('test_sce_mean_weight_ii_3d_log_prob_cpu')
backend_test.exclude('test_gather_elements_negative_indices_cpu')
backend_test.exclude('test_sce_NCd1d2d3d4d5_mean_weight_log_prob_cpu')
backend_test.exclude('test_sce_NCd1_mean_weight_negative_ii_log_prob_cpu')
backend_test.exclude('test_nllloss_NCd1d2_no_weight_reduction_mean_ii_expanded_cpu')
backend_test.exclude('test_sce_NCd1d2d3d4d5_mean_weight_cpu')
backend_test.exclude('test_nllloss_NCd1d2d3d4d5_mean_weight_expanded_cpu')
backend_test.exclude('test_nllloss_NCd1_mean_weight_negative_ii_expanded_cpu')
backend_test.exclude('test_sce_mean_weight_ii_4d_log_prob_cpu')
backend_test.exclude('test_nllloss_NCd1d2_with_weight_reduction_mean_expanded_cpu')
backend_test.exclude('test_nllloss_NCd1_weight_ii_expanded_cpu')
backend_test.exclude('test_nllloss_NCd1d2_with_weight_reduction_sum_ii_expanded_cpu')
backend_test.exclude('test_nllloss_NCd1d2_with_weight_reduction_sum_expanded_cpu')
backend_test.exclude('test_nllloss_NCd1d2_expanded_cpu')
backend_test.exclude('test_nllloss_NCd1d2_reduction_sum_expanded_cpu')
backend_test.exclude('test_nllloss_NCd1d2d3d4d5_none_no_weight_expanded_cpu')
backend_test.exclude('test_nllloss_NCd1d2d3_sum_weight_high_ii_expanded_cpu')
backend_test.exclude('test_nllloss_NCd1d2_reduction_mean_expanded_cpu')
backend_test.exclude('test_nllloss_NCd1d2_with_weight_expanded_cpu')
globals().update(backend_test.enable_report().test_cases)
if __name__ == '__main__':

4
test/test_ops.py
View File

@@ -1204,6 +1204,7 @@ class TestOps(unittest.TestCase):
i, j, k, o, p = [Tensor(tor.detach().numpy().astype(np.int32), dtype=dtypes.int32, requires_grad=False) for tor in [a,b,c,d,e]]
return a,b,c,d,e,i,j,k,o,p
@unittest.skipIf(CI and Device.DEFAULT in ["CUDA", "GPU", "CLANG"], "broken in CI")
def test_slice_fancy_indexing_no_dim_collapse(self):
a,b,c,d,e,i,j,k,o,p = self._get_index_randoms()
# no dim collapse from int or dim injection from None
@@ -1213,6 +1214,7 @@ class TestOps(unittest.TestCase):
helper_test_op([(2,5,6,5,3,4)], lambda x: x[a,...,e], lambda x: x[i,...,p])
helper_test_op([(2,5,6,5,3,4)], lambda x: x[...,c,:,e], lambda x: x[...,k,:,p])
@unittest.skipIf(CI and Device.DEFAULT in ["CUDA", "GPU", "CLANG"], "broken in CI")
def test_slice_fancy_indexing_dim_collapse_int(self):
a,b,c,d,e,i,j,k,o,p = self._get_index_randoms()
# dim collapse from int
@@ -1222,6 +1224,7 @@ class TestOps(unittest.TestCase):
helper_test_op([(2,5,6,5,3,4)], lambda x: x[a,2,2,2,e], lambda x: x[i,2,2,2,p])
helper_test_op([(2,5,6,5,3,4)], lambda x: x[1,:,3:11:2,d,0:2], lambda x: x[1,:,3:11:2,o,0:2])
@unittest.skipIf(CI and Device.DEFAULT in ["CUDA", "GPU", "CLANG"], "broken in CI")
def test_slice_fancy_indexing_dim_inject_none(self):
a,b,c,d,e,i,j,k,o,p = self._get_index_randoms()
# dim injection from None
@@ -1231,6 +1234,7 @@ class TestOps(unittest.TestCase):
helper_test_op([(2,5,6,5,3,4)], lambda x: x[None,b,c,d,None], lambda x: x[None,j,k,o,None])
helper_test_op([(2,5,6,5,3,4)], lambda x: x[a,:,None,d,e], lambda x: x[i,:,None,o,p])
@unittest.skipIf(CI and Device.DEFAULT in ["CUDA", "GPU", "CLANG"], "broken in CI")
def test_slice_fancy_indexing_dim_inject_and_collapse(self):
a,b,c,d,e,i,j,k,o,p = self._get_index_randoms()
# dim injection and collapse