From a5f5330d9184407ac2a34138eb751a401d0af4b8 Mon Sep 17 00:00:00 2001 From: chenyu Date: Wed, 19 Jul 2023 12:05:45 -0400 Subject: [PATCH] Add Fuzz Test symbolic / shapetracker to CI. (#1278) * Fuzz test symbolic and shapetracker This reverts commit d5773ddebff54c1ff608838076f0b4ff126b8aa8. * mess again * no tail * test shapetracker too * Revert mess and enable all tests * removed leftover --- .github/workflows/test.yml | 4 +++ test/external/fuzz_shapetracker.py | 21 +++++++-------- test/external/fuzz_symbolic.py | 41 +++++++++++++++--------------- test/unit/test_shapetracker.py | 7 +++-- 4 files changed, 39 insertions(+), 34 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 41f40a14ab..8f20b2d1cc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,6 +55,10 @@ jobs: run: awk '/```python/{flag=1;next}/```/{flag=0}flag' docs/quickstart.md > quickstart.py && PYTHONPATH=. python3 quickstart.py - name: Run Pytest run: python -m pytest -s -v -n=auto test/ + - name: Fuzz Test symbolic + run: DEBUG=1 python test/external/fuzz_symbolic.py + - name: Fuzz Test shapetracker + run: PYTHONPATH="." DEBUG=1 python test/external/fuzz_shapetracker.py testwebgpu: name: WebGPU Tests diff --git a/test/external/fuzz_shapetracker.py b/test/external/fuzz_shapetracker.py index 1e82dd5bca..cb012c2f4e 100644 --- a/test/external/fuzz_shapetracker.py +++ b/test/external/fuzz_shapetracker.py @@ -1,17 +1,19 @@ import random +from tinygrad.helpers import DEBUG from test.unit.test_shapetracker import CheckingShapeTracker +random.seed(42) def do_permute(st): perm = list(range(0, len(st.shape))) random.shuffle(perm) perm = tuple(perm) - print("st.permute(", perm, ")") + if DEBUG >= 1: print("st.permute(", perm, ")") st.permute(perm) def do_pad(st): c = random.randint(0, len(st.shape)-1) pad = tuple((random.randint(0,2), random.randint(0,2)) if i==c else (0,0) for i in range(len(st.shape))) - print("st.pad(", pad, ")") + if DEBUG >= 1: print("st.pad(", pad, ")") st.pad(pad) def do_reshape_split_one(st): @@ -19,14 +21,14 @@ def do_reshape_split_one(st): poss = [n for n in [1,2,3,4,5] if st.shape[c]%n == 0] spl = random.choice(poss) shp = st.shape[0:c] + (st.shape[c]//spl, spl) + st.shape[c+1:] - print("st.reshape(", shp, ")") + if DEBUG >= 1: print("st.reshape(", shp, ")") st.reshape(shp) def do_reshape_combine_two(st): if len(st.shape) < 2: return c = random.randint(0, len(st.shape)-2) shp = st.shape[:c] + (st.shape[c] * st.shape[c+1], ) + st.shape[c+2:] - print("st.reshape(", shp, ")") + if DEBUG >= 1: print("st.reshape(", shp, ")") st.reshape(shp) def do_shrink(st): @@ -34,13 +36,13 @@ def do_shrink(st): while 1: shrink = tuple((random.randint(0,s), random.randint(0,s)) if i == c else (0,s) for i,s in enumerate(st.shape)) if all(x= 1: print("st.shrink(", shrink, ")") st.shrink(shrink) def do_stride(st): c = random.randint(0, len(st.shape)-1) stride = tuple(random.choice([-2,-1,2]) if i==c else 1 for i in range(len(st.shape))) - print("st.stride(", stride, ")") + if DEBUG >= 1: print("st.stride(", stride, ")") st.stride(stride) def do_expand(st): @@ -48,13 +50,12 @@ def do_expand(st): if len(c) == 0: return c = random.choice(c) expand = tuple(random.choice([2,3,4]) if i==c else s for i,s in enumerate(st.shape)) - print("st.expand(", expand, ")") + if DEBUG >= 1: print("st.expand(", expand, ")") st.expand(expand) if __name__ == "__main__": ops = [do_permute, do_pad, do_shrink, do_reshape_split_one, do_reshape_combine_two, do_stride, do_expand] - while 1: - st = CheckingShapeTracker((3, 3, 3)) + for _ in range(200): + st = CheckingShapeTracker((random.randint(2, 10), random.randint(2, 10), random.randint(2, 10))) for i in range(8): random.choice(ops)(st) - #st.simplify() st.assert_same() diff --git a/test/external/fuzz_symbolic.py b/test/external/fuzz_symbolic.py index 7b565c467f..626cf47a68 100644 --- a/test/external/fuzz_symbolic.py +++ b/test/external/fuzz_symbolic.py @@ -1,5 +1,8 @@ +import itertools import random +from tinygrad.helpers import DEBUG from tinygrad.shape.symbolic import Variable +random.seed(42) def add_v(expr, rng=None): if rng is None: rng = random.randint(0,2) @@ -30,13 +33,14 @@ def ge(expr, rng=None): return expr >= rng, rng if __name__ == "__main__": - ops = [add_v, div, mul, add_num] - while 1: - u1 = Variable("v1", 0, 2) - u2 = Variable("v2", 0, 3) - u3 = Variable("v3", 0, 4) + ops = [add_v, div, mul, add_num, mod] + for _ in range(1000): + upper_bounds = [*list(range(1, 10)), 16, 32, 64, 128, 256] + u1 = Variable("v1", 0, random.choice(upper_bounds)) + u2 = Variable("v2", 0, random.choice(upper_bounds)) + u3 = Variable("v3", 0, random.choice(upper_bounds)) v = [u1,u2,u3] - tape = [random.choice(ops) for _ in range(20)] + tape = [random.choice(ops) for _ in range(random.randint(2, 30))] # 10% of the time, add a less than or greater than if random.random() < 0.05: tape.append(lt) elif random.random() < 0.05: tape.append(ge) @@ -44,18 +48,15 @@ if __name__ == "__main__": rngs = [] for t in tape: expr, rng = t(expr) - print(t.__name__, rng) + if DEBUG >= 1: print(t.__name__, rng) rngs.append(rng) - print(expr) - for v1 in range(u1.min, u1.max+1): - for v2 in range(u2.min, u2.max+1): - for v3 in range(u3.min, u3.max+1): - v = [v1,v2,v3] - rn = 0 - for t,r in zip(tape, rngs): - rn, _ = t(rn, r) - num = eval(expr.render()) - assert num == rn, f"mismatch at {v1} {v2} {v3}, {num} != {rn}" - #print(v1, v2, v3, num, rn) - - + if DEBUG >=1: print(expr) + space = list(itertools.product(range(u1.min, u1.max+1), range(u2.min, u2.max+1), range(u3.min, u3.max+1))) + volume = len(space) + for (v1, v2, v3) in random.sample(space, min(100, volume)): + v = [v1,v2,v3] + rn = 0 + for t,r in zip(tape, rngs): rn, _ = t(rn, r) + num = eval(expr.render()) + assert num == rn, f"mismatched {expr.render()} at {v1=} {v2=} {v3=} = {num} != {rn}" + if DEBUG >= 1: print(f"matched {expr.render()} at {v1=} {v2=} {v3=} = {num} == {rn}") diff --git a/test/unit/test_shapetracker.py b/test/unit/test_shapetracker.py index 7e62227f7f..b1e5e1d3e7 100644 --- a/test/unit/test_shapetracker.py +++ b/test/unit/test_shapetracker.py @@ -1,9 +1,8 @@ #!/usr/bin/env python import unittest import numpy as np -from tinygrad.helpers import prod, all_same -from tinygrad.shape.shapetracker import ShapeTracker, View, merge_views, get_contraction -from tinygrad.codegen.linearizer import to_image_idx +from tinygrad.helpers import prod, DEBUG +from tinygrad.shape.shapetracker import ShapeTracker, View, get_contraction def shapetracker_getitem(st, val): locals = {"idx": val, "valid": 1} @@ -63,7 +62,7 @@ class CheckingShapeTracker: x = [shapetracker_getitem(self.st, i) for i in range(prod(self.st.shape))] y = [self[i] for i in range(prod(self.shape))] idx, valid = self.st.expr_node() - print(x, y, self.st.shape, self.shape, idx.render(), valid.render(), self.st) + if DEBUG >= 1: print(x, y, self.st.shape, self.shape, idx.render(), valid.render(), self.st) assert self.st.shape == self.shape assert x == y, f"mismatch shapetracker:{x} real:{y}"