github/tinygrad
1
1
Fork 0
mirror of https://github.com/tinygrad/tinygrad.git synced 2026-01-08 22:48:25 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix shape of Jacobian to (n_out, n_in)

Browse Source
This commit is contained in:
0xNaN
2020-10-21 23:28:36 +02:00
parent 1561d3b9c0
commit 7458089b3b
1 changed files with 6 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
tinygrad/gradcheck.py
View File

@@ -16,7 +16,7 @@ def jacobian(model, input):
ji = input.data.reshape(-1).shape[-1]
jo = output.data.reshape(-1).shape[-1]
J = np.zeros((ji, jo))
J = np.zeros((jo,ji))
for o in range(jo):
# tinygrad doesn't support slicing, tiny-hack to select
@@ -25,7 +25,7 @@ def jacobian(model, input):
o_scalar.backward()
for i, grad in enumerate(input.grad.reshape(-1)):
J[i][o] = grad
J[o,i] = grad
return J
def mask_like(like, mask_inx, mask_value = 1.0):
@@ -52,10 +52,10 @@ def numerical_jacobian(model, input, eps = 1e-6):
ji = input.data.reshape(-1).shape[-1]
jo = output.data.reshape(-1).shape[-1]
NJ = np.zeros((ji, jo))
NJ = np.zeros((jo, ji))
for i in range(ji):
for o in range(jo):
for o in range(jo):
for i in range(ji):
eps_perturb = mask_like(input.data, i, mask_value = eps)
output_perturb_add = model(Tensor(input.data + eps_perturb)).data.reshape(-1)[o]
@@ -63,7 +63,7 @@ def numerical_jacobian(model, input, eps = 1e-6):
grad_approx = ((output_perturb_add) - (output_perturb_sub)) / (2*eps)
NJ[i,o] = grad_approx
NJ[o,i] = grad_approx
return NJ
def gradcheck(model, input, eps = 1e-06, atol = 1e-5, rtol = 0.001):