github/tinygrad
1
1
Fork 0
mirror of https://github.com/tinygrad/tinygrad.git synced 2026-01-09 15:08:02 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #10 from TheMrGhostman/master

Browse Source 
fixing adam
This commit is contained in:
George Hotz
2020-10-21 07:37:31 -07:00
committed by GitHub
parent 3a111c832c fc6f355c6a
commit 9172bc9c38
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tinygrad/optim.py
View File

@@ -27,8 +27,8 @@ class Adam(Optimizer):
self.v = [np.zeros_like(t.data) for t in self.params] self.v = [np.zeros_like(t.data) for t in self.params]
def step(self): def step(self):
self.t += 1
for i,t in enumerate(self.params): for i,t in enumerate(self.params):
self.t += 1
self.m[i] = self.b1 * self.m[i] + (1 - self.b1) * t.grad self.m[i] = self.b1 * self.m[i] + (1 - self.b1) * t.grad
self.v[i] = self.b2 * self.v[i] + (1 - self.b2) * np.square(t.grad) self.v[i] = self.b2 * self.v[i] + (1 - self.b2) * np.square(t.grad)
mhat = self.m[i] / (1. - self.b1**self.t) mhat = self.m[i] / (1. - self.b1**self.t)