github/tinygrad
1
1
Fork 0
mirror of https://github.com/tinygrad/tinygrad.git synced 2026-01-09 15:08:02 -05:00
Code Issues Packages Projects Releases Wiki Activity

update rsmprop

Browse Source
This commit is contained in:
f0ti
2020-10-23 14:46:45 +02:00
parent c5f726ec2e
commit 0b87aaca1e
3 changed files with 12 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
README.md
View File

@@ -39,13 +39,13 @@ print(y.grad) # dz/dy
### Neural networks? ### Neural networks?
It turns out, a decent autograd tensor library is 90% of what you need for neural networks. Add an optimizer (SGD and Adam implemented) from tinygrad.optim, write some boilerplate minibatching code, and you have all you need. It turns out, a decent autograd tensor library is 90% of what you need for neural networks. Add an optimizer (SGD, RMSprop and Adam implemented) from tinygrad.optim, write some boilerplate minibatching code, and you have all you need.
### Neural network example (from test/test_mnist.py) ### Neural network example (from test/test_mnist.py)
```python ```python
from tinygrad.tensor import Tensor from tinygrad.tensor import Tensor
import tinygrad.optim as tinygrad_optim import tinygrad.optim as optim
from tinygrad.utils import layer_init_uniform from tinygrad.utils import layer_init_uniform
class TinyBobNet: class TinyBobNet:
@@ -57,8 +57,7 @@ class TinyBobNet:
return x.dot(self.l1).relu().dot(self.l2).logsoftmax() return x.dot(self.l1).relu().dot(self.l2).logsoftmax()
model = TinyBobNet() model = TinyBobNet()
optim = tinygrad_optim.SGD([model.l1, model.l2], lr=0.001) # or optim = optim.SGD([model.l1, model.l2], lr=0.001)
optim = tinygrad_optim.RMSprop([model.l1, model.l2], lr=0.001)
# ... and complete like pytorch, with (x,y) data # ... and complete like pytorch, with (x,y) data

16
test/test_mnist.py
View File

@@ -4,7 +4,7 @@ import unittest
import numpy as np import numpy as np
from tinygrad.tensor import Tensor from tinygrad.tensor import Tensor
from tinygrad.utils import layer_init_uniform, fetch_mnist from tinygrad.utils import layer_init_uniform, fetch_mnist
import tinygrad.optim as tinygrad_optim import tinygrad.optim as optim
from tqdm import trange from tqdm import trange
np.random.seed(1337) np.random.seed(1337)
@@ -78,20 +78,20 @@ class TestMNIST(unittest.TestCase):
# models # models
model = TinyConvNet() model = TinyConvNet()
optim = tinygrad_optim.Adam([model.c1, model.l1, model.l2], lr=0.001) optimizer = optim.Adam([model.c1, model.l1, model.l2], lr=0.001)
steps = 400 steps = 400
train(model, optim, steps) train(model, optimizer, steps)
evaluate(model) evaluate(model)
model = TinyBobNet() model = TinyBobNet()
optim = tinygrad_optim.SGD([model.l1, model.l2], lr=0.001)
steps = 1000 steps = 1000
train(model, optim, steps) optimizer = optim.SGD([model.l1, model.l2], lr=0.001)
train(model, optimizer, steps)
evaluate(model) evaluate(model)
# RMSprop model = TinyBobNet()
optim = tinygrad_optim.RMSprop([model.l1, model.l2], lr=0.001) optimizer = optim.RMSprop([model.l1, model.l2], lr=0.001)
train(model, optim, steps) train(model, optimizer, steps)
evaluate(model) evaluate(model)
if __name__ == '__main__': if __name__ == '__main__':

2
tinygrad/optim.py
View File

@@ -37,7 +37,7 @@ class Adam(Optimizer):
# fill the 20% uncertainty of the above optim # fill the 20% uncertainty of the above optim
class RMSprop(Optimizer): class RMSprop(Optimizer):
def __init__(self, params, lr=0.001, decay=0.9, eps=1e-6): def __init__(self, params, lr=0.001, decay=0.9, eps=1e-8):
super(RMSprop, self).__init__(params) super(RMSprop, self).__init__(params)
self.lr = lr self.lr = lr
self.decay = decay self.decay = decay