mirror of
https://github.com/tinygrad/tinygrad.git
synced 2026-01-24 14:28:09 -05:00
f08187526f
* Fix examples * Remove training in parameters * Simplify a bit * Remove extra import * Fix linter errors * factor out Device * NumPy-like semantics for Tensor.__getitem__ (#506) * Rewrote Tensor.__getitem__ to fix negative indices and add support for np.newaxis/None * Fixed pad2d * mypy doesn't know about mlops methods * normal python behavior for out-of-bounds slicing * type: ignore * inlined idxfix * added comment for __getitem__ * Better comments, better tests, and fixed bug in np.newaxis * update cpu and torch to hold buffers (#542) * update cpu and torch to hold buffers * save lines, and probably faster * Mypy fun (#541) * mypy fun * things are just faster * running fast * mypy is fast * compile.sh * no gpu hack * refactor ops_cpu and ops_torch to not subclass * make weak buffer work * tensor works * fix test failing * cpu/torch cleanups * no or operator on dict in python 3.8 * that was junk * fix warnings * comment and touchup * dyn add of math ops * refactor ops_cpu and ops_torch to not share code * nn/optim.py compiles now * Reorder imports * call mkdir only if directory doesn't exist --------- Co-authored-by: George Hotz <geohot@gmail.com> Co-authored-by: Mitchell Goff <mitchellgoffpc@gmail.com> Co-authored-by: George Hotz <72895+geohot@users.noreply.github.com>
104 lines
2.9 KiB
Python
104 lines
2.9 KiB
Python
import traceback
|
|
import time
|
|
from multiprocessing import Process, Queue
|
|
import numpy as np
|
|
from tqdm import trange
|
|
import tinygrad.nn.optim as optim
|
|
from tinygrad.helpers import getenv
|
|
from tinygrad.tensor import Tensor
|
|
from datasets import fetch_cifar
|
|
from datasets.imagenet import fetch_batch
|
|
from extra.utils import get_parameters
|
|
from models.efficientnet import EfficientNet
|
|
|
|
class TinyConvNet:
|
|
def __init__(self, classes=10):
|
|
conv = 3
|
|
inter_chan, out_chan = 8, 16 # for speed
|
|
self.c1 = Tensor.uniform(inter_chan,3,conv,conv)
|
|
self.c2 = Tensor.uniform(out_chan,inter_chan,conv,conv)
|
|
self.l1 = Tensor.uniform(out_chan*6*6, classes)
|
|
|
|
def forward(self, x):
|
|
x = x.conv2d(self.c1).relu().max_pool2d()
|
|
x = x.conv2d(self.c2).relu().max_pool2d()
|
|
x = x.reshape(shape=[x.shape[0], -1])
|
|
return x.dot(self.l1)
|
|
|
|
if __name__ == "__main__":
|
|
IMAGENET = getenv("IMAGENET")
|
|
classes = 1000 if IMAGENET else 10
|
|
|
|
TINY = getenv("TINY")
|
|
TRANSFER = getenv("TRANSFER")
|
|
if TINY:
|
|
model = TinyConvNet(classes)
|
|
elif TRANSFER:
|
|
model = EfficientNet(getenv("NUM", 0), classes, has_se=True)
|
|
model.load_from_pretrained()
|
|
else:
|
|
model = EfficientNet(getenv("NUM", 0), classes, has_se=False)
|
|
|
|
parameters = get_parameters(model)
|
|
print("parameter count", len(parameters))
|
|
optimizer = optim.Adam(parameters, lr=0.001)
|
|
|
|
BS, steps = getenv("BS", 64 if TINY else 16), getenv("STEPS", 2048)
|
|
print(f"training with batch size {BS} for {steps} steps")
|
|
|
|
if IMAGENET:
|
|
def loader(q):
|
|
while 1:
|
|
try:
|
|
q.put(fetch_batch(BS))
|
|
except Exception:
|
|
traceback.print_exc()
|
|
q = Queue(16)
|
|
for i in range(2):
|
|
p = Process(target=loader, args=(q,))
|
|
p.daemon = True
|
|
p.start()
|
|
else:
|
|
X_train, Y_train = fetch_cifar()
|
|
|
|
Tensor.training = True
|
|
for i in (t := trange(steps)):
|
|
if IMAGENET:
|
|
X, Y = q.get(True)
|
|
else:
|
|
samp = np.random.randint(0, X_train.shape[0], size=(BS))
|
|
X, Y = X_train[samp], Y_train[samp]
|
|
|
|
st = time.time()
|
|
out = model.forward(Tensor(X.astype(np.float32), requires_grad=False))
|
|
fp_time = (time.time()-st)*1000.0
|
|
|
|
y = np.zeros((BS,classes), np.float32)
|
|
y[range(y.shape[0]),Y] = -classes
|
|
y = Tensor(y, requires_grad=False)
|
|
loss = out.logsoftmax().mul(y).mean()
|
|
|
|
optimizer.zero_grad()
|
|
|
|
st = time.time()
|
|
loss.backward()
|
|
bp_time = (time.time()-st)*1000.0
|
|
|
|
st = time.time()
|
|
optimizer.step()
|
|
opt_time = (time.time()-st)*1000.0
|
|
|
|
st = time.time()
|
|
loss = loss.cpu().data
|
|
cat = np.argmax(out.cpu().data, axis=1)
|
|
accuracy = (cat == Y).mean()
|
|
finish_time = (time.time()-st)*1000.0
|
|
|
|
# printing
|
|
t.set_description("loss %.2f accuracy %.2f -- %.2f + %.2f + %.2f + %.2f = %.2f" %
|
|
(loss, accuracy,
|
|
fp_time, bp_time, opt_time, finish_time,
|
|
fp_time + bp_time + opt_time + finish_time))
|
|
|
|
del out, y, loss
|