mirror of
https://github.com/tinygrad/tinygrad.git
synced 2026-01-09 15:08:02 -05:00
imagenet loader minor cleanups
This commit is contained in:
1
.gitignore
vendored
1
.gitignore
vendored
@@ -28,5 +28,6 @@ datasets/squad/
|
||||
datasets/img_align_celeba*
|
||||
datasets/open-images-v6-mlperf
|
||||
datasets/kits/
|
||||
datasets/COCO/
|
||||
datasets/audio*
|
||||
venv
|
||||
|
||||
@@ -28,13 +28,12 @@ def image_load(fn):
|
||||
ret = np.array(img)
|
||||
return ret
|
||||
|
||||
def iterate(bs=32, val=True, shuffle=True, no_image=False):
|
||||
def iterate(bs=32, val=True, shuffle=True):
|
||||
files = get_val_files() if val else get_train_files()
|
||||
#files = files[:100]
|
||||
order = list(range(0, len(files)))
|
||||
if shuffle: random.shuffle(order)
|
||||
from multiprocessing import Pool
|
||||
p = Pool(32)
|
||||
p = Pool(16)
|
||||
for i in range(0, len(files), bs):
|
||||
X = p.map(image_load, [files[i] for i in order[i:i+bs]])
|
||||
Y = [cir[files[i].split("/")[-2]] for i in order[i:i+bs]]
|
||||
|
||||
Reference in New Issue
Block a user