imagenet loader minor cleanups

This commit is contained in:
George Hotz
2023-06-28 05:08:09 +00:00
parent 6ec0a24706
commit 1f5d45ca8c
2 changed files with 3 additions and 3 deletions

1
.gitignore vendored
View File

@@ -28,5 +28,6 @@ datasets/squad/
datasets/img_align_celeba*
datasets/open-images-v6-mlperf
datasets/kits/
datasets/COCO/
datasets/audio*
venv

View File

@@ -28,13 +28,12 @@ def image_load(fn):
ret = np.array(img)
return ret
def iterate(bs=32, val=True, shuffle=True, no_image=False):
def iterate(bs=32, val=True, shuffle=True):
files = get_val_files() if val else get_train_files()
#files = files[:100]
order = list(range(0, len(files)))
if shuffle: random.shuffle(order)
from multiprocessing import Pool
p = Pool(32)
p = Pool(16)
for i in range(0, len(files), bs):
X = p.map(image_load, [files[i] for i in order[i:i+bs]])
Y = [cir[files[i].split("/")[-2]] for i in order[i:i+bs]]