goat progress bar

extra/utils.py

@@ -1,5 +1,6 @@
 from tinygrad.tensor import Tensor
 import pickle
+from tqdm import tqdm
 import numpy as np
 from tinygrad.helpers import prod
 
@@ -14,10 +15,10 @@ def fetch(url):
     with open(fp, "rb") as f:
       dat = f.read()
   else:
-    print("fetching %s" % url)
-    r = requests.get(url)
+    r = requests.get(url, stream=True)
     assert r.status_code == 200
-    dat = r.content
+    progress_bar = tqdm(total=int(r.headers.get('content-length', 0)), unit='B', unit_scale=True, desc=url)
+    dat = b''.join(x for x in r.iter_content(chunk_size=16384) if progress_bar.update(len(x)) or True)
     with open(fp+".tmp", "wb") as f:
       f.write(dat)
     os.rename(fp+".tmp", fp)