From 5e4c1599e8cf6447a26f5246ffc79f9c8e702037 Mon Sep 17 00:00:00 2001
From: steve <steve@reddit.com>
Date: Mon, 7 Jul 2008 23:24:12 -0700
Subject: [PATCH] only cache the size requests

---
 r2/r2/lib/scraper.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/r2/r2/lib/scraper.py b/r2/r2/lib/scraper.py
index 9be667874..d7690fd97 100644
--- a/r2/r2/lib/scraper.py
+++ b/r2/r2/lib/scraper.py
@@ -55,7 +55,6 @@ def clean_url(url):
     url = ''.join([urllib.quote(c) if ord(c) >= 127 else c for c in url])
     return url
 
-@memoize('media.fetch_url')
 def fetch_url(url, referer = None, retries = 1, dimension = False):
     cur_try = 0
     log.debug('fetching: %s' % url)
@@ -114,6 +113,10 @@ def fetch_url(url, referer = None, retries = 1, dimension = False):
             if 'open_req' in locals():
                 open_req.close()
 
+@memoize('media.fetch_size')
+def fetch_size(url, referer = None, retries = 1):
+    return fetch_url(url, referer, retries, dimension = True)
+
 class Scraper:
     def __init__(self, url):
         self.url = url
@@ -148,7 +151,7 @@ class Scraper:
         max_url = None
 
         for image_url in self.image_urls():
-            size = fetch_url(image_url, referer = self.url, dimension = True)
+            size = fetch_size(image_url, referer = self.url)
             if not size:
                 continue