diff --git a/r2/r2/lib/app_globals.py b/r2/r2/lib/app_globals.py index 227850852..1e2a2e254 100644 --- a/r2/r2/lib/app_globals.py +++ b/r2/r2/lib/app_globals.py @@ -148,10 +148,10 @@ class Globals(object): #setup the logger self.log = logging.getLogger('reddit') + self.log.addHandler(logging.StreamHandler()) if self.debug: self.log.setLevel(logging.DEBUG) - self.log.addHandler(logging.StreamHandler()) - + def __del__(self): """ Put any cleanup code to be run when the application finally exits diff --git a/r2/r2/lib/media.py b/r2/r2/lib/media.py index ce4e072ef..29c77268d 100644 --- a/r2/r2/lib/media.py +++ b/r2/r2/lib/media.py @@ -57,15 +57,20 @@ def make_link_info_job(results, link, useragent): link's thumbnail and media object. Places the result in the results dict""" def job(): - scraper = make_scraper(link.url) + try: + scraper = make_scraper(link.url) - thumbnail = scraper.thumbnail() - media_object = scraper.media_object() + thumbnail = scraper.thumbnail() + media_object = scraper.media_object() - if thumbnail: - upload_thumb(link, thumbnail) + if thumbnail: + upload_thumb(link, thumbnail) + + results[link] = (thumbnail, media_object) + except: + log.warning('error fetching %s %s' % (link._fullname, link.url)) + raise - results[link] = (thumbnail, media_object) return job def update_link(link, thumbnail, media_object): diff --git a/r2/r2/lib/scraper.py b/r2/r2/lib/scraper.py index 5ee4751f1..251a2d0c0 100644 --- a/r2/r2/lib/scraper.py +++ b/r2/r2/lib/scraper.py @@ -69,6 +69,9 @@ def fetch_url(url, referer = None, retries = 1, dimension = False): content = open_req.read() content_type = open_req.headers.get('content-type') + if not content_type: + return nothing + if 'image' in content_type: p = ImageFile.Parser() new_data = content diff --git a/r2/r2/lib/utils/http_utils.py b/r2/r2/lib/utils/http_utils.py index 5613fa488..4b0e49fb6 100644 --- a/r2/r2/lib/utils/http_utils.py +++ b/r2/r2/lib/utils/http_utils.py @@ -12,7 +12,10 @@ def read_http_date(date_str): try: date = datetime.strptime(date_str, DATE_RFC850) except ValueError: - date = datetime.strptime(date_str, DATE_ANSI) + try: + date = datetime.strptime(date_str, DATE_ANSI) + except ValueError: + return None date = date.replace(tzinfo = pytz.timezone('GMT')) return date diff --git a/r2/r2/templates/link.html b/r2/r2/templates/link.html index 837f11b7a..f9b36da70 100644 --- a/r2/r2/templates/link.html +++ b/r2/r2/templates/link.html @@ -197,7 +197,7 @@ <%def name="thumbnail()"> %if thing.thumbnail: <%call expr="make_link('thumbnail', 'thumbnail')"> - thumbnail for ${thing._fullname} + %endif