From 40b3093552885d9eabc251a835f844da9ed7a4e2 Mon Sep 17 00:00:00 2001 From: Chad Birch Date: Wed, 30 Jan 2013 13:41:49 -0800 Subject: [PATCH] Use a User-Agent for suggest title urlopen() This will fix the 429 errors on reddit urls (rate-limiting ourselves) --- r2/r2/lib/utils/utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/r2/r2/lib/utils/utils.py b/r2/r2/lib/utils/utils.py index fe8cf3535..c593aad23 100644 --- a/r2/r2/lib/utils/utils.py +++ b/r2/r2/lib/utils/utils.py @@ -27,7 +27,7 @@ import ConfigParser import codecs from urllib import unquote_plus -from urllib2 import urlopen +from urllib2 import urlopen, Request from urlparse import urlparse, urlunparse import signal from copy import deepcopy @@ -255,7 +255,10 @@ def get_title(url): return None try: - opener = urlopen(url, timeout=15) + req = Request(url) + if g.useragent: + req.add_header('User-Agent', g.useragent) + opener = urlopen(req, timeout=15) # determine the encoding of the response for param in opener.info().getplist():