Use a User-Agent for suggest title urlopen()

This will fix the 429 errors on reddit urls (rate-limiting ourselves)
This commit is contained in:
Chad Birch
2013-01-30 13:41:49 -08:00
committed by Neil Williams
parent c04d8a6ff6
commit 40b3093552

View File

@@ -27,7 +27,7 @@ import ConfigParser
import codecs
from urllib import unquote_plus
from urllib2 import urlopen
from urllib2 import urlopen, Request
from urlparse import urlparse, urlunparse
import signal
from copy import deepcopy
@@ -255,7 +255,10 @@ def get_title(url):
return None
try:
opener = urlopen(url, timeout=15)
req = Request(url)
if g.useragent:
req.add_header('User-Agent', g.useragent)
opener = urlopen(req, timeout=15)
# determine the encoding of the response
for param in opener.info().getplist():