diff --git a/r2/r2/lib/media.py b/r2/r2/lib/media.py index 3bbc8851f..bd7d4c410 100644 --- a/r2/r2/lib/media.py +++ b/r2/r2/lib/media.py @@ -52,6 +52,7 @@ from r2.lib.utils import ( UrlParser, coerce_url_to_protocol, domain, + get_requests_resp_json, ) from r2.models.link import Link from r2.models.media_cache import ( @@ -626,7 +627,8 @@ class _EmbedlyScraper(Scraper): @memoize("media.embedly_services2", time=3600) def _fetch_embedly_service_data(): - return requests.get("https://api.embed.ly/1/services/python").json + resp = requests.get("https://api.embed.ly/1/services/python") + return get_requests_resp_json(resp) def _fetch_embedly_services(): diff --git a/r2/r2/lib/utils/http_utils.py b/r2/r2/lib/utils/http_utils.py index b0eb22164..0030743ff 100644 --- a/r2/r2/lib/utils/http_utils.py +++ b/r2/r2/lib/utils/http_utils.py @@ -28,6 +28,7 @@ DATE_RFC850 = '%A, %d-%b-%y %H:%M:%S %Z' DATE_RFC3339 = "%Y-%m-%dT%H:%M:%SZ" DATE_ANSI = '%a %b %d %H:%M:%S %Y' + def read_http_date(date_str): try: date = datetime.strptime(date_str, DATE_RFC822) @@ -42,9 +43,18 @@ def read_http_date(date_str): date = date.replace(tzinfo = pytz.timezone('GMT')) return date + def http_date_str(date): date = date.astimezone(pytz.timezone('GMT')) return date.strftime(DATE_RFC822) + def rfc3339_date_str(date): return date.strftime(DATE_RFC3339) + + +def get_requests_resp_json(resp): + """Kludge so we can use `requests` versions below or above 1.x""" + if callable(resp.json): + return resp.json() + return resp.json diff --git a/scripts/inject_test_data.py b/scripts/inject_test_data.py index 408387456..551c2c746 100644 --- a/scripts/inject_test_data.py +++ b/scripts/inject_test_data.py @@ -35,7 +35,7 @@ from pylons import g from r2.lib.db import queries from r2.lib import amqp -from r2.lib.utils import weighted_lottery +from r2.lib.utils import weighted_lottery, get_requests_resp_json from r2.models import Account, NotFound, register, Subreddit, Link, Comment @@ -121,7 +121,7 @@ def fetch_listing(path, limit=1000, batch_size=100): response = session.get(base_url, params=params) response.raise_for_status() - listing = response.json["data"] + listing = get_requests_resp_json(response)["data"] for child in listing["children"]: yield child["data"] count += 1