From ce578353940f60309ee7eda5d9ed7327ff4ff2c0 Mon Sep 17 00:00:00 2001 From: Neil Williams Date: Tue, 26 Feb 2013 17:06:43 -0800 Subject: [PATCH] Avoid loading pycountry in the app. It takes a tonne of memory. If the countries.json file is present (generated by "make countries") then the app will prefer to use it saving itself megabytes of space. --- r2/Makefile | 14 ++++++++++++ r2/r2/lib/app_globals.py | 11 +++++++++ r2/r2/lib/countries.py | 39 ++++++++++++++++++++++++++++++++ r2/r2/lib/pages/pages.py | 4 +--- r2/r2/lib/validator/validator.py | 7 +++--- r2/r2/templates/paymentform.html | 14 +++++++----- 6 files changed, 76 insertions(+), 13 deletions(-) create mode 100644 r2/r2/lib/countries.py diff --git a/r2/Makefile b/r2/Makefile index 377175b78..0c8b1be07 100644 --- a/r2/Makefile +++ b/r2/Makefile @@ -235,3 +235,17 @@ clean_gzip: rm -f $(GZIPPED) $(shell rm -f $(DEFS_FILE)) + +#################### country list +COUNTRY_FILE := $(STATIC_BUILD_DIR)/countries.json + +countries: $(COUNTRY_FILE) + +$(STATIC_BUILD_DIR): + mkdir -p $(STATIC_BUILD_DIR) + +$(COUNTRY_FILE): $(STATIC_BUILD_DIR) + $(PYTHON) r2/lib/countries.py > $(COUNTRY_FILE) + +clean_countries: + rm -f $(COUNTRY_FILE) diff --git a/r2/r2/lib/app_globals.py b/r2/r2/lib/app_globals.py index ab2742fcf..110e57858 100755 --- a/r2/r2/lib/app_globals.py +++ b/r2/r2/lib/app_globals.py @@ -53,6 +53,7 @@ from r2.lib.cache import ( ) from r2.lib.configparse import ConfigValue, ConfigValueParser from r2.lib.contrib import ipaddress +from r2.lib.countries import get_countries_and_codes from r2.lib.lock import make_lock_factory from r2.lib.manager import db_manager from r2.lib.plugin import PluginLoader @@ -359,6 +360,16 @@ class Globals(object): csslog = logging.getLogger("cssutils") cssutils.log.setLog(csslog) + # load the country list + countries_file_path = os.path.join(static_files, "countries.json") + try: + with open(countries_file_path) as handle: + self.countries = json.load(handle) + self.log.debug("Using countries.json.") + except IOError: + self.log.warning("Couldn't find countries.json. Using pycountry.") + self.countries = get_countries_and_codes() + if not self.media_domain: self.media_domain = self.domain if self.media_domain == self.domain: diff --git a/r2/r2/lib/countries.py b/r2/r2/lib/countries.py new file mode 100644 index 000000000..f31cfc031 --- /dev/null +++ b/r2/r2/lib/countries.py @@ -0,0 +1,39 @@ +# The contents of this file are subject to the Common Public Attribution +# License Version 1.0. (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# http://code.reddit.com/LICENSE. The License is based on the Mozilla Public +# License Version 1.1, but Sections 14 and 15 have been added to cover use of +# software over a computer network and provide for limited attribution for the +# Original Developer. In addition, Exhibit A has been modified to be consistent +# with Exhibit B. +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for +# the specific language governing rights and limitations under the License. +# +# The Original Code is reddit. +# +# The Original Developer is the Initial Developer. The Initial Developer of +# the Original Code is reddit Inc. +# +# All portions of the code written by reddit are Copyright (c) 2006-2012 reddit +# Inc. All Rights Reserved. +############################################################################### + +import json + + +def get_countries_and_codes(): + """Return a dict of ISO Alpha2 country codes to country names.""" + + # avoid importing this until this function is called since we're going to + # try our best to not import this at all in the app. it takes a tonne of + # memory! + import pycountry + + return {x.alpha2: x.name for x in pycountry.countries} + + +if __name__ == "__main__": + # Print out the country dict in JSON format for use in the Makefile. + print json.dumps(get_countries_and_codes(), indent=2, sort_keys=True) diff --git a/r2/r2/lib/pages/pages.py b/r2/r2/lib/pages/pages.py index 39e7a8a4c..a500664cf 100755 --- a/r2/r2/lib/pages/pages.py +++ b/r2/r2/lib/pages/pages.py @@ -60,7 +60,7 @@ from r2.lib.utils import trunc_string as _truncate, to_date from r2.lib.filters import safemarkdown import sys, random, datetime, calendar, simplejson, re, time -import pycountry, time +import time from itertools import chain from urllib import quote @@ -3500,8 +3500,6 @@ class RedditAds(Templated): class PaymentForm(Templated): def __init__(self, link, campaign, **kw): - self.countries = [pycountry.countries.get(name=n) - for n in g.allowed_pay_countries] self.link = promote.wrap_promoted(link) self.campaign = promote.get_renderable_campaigns(link, campaign) Templated.__init__(self, **kw) diff --git a/r2/r2/lib/validator/validator.py b/r2/r2/lib/validator/validator.py index 321ac664e..b4dd68e8c 100644 --- a/r2/r2/lib/validator/validator.py +++ b/r2/r2/lib/validator/validator.py @@ -46,7 +46,6 @@ from copy import copy from datetime import datetime, timedelta from curses.ascii import isprint import re, inspect -import pycountry from itertools import chain from functools import wraps @@ -1856,8 +1855,8 @@ class ValidAddress(Validator): elif not country: self.set_error(_("please pick a country"), "country") else: - country = pycountry.countries.get(alpha2=country) - if country.name not in self.allowed_countries: + country_name = g.countries.get(country) + if country_name not in self.allowed_countries: self.set_error(_("Our ToS don't cover your country (yet). Sorry."), "country") # Make sure values don't exceed max length defined in the authorize.net @@ -1882,7 +1881,7 @@ class ValidAddress(Validator): company = company or "", address = address, city = city, state = state, - zip = zipCode, country = country.name, + zip = zipCode, country = country_name, phoneNumber = phoneNumber or "") class ValidCard(Validator): diff --git a/r2/r2/templates/paymentform.html b/r2/r2/templates/paymentform.html index dd67ff194..a84d6e06d 100644 --- a/r2/r2/templates/paymentform.html +++ b/r2/r2/templates/paymentform.html @@ -157,12 +157,14 @@ ${unsafe(js.use('sponsored'))} %elif field == "country": ## TODO: pycountry does country name i18n %else: