diff --git a/r2/example.ini b/r2/example.ini index a880e25dc..3f483cd67 100644 --- a/r2/example.ini +++ b/r2/example.ini @@ -475,7 +475,9 @@ sr_dropdown_threshold = 15 # seconds of each other comment_visits_period = 600 -#user-agents to rate-limit +# user-agents to rate-limit with number of requests every ten seconds +# To limit GoogleBot to 10 requests and AppEngine to 2 requests and AppEngine to 2 requests every 10 seconds +# agents = googlebot:10, appengine:2 agents = # subreddit ratelimits sr_banned_quota = 10000 diff --git a/r2/r2/controllers/reddit_base.py b/r2/r2/controllers/reddit_base.py index 6535d7b40..a9d25d7ab 100644 --- a/r2/r2/controllers/reddit_base.py +++ b/r2/r2/controllers/reddit_base.py @@ -557,15 +557,15 @@ def set_colors(): c.bordercolor = request.get.get('bordercolor') -def ratelimit_agent(agent): - SLICE_SIZE = 10 - slice, remainder = map(int, divmod(time.time(), SLICE_SIZE)) - time_slice = time.gmtime(slice * SLICE_SIZE) +def ratelimit_agent(agent, limit=10, slice_size=10): + slice_size = min(slice_size, 60) + slice, remainder = map(int, divmod(time.time(), slice_size)) + time_slice = time.gmtime(slice * slice_size) key = "rate_agent_" + agent + time.strftime("_%S", time_slice) - g.cache.add(key, 0, time=SLICE_SIZE + 1) - if g.cache.incr(key) > SLICE_SIZE: - request.environ['retry_after'] = SLICE_SIZE - remainder + g.cache.add(key, 0, time=slice_size + 1) + if g.cache.incr(key) > limit: + request.environ['retry_after'] = slice_size - remainder abort(429) appengine_re = re.compile(r'AppEngine-Google; \(\+http://code.google.com/appengine; appid: (?:dev|s)~([a-z0-9-]{6,30})\)\Z') @@ -583,9 +583,9 @@ def ratelimit_agents(): return user_agent = user_agent.lower() - for s in g.agents: - if s and user_agent and s in user_agent: - ratelimit_agent(s) + for agent, limit in g.agents.iteritems(): + if agent in user_agent: + ratelimit_agent(agent, limit) return def ratelimit_throttled(): diff --git a/r2/r2/lib/app_globals.py b/r2/r2/lib/app_globals.py index bdeddcad2..7152d4aea 100755 --- a/r2/r2/lib/app_globals.py +++ b/r2/r2/lib/app_globals.py @@ -175,7 +175,6 @@ class Globals(object): 'admins', 'sponsors', 'automatic_reddits', - 'agents', 'allowed_css_linked_domains', 'authorized_cnames', 'hardcache_categories', @@ -189,6 +188,10 @@ class Globals(object): 'sample_multis', ], + ConfigValue.dict(ConfigValue.str, ConfigValue.int): [ + 'agents', + ], + ConfigValue.str: [ 'wiki_page_registration_info', 'wiki_page_privacy_policy',