Add statsd client and sampling of HTTP service times.

r2/r2/controllers/reddit_base.py

@@ -25,7 +25,7 @@ from pylons.controllers.util import abort, redirect_to
 from pylons.i18n import _
 from pylons.i18n.translation import LanguageError
 from r2.lib.base import BaseController, proxyurl
-from r2.lib import pages, utils, filters, amqp
+from r2.lib import pages, utils, filters, amqp, stats
 from r2.lib.utils import http_utils, is_subdomain, UniqueIterator, ip_and_slash16
 from r2.lib.cache import LocalCache, make_key, MemcachedError
 import random as rand
@@ -631,22 +631,20 @@ class MinimalController(BaseController):
                                     domain  = v.domain,
                                     expires = v.expires)
 
-        if g.usage_sampling <= 0.0:
-            return
+        end_time = datetime.now(g.tz)
 
+        if ('pylons.routes_dict' in request.environ and
+            'action' in request.environ['pylons.routes_dict']):
+            action = str(request.environ['pylons.routes_dict']['action'])
+        else:
+            action = "unknown"
+            log_text("unknown action", "no action for %r" % path_info,
+                     "warning")
         if g.usage_sampling >= 1.0 or rand.random() < g.usage_sampling:
-            if ('pylons.routes_dict' in request.environ and
-                'action' in request.environ['pylons.routes_dict']):
-                action = str(request.environ['pylons.routes_dict']['action'])
-            else:
-                action = "unknown"
-                log_text("unknown action",
-                         "no action for %r" % path_info,
-                         "warning")
 
             amqp.add_kw("usage_q",
                         start_time = c.start_time,
-                        end_time = datetime.now(g.tz),
+                        end_time = end_time,
                         sampling_rate = g.usage_sampling,
                         action = action)
 
@@ -656,6 +654,10 @@ class MinimalController(BaseController):
         # around taking up memory
         g.reset_caches()
 
+        # push data to statsd
+        g.stats.transact(action, (end_time - c.start_time).total_seconds())
+        
+
     def abort404(self):
         abort(404, "not found")
 

r2/r2/lib/app_globals.py

@@ -36,6 +36,7 @@ from r2.lib.db.stats import QueryStats
 from r2.lib.translation import get_active_langs
 from r2.lib.lock import make_lock_factory
 from r2.lib.manager import db_manager
+from r2.lib.stats import Stats
 
 class Globals(object):
 
@@ -72,6 +73,7 @@ class Globals(object):
     float_props = ['min_promote_bid',
                    'max_promote_bid',
                    'usage_sampling',
+                   'statsd_sample_rate',
                    ]
 
     bool_props = ['debug', 'translator',
@@ -367,6 +369,9 @@ class Globals(object):
                            (self.reddit_host, self.reddit_pid,
                             self.short_version, datetime.now()))
 
+        self.stats = Stats(global_conf.get('statsd_addr'),
+                           global_conf.get('statsd_sample_rate'))
+
 
     @staticmethod
     def to_bool(x):

r2/r2/lib/stats.py

@@ -0,0 +1,26 @@
+class Stats:
+    def __init__(self, addr, sample_rate):
+        if addr:
+            import statsd
+            self.statsd = statsd
+            self.host, port = addr.split(':')
+            self.port = int(port)
+            self.sample_rate = sample_rate
+            self.connection = self.statsd.connection.Connection(
+                self.host, self.port, self.sample_rate)
+        else:
+            self.host = None
+            self.port = None
+            self.sample_rate = None
+            self.connection = None
+
+    def get_timer(self, name):
+        if self.connection:
+            return self.statsd.timer.Timer(name, self.connection)
+        else:
+            return None
+
+    def transact(self, action, service_time_sec):
+        timer = self.get_timer('service_time')
+        if timer:
+            timer.send(action, service_time_sec)