github/reddit
1
1
Fork 0
mirror of https://github.com/reddit-archive/reddit.git synced 2026-01-31 09:48:01 -05:00
Code Issues Packages Projects Releases Wiki Activity

Record cache hit/miss stats.

Browse Source
This commit is contained in:
Logan Hanks
2011-11-10 10:59:55 -08:00
parent 7fed17d014
commit c1347f6d27
3 changed files with 54 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
r2/r2/lib/app_globals.py
View File

@@ -36,7 +36,7 @@ from r2.lib.db.stats import QueryStats
from r2.lib.translation import get_active_langs
from r2.lib.lock import make_lock_factory
from r2.lib.manager import db_manager
from r2.lib.stats import Stats
from r2.lib.stats import Stats, CacheStats
class Globals(object):
@@ -210,7 +210,7 @@ class Globals(object):
else LocalCache)
num_mc_clients = self.num_mc_clients
self.cache_chains = []
self.cache_chains = {}
self.memcache = CMemcache(self.memcaches, num_clients = num_mc_clients)
self.make_lock = make_lock_factory(self.memcache)
@@ -234,7 +234,7 @@ class Globals(object):
memcache = perma_memcache,
lock_factory = self.make_lock)
self.cache_chains.append(self.permacache)
self.cache_chains.update(permacache=self.permacache)
# hardcache is done after the db info is loaded, and then the
# chains are reset to use the appropriate initial entries
@@ -245,21 +245,21 @@ class Globals(object):
self.memcache)
else:
self.cache = MemcacheChain((localcache_cls(), self.memcache))
self.cache_chains.append(self.cache)
self.cache_chains.update(cache=self.cache)
self.rendercache = MemcacheChain((localcache_cls(),
CMemcache(self.rendercaches,
noreply=True, no_block=True,
num_clients = num_mc_clients)))
self.cache_chains.append(self.rendercache)
self.cache_chains.update(rendercache=self.rendercache)
self.servicecache = MemcacheChain((localcache_cls(),
CMemcache(self.servicecaches,
num_clients = num_mc_clients)))
self.cache_chains.append(self.servicecache)
self.cache_chains.update(servicecache=self.servicecache)
self.thing_cache = CacheChain((localcache_cls(),))
self.cache_chains.append(self.thing_cache)
self.cache_chains.update(thing_cache=self.thing_cache)
#load the database info
self.dbm = self.load_db_params(global_conf)
@@ -269,16 +269,20 @@ class Globals(object):
self.memcache,
HardCache(self)),
cache_negative_results = True)
self.cache_chains.append(self.hardcache)
self.cache_chains.update(hardcache=self.hardcache)
self.stats = Stats(global_conf.get('statsd_addr'),
global_conf.get('statsd_sample_rate'))
# I know this sucks, but we need non-request-threads to be
# able to reset the caches, so we need them be able to close
# around 'cache_chains' without being able to call getattr on
# 'g'
cache_chains = self.cache_chains[::]
cache_chains = self.cache_chains.copy()
def reset_caches():
for chain in cache_chains:
for name, chain in cache_chains.iteritems():
chain.reset()
chain.stats = CacheStats(self.stats, name)
self.reset_caches = reset_caches
self.reset_caches()
@@ -369,10 +373,6 @@ class Globals(object):
(self.reddit_host, self.reddit_pid,
self.short_version, datetime.now()))
self.stats = Stats(global_conf.get('statsd_addr'),
global_conf.get('statsd_sample_rate'))
@staticmethod
def to_bool(x):
return (x.lower() == 'true') if x else None

10
r2/r2/lib/cache.py
View File

@@ -323,6 +323,7 @@ class CacheChain(CacheUtils, local):
def __init__(self, caches, cache_negative_results=False):
self.caches = caches
self.cache_negative_results = cache_negative_results
self.stats = None
def make_set_fn(fn_name):
def fn(self, *a, **kw):
@@ -361,6 +362,9 @@ class CacheChain(CacheUtils, local):
val = c.get(key)
if val is not None:
if self.stats:
self.stats.cache_hit()
#update other caches
for d in self.caches:
if c is d:
@@ -373,6 +377,8 @@ class CacheChain(CacheUtils, local):
return val
#didn't find anything
if self.stats:
self.stats.cache_miss()
if self.cache_negative_results:
for c in self.caches[:-1]:
@@ -414,6 +420,10 @@ class CacheChain(CacheUtils, local):
for (k, v) in out.iteritems()
if v != NoneResult)
if self.stats:
self.stats.cache_hit(len(out))
self.stats.cache_miss(len(need))
return out
def __repr__(self):

30
r2/r2/lib/stats.py
View File

@@ -1,5 +1,6 @@
import time
from r2.lib import cache
from r2.lib import utils
class Stats:
@@ -29,6 +30,17 @@ class Stats:
if timer:
timer.send(action, service_time_sec)
def get_counter(self, name):
if self.connection:
return self.statsd.counter.Counter(name, self.connection)
else:
return None
def cache_count(self, name, delta=1):
counter = self.get_counter('cache')
if counter:
counter.increment(name, delta=delta)
def amqp_processor(self, processor):
"""Decorator for recording stats for amqp queue consumers/handlers."""
def wrap_processor(msgs, *args):
@@ -44,3 +56,21 @@ class Stats:
self.transact('amqp.%s' % msg.delivery_info['routing_key'],
service_time)
return wrap_processor
class CacheStats:
def __init__(self, parent, cache_name):
self.parent = parent
self.cache_name = cache_name
self.hit_stat_name = '%s.hit' % self.cache_name
self.miss_stat_name = '%s.miss' % self.cache_name
self.total_stat_name = '%s.total' % self.cache_name
def cache_hit(self, delta=1):
if delta:
self.parent.cache_count(self.hit_stat_name, delta=delta)
self.parent.cache_count(self.total_stat_name, delta=delta)
def cache_miss(self, delta=1):
if delta:
self.parent.cache_count(self.miss_stat_name, delta=delta)
self.parent.cache_count(self.total_stat_name, delta=delta)