tool to find average cache object lifetime

r2/r2/lib/utils/__init__.py

@@ -22,6 +22,7 @@
 from utils import *
 from http_utils import *
 from thing_utils import *
+from cmd_utils import *
 
 try:
     from r2admin.lib.admin_utils import *

r2/r2/lib/utils/cmd_utils.py

@@ -0,0 +1,82 @@
+# The contents of this file are subject to the Common Public Attribution
+# License Version 1.0. (the "License"); you may not use this file except in
+# compliance with the License. You may obtain a copy of the License at
+# http://code.reddit.com/LICENSE. The License is based on the Mozilla Public
+# License Version 1.1, but Sections 14 and 15 have been added to cover use of
+# software over a computer network and provide for limited attribution for the
+# Original Developer. In addition, Exhibit A has been modified to be consistent
+# with Exhibit B.
+# 
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for
+# the specific language governing rights and limitations under the License.
+# 
+# The Original Code is Reddit.
+# 
+# The Original Developer is the Initial Developer.  The Initial Developer of the
+# Original Code is CondeNet, Inc.
+# 
+# All portions of the code written by CondeNet are Copyright (c) 2006-2008
+# CondeNet, Inc. All Rights Reserved.
+################################################################################
+"""
+Contains utilities intended to be run from a command line
+"""
+
+from time import sleep
+import sys
+    
+def bench_cache_lifetime(minutes):
+    "Attempts to find how long a given memcached key can be expected to live"
+
+    from pylons import g
+
+    # we'll create an independent connection to memcached for this
+    # test
+    mc = Memcache(g.memcaches)
+
+    # set N keys, and tell them not to live for longer than this test
+    mc.set_multi(dict( ('bench_cache_%d' % x, x)
+                       for x in xrange(minutes) ),
+                 time=minutes*60)
+
+    # and every minute, check to see that the keys are still present,
+    # until we find one missing
+    for x in xrange(minutes):
+        if mc.get('bench_cache_%d' % x, None) is not None:
+            sleep(60)
+        else:
+            return x-1
+    else:
+        # we're out of numbers, and we didn't find any missing
+        # keys. Since we only set N keys, we can't check for anything
+        # else
+        print (("Cache lifetime is greater than %d minutes. Try again with a"+
+                " higher 'minutes' value") % minutes)
+        return None
+
+def bench_cache_lifetime_multi(attempts=10, minutes=60*24):
+    """
+    Attempts to find the minimum, maximum, and average cache key lifetime
+
+    Example:
+        paster run production.ini r2/lib/utils/cmd_utils.py -c "bench_cache_lifetime_multi()"
+    """
+    total = 0
+    attempts_so_far = 0
+    minimum = sys.maxint
+    maximum = 0
+
+    for x in xrange(attempts):
+        this_attempt = bench_cache_lifetime(minutes)
+        maximum = max(this_attempt, maximum)
+        minimum = min(this_attempt, minimum)
+
+        total += this_attempt
+        attempts_so_far += 1
+        mean = float(total)/float(attempts_so_far)
+
+        print ("Attempt #%d of %d: %d; min=%d, max=%d, mean=%.2f"
+               % (x+1, attempts, this_attempt, minimum, maximum, mean))
+
+    return (minimum, maximum, mean)