From 83e63699f9656371f0bcfb7338ff350b3cb9a58d Mon Sep 17 00:00:00 2001 From: Neil Williams Date: Fri, 2 Nov 2012 17:31:44 -0700 Subject: [PATCH] Remove unused usage_q code. usage_q used to monitor the volume of various types of request and how long it took to generate the response for them. This has been replaced by graphite-based stats collection. --- r2/r2/config/queues.py | 1 - r2/r2/config/routing.py | 2 - r2/r2/controllers/__init__.py | 1 - r2/r2/controllers/reddit_base.py | 15 --- r2/r2/controllers/usage.py | 35 ------ r2/r2/lib/app_globals.py | 1 - r2/r2/lib/menus.py | 1 - r2/r2/lib/pages/admin_pages.py | 1 - r2/r2/lib/pages/pages.py | 179 ----------------------------- r2/r2/public/static/css/reddit.css | 37 ------ r2/r2/templates/adminusage.html | 78 ------------- 11 files changed, 351 deletions(-) delete mode 100644 r2/r2/controllers/usage.py delete mode 100644 r2/r2/templates/adminusage.html diff --git a/r2/r2/config/queues.py b/r2/r2/config/queues.py index e76a509fb..73cf0b176 100644 --- a/r2/r2/config/queues.py +++ b/r2/r2/config/queues.py @@ -81,7 +81,6 @@ def declare_queues(g): "vote_comment_q": MessageQueue(bind_to_self=True), "vote_fastlane_q": MessageQueue(bind_to_self=True), "log_q": MessageQueue(bind_to_self=True), - "usage_q": MessageQueue(bind_to_self=True, durable=False), "cloudsearch_changes": MessageQueue(bind_to_self=True), "update_promos_q": MessageQueue(bind_to_self=True), }) diff --git a/r2/r2/config/routing.py b/r2/r2/config/routing.py index 671277c05..d2f891f28 100644 --- a/r2/r2/config/routing.py +++ b/r2/r2/config/routing.py @@ -88,8 +88,6 @@ def make_map(): mc('/feedback', controller='feedback', action='feedback') mc('/ad_inq', controller='feedback', action='ad_inq') - mc('/admin/usage', controller='usage') - # Used for editing ads mc('/admin/ads', controller='ads') mc('/admin/ads/:adcn/:action', controller='ads', diff --git a/r2/r2/controllers/__init__.py b/r2/r2/controllers/__init__.py index 08a1b86f5..ff5afe101 100644 --- a/r2/r2/controllers/__init__.py +++ b/r2/r2/controllers/__init__.py @@ -64,7 +64,6 @@ def load_controllers(): from toolbar import ToolbarController from awards import AwardsController from ads import AdsController - from usage import UsageController from errorlog import ErrorlogController from promotecontroller import PromoteController from mediaembed import MediaembedController diff --git a/r2/r2/controllers/reddit_base.py b/r2/r2/controllers/reddit_base.py index 6a6fe2f57..b640e25f6 100644 --- a/r2/r2/controllers/reddit_base.py +++ b/r2/r2/controllers/reddit_base.py @@ -722,21 +722,6 @@ class MinimalController(BaseController): request.path != '/validuser'): c.user.update_last_visit(c.start_time) - if ('pylons.routes_dict' in request.environ and - 'action' in request.environ['pylons.routes_dict']): - action = str(request.environ['pylons.routes_dict']['action']) - else: - action = "unknown" - log_text("unknown action", "no action for %r" % path_info, - "warning") - if g.usage_sampling >= 1.0 or rand.random() < g.usage_sampling: - - amqp.add_kw("usage_q", - start_time = c.start_time, - end_time = end_time, - sampling_rate = g.usage_sampling, - action = action) - check_request(end_time) # this thread is probably going to be reused, but it could be diff --git a/r2/r2/controllers/usage.py b/r2/r2/controllers/usage.py deleted file mode 100644 index e6ceec501..000000000 --- a/r2/r2/controllers/usage.py +++ /dev/null @@ -1,35 +0,0 @@ -# The contents of this file are subject to the Common Public Attribution -# License Version 1.0. (the "License"); you may not use this file except in -# compliance with the License. You may obtain a copy of the License at -# http://code.reddit.com/LICENSE. The License is based on the Mozilla Public -# License Version 1.1, but Sections 14 and 15 have been added to cover use of -# software over a computer network and provide for limited attribution for the -# Original Developer. In addition, Exhibit A has been modified to be consistent -# with Exhibit B. -# -# Software distributed under the License is distributed on an "AS IS" basis, -# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for -# the specific language governing rights and limitations under the License. -# -# The Original Code is reddit. -# -# The Original Developer is the Initial Developer. The Initial Developer of -# the Original Code is reddit Inc. -# -# All portions of the code written by reddit are Copyright (c) 2006-2012 reddit -# Inc. All Rights Reserved. -############################################################################### - -from pylons import request, g -from reddit_base import RedditController -from r2.lib.pages import AdminPage, AdminUsage -from validator import * - -class UsageController(RedditController): - - @validate(VAdmin()) - def GET_index(self): - res = AdminPage(content = AdminUsage(), - show_sidebar = False, - title = 'usage').render() - return res diff --git a/r2/r2/lib/app_globals.py b/r2/r2/lib/app_globals.py index 629a97f08..8c3bbd122 100755 --- a/r2/r2/lib/app_globals.py +++ b/r2/r2/lib/app_globals.py @@ -129,7 +129,6 @@ class Globals(object): ConfigValue.float: [ 'min_promote_bid', 'max_promote_bid', - 'usage_sampling', 'statsd_sample_rate', 'querycache_prune_chance', ], diff --git a/r2/r2/lib/menus.py b/r2/r2/lib/menus.py index 4ee891cb8..038675b99 100644 --- a/r2/r2/lib/menus.py +++ b/r2/r2/lib/menus.py @@ -152,7 +152,6 @@ menu = MenuHandler(hot = _('hot'), errors = _("errors"), awards = _("awards"), ads = _("ads"), - usage = _("usage"), promoted = _("promoted"), reporters = _("reporters"), reports = _("reported links"), diff --git a/r2/r2/lib/pages/admin_pages.py b/r2/r2/lib/pages/admin_pages.py index d240a4eee..91072d709 100644 --- a/r2/r2/lib/pages/admin_pages.py +++ b/r2/r2/lib/pages/admin_pages.py @@ -54,7 +54,6 @@ class AdminPage(Reddit): buttons.append(NavButton(menu.awards, "ads")) buttons.append(NavButton(menu.awards, "awards")) buttons.append(NavButton(menu.errors, "error log")) - buttons.append(NavButton(menu.usage, "usage stats")) admin_menu = NavMenu(buttons, title='show', base_path = '/admin', type="lightdrop") diff --git a/r2/r2/lib/pages/pages.py b/r2/r2/lib/pages/pages.py index 735166d9e..6a72fa6ce 100755 --- a/r2/r2/lib/pages/pages.py +++ b/r2/r2/lib/pages/pages.py @@ -2306,185 +2306,6 @@ class AdminAwardWinners(Templated): trophies = Trophy.by_award(award) Templated.__init__(self, award = award, trophies = trophies) -class AdminUsage(Templated): - """The admin page for viewing usage stats""" - def __init__(self): - hcb = g.hardcache.backend - - self.actions = {} - triples = set() # sorting key - daily_stats = {} - - idses = hcb.ids_by_category("profile_count", limit=10000) - counts = g.hardcache.get_multi(prefix="profile_count-", keys=idses) - elapseds = g.hardcache.get_multi(prefix="profile_elapsed-", keys=idses) - - # The next three code paragraphs are for the case where we're - # rendering the current period and trying to decide what load class - # to use. For example, if today's number of hits equals yesterday's, - # and we're 23:59 into the day, that's totally normal. But if we're - # only 12 hours into the day, that's twice what we'd expect. So - # we're going to scale the current period by the percent of the way - # into the period that we are. - # - # If we're less than 5% of the way into the period, we skip this - # step. This both avoids Div0 errors and keeps us from extrapolating - # ridiculously from a tiny sample size. - - now = c.start_time.astimezone(g.display_tz) - t_midnight = trunc_time(now, hours=24, mins=60) - t_hour = trunc_time(now, mins=60) - t_5min = trunc_time(now, mins=5) - - offset_day = (now - t_midnight).seconds / 86400.0 - offset_hour = (now - t_hour).seconds / 3600.0 - offset_5min = (now - t_5min).seconds / 300.0 - - this_day = t_midnight.strftime("%Y/%m/%d_xx:xx") - this_hour = t_hour.strftime("%Y/%m/%d_%H:xx") - this_5min = t_5min.strftime("%Y/%m/%d_%H:%M") - - for ids in idses: - time, action = ids.split("-") - - # coltype strings are carefully chosen to sort alphabetically - # in the order that they do - - if time.endswith("xx:xx"): - coltype = 'Day' - factor = 1.0 - label = time[5:10] # MM/DD - if time == this_day and offset_day > 0.05: - factor /= offset_day - elif time.endswith(":xx"): - coltype = 'Hour' - factor = 24.0 - label = time[11:] # HH:xx - if time == this_hour and offset_hour > 0.05: - factor /= offset_hour - else: - coltype = 'five-min' - factor = 288.0 # number of five-minute periods in a day - label = time[11:] # HH:MM - if time == this_5min and offset_5min > 0.05: - factor /= offset_5min - - count = counts.get(ids, None) - if count is None or count == 0: - log_text("usage count=None", "For %r, it's %r" % (ids, count), "error") - continue - - # Elapsed in hardcache is in hundredths of a second. - # Multiply it by 100 so from this point forward, we're - # dealing with seconds -- as floats with two decimal - # places of precision. Similarly, round the average - # to two decimal places. - elapsed = elapseds.get(ids, 0) / 100.0 - average = int(100.0 * elapsed / count) / 100.0 - - # Again, the "triple" tuples are a sorting key for the columns - triples.add( (coltype, time, label) ) - - if coltype == 'Day': - daily_stats.setdefault(action, []).append( - (count, elapsed, average) - ) - - self.actions.setdefault(action, {}) - self.actions[action][label] = dict(count=count, elapsed=elapsed, - average=average, - factor=factor, - classes = {}) - - # Figure out what a typical day looks like. For each action, - # look at the daily stats and record the median. - for action in daily_stats.keys(): - if len(daily_stats[action]) < 2: - # This is a new action. No point in guessing what normal - # load for it looks like. - continue - med = {} - med["count"] = median([ x[0] for x in daily_stats[action] ]) - med["elapsed"] = median([ x[1] for x in daily_stats[action] ]) - med["average"] = median([ x[2] for x in daily_stats[action] ]) - - # For the purposes of load classes, round the baseline count up - # to 5000 times per day, the elapsed to 30 minutes per day, and - # the average to 0.10 seconds per request. This not only avoids - # division-by-zero problems but also means that if something - # went from taking 0.01 seconds per day to 0.08 seconds per day, - # we're not going to consider it an emergency. - med["count"] = max(5000, med["count"]) - med["elapsed"] = max(1800.0, med["elapsed"]) - med["average"] = max(0.10, med["average"]) - -# print "Median count for %s is %r" % (action, med["count"]) - - for d in self.actions[action].values(): - ice_cold = False - for category in ("elapsed", "count", "average"): - if category == "average": - scaled = d[category] - else: - scaled = d[category] * d["factor"] - - if category == "elapsed" and scaled < 5 * 60: - # If we're spending less than five mins a day - # on this operation, consider it ice cold regardless - # of how much of an outlier it is - ice_cold = True - - if ice_cold: - d["classes"][category] = "load0" - continue - - if med[category] <= 0: - d["classes"][category] = "load9" - continue - - ratio = scaled / med[category] - if ratio > 5.0: - d["classes"][category] = "load9" - elif ratio > 3.0: - d["classes"][category] = "load8" - elif ratio > 2.0: - d["classes"][category] = "load7" - elif ratio > 1.5: - d["classes"][category] = "load6" - elif ratio > 1.1: - d["classes"][category] = "load5" - elif ratio > 0.9: - d["classes"][category] = "load4" - elif ratio > 0.75: - d["classes"][category] = "load3" - elif ratio > 0.5: - d["classes"][category] = "load2" - elif ratio > 0.10: - d["classes"][category] = "load1" - else: - d["classes"][category] = "load0" - - # Build a list called labels that gives the template a sorting - # order for the columns. - self.labels = [] - # Keep track of how many times we've seen a granularity (i.e., coltype) - # so we can hide any that come after the third - coltype_counts = {} - # sort actions by whatever will end up as the first column - action_sorting_column = None - for coltype, time, label in sorted(triples, reverse=True): - if action_sorting_column is None: - action_sorting_column = label - coltype_counts.setdefault(coltype, 0) - coltype_counts[coltype] += 1 - self.labels.append( (label, coltype_counts[coltype] > 3) ) - - self.action_order = sorted(self.actions.keys(), reverse=True, - key = lambda x: - self.actions[x].get(action_sorting_column, {"elapsed":0})["elapsed"]) - - Templated.__init__(self) - class Ads(Templated): pass diff --git a/r2/r2/public/static/css/reddit.css b/r2/r2/public/static/css/reddit.css index 5e603b85e..ece495140 100755 --- a/r2/r2/public/static/css/reddit.css +++ b/r2/r2/public/static/css/reddit.css @@ -5123,43 +5123,6 @@ a.adminbox:hover { color: red; } -.usage-table .intersection { - color: #888; - font-family: monospace; - text-align: right; - border-left: none; - border-right: none; -} - -.usage-table .intersection span { - padding: 1px 3px 0 2px; -} - -.usage-table .empty.intersection { - text-align: center; - color: #ccc; -} - -.usage-table .elapsed.intersection { - color: black; -} - -.usage-table .count.intersection { - color: black; -} - -.usage-table .average.intersection { - color: black; - border-right: solid #cdcdcd 1px; -} - -.usage-table .empty.intersection, .usage-table .average.intersection { - padding-left: 0; - margin-left: 0; - border-right: solid #cdcdcd 1px; - padding-right: 5px; -} - a.pretty-button:hover { text-decoration: none !important } .pretty-button { diff --git a/r2/r2/templates/adminusage.html b/r2/r2/templates/adminusage.html deleted file mode 100644 index 32f6a3284..000000000 --- a/r2/r2/templates/adminusage.html +++ /dev/null @@ -1,78 +0,0 @@ -## The contents of this file are subject to the Common Public Attribution -## License Version 1.0. (the "License"); you may not use this file except in -## compliance with the License. You may obtain a copy of the License at -## http://code.reddit.com/LICENSE. The License is based on the Mozilla Public -## License Version 1.1, but Sections 14 and 15 have been added to cover use of -## software over a computer network and provide for limited attribution for the -## Original Developer. In addition, Exhibit A has been modified to be -## consistent with Exhibit B. -## -## Software distributed under the License is distributed on an "AS IS" basis, -## WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for -## the specific language governing rights and limitations under the License. -## -## The Original Code is reddit. -## -## The Original Developer is the Initial Developer. The Initial Developer of -## the Original Code is reddit Inc. -## -## All portions of the code written by reddit are Copyright (c) 2006-2012 -## reddit Inc. All Rights Reserved. -############################################################################### - -<%def name="intersection(d, hidden)"> - %if d is None: - - — - - %else: - %for cls in ("elapsed", "slash", "count", "equals", "average"): - - %if cls == "slash": - / - %elif cls == "equals": - = - %else: - - %if cls == 'count': - ${d[cls]} - %else: - ${"%0.2f" % d[cls]} - %endif - - %endif - - %endfor - %endif - - - - - - %for label, hidden in thing.labels: - - %endfor - - -%for action in thing.action_order: - - - %for label, hidden in thing.labels: - ${intersection(thing.actions[action].get(label), hidden)} - %endfor - -%endfor - -
action${label}
${action}