github/reddit
1
1
Fork 0
mirror of https://github.com/reddit-archive/reddit.git synced 2026-01-26 07:19:25 -05:00
Code Issues Packages Projects Releases Wiki Activity

Remove unused usage_q code.

Browse Source 
usage_q used to monitor the volume of various types of request and how
long it took to generate the response for them.  This has been replaced
by graphite-based stats collection.
This commit is contained in:
Neil Williams
2012-11-02 17:31:44 -07:00
parent e93dede2d1
commit 83e63699f9
11 changed files with 0 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
r2/r2/config/queues.py
View File

@@ -81,7 +81,6 @@ def declare_queues(g):
"vote_comment_q": MessageQueue(bind_to_self=True),
"vote_fastlane_q": MessageQueue(bind_to_self=True),
"log_q": MessageQueue(bind_to_self=True),
"usage_q": MessageQueue(bind_to_self=True, durable=False),
"cloudsearch_changes": MessageQueue(bind_to_self=True),
"update_promos_q": MessageQueue(bind_to_self=True),
})

2
r2/r2/config/routing.py
View File

@@ -88,8 +88,6 @@ def make_map():
mc('/feedback', controller='feedback', action='feedback')
mc('/ad_inq', controller='feedback', action='ad_inq')
mc('/admin/usage', controller='usage')
# Used for editing ads
mc('/admin/ads', controller='ads')
mc('/admin/ads/:adcn/:action', controller='ads',

1
r2/r2/controllers/__init__.py
View File

@@ -64,7 +64,6 @@ def load_controllers():
from toolbar import ToolbarController
from awards import AwardsController
from ads import AdsController
from usage import UsageController
from errorlog import ErrorlogController
from promotecontroller import PromoteController
from mediaembed import MediaembedController

15
r2/r2/controllers/reddit_base.py
View File

@@ -722,21 +722,6 @@ class MinimalController(BaseController):
request.path != '/validuser'):
c.user.update_last_visit(c.start_time)
if ('pylons.routes_dict' in request.environ and
'action' in request.environ['pylons.routes_dict']):
action = str(request.environ['pylons.routes_dict']['action'])
else:
action = "unknown"
log_text("unknown action", "no action for %r" % path_info,
"warning")
if g.usage_sampling >= 1.0 or rand.random() < g.usage_sampling:
amqp.add_kw("usage_q",
start_time = c.start_time,
end_time = end_time,
sampling_rate = g.usage_sampling,
action = action)
check_request(end_time)
# this thread is probably going to be reused, but it could be

35
r2/r2/controllers/usage.py
View File

@@ -1,35 +0,0 @@
# The contents of this file are subject to the Common Public Attribution
# License Version 1.0. (the "License"); you may not use this file except in
# compliance with the License. You may obtain a copy of the License at
# http://code.reddit.com/LICENSE. The License is based on the Mozilla Public
# License Version 1.1, but Sections 14 and 15 have been added to cover use of
# software over a computer network and provide for limited attribution for the
# Original Developer. In addition, Exhibit A has been modified to be consistent
# with Exhibit B.
#
# Software distributed under the License is distributed on an "AS IS" basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for
# the specific language governing rights and limitations under the License.
#
# The Original Code is reddit.
#
# The Original Developer is the Initial Developer. The Initial Developer of
# the Original Code is reddit Inc.
#
# All portions of the code written by reddit are Copyright (c) 2006-2012 reddit
# Inc. All Rights Reserved.
###############################################################################
from pylons import request, g
from reddit_base import RedditController
from r2.lib.pages import AdminPage, AdminUsage
from validator import *
class UsageController(RedditController):
@validate(VAdmin())
def GET_index(self):
res = AdminPage(content = AdminUsage(),
show_sidebar = False,
title = 'usage').render()
return res

1
r2/r2/lib/app_globals.py
View File

@@ -129,7 +129,6 @@ class Globals(object):
ConfigValue.float: [
'min_promote_bid',
'max_promote_bid',
'usage_sampling',
'statsd_sample_rate',
'querycache_prune_chance',
],

1
r2/r2/lib/menus.py
View File

@@ -152,7 +152,6 @@ menu = MenuHandler(hot = _('hot'),
errors = _("errors"),
awards = _("awards"),
ads = _("ads"),
usage = _("usage"),
promoted = _("promoted"),
reporters = _("reporters"),
reports = _("reported links"),

1
r2/r2/lib/pages/admin_pages.py
View File

@@ -54,7 +54,6 @@ class AdminPage(Reddit):
buttons.append(NavButton(menu.awards, "ads"))
buttons.append(NavButton(menu.awards, "awards"))
buttons.append(NavButton(menu.errors, "error log"))
buttons.append(NavButton(menu.usage, "usage stats"))
admin_menu = NavMenu(buttons, title='show', base_path = '/admin',
type="lightdrop")

179
r2/r2/lib/pages/pages.py
View File

@@ -2306,185 +2306,6 @@ class AdminAwardWinners(Templated):
trophies = Trophy.by_award(award)
Templated.__init__(self, award = award, trophies = trophies)
class AdminUsage(Templated):
"""The admin page for viewing usage stats"""
def __init__(self):
hcb = g.hardcache.backend
self.actions = {}
triples = set() # sorting key
daily_stats = {}
idses = hcb.ids_by_category("profile_count", limit=10000)
counts = g.hardcache.get_multi(prefix="profile_count-", keys=idses)
elapseds = g.hardcache.get_multi(prefix="profile_elapsed-", keys=idses)
# The next three code paragraphs are for the case where we're
# rendering the current period and trying to decide what load class
# to use. For example, if today's number of hits equals yesterday's,
# and we're 23:59 into the day, that's totally normal. But if we're
# only 12 hours into the day, that's twice what we'd expect. So
# we're going to scale the current period by the percent of the way
# into the period that we are.
#
# If we're less than 5% of the way into the period, we skip this
# step. This both avoids Div0 errors and keeps us from extrapolating
# ridiculously from a tiny sample size.
now = c.start_time.astimezone(g.display_tz)
t_midnight = trunc_time(now, hours=24, mins=60)
t_hour = trunc_time(now, mins=60)
t_5min = trunc_time(now, mins=5)
offset_day = (now - t_midnight).seconds / 86400.0
offset_hour = (now - t_hour).seconds / 3600.0
offset_5min = (now - t_5min).seconds / 300.0
this_day = t_midnight.strftime("%Y/%m/%d_xx:xx")
this_hour = t_hour.strftime("%Y/%m/%d_%H:xx")
this_5min = t_5min.strftime("%Y/%m/%d_%H:%M")
for ids in idses:
time, action = ids.split("-")
# coltype strings are carefully chosen to sort alphabetically
# in the order that they do
if time.endswith("xx:xx"):
coltype = 'Day'
factor = 1.0
label = time[5:10] # MM/DD
if time == this_day and offset_day > 0.05:
factor /= offset_day
elif time.endswith(":xx"):
coltype = 'Hour'
factor = 24.0
label = time[11:] # HH:xx
if time == this_hour and offset_hour > 0.05:
factor /= offset_hour
else:
coltype = 'five-min'
factor = 288.0 # number of five-minute periods in a day
label = time[11:] # HH:MM
if time == this_5min and offset_5min > 0.05:
factor /= offset_5min
count = counts.get(ids, None)
if count is None or count == 0:
log_text("usage count=None", "For %r, it's %r" % (ids, count), "error")
continue
# Elapsed in hardcache is in hundredths of a second.
# Multiply it by 100 so from this point forward, we're
# dealing with seconds -- as floats with two decimal
# places of precision. Similarly, round the average
# to two decimal places.
elapsed = elapseds.get(ids, 0) / 100.0
average = int(100.0 * elapsed / count) / 100.0
# Again, the "triple" tuples are a sorting key for the columns
triples.add( (coltype, time, label) )
if coltype == 'Day':
daily_stats.setdefault(action, []).append(
(count, elapsed, average)
)
self.actions.setdefault(action, {})
self.actions[action][label] = dict(count=count, elapsed=elapsed,
average=average,
factor=factor,
classes = {})
# Figure out what a typical day looks like. For each action,
# look at the daily stats and record the median.
for action in daily_stats.keys():
if len(daily_stats[action]) < 2:
# This is a new action. No point in guessing what normal
# load for it looks like.
continue
med = {}
med["count"] = median([ x[0] for x in daily_stats[action] ])
med["elapsed"] = median([ x[1] for x in daily_stats[action] ])
med["average"] = median([ x[2] for x in daily_stats[action] ])
# For the purposes of load classes, round the baseline count up
# to 5000 times per day, the elapsed to 30 minutes per day, and
# the average to 0.10 seconds per request. This not only avoids
# division-by-zero problems but also means that if something
# went from taking 0.01 seconds per day to 0.08 seconds per day,
# we're not going to consider it an emergency.
med["count"] = max(5000, med["count"])
med["elapsed"] = max(1800.0, med["elapsed"])
med["average"] = max(0.10, med["average"])
# print "Median count for %s is %r" % (action, med["count"])
for d in self.actions[action].values():
ice_cold = False
for category in ("elapsed", "count", "average"):
if category == "average":
scaled = d[category]
else:
scaled = d[category] * d["factor"]
if category == "elapsed" and scaled < 5 * 60:
# If we're spending less than five mins a day
# on this operation, consider it ice cold regardless
# of how much of an outlier it is
ice_cold = True
if ice_cold:
d["classes"][category] = "load0"
continue
if med[category] <= 0:
d["classes"][category] = "load9"
continue
ratio = scaled / med[category]
if ratio > 5.0:
d["classes"][category] = "load9"
elif ratio > 3.0:
d["classes"][category] = "load8"
elif ratio > 2.0:
d["classes"][category] = "load7"
elif ratio > 1.5:
d["classes"][category] = "load6"
elif ratio > 1.1:
d["classes"][category] = "load5"
elif ratio > 0.9:
d["classes"][category] = "load4"
elif ratio > 0.75:
d["classes"][category] = "load3"
elif ratio > 0.5:
d["classes"][category] = "load2"
elif ratio > 0.10:
d["classes"][category] = "load1"
else:
d["classes"][category] = "load0"
# Build a list called labels that gives the template a sorting
# order for the columns.
self.labels = []
# Keep track of how many times we've seen a granularity (i.e., coltype)
# so we can hide any that come after the third
coltype_counts = {}
# sort actions by whatever will end up as the first column
action_sorting_column = None
for coltype, time, label in sorted(triples, reverse=True):
if action_sorting_column is None:
action_sorting_column = label
coltype_counts.setdefault(coltype, 0)
coltype_counts[coltype] += 1
self.labels.append( (label, coltype_counts[coltype] > 3) )
self.action_order = sorted(self.actions.keys(), reverse=True,
key = lambda x:
self.actions[x].get(action_sorting_column, {"elapsed":0})["elapsed"])
Templated.__init__(self)
class Ads(Templated):
pass

37
r2/r2/public/static/css/reddit.css
View File

@@ -5123,43 +5123,6 @@ a.adminbox:hover {
color: red;
}
.usage-table .intersection {
color: #888;
font-family: monospace;
text-align: right;
border-left: none;
border-right: none;
}
.usage-table .intersection span {
padding: 1px 3px 0 2px;
}
.usage-table .empty.intersection {
text-align: center;
color: #ccc;
}
.usage-table .elapsed.intersection {
color: black;
}
.usage-table .count.intersection {
color: black;
}
.usage-table .average.intersection {
color: black;
border-right: solid #cdcdcd 1px;
}
.usage-table .empty.intersection, .usage-table .average.intersection {
padding-left: 0;
margin-left: 0;
border-right: solid #cdcdcd 1px;
padding-right: 5px;
}
a.pretty-button:hover { text-decoration: none !important }
.pretty-button {

78
r2/r2/templates/adminusage.html
View File

@@ -1,78 +0,0 @@
## The contents of this file are subject to the Common Public Attribution
## License Version 1.0. (the "License"); you may not use this file except in
## compliance with the License. You may obtain a copy of the License at
## http://code.reddit.com/LICENSE. The License is based on the Mozilla Public
## License Version 1.1, but Sections 14 and 15 have been added to cover use of
## software over a computer network and provide for limited attribution for the
## Original Developer. In addition, Exhibit A has been modified to be
## consistent with Exhibit B.
##
## Software distributed under the License is distributed on an "AS IS" basis,
## WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for
## the specific language governing rights and limitations under the License.
##
## The Original Code is reddit.
##
## The Original Developer is the Initial Developer. The Initial Developer of
## the Original Code is reddit Inc.
##
## All portions of the code written by reddit are Copyright (c) 2006-2012
## reddit Inc. All Rights Reserved.
###############################################################################
<%def name="intersection(d, hidden)">
%if d is None:
<td class="empty intersection" colspan="5"
%if hidden:
style="display:none"
%endif
>
<span>&mdash;</span>
</td>
%else:
%for cls in ("elapsed", "slash", "count", "equals", "average"):
<td class="${cls} intersection"
%if hidden:
style="display:none"
%endif
>
%if cls == "slash":
/
%elif cls == "equals":
=
%else:
<span class="${d['classes'].get(cls, 'load0')}">
%if cls == 'count':
${d[cls]}
%else:
${"%0.2f" % d[cls]}
%endif
</span>
%endif
</td>
%endfor
%endif
</%def>
<table class="usage-table lined-table">
<tr>
<th>action</th>
%for label, hidden in thing.labels:
<th colspan="5"
%if hidden:
style="display:none"
%endif
>${label}</th>
%endfor
</tr>
%for action in thing.action_order:
<tr>
<td>${action}</td>
%for label, hidden in thing.labels:
${intersection(thing.actions[action].get(label), hidden)}
%endfor
</tr>
%endfor
</table>