mirror of
https://github.com/reddit-archive/reddit.git
synced 2026-01-24 22:38:09 -05:00
Remove all remaining traces of the unused service cache.
This commit is contained in:
Neil Williams
113
r2/draw_load.py
113
r2/draw_load.py
@@ -1,113 +0,0 @@
|
||||
from __future__ import with_statement
|
||||
import Image, ImageDraw
|
||||
|
||||
colors = [
|
||||
"#FFFFFF", "#f0f5FF",
|
||||
"#E2ECFF", "#d6f5cb",
|
||||
"#CAFF98", "#e4f484",
|
||||
"#FFEA71", "#ffdb81",
|
||||
"#FF9191", "#FF0000"]
|
||||
|
||||
def get_load_level(host, nlevels = 8):
|
||||
# default number of cpus shall be 1
|
||||
ncpus = getattr(host, "ncpu", 1)
|
||||
# color code in nlevel levels
|
||||
return _load_int(host.load(), ncpus, nlevels = 8)
|
||||
|
||||
def _load_int(current, max_val, nlevels = 8):
|
||||
i = min(max(int(nlevels*current/max_val+0.4), 0),nlevels+1)
|
||||
return colors[i]
|
||||
|
||||
def draw_load(row_size = 12, width = 200, out_file = "/tmp/load.png"):
|
||||
from r2.lib import services
|
||||
|
||||
a = services.AppServiceMonitor()
|
||||
hosts = list(a)
|
||||
|
||||
number = (len([x for x in hosts if x.services]) +
|
||||
len([x for x in hosts if x.database]) +
|
||||
sum(len(x.queue.queues) for x in hosts if x.queue)) + 14
|
||||
|
||||
im = Image.new("RGB", (width, number * row_size + 3))
|
||||
draw = ImageDraw.Draw(im)
|
||||
def draw_box(label, color, center = False):
|
||||
ypos = draw_box.ypos
|
||||
xpos = 1
|
||||
if center:
|
||||
w, h = draw.textsize(label)
|
||||
xpos = (width - w) / 2
|
||||
draw.rectangle(((1, ypos), (width-2, ypos + row_size)), color)
|
||||
draw.text((xpos,ypos+1), label, fill = "#000000")
|
||||
draw_box.ypos += row_size
|
||||
draw_box.ypos = 0
|
||||
|
||||
draw_box(" ==== DATABASES ==== ", "#BBBBBB", center = True)
|
||||
for host in hosts:
|
||||
if host.database or host.host.startswith('vote'):
|
||||
draw_box(" %s load: %s" % (host.host, host.load()),
|
||||
get_load_level(host))
|
||||
|
||||
draw_box(" ==== SERVICES ==== ", "#BBBBBB", center = True)
|
||||
for host in hosts:
|
||||
if host.host.startswith('app'):
|
||||
draw_box(" %s load: %s" % (host.host, host.load()),
|
||||
get_load_level(host))
|
||||
|
||||
draw_box(" ==== BUTTONS ==== ", "#BBBBBB", center = True)
|
||||
for host in hosts:
|
||||
if host.host.startswith('button'):
|
||||
draw_box(" %s load: %s" % (host.host, host.load()),
|
||||
get_load_level(host))
|
||||
|
||||
draw_box(" ==== MEDIA ==== ", "#BBBBBB", center = True)
|
||||
for host in hosts:
|
||||
if host.host.startswith('media'):
|
||||
draw_box(" %s load: %s" % (host.host, host.load()),
|
||||
get_load_level(host))
|
||||
draw_box(" ==== SEARCH ==== ", "#BBBBBB", center = True)
|
||||
for host in hosts:
|
||||
if host.host.startswith('search'):
|
||||
draw_box(" %s load: %s" % (host.host, host.load()),
|
||||
get_load_level(host))
|
||||
|
||||
draw_box(" ==== CACHES ==== ", "#BBBBBB", center = True)
|
||||
for host in hosts:
|
||||
if (host.host.startswith('cache') or host.host.startswith('pmc') or
|
||||
host.host.startswith('url')):
|
||||
draw_box(" %s load: %s" % (host.host, host.load()),
|
||||
get_load_level(host))
|
||||
|
||||
draw_box(" ==== QUEUES ==== ", "#BBBBBB", center = True)
|
||||
for host in hosts:
|
||||
if host.queue:
|
||||
draw_box(" %s load: %s" % (host.host, host.load()),
|
||||
get_load_level(host))
|
||||
for name, data in host.queue:
|
||||
max_len = host.queue.max_length(name)
|
||||
draw_box(" %16s: %5s / %5s" % (name, data(), max_len),
|
||||
_load_int(data(), max_len))
|
||||
|
||||
with open(out_file, 'w') as handle:
|
||||
im.save(handle, "PNG")
|
||||
|
||||
|
||||
def merge_images(out_file, file_names):
|
||||
images = []
|
||||
width = 0
|
||||
height = 0
|
||||
for f in file_names:
|
||||
images.append(Image.open(f))
|
||||
w, h = images[-1].size
|
||||
width = max(w, width)
|
||||
height += h
|
||||
|
||||
total = Image.new("RGB", (width, height))
|
||||
height = 0
|
||||
for im in images:
|
||||
total.paste(im, (0, height))
|
||||
w, h = im.size
|
||||
height += h
|
||||
|
||||
with open(out_file, 'w') as handle:
|
||||
total.save(out_file)
|
||||
|
||||
@@ -85,8 +85,6 @@ timezone = UTC
|
||||
display_timezone = MST
|
||||
# secret key for accessing /shutdown
|
||||
shutdown_secret = 12345
|
||||
# list of servers that the service monitor will care about
|
||||
monitored_servers = reddit, localhost
|
||||
# https api endpoint (must be g.domain or a subdomain of g.domain)
|
||||
https_endpoint =
|
||||
# name of the cookie to drop with login information
|
||||
@@ -110,8 +108,6 @@ stalecaches =
|
||||
# render caches (the second is "remote" and the local is optional but in the same format)
|
||||
local_rendercache =
|
||||
rendercaches = 127.0.0.1:11211
|
||||
# cache for storing service monitor information
|
||||
servicecaches = 127.0.0.1:11211
|
||||
|
||||
# -- permacache options --
|
||||
# permacache is memcaches -> cassanda -> memcachedb
|
||||
|
||||
@@ -104,11 +104,9 @@ class Globals(object):
|
||||
'memcaches',
|
||||
'permacache_memcaches',
|
||||
'rendercaches',
|
||||
'servicecaches',
|
||||
'cassandra_seeds',
|
||||
'admins',
|
||||
'sponsors',
|
||||
'monitored_servers',
|
||||
'automatic_reddits',
|
||||
'agents',
|
||||
'allowed_css_linked_domains',
|
||||
@@ -278,11 +276,6 @@ class Globals(object):
|
||||
num_clients = num_mc_clients)))
|
||||
self.cache_chains.update(rendercache=self.rendercache)
|
||||
|
||||
self.servicecache = MemcacheChain((localcache_cls(),
|
||||
CMemcache(self.servicecaches,
|
||||
num_clients = num_mc_clients)))
|
||||
self.cache_chains.update(servicecache=self.servicecache)
|
||||
|
||||
self.thing_cache = CacheChain((localcache_cls(),))
|
||||
self.cache_chains.update(thing_cache=self.thing_cache)
|
||||
|
||||
@@ -400,8 +393,6 @@ class Globals(object):
|
||||
return (x.strip() for x in v.split(delim) if x)
|
||||
|
||||
def load_db_params(self, gc):
|
||||
from r2.lib.services import get_db_load
|
||||
|
||||
self.databases = tuple(self.to_iter(gc['databases']))
|
||||
self.db_params = {}
|
||||
if not self.databases:
|
||||
@@ -425,10 +416,7 @@ class Globals(object):
|
||||
if params['max_overflow'] == "*":
|
||||
params['max_overflow'] = self.db_pool_overflow_size
|
||||
|
||||
ip = params['db_host']
|
||||
ip_loads = get_db_load(self.servicecache, ip)
|
||||
if ip not in ip_loads or ip_loads[ip][0] < 1000:
|
||||
dbm.setup_db(db_name, g_override=self, **params)
|
||||
dbm.setup_db(db_name, g_override=self, **params)
|
||||
self.db_params[db_name] = params
|
||||
|
||||
dbm.type_db = dbm.get_engine(gc['type_db'])
|
||||
|
||||
@@ -78,13 +78,8 @@ class db_manager:
|
||||
yield name, (type1_name, type2_name, engines)
|
||||
|
||||
def mark_dead(self, engine, g_override=None):
|
||||
from r2.lib import services
|
||||
logger.error("db_manager: marking connection dead: %r" % engine)
|
||||
self.dead[engine] = time.time()
|
||||
if g_override is None:
|
||||
services.AppServiceMonitor.mark_db_down(engine.url.host)
|
||||
else:
|
||||
services.mark_db_down(g_override.servicecache, engine.url.host)
|
||||
|
||||
def test_engine(self, engine, g_override=None):
|
||||
try:
|
||||
@@ -106,10 +101,7 @@ class db_manager:
|
||||
return [self._engines[name] for name in names if name in self._engines]
|
||||
|
||||
def get_read_table(self, tables):
|
||||
from r2.lib.services import AppServiceMonitor
|
||||
# short-cut for only one element
|
||||
if len(tables) == 1:
|
||||
return tables[0]
|
||||
|
||||
return random.choice(list(tables))
|
||||
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
# The contents of this file are subject to the Common Public Attribution
|
||||
# License Version 1.0. (the "License"); you may not use this file except in
|
||||
# compliance with the License. You may obtain a copy of the License at
|
||||
# http://code.reddit.com/LICENSE. The License is based on the Mozilla Public
|
||||
# License Version 1.1, but Sections 14 and 15 have been added to cover use of
|
||||
# software over a computer network and provide for limited attribution for the
|
||||
# Original Developer. In addition, Exhibit A has been modified to be consistent
|
||||
# with Exhibit B.
|
||||
#
|
||||
# Software distributed under the License is distributed on an "AS IS" basis,
|
||||
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for
|
||||
# the specific language governing rights and limitations under the License.
|
||||
#
|
||||
# The Original Code is Reddit.
|
||||
#
|
||||
# The Original Developer is the Initial Developer. The Initial Developer of the
|
||||
# Original Code is CondeNet, Inc.
|
||||
#
|
||||
# All portions of the code written by CondeNet are Copyright (c) 2006-2010
|
||||
# CondeNet, Inc. All Rights Reserved.
|
||||
################################################################################
|
||||
import os, re, sys, time, smtplib
|
||||
from r2.lib.services import AppServiceMonitor
|
||||
from pylons import g
|
||||
|
||||
nerds_email = g.nerds_email
|
||||
smtp_server = g.smtp_server
|
||||
|
||||
def Alert(restart_list = ['MEM','CPU'],
|
||||
alert_recipients = [nerds_email],
|
||||
alert_sender = nerds_email,
|
||||
cpu_limit = 99, mem_limit = 5,
|
||||
smtpserver = smtp_server, test = False):
|
||||
|
||||
p = re.compile("newreddit(\d+)")
|
||||
cache_key = 'already_alerted_'
|
||||
for host in AppServiceMonitor(g.monitored_servers):
|
||||
for service in host:
|
||||
# cpu values
|
||||
cpu = [service.cpu(x) for x in (0, 5, 60, 300)]
|
||||
|
||||
output = "\nCPU: " + ' '.join("%6.2f%%" % x for x in cpu)
|
||||
output += "\nMEMORY: %6.2f%%" % service.mem()
|
||||
|
||||
service_name = "%s %s" % (host.host, service.name)
|
||||
|
||||
# is this service pegged?
|
||||
mem_pegged = ('MEM' in restart_list and service.mem() > mem_limit)
|
||||
need_restart = (('CPU' in restart_list and
|
||||
all(x >= cpu_limit for x in cpu)) or mem_pegged)
|
||||
|
||||
|
||||
if (need_restart):
|
||||
mesg = ("To: " + ', '.join(alert_recipients) +
|
||||
"\nSubject: " + service_name +" needs attention\n\n"
|
||||
+ service_name
|
||||
+ (" is out of mem: " if mem_pegged else " is pegged:" )
|
||||
+ output)
|
||||
m = p.match(service.name)
|
||||
# If we can restart this process, we do it here
|
||||
if m:
|
||||
proc_number = str(m.groups()[0])
|
||||
cmd = "/usr/local/bin/push -h " + \
|
||||
host.host + " -r " + proc_number
|
||||
if test:
|
||||
print ("would have restarted the app with command '%s'"
|
||||
% cmd)
|
||||
else:
|
||||
result = os.popen3(cmd)[2].read()
|
||||
# We override the other message to show we restarted it
|
||||
mesg = ("To: nerds@gmail.com\n" +
|
||||
"Subject: " + "Process " +
|
||||
proc_number + " on " + host.host +
|
||||
" was automatically restarted " +
|
||||
"due to the following:\n\n" +
|
||||
output + "\n\n" +
|
||||
"Here was the output:\n" + result)
|
||||
# Uncomment this to disable restart messages
|
||||
#mesg = ""
|
||||
last_alerted = g.rendercache.get(cache_key + service_name) or 0
|
||||
#last_alerted = 0
|
||||
if mesg is not "":
|
||||
if test:
|
||||
print "would have sent email\n '%s'" % mesg
|
||||
elif (time.time() - last_alerted > 300):
|
||||
g.rendercache.set(cache_key + service_name, time.time())
|
||||
session = smtplib.SMTP(smtpserver)
|
||||
smtpresult = session.sendmail(alert_sender,
|
||||
alert_recipients,
|
||||
mesg)
|
||||
session.quit()
|
||||
|
||||
def Run(srvname, *a, **kw):
|
||||
args = {}
|
||||
if kw.has_key('queue_length_max'):
|
||||
args['queue_length_max'] = kw.pop('queue_length_max')
|
||||
AppServiceMonitor(**args).monitor(srvname, *a, **kw)
|
||||
|
||||
def Test(num, load = 1., pid = 0):
|
||||
services = Services()
|
||||
for i in xrange(num):
|
||||
name = 'testproc' + str(i)
|
||||
p = i or pid
|
||||
services.add(name, p, "10")
|
||||
|
||||
services.track(p, 100. * (i+1) / (num),
|
||||
20. * (i+1) / num, 1.)
|
||||
services.load = load
|
||||
services.set_cache()
|
||||
|
||||
if __name__ == '__main__':
|
||||
Run(sys.argv[1:] if sys.argv[1:] else ['newreddit'])
|
||||
Reference in New Issue
Block a user