From b7b2b8aef44e1b01872a74f8d86eec09ccfde845 Mon Sep 17 00:00:00 2001 From: steve Date: Sat, 6 Sep 2008 17:17:43 -0700 Subject: [PATCH] enable the query cache for profile pages --- r2/r2/config/routing.py | 4 +- r2/r2/controllers/__init__.py | 1 + r2/r2/controllers/api.py | 27 +++-- r2/r2/controllers/listingcontroller.py | 138 +++++++++++++++++++++---- r2/r2/lib/app_globals.py | 2 +- r2/r2/lib/db/queries.py | 74 ++++++++----- r2/r2/lib/db/query_queue.py | 2 +- r2/r2/lib/db/tdb_sql.py | 2 +- 8 files changed, 194 insertions(+), 56 deletions(-) diff --git a/r2/r2/config/routing.py b/r2/r2/config/routing.py index 2bafafdf0..5ac86b30f 100644 --- a/r2/r2/config/routing.py +++ b/r2/r2/config/routing.py @@ -74,8 +74,8 @@ def make_map(global_conf={}, app_conf={}): mc('/admin/i18n/:action', controller='i18n') mc('/admin/i18n/:action/:lang', controller='i18n') - mc('/user/:username/:location', controller='front', action='user', - location='overview') + mc('/user/:username/:where', controller='user', action='listing', + where='overview') mc('/prefs/:location', controller='front', action='prefs', location='options') diff --git a/r2/r2/controllers/__init__.py b/r2/r2/controllers/__init__.py index 2cf4e9dc8..145d06943 100644 --- a/r2/r2/controllers/__init__.py +++ b/r2/r2/controllers/__init__.py @@ -30,6 +30,7 @@ from listingcontroller import MessageController from listingcontroller import RedditsController from listingcontroller import MyredditsController from listingcontroller import RandomrisingController +from listingcontroller import UserController from feedback import FeedbackController from front import FrontController diff --git a/r2/r2/controllers/api.py b/r2/r2/controllers/api.py index 3eede94ac..30c61d695 100644 --- a/r2/r2/controllers/api.py +++ b/r2/r2/controllers/api.py @@ -173,6 +173,9 @@ class ApiController(RedditController): res._update('to', value='') res._update('subject', value='') res._update('message', value='') + + if g.write_query_queue: + queries.new_message(m) else: res._update('success', innerHTML='') @@ -630,6 +633,12 @@ class ApiController(RedditController): set_last_modified(c.user, 'commented') set_last_modified(link, 'comments') + #update the queries + if g.write_query_queue: + if is_message: + queries.new_message(item) + else: + queries.new_comment(item) #set the ratelimiter if should_ratelimit: @@ -957,18 +966,18 @@ class ApiController(RedditController): VModhash(), thing = VByName('id')) def POST_save(self, res, thing): - user = c.user - thing._save(user) - + thing._save(c.user) + if g.write_query_queue: + queries.new_savehide(c.user, 'save') @Json @validate(VUser(), VModhash(), thing = VByName('id')) def POST_unsave(self, res, thing): - user = c.user - thing._unsave(user) - + thing._unsave(c.user) + if g.write_query_queue: + queries.new_savehide(c.user, 'save') @Json @validate(VUser(), @@ -976,7 +985,8 @@ class ApiController(RedditController): thing = VByName('id')) def POST_hide(self, res, thing): thing._hide(c.user) - + if g.write_query_queue: + queries.new_savehide(c.user, 'hide') @Json @validate(VUser(), @@ -984,6 +994,9 @@ class ApiController(RedditController): thing = VByName('id')) def POST_unhide(self, res, thing): thing._unhide(c.user) + if g.write_query_queue: + queries.new_savehide(c.user, 'hide') + @Json @validate(link = VByName('link_id'), diff --git a/r2/r2/controllers/listingcontroller.py b/r2/r2/controllers/listingcontroller.py index a1b734278..2455b1be0 100644 --- a/r2/r2/controllers/listingcontroller.py +++ b/r2/r2/controllers/listingcontroller.py @@ -29,10 +29,13 @@ from r2.lib.rising import get_rising from r2.lib.wrapped import Wrapped from r2.lib.normalized_hot import normalized_hot, get_hot from r2.lib.recommendation import get_recommended -from r2.lib.db.thing import Query +from r2.lib.db.thing import Query, Merge, Relations +from r2.lib.db import queries from r2.lib.strings import Score from r2.lib import organic -from r2.lib.utils import iters +from r2.lib.utils import iters, check_cheating + +from admin import admin_profile_query from pylons.i18n import _ @@ -67,6 +70,9 @@ class ListingController(RedditController): # class (probably a subclass of Reddit) to use to render the page. render_cls = Reddit + #extra parameters to send to the render_cls constructor + render_params = {} + @property def menus(self): """list of menus underneat the header (e.g., sort, time, kind, @@ -91,7 +97,8 @@ class ListingController(RedditController): show_sidebar = self.show_sidebar, nav_menus = self.menus, title = self.title(), - infotext = self.infotext).render() + infotext = self.infotext, + **self.render_params).render() return res @@ -230,16 +237,18 @@ class HotController(FixListing, ListingController): return ListingController.GET_listing(self, **env) class SavedController(ListingController): - prewrap_fn = lambda self, x: x._thing2 where = 'saved' skip = False title_text = _('saved') def query(self): - q = SaveHide._query(SaveHide.c._thing1_id == c.user._id, - SaveHide.c._name == 'save', - sort = desc('_date'), - eager_load = True, thing_data = True) + q = queries.get_saved(c.user) + if g.use_query_cache: + q = q.fetch() + else: + self.prewrap_fn = q.filter + q = q.query + return q @validate(VUser()) @@ -340,6 +349,96 @@ class RecommendedController(ListingController): self.sort = sort return ListingController.GET_listing(self, **env) +class UserController(ListingController): + render_cls = ProfilePage + skip = False + + def title(self): + titles = {'overview': _("overview for %(user)s"), + 'comments': _("comments by %(user)s"), + 'submitted': _("submitted by %(user)s"), + 'liked': _("liked by %(user)s"), + 'disliked': _("disliked by %(user)s"), + 'hidden': _("hidden by %(user)s")} + title = titles.get(self.where, _('profile for %(user)s')) \ + % dict(user = self.vuser.name, site = c.site.name) + return title + + def query(self): + q = None + if self.where == 'overview': + self.check_modified(self.vuser, 'overview') + if g.use_query_cache: + q = queries.get_overview(self.vuser, 'new', 'all') + else: + links = Link._query(Link.c.author_id == self.vuser._id, + Link.c._spam == (True, False)) + comments = Comment._query(Comment.c.author_id == self.vuser._id, + Comment.c._spam == (True, False)) + q = Merge((links, comments), sort = desc('_date'), data = True) + elif self.where == 'comments': + self.check_modified(self.vuser, 'commented') + q = queries.get_comments(self.vuser, 'new', 'all') + elif self.where == 'submitted': + self.check_modified(self.vuser, 'submitted') + q = queries.get_submitted(self.vuser, 'new', 'all') + elif self.where in ('liked', 'disliked'): + self.check_modified(self.vuser, self.where) + if self.where == 'liked': + q = queries.get_liked(self.vuser) + else: + q = queries.get_disliked(self.vuser) + + elif self.where == 'hidden': + q = queries.get_hidden(self.vuser) + + elif c.user_is_admin: + q, self.prewrap_fn = admin_profile_query(self.vuser, + self.where, + desc('_date')) + + #QUERIES HACK + if isinstance(q, queries.CachedResults): + if g.use_query_cache: + q = q.fetch() + elif isinstance(q.query, Relations): + self.prewrap_fn = q.filter + q = q.query + else: + q = q.query + + if q is None: + return self.abort404() + + return q + + @validate(vuser = VExistingUname('username')) + def GET_listing(self, where, vuser, **env): + self.where = where + + # the validator will ensure that vuser is a valid account + if not vuser: + return self.abort404() + + # hide spammers profile pages + if (not c.user_is_loggedin or + (c.user._id != vuser._id and not c.user_is_admin)) \ + and vuser._spam: + return self.abort404() + + if (where not in ('overview', 'submitted', 'comments') + and not votes_visible(vuser)): + return self.abort404() + + check_cheating('user') + + self.vuser = vuser + self.render_params = {'user' : vuser} + c.profilepage = True + + return ListingController.GET_listing(self, **env) + + class MessageController(ListingController): show_sidebar = False render_cls = MessagePage @@ -364,10 +463,13 @@ class MessageController(ListingController): def query(self): if self.where == 'inbox': - q = Inbox._query(Inbox.c._thing1_id == c.user._id, - eager_load = True, - thing_data = True) - self.prewrap_fn = lambda x: x._thing2 + if g.use_query_cache: + q = queries.get_inbox(c.user) + else: + q = Inbox._query(Inbox.c._thing1_id == c.user._id, + eager_load = True, + thing_data = True) + self.prewrap_fn = lambda x: x._thing2 #reset the inbox if c.have_messages: @@ -375,16 +477,14 @@ class MessageController(ListingController): c.user._commit() elif self.where == 'sent': - q = Message._query(Message.c.author_id == c.user._id, - Message.c._spam == (True, False)) + q = queries.get_sent(c.user) + if g.use_query_cache: + q = q.fetch() + else: + q = q.query - q._sort = desc('_date') return q - def content(self): - self.page = self.listing_obj - return self.page - @validate(VUser()) def GET_listing(self, where, **env): self.where = where diff --git a/r2/r2/lib/app_globals.py b/r2/r2/lib/app_globals.py index 1926b3848..2d77ab5a7 100644 --- a/r2/r2/lib/app_globals.py +++ b/r2/r2/lib/app_globals.py @@ -41,7 +41,7 @@ class Globals(object): 'num_comments', 'max_comments', 'num_side_reddits', - 'num_precompute_workers', + 'num_query_queue_workers', ] bool_props = ['debug', 'translator', diff --git a/r2/r2/lib/db/queries.py b/r2/r2/lib/db/queries.py index 97c2ca8fe..21eaacf2f 100644 --- a/r2/r2/lib/db/queries.py +++ b/r2/r2/lib/db/queries.py @@ -67,7 +67,8 @@ class CachedResults(object): item = self.filter(i) l = [item._fullname] for col in sort_cols: - attr = getattr(item, col) + #take the property of the original + attr = getattr(i, col) #convert dates to epochs to take less space if isinstance(attr, datetime): attr = epoch_seconds(attr) @@ -149,7 +150,9 @@ def user_rel_query(rel, user, name): rel.c._t2_deleted == False, rel.c._name == name, sort = desc('_date'), - eager_load = True) + eager_load = True, + #thing_data = True + ) return CachedResults(q, filter_thing2) @@ -219,20 +222,18 @@ def add_queries(queries): ## actions to update the correct listings. def new_link(link): sr = Subreddit._byID(link.sr_id) - #author = Account._byID(link.author_id) + author = Account._byID(link.author_id) results = all_queries(get_links, sr, ('hot', 'new', 'old'), ['all']) results.extend(all_queries(get_links, sr, ('top', 'controversial'), db_times.keys())) - #results.extend(all_queries(get_submitted, author, - # db_sorts.keys(), - # db_times.keys())) + results.append(get_submitted(author, 'new', 'all')) add_queries(results) def new_comment(comment): author = Account._byID(comment.author_id) - results = all_queries(get_comments, author, db_sorts.keys(), db_times.keys()) - + results = [get_comments(author, 'new', 'all')] + if hasattr(comment, 'parent_id'): parent = Comment._byID(comment.parent_id, data = True) parent_author = Account._byID(parent.author_id) @@ -252,8 +253,8 @@ def new_vote(vote): results.extend(all_queries(get_links, sr, ('top', 'controversial'), db_times.keys())) #must update both because we don't know if it's a changed vote - #results.append(get_liked(user)) - #results.append(get_disliked(user)) + results.append(get_liked(user)) + results.append(get_disliked(user)) add_queries(results) @@ -266,11 +267,10 @@ def new_message(message): add_queries(results) -def new_savehide(savehide): - user = savehide._thing1 - if savehide._name == 'save': +def new_savehide(user, action): + if action == 'save': results = [get_saved(user)] - else: + elif action == 'hide': results = [get_hidden(user)] add_queries(results) @@ -283,16 +283,40 @@ def add_all_srs(): add_queries(all_queries(get_links, sr, ('top', 'controversial'), db_times.keys())) def add_all_users(): - """Adds every profile-page query for every user to the queue""" q = Account._query(sort = asc('_date')) for user in fetch_things2(q): - queries = [] - queries.extend(all_queries(get_submitted, user, db_sorts.keys(), db_times.keys())) - queries.extend(all_queries(get_comments, user, db_sorts.keys(), db_times.keys())) - queries.append(get_inbox_messages(user)) - queries.append(get_inbox_comments(user)) - queries.append(get_saved(user)) - queries.append(get_hidden(user)) - queries.append(get_liked(user)) - queries.append(get_disliked(user)) - add_queries(queries) + results = [get_inbox_messages(user), + get_inbox_comments(user), + get_sent(user), + get_liked(user), + get_disliked(user), + get_saved(user), + get_hidden(user), + get_submitted(user, 'new', 'all'), + get_comments(user, 'new', 'all')] + add_queries(results) + +def compute_all_inboxes(): + q = Account._query(sort = asc('_date')) + for user in fetch_things2(q): + get_inbox_messages(user).update() + get_inbox_comments(user).update() + get_sent(user).update() + +def compute_all_liked(): + q = Account._query(sort = asc('_date')) + for user in fetch_things2(q): + get_liked(user).update() + get_disliked(user).update() + +def compute_all_saved(): + q = Account._query(sort = asc('_date')) + for user in fetch_things2(q): + get_saved(user).update() + get_hidden(user).update() + +def compute_all_user_pages(): + q = Account._query(sort = asc('_date')) + for user in fetch_things2(q): + get_submitted(user, 'new', 'all').update() + get_comments(user, 'new', 'all').update() diff --git a/r2/r2/lib/db/query_queue.py b/r2/r2/lib/db/query_queue.py index 9d53b7d2b..6b61f1743 100644 --- a/r2/r2/lib/db/query_queue.py +++ b/r2/r2/lib/db/query_queue.py @@ -102,7 +102,7 @@ def run(): #limit the total number of jobs in the WorkQueue. we don't #need to load the entire db queue right away (the db queue can #get quite large). - if wq.jobs.qsize() < 2 * num_workers: + if len(running) < 2 * num_workers: with running_lock: iden, pickled_cr = get_query() if pickled_cr is not None: diff --git a/r2/r2/lib/db/tdb_sql.py b/r2/r2/lib/db/tdb_sql.py index d319f7368..f7b6aa013 100644 --- a/r2/r2/lib/db/tdb_sql.py +++ b/r2/r2/lib/db/tdb_sql.py @@ -35,7 +35,7 @@ import logging log_format = logging.Formatter('sql: %(message)s') settings = storage() -settings.DEBUG = False +settings.DEBUG = g.debug settings.DB_CREATE_TABLES = True settings.DB_APP_NAME = 'reddit'