+ 
@@ -65,7 +67,7 @@ redditbroke = \
-''' +''' toofast = \ ''' @@ -85,7 +87,7 @@ class ErrorController(RedditController): This behaviour can be altered by changing the parameters to the ErrorDocuments middleware in your config/middleware.py file. """ - allowed_render_styles = ('html', 'xml', 'js', 'embed', '') + allowed_render_styles = ('html', 'xml', 'js', 'embed', '', 'api') def __before__(self): try: c.error_page = True @@ -150,19 +152,27 @@ class ErrorController(RedditController): c.cookies = Cookies() code = request.GET.get('code', '') + try: + code = int(code) + except ValueError: + code = 404 srname = request.GET.get('srname', '') takedown = request.GET.get('takedown', "") if srname: c.site = Subreddit._by_name(srname) if c.render_style not in self.allowed_render_styles: - return str(int(code)) + c.response.content = str(int(code)) + return c.response + elif c.render_style == "api": + c.response.content = "{error: %s}" % code + return c.response elif takedown and code == '404': link = Link._by_fullname(takedown) return pages.TakedownPage(link).render() elif code == '403': return self.send403() elif code == '500': - return redditbroke % rand_strings.sadmessages + return redditbroke % (rand.randint(1,NUM_FAILIENS), rand_strings.sadmessages) elif code == '503': return self.send503() elif code == '304': @@ -193,7 +203,7 @@ def handle_awful_failure(fail_text): import traceback g.log.error("FULLPATH: %s" % fail_text) g.log.error(traceback.format_exc()) - return redditbroke % fail_text + return redditbroke % (rand.randint(1,NUM_FAILIENS), fail_text) except: # we are doomed. Admit defeat return "This is an error that should never occur. You win." diff --git a/r2/r2/controllers/front.py b/r2/r2/controllers/front.py index b5cdb5f3a..28d36bb17 100644 --- a/r2/r2/controllers/front.py +++ b/r2/r2/controllers/front.py @@ -185,11 +185,6 @@ class FrontController(RedditController): if limit is not None and 0 < limit < g.max_comments: num = limit - builder = CommentBuilder(article, CommentSortMenu.operator(sort), - comment, context, **kw) - listing = NestedListing(builder, num = num, - parent_name = article._fullname) - displayPane = PaneStack() # if permalink page, add that message first to the content @@ -206,7 +201,8 @@ class FrontController(RedditController): cloneable = True)) # finally add the comment listing - displayPane.append(listing.listing()) + displayPane.append(CommentPane(article, CommentSortMenu.operator(sort), + comment, context, num, **kw)) loc = None if c.focal_comment or context is not None else 'comments' @@ -337,8 +333,10 @@ class FrontController(RedditController): elif is_moderator and location == 'banned': pane = BannedList(editable = is_moderator) elif (location == 'contributors' and - (c.site.type != 'public' or - (c.user_is_loggedin and c.site.use_whitelist and + # On public reddits, only moderators can see the whitelist. + # On private reddits, all contributors can see each other. + (c.site.type != 'public' or + (c.user_is_loggedin and (c.site.is_moderator(c.user) or c.user_is_admin)))): pane = ContributorList(editable = is_moderator) elif (location == 'stylesheet' @@ -855,7 +853,7 @@ class FormsController(RedditController): #check like this because c.user_is_admin is still false if not c.user.name in g.admins: return self.abort404() - self.login(c.user, admin = True) + self.login(c.user, admin = True, rem = True) return self.redirect(dest) @validate(VAdmin(), @@ -864,7 +862,7 @@ class FormsController(RedditController): """disable admin interaction with site.""" if not c.user.name in g.admins: return self.abort404() - self.login(c.user, admin = False) + self.login(c.user, admin = False, rem = True) return self.redirect(dest) def GET_validuser(self): diff --git a/r2/r2/controllers/listingcontroller.py b/r2/r2/controllers/listingcontroller.py index 8c31c6ecc..d91de7df7 100644 --- a/r2/r2/controllers/listingcontroller.py +++ b/r2/r2/controllers/listingcontroller.py @@ -414,7 +414,8 @@ class RandomrisingController(ListingController): if isinstance(links, Query): links._limit = 200 links = [x._fullname for x in links] - + + links = list(links) random.shuffle(links) return links @@ -577,7 +578,7 @@ class MessageController(ListingController): wouldkeep = item.keep_item(item) # don't show user their own unread stuff if ((self.where == 'unread' or self.subwhere == 'unread') - and item.author_id == c.user._id): + and (item.author_id == c.user._id or not item.new)): return False return wouldkeep return keep @@ -729,7 +730,7 @@ class MyredditsController(ListingController): @property def menus(self): buttons = (NavButton(plurals.subscriber, 'subscriber'), - NavButton(plurals.contributor, 'contributor'), + NavButton(getattr(plurals, "approved submitter"), 'contributor'), NavButton(plurals.moderator, 'moderator')) return [NavMenu(buttons, base_path = '/reddits/mine/', diff --git a/r2/r2/controllers/promotecontroller.py b/r2/r2/controllers/promotecontroller.py index 62ba61170..4b4d0332b 100644 --- a/r2/r2/controllers/promotecontroller.py +++ b/r2/r2/controllers/promotecontroller.py @@ -185,8 +185,7 @@ class PromoteController(ListingController): changed = False # live items can only be changed by a sponsor, and also # pay the cost of de-approving the link - trusted = c.user_is_sponsor or \ - getattr(c.user, "trusted_sponsor", False) + trusted = c.user_is_sponsor or c.user.trusted_sponsor if not promote.is_promoted(l) or trusted: if title and title != l.title: l.title = title @@ -198,6 +197,8 @@ class PromoteController(ListingController): # only trips if the title and url are changed by a non-sponsor if changed and not promote.is_unpaid(l): promote.unapprove_promotion(l) + if trusted and promote.is_unapproved(l): + promote.accept_promotion(l) if c.user_is_sponsor: l.maximum_clicks = max_clicks @@ -455,7 +456,8 @@ class PromoteController(ListingController): if any(errors.values()): return UploadedImage("", "", "upload", errors = errors).render() else: - if not c.user_is_sponsor and not promote.is_unpaid(link): + if (not c.user_is_sponsor and not c.user.trusted_sponsor and + not promote.is_unpaid(link)): promote.unapprove_promotion(link) return UploadedImage(_('saved'), thumbnail_url(link), "", errors = errors).render() diff --git a/r2/r2/controllers/reddit_base.py b/r2/r2/controllers/reddit_base.py index f9400bb8f..6060bbd45 100644 --- a/r2/r2/controllers/reddit_base.py +++ b/r2/r2/controllers/reddit_base.py @@ -237,11 +237,12 @@ def set_subreddit(): sr_name = request.environ.get("subreddit", request.POST.get('r')) domain = request.environ.get("domain") + c.site = Default if not sr_name: #check for cnames sub_domain = request.environ.get('sub_domain') - sr = Subreddit._by_domain(sub_domain) if sub_domain else None - c.site = sr or Default + if sub_domain and not sub_domain.endswith(g.media_domain): + c.site = Subreddit._by_domain(sub_domain) or Default elif sr_name == 'r': #reddits c.site = Sub @@ -258,9 +259,9 @@ def set_subreddit(): else: c.site = Subreddit._by_name(sr_name) except NotFound: - c.site = Default - if chksrname(sr_name): - redirect_to("/reddits/create?name=%s" % sr_name) + sr_name = chksrname(sr_name) + if sr_name: + redirect_to("/reddits/search?q=%s" % sr_name) elif not c.error_page: abort(404, "not found") #if we didn't find a subreddit, check for a domain listing @@ -288,7 +289,7 @@ def set_content_type(): user = valid_feed(request.GET.get("user"), request.GET.get("feed"), request.path) - if user: + if user and not g.read_only_mode: c.user = user c.user_is_loggedin = True @@ -573,6 +574,13 @@ class MinimalController(BaseController): sampling_rate = g.usage_sampling, action = action) + def abort404(self): + abort(404, "not found") + + def abort403(self): + abort(403, "forbidden") + + class RedditController(MinimalController): @@ -603,6 +611,8 @@ class RedditController(MinimalController): c.response_wrappers = [] c.firsttime = firsttime() + + (c.user, maybe_admin) = \ valid_cookie(c.cookies[g.login_cookie].value if g.login_cookie in c.cookies @@ -709,12 +719,6 @@ class RedditController(MinimalController): if modified_since and modified_since >= last_modified: abort(304, 'not modified') - def abort404(self): - abort(404, "not found") - - def abort403(self): - abort(403, "forbidden") - def sendpng(self, string): c.response_content_type = 'image/png' c.response.content = string diff --git a/r2/r2/controllers/validator/validator.py b/r2/r2/controllers/validator/validator.py index f2303b4fd..8bdeccd44 100644 --- a/r2/r2/controllers/validator/validator.py +++ b/r2/r2/controllers/validator/validator.py @@ -1156,10 +1156,6 @@ class ValidIP(Validator): self.set_error(errors.BANNED_IP) return request.ip -class VOkayDomain(Validator): - def run(self, url): - return is_banned_domain(url) - class VDate(Validator): """ Date checker that accepts string inputs in %m/%d/%Y format. diff --git a/r2/r2/lib/app_globals.py b/r2/r2/lib/app_globals.py index 3e926ac74..e8b880cb1 100644 --- a/r2/r2/lib/app_globals.py +++ b/r2/r2/lib/app_globals.py @@ -36,6 +36,7 @@ class Globals(object): int_props = ['page_cache_time', 'solr_cache_time', + 'num_mc_clients', 'MIN_DOWN_LINK', 'MIN_UP_KARMA', 'MIN_DOWN_KARMA', @@ -73,16 +74,17 @@ class Globals(object): 'disallow_db_writes', 'exception_logging', 'amqp_logging', + 'read_only_mode', ] tuple_props = ['memcaches', - 'rec_cache', 'rendercaches', 'local_rendercache', 'servicecaches', 'permacache_memcaches', 'cassandra_seeds', - 'permacaches', + 'url_caches', + 'url_seeds', 'admins', 'sponsors', 'monitored_servers', @@ -138,55 +140,30 @@ class Globals(object): localcache_cls = (SelfEmptyingCache if self.running_as_script else LocalCache) - num_mc_clients = 2# if self.running_as_script else 10 + num_mc_clients = self.num_mc_clients - c_mc = CMemcache(self.memcaches, num_clients = num_mc_clients, legacy=True) - rmc = CMemcache(self.rendercaches, num_clients = num_mc_clients, - noreply=True, no_block=True) - lrmc = None - if self.local_rendercache: - lrmc = CMemcache(self.local_rendercache, - num_clients = num_mc_clients, - noreply=True, no_block=True) - smc = CMemcache(self.servicecaches, num_clients = num_mc_clients) - rec_cache = None # we're not using this for now + self.cache_chains = [] - pmc_chain = (localcache_cls(),) - if self.permacache_memcaches: - pmc_chain += (CMemcache(self.permacache_memcaches, - num_clients=num_mc_clients, - legacy=True),) - if self.cassandra_seeds: - self.cassandra_seeds = list(self.cassandra_seeds) - random.shuffle(self.cassandra_seeds) - pmc_chain += (CassandraCache('permacache', 'permacache', - self.cassandra_seeds),) - if self.permacaches: - pmc_chain += (CMemcache(self.permacaches, - num_clients=num_mc_clients, - legacy=True),) - if len(pmc_chain) == 1: - print 'Warning: proceding without a permacache' + self.permacache = self.init_cass_cache('permacache', + self.permacache_memcaches, + self.cassandra_seeds) - self.permacache = CassandraCacheChain(pmc_chain, cache_negative_results = True) + self.urlcache = self.init_cass_cache('urls', + self.url_caches, + self.url_seeds) # hardcache is done after the db info is loaded, and then the # chains are reset to use the appropriate initial entries - self.memcache = c_mc # we'll keep using this one for locks - # intermediately + self.cache = self.init_memcached(self.memcaches) + self.memcache = self.cache.caches[1] # used by lock.py - self.cache = MemcacheChain((localcache_cls(), c_mc)) - if lrmc: - self.rendercache = MemcacheChain((localcache_cls(), lrmc, rmc)) - else: - self.rendercache = MemcacheChain((localcache_cls(), rmc)) - self.servicecache = MemcacheChain((localcache_cls(), smc)) - self.rec_cache = rec_cache + self.rendercache = self.init_memcached(self.rendercaches, + noreply=True, no_block=True) + + self.servicecache = self.init_memcached(self.servicecaches) self.make_lock = make_lock_factory(self.memcache) - cache_chains = [self.cache, self.permacache, self.rendercache, - self.servicecache] # set default time zone if one is not set tz = global_conf.get('timezone') @@ -199,15 +176,17 @@ class Globals(object): self.dbm = self.load_db_params(global_conf) # can't do this until load_db_params() has been called - self.hardcache = HardcacheChain((localcache_cls(), c_mc, + self.hardcache = HardcacheChain((localcache_cls(), + self.memcache, HardCache(self)), cache_negative_results = True) - cache_chains.append(self.hardcache) + self.cache_chains.append(self.hardcache) # I know this sucks, but we need non-request-threads to be # able to reset the caches, so we need them be able to close # around 'cache_chains' without being able to call getattr on # 'g' + cache_chains = self.cache_chains[::] def reset_caches(): for chain in cache_chains: chain.reset() @@ -317,7 +296,37 @@ class Globals(object): self.version = self.short_version = '(unknown)' if self.log_start: - self.log.error("reddit app started %s at %s" % (self.short_version, datetime.now())) + self.log.error("reddit app %s:%s started %s at %s" % (self.reddit_host, self.reddit_pid, + self.short_version, datetime.now())) + + def init_memcached(self, caches, **kw): + return self.init_cass_cache(None, caches, None, memcached_kw = kw) + + def init_cass_cache(self, cluster, caches, cassandra_seeds, + memcached_kw = {}, cassandra_kw = {}): + localcache_cls = (SelfEmptyingCache if self.running_as_script + else LocalCache) + + pmc_chain = (localcache_cls(),) + + # if caches, append + if caches: + pmc_chain += (CMemcache(caches, num_clients=self.num_mc_clients, + **memcached_kw),) + + # if seeds, append + if cassandra_seeds: + cassandra_seeds = list(cassandra_seeds) + random.shuffle(cassandra_seeds) + pmc_chain += (CassandraCache(cluster, cluster, cassandra_seeds, + **cassandra_kw),) + mc = CassandraCacheChain(pmc_chain, cache_negative_results = True) + else: + mc = MemcacheChain(pmc_chain) + + self.cache_chains.append(mc) + return mc + @staticmethod def to_bool(x): diff --git a/r2/r2/lib/cache.py b/r2/r2/lib/cache.py index 13c7b30bb..9c519f8a3 100644 --- a/r2/r2/lib/cache.py +++ b/r2/r2/lib/cache.py @@ -271,7 +271,7 @@ class HardCache(CacheUtils): return category, ids def set(self, key, val, time=0): - if val is NoneResult: + if val == NoneResult: # NoneResult caching is for other parts of the chain return @@ -296,7 +296,7 @@ class HardCache(CacheUtils): def set_multi(self, keys, prefix='', time=0): for k,v in keys.iteritems(): - if v is not NoneResult: + if v != NoneResult: self.set(prefix+str(k), v, time=time) def get(self, key, default=None): @@ -434,7 +434,7 @@ class CacheChain(CacheUtils, local): break # so we don't set caches later in the chain d.set(key, val) - if self.cache_negative_results and val is NoneResult: + if self.cache_negative_results and val == NoneResult: return default else: return val @@ -475,14 +475,14 @@ class CacheChain(CacheUtils, local): need = need - set(r.keys()) if need and self.cache_negative_results: - d = dict( (key,NoneResult) for key in need) + d = dict((key, NoneResult) for key in need) for c in self.caches: c.set_multi(d) if self.cache_negative_results: filtered_out = {} for k,v in out.iteritems(): - if v is not NoneResult: + if v != NoneResult: filtered_out[k] = v out = filtered_out diff --git a/r2/r2/lib/db/queries.py b/r2/r2/lib/db/queries.py index b4d2bf0c9..f5d4a735f 100644 --- a/r2/r2/lib/db/queries.py +++ b/r2/r2/lib/db/queries.py @@ -201,8 +201,10 @@ class CachedResults(object): def insert(self, items): """Inserts the item into the cached data. This only works under certain criteria, see can_insert.""" + self._insert_tuples([self.make_item_tuple(item) for item in tup(items)]) + + def _insert_tuples(self, t): self.fetch() - t = [ self.make_item_tuple(item) for item in tup(items) ] # insert the new items, remove the duplicates (keeping the one # being inserted over the stored value if applicable), and @@ -676,6 +678,8 @@ def new_vote(vote): sr.last_valid_vote = datetime.now(g.tz) sr._commit() + + vote._fast_query_timestamp_touch(user) #must update both because we don't know if it's a changed vote if vote._name == '1': @@ -713,7 +717,7 @@ def set_unread(message, to, unread): kw = dict(insert_items = i) if unread else dict(delete_items = i) add_queries([get_unread_subreddit_messages(i._thing1)], **kw) else: - for i in Inbox.set_unread(message, unread): + for i in Inbox.set_unread(message, unread, to = to): kw = dict(insert_items = i) if unread else dict(delete_items = i) if i._name == 'selfreply': add_queries([get_unread_selfreply(i._thing1)], **kw) @@ -992,34 +996,6 @@ def get_likes(user, items): for k, v in r.iteritems(): res[keys[k]] = v - # performance hack: if their last vote came in before this thing - # was created, they can't possibly have voted on it - cantexist = {} - for item in items: - if (user, item) in res: - continue - - last_vote_attr_name = 'last_vote_' + item.__class__.__name__ - last_vote = getattr(user, last_vote_attr_name, None) - if not last_vote: - continue - - try: - if last_vote < item._date: - res[(user, item)] = '0' - cantexist[prequeued_vote_key(user, item)] = '0' - except TypeError: - g.log.error("user %s has a broken %s? (%r)" - % (user._id, last_vote_attr_name, last_vote)) - # accounts for broken last_vote properties - pass - - # this is a bit dodgy, but should save us from having to reload - # all of the votes on pages they've already loaded as soon as they - # cast a new vote - if cantexist: - g.cache.set_multi(cantexist) - # now hit the vote db with the remainder likes = Vote.likes(user, [i for i in items if (user, i) not in res]) diff --git a/r2/r2/lib/db/sorts.py b/r2/r2/lib/db/sorts.py index 498e63d9d..0b22c5a1f 100644 --- a/r2/r2/lib/db/sorts.py +++ b/r2/r2/lib/db/sorts.py @@ -26,8 +26,9 @@ from pylons import g epoch = datetime(1970, 1, 1, tzinfo = g.tz) def epoch_seconds(date): - """Returns the number of seconds from the epoch to date. Should match - the number returned by the equivalent function in postgres.""" + """Returns the number of seconds from the epoch to date. Should + match the number returned by the equivalent function in + postgres.""" td = date - epoch return td.days * 86400 + td.seconds + (float(td.microseconds) / 1000000) @@ -35,11 +36,13 @@ def score(ups, downs): return ups - downs def hot(ups, downs, date): + return _hot(ups, downs, epoch_seconds(date)) +def _hot(ups, downs, date): """The hot formula. Should match the equivalent function in postgres.""" s = score(ups, downs) order = log(max(abs(s), 1), 10) sign = 1 if s > 0 else -1 if s < 0 else 0 - seconds = epoch_seconds(date) - 1134028003 + seconds = date - 1134028003 return round(order + sign * seconds / 45000, 7) def controversy(ups, downs): diff --git a/r2/r2/lib/db/thing.py b/r2/r2/lib/db/thing.py index 02de55bd8..edc00425c 100644 --- a/r2/r2/lib/db/thing.py +++ b/r2/r2/lib/db/thing.py @@ -708,9 +708,18 @@ def Relation(type1, type2, denorm1 = None, denorm2 = None): self._name = 'un' + self._name @classmethod - def _fast_query(cls, thing1s, thing2s, name, data=True, eager_load=True): - """looks up all the relationships between thing1_ids and thing2_ids - and caches them""" + def _fast_query_timestamp_touch(cls, thing1): + assert thing1._loaded + timestamp_dict = getattr(thing1, 'fast_query_timestamp', {}).copy() + timestamp_dict[cls._type_name] = datetime.now(g.tz) + thing1.fast_query_timestamp = timestamp_dict + thing1._commit() + + @classmethod + def _fast_query(cls, thing1s, thing2s, name, data=True, eager_load=True, + timestamp_optimize = False): + """looks up all the relationships between thing1_ids and + thing2_ids and caches them""" prefix = thing_prefix(cls.__name__) thing1_dict = dict((t._id, t) for t in tup(thing1s)) @@ -721,16 +730,46 @@ def Relation(type1, type2, denorm1 = None, denorm2 = None): name = tup(name) + def can_skip_lookup(t1, t2, name): + # we can't possibly have voted on things that were + # created after the last time we voted. for relations + # that have an invariant like this we can avoid doing + # these lookups as long as the relation takes + # responsibility for keeping the timestamp up-to-date + thing1 = thing1_dict[t1] + thing2 = thing2_dict[t2] + + # check to see if we have the history information + if not thing1._loaded: + return False + if not hasattr(thing1, 'fast_query_timestamp'): + return False + + last_done = thing1.fast_query_timestamp.get(cls._type_name,None) + + if not last_done: + return False + + if thing2._date > last_done: + return True + + return False + + # permute all of the pairs pairs = set((x, y, n) for x in thing1_ids for y in thing2_ids for n in name) def items_db(pairs): + rel_ids = {} + t1_ids = set() t2_ids = set() names = set() for t1, t2, name in pairs: + if timestamp_optimize and can_skip_lookup(t1, t2, name): + continue t1_ids.add(t1) t2_ids.add(t2) names.add(name) @@ -741,21 +780,21 @@ def Relation(type1, type2, denorm1 = None, denorm2 = None): eager_load = eager_load, data = data) - rel_ids = {} for rel in q: #TODO an alternative for multiple #relations with the same keys #l = rel_ids.setdefault((rel._thing1_id, rel._thing2_id), []) #l.append(rel._id) rel_ids[(rel._thing1_id, rel._thing2_id, rel._name)] = rel._id - + for p in pairs: if p not in rel_ids: rel_ids[p] = None - + return rel_ids res = sgm(cache, pairs, items_db, prefix) + #convert the keys back into objects # populate up the local-cache in batch @@ -1194,14 +1233,15 @@ def MultiRelation(name, *relations): return Merge(queries) @classmethod - def _fast_query(cls, sub, obj, name, data=True, eager_load=True): + def _fast_query(cls, sub, obj, name, data=True, eager_load=True, + timestamp_optimize = False): #divide into types def type_dict(items): types = {} for i in items: types.setdefault(i.__class__, []).append(i) return types - + sub_dict = type_dict(tup(sub)) obj_dict = type_dict(tup(obj)) @@ -1211,7 +1251,8 @@ def MultiRelation(name, *relations): t1, t2 = types if sub_dict.has_key(t1) and obj_dict.has_key(t2): res.update(rel._fast_query(sub_dict[t1], obj_dict[t2], name, - data = data, eager_load=eager_load)) + data = data, eager_load=eager_load, + timestamp_optimize = timestamp_optimize)) return res diff --git a/r2/r2/lib/log.py b/r2/r2/lib/log.py index d6087bc76..92b0398b5 100644 --- a/r2/r2/lib/log.py +++ b/r2/r2/lib/log.py @@ -46,7 +46,8 @@ def log_exception(e, e_type, e_value, e_traceback): d['traceback'] = traceback.extract_tb(e_traceback) d['exception_type'] = e.__class__.__name__ - d['exception_desc'] = str(e) + s = str(e) + d['exception_desc'] = s[:10000] amqp.add_item(Q, pickle.dumps(d)) diff --git a/r2/r2/lib/menus.py b/r2/r2/lib/menus.py index ce9e156e7..15e87cbf7 100644 --- a/r2/r2/lib/menus.py +++ b/r2/r2/lib/menus.py @@ -127,7 +127,7 @@ menu = MenuHandler(hot = _('hot'), edit = _("edit this reddit"), moderators = _("edit moderators"), modmail = _("moderator mail"), - contributors = _("edit contributors"), + contributors = _("edit approved submitters"), banned = _("ban users"), banusers = _("ban users"), diff --git a/r2/r2/lib/migrate.py b/r2/r2/lib/migrate.py index a47b4322b..a16516957 100644 --- a/r2/r2/lib/migrate.py +++ b/r2/r2/lib/migrate.py @@ -380,3 +380,26 @@ def shorten_byurl_keys(): for (old_key, value) in old.iteritems()) g.permacache.set_multi(new) + +def prime_url_cache(f, verbosity = 10000): + import gzip, time + from pylons import g + handle = gzip.open(f, 'rb') + counter = 0 + start_time = time.time() + for line in handle: + try: + tid, key, url, kind = line.split('|') + tid = int(tid) + if url.lower() != "self": + key = Link.by_url_key(url) + link_ids = g.urlcache.get(key) or [] + if tid not in link_ids: + link_ids.append(tid) + g.urlcache.set(key, link_ids) + except ValueError: + print "FAIL: %s" % line + counter += 1 + if counter % verbosity == 0: + print "%6d: %s" % (counter, line) + print "--> doing %5.2f / s" % (float(counter) / (time.time() - start_time)) diff --git a/r2/r2/lib/mr_tools.py b/r2/r2/lib/mr_tools.py new file mode 100755 index 000000000..dbcae581b --- /dev/null +++ b/r2/r2/lib/mr_tools.py @@ -0,0 +1,171 @@ +import sys + +class LineReader(object): + """A simple class to read lines from a File (like stdin) that + supports pushing lines back into the buffer""" + def __init__(self, stream): + self.stream = stream + self.pushed_back = [] + + def readline(self): + if self.pushed_back: + return self.pushed_back.pop() + else: + return self.stream.readline() + + def push_back(self, line): + self.pushed_back.append(line) + +def in_chunks(it, size=25): + chunk = [] + it = iter(it) + try: + while True: + chunk.append(it.next()) + if len(chunk) >= size: + yield chunk + chunk = [] + except StopIteration: + if chunk: + yield chunk + +def valiter(key, lr, firstline): + line = firstline + while line: + linevals = line.strip('\n').split('\t') + readkey, vals = linevals[0], linevals[1:] + if readkey == key: + yield vals + line = lr.readline() + else: + lr.push_back(line) + line = None + +def keyiter(stream): + lr = LineReader(stream) + + line = lr.readline() + while line: + key = line.strip('\n').split('\t',1)[0] + + vi = valiter(key, lr, line) + yield key, vi + # read the rest of the valueiter before reading any more lines + try: + while vi.next(): + pass + except StopIteration: + pass + + line = lr.readline() + +def status(msg, **opts): + if opts: + msg = msg % opts + sys.stderr.write("%s\n" % msg) + +def emit(vals): + print '\t'.join(str(val) for val in vals) + +def emit_all(vals): + for val in vals: + emit(val) + +class Storage(dict): + def __getattr__(self, attr): + return self[attr] + +def format_dataspec(msg, specs): + # spec() =:= name | (name, fn) + # specs =:= [ spec() ] + ret = Storage() + for val, spec in zip(msg, specs): + if isinstance(spec, basestring): + name = spec + ret[name] = val + else: + name, fn = spec + ret[name] = fn(val) + return Storage(**ret) + +class dataspec_m(object): + def __init__(self, *specs): + self.specs = specs + + def __call__(self, fn): + specs = self.specs + def wrapped_fn(args): + return fn(format_dataspec(args, specs)) + return wrapped_fn + +class dataspec_r(object): + def __init__(self, *specs): + self.specs = specs + + def __call__(self, fn): + specs = self.specs + def wrapped_fn(key, msgs): + return fn(key, ( format_dataspec(msg, specs) + for msg in msgs )) + return wrapped_fn + +def mr_map(process, fd = sys.stdin): + for line in fd: + vals = line.strip('\n').split('\t') + for res in process(vals): + emit(res) + +def mr_reduce(process, fd = sys.stdin): + for key, vals in keyiter(fd): + for res in process(key, vals): + emit(res) + +def mr_foldl(process, init, emit = False, fd = sys.stdin): + acc = init + for key, vals in keyiter(fd): + acc = process(key, vals, acc) + + if emit: + emit(acc) + + return acc + +def mr_max(process, idx = 0, num = 10, emit = False, fd = sys.stdin): + """a reducer that, in the process of reduction, only returns the + top N results""" + maxes = [] + for key, vals in keyiter(fd): + for newvals in in_chunks(process(key, vals)): + for val in newvals: + if len(maxes) < num or newval[idx] > maxes[-1][idx]: + maxes.append(newval) + maxes.sort(reverse=True) + maxes = maxes[:num] + + if emit: + emit_all(maxes) + + return maxes + +def mr_reduce_max_per_key(sort_key, post = None, num = 10, fd = sys.stdin): + def process(key, vals): + maxes = [] + for val_chunk in in_chunks(vals, num): + maxes.extend(val_chunk) + maxes.sort(reverse=True, key=sort_key) + maxes = maxes[:num] + if post: + # if we were passed a "post" function, he takes + # responsibility for emitting + post(key, maxes) + else: + for item in maxes: + yield [key] + item + + return mr_reduce(process, fd = fd) + +if __name__ == '__main__': + for key, vals in keyiter(sys.stdin): + print key, vals + for val in vals: + print '\t', val diff --git a/r2/r2/lib/pages/pages.py b/r2/r2/lib/pages/pages.py index f010ad83a..d99dd8e33 100644 --- a/r2/r2/lib/pages/pages.py +++ b/r2/r2/lib/pages/pages.py @@ -129,7 +129,9 @@ class Reddit(Templated): #add the infobar self.infobar = None if self.show_firsttext and not infotext: - if c.firsttime == 'iphone': + if g.read_only_mode: + infotext = strings.read_only_msg + elif c.firsttime == 'iphone': infotext = strings.iphone_first elif c.firsttime and c.site.firsttext: infotext = c.site.firsttext @@ -156,9 +158,9 @@ class Reddit(Templated): if c.site.type != 'public': buttons.append(NamedButton('contributors', css_class = 'reddit-contributors')) - elif (c.user_is_loggedin and c.site.use_whitelist and + elif (c.user_is_loggedin and (c.site.is_moderator(c.user) or c.user_is_admin)): - buttons.append(NavButton(menu.whitelist, "contributors", + buttons.append(NavButton(menu.contributors, "contributors", css_class = 'reddit-contributors')) buttons.extend([ @@ -184,7 +186,7 @@ class Reddit(Templated): if self.searchbox: ps.append(SearchForm()) - if not c.user_is_loggedin and self.loginbox: + if not c.user_is_loggedin and self.loginbox and not g.read_only_mode: ps.append(LoginFormWide()) if not isinstance(c.site, FakeSubreddit): @@ -456,7 +458,7 @@ class SubredditInfoBar(CachedTemplate): def nav(self): buttons = [NavButton(plurals.moderators, 'moderators')] if self.type != 'public': - buttons.append(NavButton(plurals.contributors, 'contributors')) + buttons.append(NavButton(getattr(plurals, "approved submitters"), 'contributors')) if self.is_moderator or self.is_admin: buttons.extend([ @@ -783,6 +785,107 @@ class LinkInfoPage(Reddit): rb.insert(1, LinkInfoBar(a = self.link)) return rb +class CommentPane(Templated): + def cache_key(self): + return "_".join(map(str, ["commentpane", self.article._fullname, + self.article.num_comments, + self.sort, self.num, c.lang, + self.can_reply, c.render_style])) + + def __init__(self, article, sort, comment, context, num, **kw): + # keys: lang, num, can_reply, render_style + # disable: admin + + from r2.models import CommentBuilder, NestedListing + from r2.controllers.reddit_base import UnloggedUser + + self.sort = sort + self.num = num + self.article = article + + # don't cache on permalinks or contexts, and keep it to html + try_cache = not comment and not context and (c.render_style == "html") + self.can_reply = False + if try_cache and c.user_is_loggedin: + sr = article.subreddit_slow + c.can_reply = self.can_reply = sr.can_comment(c.user) + # don't cache if the current user can ban comments in the listing + try_cache = not sr.can_ban(c.user) + # don't cache for users with custom hide threshholds + try_cache = (c.user.pref_min_comment_score == + Account._defaults["pref_min_comment_score"]) + + def renderer(): + builder = CommentBuilder(article, sort, comment, context, **kw) + listing = NestedListing(builder, num = num, + parent_name = article._fullname) + return listing.listing() + + # generate the listing we would make for this user if caching is disabled. + my_listing = renderer() + + # for now, disable the cache if the user happens to be an author of anything. + if try_cache: + for t in self.listing_iter(my_listing): + if getattr(t, "is_author", False): + try_cache = False + break + + if try_cache: + # try to fetch the comment tree from the caceh + key = self.cache_key() + self.rendered = g.cache.get(key) + if not self.rendered: + # spoof an unlogged in user + user = c.user + logged_in = c.user_is_loggedin + try: + c.user = UnloggedUser([c.lang]) + c.user_is_loggedin = False + + # render as if not logged in (but possibly with reply buttons) + self.rendered = renderer().render() + g.cache.set(key, self.rendered, time = 30) + + finally: + # undo the spoofing + c.user = user + c.user_is_loggedin = logged_in + + # figure out what needs to be updated on the listing + likes = [] + dislikes = [] + is_friend = set() + for t in self.listing_iter(my_listing): + if not hasattr(t, "likes"): + # this is for MoreComments and MoreRecursion + continue + if getattr(t, "friend", False): + is_friend.add(t.author._fullname) + if t.likes: + likes.append(t._fullname) + if t.likes is False: + dislikes.append(t._fullname) + self.rendered += ThingUpdater(likes = likes, + dislikes = dislikes, + is_friend = is_friend).render() + g.log.debug("using comment page cache") + else: + self.rendered = my_listing.render() + + def listing_iter(self, l): + for t in l: + yield t + for x in self.listing_iter(getattr(t, "child", [])): + yield x + + def render(self, *a, **kw): + return self.rendered + +class ThingUpdater(Templated): + pass + + class LinkInfoBar(Templated): """Right box for providing info about a link.""" def __init__(self, a = None): @@ -978,7 +1081,7 @@ class RedditError(BoringPage): class Reddit404(BoringPage): site_tracking = False def __init__(self): - ch=random.choice(['a','b','c','d','e']) + ch=random.choice(['a','b','c']) BoringPage.__init__(self, _("page not found"), loginbox=False, show_sidebar = False, content=UnfoundPage(ch)) @@ -996,9 +1099,10 @@ class ErrorPage(Templated): class Profiling(Templated): """Debugging template for code profiling using built in python library (only used in middleware)""" - def __init__(self, header = '', table = [], caller = [], callee = [], path = ''): + def __init__(self, header = '', table = [], caller = [], callee = [], + path = '', sort_order = ''): Templated.__init__(self, header = header, table = table, caller = caller, - callee = callee, path = path) + callee = callee, path = path) class Over18(Templated): """The creepy 'over 18' check page for nsfw content.""" @@ -1851,6 +1955,7 @@ class WrappedUser(CachedTemplate): karma = karma, ip_span = ip_span, context_deleted = context_deleted, + fullname = user._fullname, user_deleted = user._deleted) # Classes for dealing with friend/moderator/contributor/banned lists @@ -1883,6 +1988,7 @@ class UserList(Templated): _class = "" destination = "friend" remove_action = "unfriend" + editable_fn = None def __init__(self, editable = True): self.editable = editable @@ -1892,8 +1998,13 @@ class UserList(Templated): """Convenience method for constructing a UserTableItem instance of the user with type, container_name, etc. of this UserList instance""" + editable = self.editable + + if self.editable_fn and not self.editable_fn(user): + editable = False + return UserTableItem(user, self.type, self.cells, self.container_name, - self.editable, self.remove_action) + editable, self.remove_action) @property def users(self, site = None): @@ -1940,13 +2051,11 @@ class ContributorList(UserList): @property def form_title(self): - if c.site.type == "public": - return _("add to whitelist") - return _('add contributor') + return _("add approved submitter") @property def table_title(self): - return _("contributors to %(reddit)s") % dict(reddit = c.site.name) + return _("approved submitters for %(reddit)s") % dict(reddit = c.site.name) def user_ids(self): return c.site.contributors @@ -1963,6 +2072,14 @@ class ModList(UserList): def table_title(self): return _("moderators to %(reddit)s") % dict(reddit = c.site.name) + def editable_fn(self, user): + if not c.user_is_loggedin: + return False + elif c.user_is_admin: + return True + else: + return c.site.can_demod(c.user, user) + def user_ids(self): return c.site.moderators @@ -2412,8 +2529,12 @@ class RedditTraffic(Traffic): if self.has_data: imp_by_day = [[] for i in range(7)] uni_by_day = [[] for i in range(7)] - dates, imps = zip(*self.impressions_day_chart) - dates, uniques = zip(*self.uniques_day_chart) + if c.site.domain: + dates, imps, foo = zip(*self.impressions_day_chart) + dates, uniques, foo = zip(*self.uniques_day_chart) + else: + dates, imps = zip(*self.impressions_day_chart) + dates, uniques = zip(*self.uniques_day_chart) self.uniques_mean = sum(map(float, uniques))/len(uniques) self.impressions_mean = sum(map(float, imps))/len(imps) for i, d in enumerate(dates): diff --git a/r2/r2/lib/pages/things.py b/r2/r2/lib/pages/things.py index ac372a91c..84cae1580 100644 --- a/r2/r2/lib/pages/things.py +++ b/r2/r2/lib/pages/things.py @@ -34,8 +34,13 @@ class PrintableButtons(Styled): def __init__(self, style, thing, show_delete = False, show_report = True, show_distinguish = False, - show_indict = False, **kw): + show_indict = False, is_link=False, **kw): show_report = show_report and c.user_is_loggedin + show_ignore = (thing.show_reports or + (thing.reveal_trial_info and not thing.show_spam)) + approval_checkmark = getattr(thing, "approval_checkmark", None) + show_approve = (thing.show_spam or show_ignore or + (is_link and approval_checkmark is None)) Styled.__init__(self, style = style, thing = thing, @@ -43,11 +48,10 @@ class PrintableButtons(Styled): can_ban = thing.can_ban, show_spam = thing.show_spam, show_reports = thing.show_reports, - show_ignore = thing.show_reports or - (thing.reveal_trial_info and not thing.show_spam), - approval_checkmark = getattr(thing, - "approval_checkmark", None), + show_ignore = show_ignore, + approval_checkmark = approval_checkmark, show_delete = show_delete, + show_approve = show_approve, show_report = show_report, show_indict = show_indict, show_distinguish = show_distinguish, @@ -64,7 +68,9 @@ class LinkButtons(PrintableButtons): is_author = (c.user_is_loggedin and thing.author and c.user.name == thing.author.name) # do we show the report button? - show_report = not is_author and report + show_report = (not is_author and + report and + getattr(thing, "promoted", None) is None) if c.user_is_admin and thing.promoted is None: show_report = False @@ -112,13 +118,13 @@ class LinkButtons(PrintableButtons): show_comments = comments, # promotion promoted = thing.promoted, + is_link = True, **kw) class CommentButtons(PrintableButtons): def __init__(self, thing, delete = True, report = True): # is the current user the author? - is_author = (c.user_is_loggedin and thing.author and - c.user.name == thing.author.name) + is_author = thing.is_author # do we show the report button? show_report = not is_author and report # do we show the delete button? diff --git a/r2/r2/lib/promote.py b/r2/r2/lib/promote.py index abe5b9122..d117ea113 100644 --- a/r2/r2/lib/promote.py +++ b/r2/r2/lib/promote.py @@ -310,7 +310,7 @@ def new_promotion(title, url, user, ip): l._commit() # set the status of the link, populating the query queue - if c.user_is_sponsor or getattr(user, "trusted_sponsor", False): + if c.user_is_sponsor or user.trusted_sponsor: set_status(l, STATUS.accepted) else: set_status(l, STATUS.unpaid) diff --git a/r2/r2/lib/strings.py b/r2/r2/lib/strings.py index fde1488e8..115e87e2d 100644 --- a/r2/r2/lib/strings.py +++ b/r2/r2/lib/strings.py @@ -84,7 +84,7 @@ string_dict = dict( msg_add_friend = dict( friend = None, moderator = _("you have been added as a moderator to [%(title)s](%(url)s)."), - contributor = _("you have been added as a contributor to [%(title)s](%(url)s)."), + contributor = _("you have been added as an approved submitter to [%(title)s](%(url)s)."), banned = _("you have been banned from posting to [%(title)s](%(url)s)."), traffic = _('you have been added to the list of users able to see [traffic for the sponsored link "%(title)s"](%(traffic_url)s).') ), @@ -92,7 +92,7 @@ string_dict = dict( subj_add_friend = dict( friend = None, moderator = _("you are a moderator"), - contributor = _("you are a contributor"), + contributor = _("you are an approved submitter"), banned = _("you've been banned"), traffic = _("you can view traffic on a promoted link") ), @@ -100,7 +100,7 @@ string_dict = dict( sr_messages = dict( empty = _('you have not subscribed to any reddits.'), subscriber = _('below are the reddits you have subscribed to'), - contributor = _('below are the reddits that you have contributor access to.'), + contributor = _('below are the reddits that you are an approved submitter on.'), moderator = _('below are the reddits that you have moderator access to.') ), @@ -135,6 +135,7 @@ string_dict = dict( search_failed = _("Our search machines are under too much load to handle your request right now. :( Sorry for the inconvenience. [Try again](%(link)s) in a little bit -- but please don't mash reload; that only makes the problem worse."), verified_quota_msg = _("You've submitted several links recently that haven't been doing very well. You'll have to wait a while before you can submit again, or [write to the moderators of this reddit](%(link)s) and ask for an exemption."), unverified_quota_msg = _("You haven't [verified your email address](%(link1)s); until you do, your submitting privileges will be severely limited. Please try again in an hour or verify your email address. If you'd like an exemption from this rule, please [write to the moderators of this reddit](%(link2)s)."), + read_only_msg = _("reddit is in \"emergency read-only mode\" right now. :( you won't be able to log in. we're sorry, and are working frantically to fix the problem."), ) class StringHandler(object): @@ -206,7 +207,7 @@ plurals = PluralManager([P_("comment", "comments"), # people P_("reader", "readers"), P_("subscriber", "subscribers"), - P_("contributor", "contributors"), + P_("approved submitter", "approved submitters"), P_("moderator", "moderators"), # time words diff --git a/r2/r2/lib/utils/__init__.py b/r2/r2/lib/utils/__init__.py index e3bcaaf57..63c53d0c9 100644 --- a/r2/r2/lib/utils/__init__.py +++ b/r2/r2/lib/utils/__init__.py @@ -27,4 +27,4 @@ from cmd_utils import * try: from r2admin.lib.admin_utils import * except ImportError: - pass + from admin_utils import * diff --git a/r2/r2/lib/utils/trial_utils.py b/r2/r2/lib/utils/trial_utils.py index dc5039342..e63fd16c3 100644 --- a/r2/r2/lib/utils/trial_utils.py +++ b/r2/r2/lib/utils/trial_utils.py @@ -60,7 +60,7 @@ def indict(defendant): rv = False if defendant._deleted: result = "already deleted" - elif hasattr(defendant, "promoted") and defendant.promoted: + elif getattr(defendant, "promoted", None) is not None: result = "it's promoted" elif hasattr(defendant, "verdict") and defendant.verdict is not None: result = "it already has a verdict" diff --git a/r2/r2/lib/utils/utils.py b/r2/r2/lib/utils/utils.py index 2025984c3..93aa90fc2 100644 --- a/r2/r2/lib/utils/utils.py +++ b/r2/r2/lib/utils/utils.py @@ -1273,7 +1273,7 @@ def progress(it, verbosity=100, key=repr, estimate=None, persec=False): except: pass - print 'Starting at %s' % (start,) + sys.stderr.write('Starting at %s\n' % (start,)) seen = 0 for item in it: @@ -1300,20 +1300,19 @@ def progress(it, verbosity=100, key=repr, estimate=None, persec=False): key_str = '' if persec and elapsed_seconds > 0: - persec_str = ' (%.2f/s)' % (seen/elapsed_seconds,) + persec_str = ' (%.2f/s)' % (float(seen)/elapsed_seconds,) else: persec_str = '' - sys.stdout.write('%s%s, %s%s%s\n' + sys.stderr.write('%s%s, %s%s%s\n' % (count_str, persec_str, elapsed, estimate_str, key_str)) - sys.stdout.flush() this_chunk = 0 yield item now = datetime.now() elapsed = now - start - print 'Processed %d items in %s..%s (%s)' % (seen, start, now, elapsed) + sys.stderr.write('Processed %d items in %s..%s (%s)\n' % (seen, start, now, elapsed)) class Hell(object): def __str__(self): diff --git a/r2/r2/models/account.py b/r2/r2/models/account.py index c817ed22b..e20c16c91 100644 --- a/r2/r2/models/account.py +++ b/r2/r2/models/account.py @@ -62,6 +62,7 @@ class Account(Thing): pref_threaded_messages = True, pref_collapse_read_messages = False, pref_private_feeds = True, + trusted_sponsor = False, reported = 0, report_made = 0, report_correct = 0, @@ -397,6 +398,9 @@ def valid_cookie(cookie): except: return (False, False) + if g.read_only_mode: + return (False, False) + try: account = Account._byID(uid, True) if account._deleted: diff --git a/r2/r2/models/admintools.py b/r2/r2/models/admintools.py index d026db594..242731c70 100644 --- a/r2/r2/models/admintools.py +++ b/r2/r2/models/admintools.py @@ -43,6 +43,9 @@ class AdminTools(object): Report.accept(new_things, True) for t in all_things: + if getattr(t, "promoted", None) is not None: + g.log.debug("Refusing to mark promotion %r as spam" % t) + continue t._spam = True ban_info = copy(getattr(t, 'ban_info', {})) ban_info.update(auto = auto, @@ -198,8 +201,6 @@ def filter_quotas(unfiltered): quotas_changed = True continue - score = item._downs - item._ups - verdict = getattr(item, "verdict", None) approved = verdict and verdict in ( 'admin-approved', 'mod-approved') @@ -211,9 +212,9 @@ def filter_quotas(unfiltered): pass elif item._deleted: pass - elif score <= 0: + elif item._score <= 0: pass - elif age < 86400 and score <= g.QUOTA_THRESHOLD and not approved: + elif age < 86400 and item._score <= g.QUOTA_THRESHOLD and not approved: pass else: quotas_changed = True diff --git a/r2/r2/models/builder.py b/r2/r2/models/builder.py index 1a8314fa9..3526f0d9f 100644 --- a/r2/r2/models/builder.py +++ b/r2/r2/models/builder.py @@ -204,7 +204,8 @@ class Builder(object): or (user and hasattr(item,'sr_id') and item.sr_id in can_ban_set)): - w.can_ban = True + if getattr(item, "promoted", None) is None: + w.can_ban = True ban_info = getattr(item, 'ban_info', {}) w.unbanner = ban_info.get('unbanner') @@ -498,7 +499,7 @@ class CommentBuilder(Builder): def get_items(self, num): r = link_comments(self.link._id) - cids, comment_tree, depth, num_children = r + cids, cid_tree, depth, num_children = r if (not isinstance(self.comment, utils.iters) and self.comment and not self.comment._id in depth): @@ -506,7 +507,7 @@ class CommentBuilder(Builder): % self.comment._id) r = link_comments(self.link._id, _update=True) - cids, comment_tree, depth, num_children = r + cids, cid_tree, depth, num_children = r if not self.comment._id in depth: g.log.error("Update didn't help. This is gonna end in tears.") @@ -520,9 +521,9 @@ class CommentBuilder(Builder): comment_dict = dict((cm._id, cm) for cm in comments) #convert tree into objects - for k, v in comment_tree.iteritems(): - comment_tree[k] = [comment_dict[cid] for cid in comment_tree[k]] - + comment_tree = {} + for k, v in cid_tree.iteritems(): + comment_tree[k] = [comment_dict[cid] for cid in cid_tree[k]] items = [] extra = {} top = None diff --git a/r2/r2/models/link.py b/r2/r2/models/link.py index c20c015f9..a1eb77d78 100644 --- a/r2/r2/models/link.py +++ b/r2/r2/models/link.py @@ -80,7 +80,7 @@ class Link(Thing, Printable): sr = None url = cls.by_url_key(url) - link_ids = g.permacache.get(url) + link_ids = g.urlcache.get(url) if link_ids: links = Link._byID(link_ids, data = True, return_dict = False) links = [l for l in links if not l._deleted] @@ -97,20 +97,20 @@ class Link(Thing, Printable): def set_url_cache(self): if self.url != 'self': key = self.by_url_key(self.url) - link_ids = g.permacache.get(key) or [] + link_ids = g.urlcache.get(key) or [] if self._id not in link_ids: link_ids.append(self._id) - g.permacache.set(key, link_ids) + g.urlcache.set(key, link_ids) def update_url_cache(self, old_url): """Remove the old url from the by_url cache then update the cache with the new url.""" if old_url != 'self': key = self.by_url_key(old_url) - link_ids = g.permacache.get(key) or [] + link_ids = g.urlcache.get(key) or [] while self._id in link_ids: link_ids.remove(self._id) - g.permacache.set(key, link_ids) + g.urlcache.set(key, link_ids) self.set_url_cache() @property @@ -142,16 +142,19 @@ class Link(Thing, Printable): @classmethod def _somethinged(cls, rel, user, link, name): - return rel._fast_query(tup(user), tup(link), name = name) + return rel._fast_query(tup(user), tup(link), name = name, + timestamp_optimize = True) def _something(self, rel, user, somethinged, name): try: saved = rel(user, self, name=name) saved._commit() - return saved except CreationError, e: return somethinged(user, self)[(user, self, name)] + rel._fast_query_timestamp_touch(user) + return saved + def _unsomething(self, user, somethinged, name): saved = somethinged(user, self)[(user, self, name)] if saved: @@ -498,6 +501,7 @@ class Comment(Thing, Printable): _defaults = dict(reported = 0, parent_id = None, moderator_banned = False, new = False, banned_before_moderator = False) + _essentials = ('link_id', 'author_id') def _markdown(self): pass @@ -583,9 +587,7 @@ class Comment(Thing, Printable): def add_props(cls, user, wrapped): from r2.lib.template_helpers import add_attr from r2.lib import promote - #fetch parent links - links = Link._byID(set(l.link_id for l in wrapped), data = True, return_dict = True) @@ -642,7 +644,7 @@ class Comment(Thing, Printable): else: item.parent_permalink = None - item.can_reply = (item.sr_id in can_reply_srs) + item.can_reply = c.can_reply or (item.sr_id in can_reply_srs) # not deleted on profile pages, @@ -915,7 +917,7 @@ class Message(Thing, Printable): # load the inbox relations for the messages to determine new-ness # TODO: query cache? - inbox = Inbox._fast_query(c.user, + inbox = Inbox._fast_query(c.user, [item.lookups[0] for item in wrapped], ['inbox', 'selfreply']) @@ -923,15 +925,17 @@ class Message(Thing, Printable): inbox = dict((m._fullname, v) for (u, m, n), v in inbox.iteritems() if v) - modinbox = ModeratorInbox._query( - ModeratorInbox.c._thing2_id == [item._id for item in wrapped], - data = True) + msgs = filter (lambda x: isinstance(x.lookups[0], Message), wrapped) + + modinbox = ModeratorInbox._fast_query(m_subreddits.values(), + msgs, + ['inbox'] ) # best to not have to eager_load the things def make_message_fullname(mid): return "t%s_%s" % (utils.to36(Message._type_id), utils.to36(mid)) modinbox = dict((make_message_fullname(v._thing2_id), v) - for v in modinbox) + for (u, m, n), v in modinbox.iteritems() if v) for item in wrapped: item.to = tos.get(item.to_id) @@ -1036,10 +1040,15 @@ class Inbox(MultiRelation('inbox', return i @classmethod - def set_unread(cls, thing, unread): + def set_unread(cls, thing, unread, to = None): inbox_rel = cls.rel(Account, thing.__class__) - inbox = inbox_rel._query(inbox_rel.c._thing2_id == thing._id, - eager_load = True) + if to: + inbox = inbox_rel._query(inbox_rel.c._thing2_id == thing._id, + eager_load = True) + else: + inbox = inbox_rel._query(inbox_rel.c._thing2_id == thing._id, + inbox_rel.c._thing1_id == to._id, + eager_load = True) res = [] for i in inbox: if i: diff --git a/r2/r2/models/populatedb.py b/r2/r2/models/populatedb.py index d30e820e8..cadfb399b 100644 --- a/r2/r2/models/populatedb.py +++ b/r2/r2/models/populatedb.py @@ -21,51 +21,53 @@ ################################################################################ from r2.models import * from r2.lib.utils import fetch_things2 +from pylons import g +from r2.lib.db import queries + import string import random -def populate(sr_name = 'reddit.com', sr_title = "reddit.com: what's new online", - num = 100): - create_accounts(num) - - a = list(Account._query(limit = 1))[0] - +def populate(num_srs = 10, num_users = 10, num_links = 100): try: - sr = Subreddit._new(name = sr_name, title = sr_title, - ip = '0.0.0.0', author_id = a._id) - sr._commit() - except SubredditExists: - pass + a = Account._by_name(g.system_user) + except NotFound: + a = register(g.system_user, "password") - create_links(num) - -def create_accounts(num): - for i in range(num): + srs = [] + for i in range(num_srs): + name = "reddit_test%d" % i + try: + sr = Subreddit._new(name = name, title = "everything about #%d"%i, + ip = '0.0.0.0', author_id = a._id) + sr._downs = 10 + sr.lang = "en" + sr._commit() + except SubredditExists: + sr = Subreddit._by_name(name) + srs.append(sr) + + accounts = [] + for i in range(num_users): name_ext = ''.join([ random.choice(string.letters) for x in range(int(random.uniform(1, 10))) ]) name = 'test_' + name_ext try: - register(name, name) + a = register(name, name) except AccountExists: - pass + a = Account._by_name(name) + accounts.append(a) -def create_links(num): - from r2.lib.db import queries - - accounts = list(Account._query(limit = num, data = True)) - subreddits = list(Subreddit._query(limit = num, data = True)) - for i in range(num): + for i in range(num_links): id = random.uniform(1,100) title = url = 'http://google.com/?q=' + str(id) user = random.choice(accounts) - sr = random.choice(subreddits) + sr = random.choice(srs) l = Link._submit(title, url, user, sr, '127.0.0.1') queries.new_link(l) queries.worker.join() - def by_url_cache(): diff --git a/r2/r2/models/subreddit.py b/r2/r2/models/subreddit.py index 314bd3677..e1ae77680 100644 --- a/r2/r2/models/subreddit.py +++ b/r2/r2/models/subreddit.py @@ -57,9 +57,8 @@ class Subreddit(Thing, Printable): valid_votes = 0, show_media = False, css_on_cname = True, - use_whitelist = False, domain = None, - over_18 = False, + over_18 = None, mod_actions = 0, sponsorship_text = "this reddit is sponsored by", sponsorship_url = None, @@ -215,9 +214,7 @@ class Subreddit(Thing, Printable): return (user and (c.user_is_admin or self.is_moderator(user) - or ((self.type in ('restricted', 'private') or - self.use_whitelist) and - self.is_contributor(user)))) + or self.is_contributor(user))) def can_give_karma(self, user): return self.is_special(user) @@ -243,6 +240,15 @@ class Subreddit(Thing, Printable): #private requires contributorship return self.is_contributor(user) or self.is_moderator(user) + def can_demod(self, bully, victim): + # This works because the is_*() functions return the relation + # when True. So we can compare the dates on the relations. + bully_rel = self.is_moderator(bully) + victim_rel = self.is_moderator(victim) + if bully_rel is None or victim_rel is None: + return False + return bully_rel._date <= victim_rel._date + @classmethod def load_subreddits(cls, links, return_dict = True): """returns the subreddits for a list of links. it also preloads the @@ -787,7 +793,7 @@ class ModSR(ModContribSR): class ContribSR(ModContribSR): name = "contrib" - title = "communities you're a contributor on" + title = "communities you're approved on" query_param = "contributor" real_path = "contrib" diff --git a/r2/r2/models/trial.py b/r2/r2/models/trial.py index 76d1945f4..b6a256dcb 100644 --- a/r2/r2/models/trial.py +++ b/r2/r2/models/trial.py @@ -101,9 +101,12 @@ class Trial(Storage): g.log.debug("not enough votes yet") return (None, koshers, spams) - # Stop showing this in the spotlight box once it has 30 votes - if total_votes >= 30: + # Stop showing this in the spotlight box once it has 20 votes + if total_votes >= 20: g.cache.set("quench_jurors-" + self.defendant._fullname, True) + quenching = True + else: + quenching = False # If a trial is less than an hour old, and votes are still trickling # in (i.e., there was one in the past five minutes), we're going to @@ -127,9 +130,20 @@ class Trial(Storage): return ("guilty", koshers, spams) elif kosher_pct > 0.66: return ("innocent", koshers, spams) + elif not quenching: + g.log.debug("not yet quenching") + return (None, koshers, spams) + # At this point, we're not showing the link to any new jurors, and + # the existing jurors haven't changed or submitted votes for several + # minutes, so we're not really expecting to get many more votes. + # Thus, lower our standards for consensus. + elif kosher_pct < 0.3999: + return ("guilty", koshers, spams) + elif kosher_pct > 0.6001: + return ("innocent", koshers, spams) elif total_votes >= 100: # This should never really happen; quenching should kick in - # after 30 votes, so new jurors won't be assigned to the + # after 20 votes, so new jurors won't be assigned to the # trial. Just in case something goes wrong, close any trials # with more than 100 votes. return ("hung jury", koshers, spams) @@ -173,7 +187,7 @@ class Trial(Storage): defs = Thing._by_fullname(all, data=True).values() if quench: - # Used for the spotlight, to filter out trials with over 30 votes; + # Used for the spotlight, to filter out trials with over 20 votes; # otherwise, hung juries would hog the spotlight for an hour as # their vote counts continued to skyrocket diff --git a/r2/r2/models/vote.py b/r2/r2/models/vote.py index 8f272e5ae..43a44cec4 100644 --- a/r2/r2/models/vote.py +++ b/r2/r2/models/vote.py @@ -102,16 +102,7 @@ class Vote(MultiRelation('vote', v._commit() g.cache.delete(queries.prequeued_vote_key(sub, obj)) - lastvote_attr_name = 'last_vote_' + obj.__class__.__name__ - try: - setattr(sub, lastvote_attr_name, datetime.now(g.tz)) - except TypeError: - # this temporarily works around an issue with timezones in - # a really hacky way. Remove me later - setattr(sub, lastvote_attr_name, None) - sub._commit() - setattr(sub, lastvote_attr_name, datetime.now(g.tz)) - sub._commit() + v._fast_query_timestamp_touch(sub) up_change, down_change = score_changes(amount, oldamount) @@ -136,7 +127,8 @@ class Vote(MultiRelation('vote', @classmethod def likes(cls, sub, obj): votes = cls._fast_query(sub, obj, ('1', '-1'), - data=False, eager_load=False) + data=False, eager_load=False, + timestamp_optimize=True) votes = dict((tuple(k[:2]), v) for k, v in votes.iteritems() if v) return votes diff --git a/r2/r2/public/static/css/reddit.css b/r2/r2/public/static/css/reddit.css index a3425018c..853aa2d6a 100644 --- a/r2/r2/public/static/css/reddit.css +++ b/r2/r2/public/static/css/reddit.css @@ -3983,4 +3983,4 @@ a.pretty-button.positive.pressed { .oatmeal img { display: block; margin: 5px auto; -} \ No newline at end of file +} diff --git a/r2/r2/public/static/js/jquery.reddit.js b/r2/r2/public/static/js/jquery.reddit.js index e287d3a45..1b6c86a61 100644 --- a/r2/r2/public/static/js/jquery.reddit.js +++ b/r2/r2/public/static/js/jquery.reddit.js @@ -254,9 +254,12 @@ rate_limit = function() { }() -$.fn.vote = function(vh, callback, event) { +$.fn.vote = function(vh, callback, event, ui_only) { /* for vote to work, $(this) should be the clicked arrow */ - if($(this).hasClass("arrow")) { + if (!reddit.logged) { + showcover(true, 'vote_' + $(this).thing_id()); + } + else if($(this).hasClass("arrow")) { var dir = ( $(this).hasClass(up_cls) ? 1 : ( $(this).hasClass(down_cls) ? -1 : 0) ); var things = $(this).all_things_by_id(); @@ -267,7 +270,7 @@ $.fn.vote = function(vh, callback, event) { var u_before = (dir == 1) ? up_cls : upmod_cls; var u_after = (dir == 1) ? upmod_cls : up_cls; arrows.filter("."+u_before).removeClass(u_before).addClass(u_after); - + var d_before = (dir == -1) ? down_cls : downmod_cls; var d_after = (dir == -1) ? downmod_cls : down_cls; arrows.filter("."+d_before).removeClass(d_before).addClass(d_after); @@ -276,20 +279,22 @@ $.fn.vote = function(vh, callback, event) { if(reddit.logged) { things.each(function() { var entry = $(this).find(".entry:first, .midcol:first"); - if(dir > 0) + if(dir > 0) entry.addClass('likes') .removeClass('dislikes unvoted'); - else if(dir < 0) + else if(dir < 0) entry.addClass('dislikes') .removeClass('likes unvoted'); else entry.addClass('unvoted') .removeClass('likes dislikes'); }); - var thing_id = things.filter(":first").thing_id(); - /* IE6 hack */ - vh += event ? "" : ("-" + thing_id); - $.request("vote", {id: thing_id, dir : dir, vh : vh}); + if(!$.defined(ui_only)) { + var thing_id = things.filter(":first").thing_id(); + /* IE6 hack */ + vh += event ? "" : ("-" + thing_id); + $.request("vote", {id: thing_id, dir : dir, vh : vh}); + } } /* execute any callbacks passed in. */ if(callback) diff --git a/r2/r2/public/static/js/reddit.js b/r2/r2/public/static/js/reddit.js index 993e98a02..18676200c 100644 --- a/r2/r2/public/static/js/reddit.js +++ b/r2/r2/public/static/js/reddit.js @@ -1283,3 +1283,22 @@ $(function() { last_click(); }); + +function friend(account_fullname) { + var ua = $(".author.id-" + account_fullname).addClass("friend") + .next(".userattrs"); + var add_braces = (!ua.html()); + + ua.html(((add_braces) ? " [" : "") + + 'F' + + ((add_braces) ? "]" : "")); +} + +function unfriend(account_fullname) { + var ua = $(".author.id-" + account_fullname).removeClass("friend") + .next(".userattrs"); + ua.find("a.friend").remove(); + if (ua.find("a").length == 0) { + ua.html(""); + } +} \ No newline at end of file diff --git a/r2/r2/public/static/reddit404a.png b/r2/r2/public/static/reddit404a.png index 3a6b89f71..a69093074 100644 Binary files a/r2/r2/public/static/reddit404a.png and b/r2/r2/public/static/reddit404a.png differ diff --git a/r2/r2/public/static/reddit404b.png b/r2/r2/public/static/reddit404b.png index a9259d13c..83443d88d 100644 Binary files a/r2/r2/public/static/reddit404b.png and b/r2/r2/public/static/reddit404b.png differ diff --git a/r2/r2/public/static/reddit404c.png b/r2/r2/public/static/reddit404c.png index 9d5acec4c..446ed9f8b 100644 Binary files a/r2/r2/public/static/reddit404c.png and b/r2/r2/public/static/reddit404c.png differ diff --git a/r2/r2/public/static/robots.txt b/r2/r2/public/static/robots.txt index cd6dbede2..23664b602 100644 --- a/r2/r2/public/static/robots.txt +++ b/r2/r2/public/static/robots.txt @@ -8,4 +8,7 @@ Disallow: /*after= Disallow: /*before= Disallow: /domain/*t= Disallow: /login +Disallow: /reddits/search +Disallow: /search +Disallow: /r/*/search Allow: / diff --git a/r2/r2/public/static/youbrokeit.png b/r2/r2/public/static/youbrokeit.png deleted file mode 100644 index d539c7ee9..000000000 Binary files a/r2/r2/public/static/youbrokeit.png and /dev/null differ diff --git a/r2/r2/public/static/youbrokeit1.png b/r2/r2/public/static/youbrokeit1.png new file mode 100644 index 000000000..e9d615a74 Binary files /dev/null and b/r2/r2/public/static/youbrokeit1.png differ diff --git a/r2/r2/public/static/youbrokeit2.png b/r2/r2/public/static/youbrokeit2.png new file mode 100644 index 000000000..7668147bc Binary files /dev/null and b/r2/r2/public/static/youbrokeit2.png differ diff --git a/r2/r2/public/static/youbrokeit3.png b/r2/r2/public/static/youbrokeit3.png new file mode 100644 index 000000000..e16659693 Binary files /dev/null and b/r2/r2/public/static/youbrokeit3.png differ diff --git a/r2/r2/templates/comment.html b/r2/r2/templates/comment.html index 8aaf3c7da..a1c0d9c36 100644 --- a/r2/r2/templates/comment.html +++ b/r2/r2/templates/comment.html @@ -117,7 +117,6 @@ ${parent.Child(not thing.collapsed)} ${parent.midcol()} - <%def name="buttons()"> ${CommentButtons(thing)} ${self.admintagline()} diff --git a/r2/r2/templates/createsubreddit.html b/r2/r2/templates/createsubreddit.html index 7f13ae92c..51d3942b9 100644 --- a/r2/r2/templates/createsubreddit.html +++ b/r2/r2/templates/createsubreddit.html @@ -130,9 +130,9 @@ function update_title(elem) {