github/reddit
1
1
Fork 0
mirror of https://github.com/reddit-archive/reddit.git synced 2026-04-05 03:00:15 -04:00
Code Issues Packages Projects Releases Wiki Activity

Significantly simplify spam and reporting, and move all of that code into the public repository

Browse Source
This commit is contained in:
ketralnis
2009-08-11 12:16:50 -07:00
parent b066246a5c
commit 35f51ea5a9
8 changed files with 226 additions and 562 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
r2/r2/controllers/api.py
View File

@@ -539,7 +539,6 @@ class ApiController(RedditController):
not (hasattr(thing, "promoted") and thing.promoted)):
Report.new(c.user, thing)
@validatedForm(VUser(),
VModhash(),
item = VByNameIfAuthor('thing_id'),
@@ -1034,29 +1033,20 @@ class ApiController(RedditController):
VSrCanBan('id'),
thing = VByName('id'))
def POST_ban(self, thing):
if not thing: return
thing.moderator_banned = not c.user_is_admin
thing.banner = c.user.name
thing._commit()
# NB: change table updated by reporting
unreport(thing, correct=True, auto=False)
admintools.spam(thing, False, not c.user_is_admin, c.user.name)
@noresponse(VUser(), VModhash(),
VSrCanBan('id'),
thing = VByName('id'))
def POST_unban(self, thing):
# NB: change table updated by reporting
if not thing: return
unreport(thing, correct=False)
admintools.unspam(thing, c.user.name)
@noresponse(VUser(), VModhash(),
VSrCanBan('id'),
thing = VByName('id'))
def POST_ignore(self, thing):
if not thing: return
# NB: change table updated by reporting
unreport(thing, correct=False)
Report.accept(thing, False)
@validatedForm(VUser(), VModhash(),
VSrCanDistinguish('id'),

181
r2/r2/lib/db/queries.py
View File

@@ -4,7 +4,7 @@ from r2.lib.db.thing import Thing, Merge
from r2.lib.db.operators import asc, desc, timeago
from r2.lib.db import query_queue
from r2.lib.db.sorts import epoch_seconds
from r2.lib.utils import fetch_things2, worker
from r2.lib.utils import fetch_things2, worker, tup
from r2.lib.solrsearch import DomainSearchQuery
from datetime import datetime
@@ -90,29 +90,31 @@ class CachedResults(object):
"""True if a item can be removed from the listing, always true for now."""
return True
def insert(self, item):
def insert(self, items):
"""Inserts the item at the front of the cached data. Assumes the query
is sorted by date descending"""
self.fetch()
t = self.make_item_tuple(item)
t = [ self.make_item_tuple(item) for item in tup(items) ]
if t not in self.data:
if self.data and t[1:] > self.data[0][1:]:
self.data.insert(0, t)
else:
self.data.append(t)
self.data.sort(key=lambda x: x[1:], reverse=True)
query_cache.set(self.iden, self.data[:precompute_limit])
if len(t) == 1 and self.data and t[0][1:] > self.data[0][1:]:
self.data.insert(0, t[0])
else:
self.data.extend(t)
self.data = list(set(self.data))
self.data.sort(key=lambda x: x[1:], reverse=True)
query_cache.set(self.iden, self.data[:precompute_limit])
def delete(self, item):
def delete(self, items):
"""Deletes an item from the cached data."""
self.fetch()
t = self.make_item_tuple(item)
changed = False
while t in self.data:
self.data.remove(t)
changed = True
for item in tup(items):
t = self.make_item_tuple(item)
while t in self.data:
self.data.remove(t)
changed = True
if changed:
query_cache.set(self.iden, self.data)
@@ -292,9 +294,9 @@ def get_sent(user):
sort = desc('_date'))
return make_results(q)
def add_queries(queries, insert_item = None, delete_item = None):
"""Adds multiple queries to the query queue. If insert_item or
delete_item is specified, the query may not need to be recomputed at
def add_queries(queries, insert_items = None, delete_items = None):
"""Adds multiple queries to the query queue. If insert_items or
delete_items is specified, the query may not need to be recomputed at
all."""
log = g.log.debug
make_lock = g.make_lock
@@ -303,14 +305,15 @@ def add_queries(queries, insert_item = None, delete_item = None):
if not isinstance(q, CachedResults):
continue
log('Adding precomputed query %s' % q)
with make_lock("add_query(%s)" % q.iden):
if insert_item and q.can_insert():
q.insert(insert_item)
elif delete_item and q.can_delete():
q.delete(delete_item)
if insert_items and q.can_insert():
log("Inserting %s into query %s" % (insert_items, q))
q.insert(insert_items)
elif delete_items and q.can_delete():
log("Deleting %s from query %s" % (delete_items, q))
q.delete(delete_items)
else:
log('Adding precomputed query %s' % q)
query_queue.add_query(q)
worker.do(_add_queries)
@@ -351,25 +354,25 @@ def new_link(link):
results.append(get_spam_links(sr))
if link._deleted:
add_queries(results, delete_item = link)
add_queries(results, delete_items = link)
else:
add_queries(results, insert_item = link)
add_queries(results, insert_items = link)
def new_comment(comment, inbox_rel):
author = Account._byID(comment.author_id)
job = [get_comments(author, 'new', 'all')]
if comment._deleted:
add_queries(job, delete_item = comment)
add_queries(job, delete_items = comment)
else:
#if comment._spam:
# sr = Subreddit._byID(comment.sr_id)
# job.append(get_spam_comments(sr))
add_queries(job, insert_item = comment)
add_queries(job, insert_items = comment)
if inbox_rel:
inbox_owner = inbox_rel._thing1
add_queries([get_inbox_comments(inbox_owner)],
insert_item = inbox_rel)
insert_items = inbox_rel)
def new_vote(vote):
user = vote._thing1
@@ -387,72 +390,103 @@ def new_vote(vote):
#must update both because we don't know if it's a changed vote
if vote._name == '1':
add_queries([get_liked(user)], insert_item = vote)
add_queries([get_disliked(user)], delete_item = vote)
add_queries([get_liked(user)], insert_items = vote)
add_queries([get_disliked(user)], delete_items = vote)
elif vote._name == '-1':
add_queries([get_liked(user)], delete_item = vote)
add_queries([get_disliked(user)], insert_item = vote)
add_queries([get_liked(user)], delete_items = vote)
add_queries([get_disliked(user)], insert_items = vote)
else:
add_queries([get_liked(user)], delete_item = vote)
add_queries([get_disliked(user)], delete_item = vote)
add_queries([get_liked(user)], delete_items = vote)
add_queries([get_disliked(user)], delete_items = vote)
def new_message(message, inbox_rel):
from_user = Account._byID(message.author_id)
to_user = Account._byID(message.to_id)
add_queries([get_sent(from_user)], insert_item = message)
add_queries([get_inbox_messages(to_user)], insert_item = inbox_rel)
add_queries([get_sent(from_user)], insert_items = message)
add_queries([get_inbox_messages(to_user)], insert_items = inbox_rel)
def new_savehide(rel):
user = rel._thing1
name = rel._name
if name == 'save':
add_queries([get_saved(user)], insert_item = rel)
add_queries([get_saved(user)], insert_items = rel)
elif name == 'unsave':
add_queries([get_saved(user)], delete_item = rel)
add_queries([get_saved(user)], delete_items = rel)
elif name == 'hide':
add_queries([get_hidden(user)], insert_item = rel)
add_queries([get_hidden(user)], insert_items = rel)
elif name == 'unhide':
add_queries([get_hidden(user)], delete_item = rel)
add_queries([get_hidden(user)], delete_items = rel)
def ban(x, auto = False):
if isinstance(x,Link):
sr = Subreddit._byID(x.sr_id)
add_queries([get_spam_links(sr)], insert_item = x)
def _by_srid(things):
"""Takes a list of things and returns them in a dict separated by
sr_id, in addition to the looked-up subreddits"""
ret = {}
#elif isinstance(x,Comment):
# sr = Subreddit._byID(x.sr_id)
# add_queries([get_spam_comments(sr)])
for thing in things:
if hasattr(thing, 'sr_id'):
ret.setdefault(thing.sr_id, []).append(thing)
def unban(x):
if isinstance(x,Link):
sr = Subreddit._byID(x.sr_id)
add_queries([get_spam_links(sr)], delete_item = x)
#elif isinstance(x,Comment):
# sr = Subreddit._byID(x.sr_id)
# add_queries([get_spam_comments(sr)])
srs = Subreddit._byID(ret.keys(), return_dict=True) if ret else {}
def new_report(report):
reporter = report._thing1
reported = report._thing2
return ret, srs
if isinstance(reported, Link):
sr = Subreddit._byID(reported.sr_id)
add_queries([get_reported_links(sr)], insert_item = reported)
#elif isinstance(reported, Comment):
# sr = Subreddit._byID(reported.sr_id)
# add_queries([get_reported_comments(sr)], insert_item = reported)
def ban(things):
by_srid, srs = _by_srid(things)
if not by_srid:
return
def clear_report(report):
reporter = report._thing1
reported = report._thing2
for sr_id, things in by_srid.iteritems():
sr = srs[sr_id]
links = [ x for x in things if isinstance(x, Link) ]
#comments = [ x for x in things if isinstance(x, Comment) ]
if isinstance(reported, Link):
sr = Subreddit._byID(reported.sr_id)
add_queries([get_reported_links(sr)], delete_item = reported)
#elif isinstance(reported, Comment):
# sr = Subreddit._byID(reported.sr_id)
# add_queries([get_reported_comments(sr)], delete_item = reported)
if links:
add_queries([get_spam_links(sr)], insert_items = links)
add_queries([get_links(sr, 'new', 'all')], delete_items = links)
#if comments:
# add_queries([get_spam_comments(sr)], insert_items = comments)
def unban(things):
by_srid, srs = _by_srid(things)
if not by_srid:
return
for sr_id, things in by_srid.iteritems():
sr = srs[sr_id]
links = [ x for x in things if isinstance(x, Link) ]
#comments = [ x for x in things if isinstance(x, Comment) ]
if links:
add_queries([get_spam_links(sr)], delete_items = links)
# put it back in 'new'
add_queries([get_links(sr, 'new', 'all')], insert_items = links)
#if comments:
# add_queries([get_spam_comments(sr)], delete_items = comments)
def new_report(thing):
if isinstance(thing, Link):
sr = Subreddit._byID(thing.sr_id)
add_queries([get_reported_links(sr)], insert_items = thing)
#elif isinstance(thing, Comment):
# sr = Subreddit._byID(thing.sr_id)
# add_queries([get_reported_comments(sr)], insert_items = thing)
def clear_reports(things):
by_srid, srs = _by_srid(things)
if not by_srid:
return
for sr_id, sr_things in by_srid.iteritems():
sr = srs[sr_id]
links = [ x for x in sr_things if isinstance(x, Link) ]
#comments = [ x for x in sr_things if isinstance(x, Comment) ]
if links:
add_queries([get_reported_links(sr)], delete_items = links)
#if comments:
# add_queries([get_reported_comments(sr)], delete_items = comments)
def add_all_ban_report_srs():
"""Adds the initial spam/reported pages to the report queue"""
@@ -477,7 +511,6 @@ def add_all_srs():
#get_reported_comments(sr),
])
def update_user(user):
if isinstance(user, str):
user = Account._by_name(user)

4
r2/r2/lib/db/thing.py
View File

@@ -612,8 +612,8 @@ def Relation(type1, type2, denorm1 = None, denorm2 = None):
and caches them"""
prefix = thing_prefix(cls.__name__)
thing1_dict = dict((t._id, t) for t in thing1s)
thing2_dict = dict((t._id, t) for t in thing2s)
thing1_dict = dict((t._id, t) for t in tup(thing1s))
thing2_dict = dict((t._id, t) for t in tup(thing2s))
thing1_ids = thing1_dict.keys()
thing2_ids = thing2_dict.keys()

12
r2/r2/lib/utils/utils.py
View File

@@ -845,7 +845,7 @@ def fetch_things(t_class,since,until,batch_fn=None,
q._after(t)
things = list(q)
def fetch_things2(query, chunk_size = 100, batch_fn = None):
def fetch_things2(query, chunk_size = 100, batch_fn = None, chunks = False):
"""Incrementally run query with a limit of chunk_size until there are
no results left. batch_fn transforms the results for each chunk
before returning."""
@@ -861,12 +861,16 @@ def fetch_things2(query, chunk_size = 100, batch_fn = None):
if batch_fn:
items = batch_fn(items)
for i in items:
yield i
if chunks:
yield items
else:
for i in items:
yield i
after = items[-1]
if not done:
query._rules = deepcopy(orig_rules)
query._after(i)
query._after(after)
items = list(query)
def set_emptying_cache():

64
r2/r2/models/admintools.py
View File

@@ -20,22 +20,62 @@
# CondeNet, Inc. All Rights Reserved.
################################################################################
from r2.lib.utils import tup
from r2.models import Report, Account
from r2.models.thing_changes import changed
from r2.lib.db import queries
from pylons import g
from datetime import datetime
from copy import copy
class AdminTools(object):
def spam(self, things, amount = 1, mark_as_spam = True, **kw):
for t in tup(things):
if mark_as_spam:
t._spam = (amount > 0)
t._commit()
def spam(self, things, auto, moderator_banned, banner, date = None, **kw):
Report.accept(things, True)
things = [ x for x in tup(things) if not x._spam ]
for t in things:
t._spam = True
ban_info = copy(getattr(t, 'ban_info', {}))
ban_info.update(auto = auto,
moderator_banned = moderator_banned,
banner = banner,
banned_at = date or datetime.now(g.tz),
**kw)
t.ban_info = ban_info
t._commit()
changed(t)
self.author_spammer(things, True)
queries.ban(things)
def report(self, thing, amount = 1):
pass
def unspam(self, things, unbanner = None):
Report.accept(things, False)
things = [ x for x in tup(things) if x._spam ]
for t in things:
ban_info = copy(getattr(t, 'ban_info', {}))
ban_info['unbanned_at'] = datetime.now(g.tz)
if unbanner:
ban_info['unbanner'] = unbanner
t.ban_info = ban_info
t._spam = False
t._commit()
changed(t)
self.author_spammer(things, False)
queries.unban(things)
def ban_info(self, thing):
return {}
def author_spammer(self, things, spam):
"""incr/decr the 'spammer' field for the author of every
passed thing"""
by_aid = {}
for thing in things:
if hasattr(thing, 'author_id'):
by_aid.setdefault(thing.author_id, []).append(thing)
def get_corrections(self, cls, min_date = None, max_date = None, limit = 50):
return []
if by_aid:
authors = Account._byID(by_aid.keys(), data=True, return_dict=True)
for aid, author_things in by_aid.iteritems():
author = authors[aid]
author._incr('spammer', len(author_things) if spam else -len(author_things))
admintools = AdminTools()
@@ -62,5 +102,5 @@ def compute_votes(wrapper, item):
try:
from r2admin.models.admintools import *
except:
except ImportError:
pass

28
r2/r2/models/builder.py
View File

@@ -117,22 +117,9 @@ class Builder(object):
#get likes/dislikes
#TODO Vote.likes should accept empty lists
likes = Vote.likes(user, items) if user and items else {}
reports = Report.fastreported(user, items) if user else {}
uid = user._id if user else None
# we'll be grabbing this in the spam processing below
if c.user_is_admin:
ban_info = admintools.ban_info([x for x in items if x._spam])
elif user and len(can_ban_set) > 0:
ban_info = admintools.ban_info(
[ x for x in items
if (x._spam
and hasattr(x,'sr_id')
and x.sr_id in can_ban_set) ])
else:
ban_info = dict()
types = {}
wrapped = []
count = 0
@@ -227,12 +214,6 @@ class Builder(object):
count += 1
# would have called it "reported", but that is already
# taken on the thing itself as "how many total
# reports". Indicates whether this user reported this
# item, and should be visible to all users
w.report_made = reports.get((user, item, Report._field))
# if the user can ban things on a given subreddit, or an
# admin, then allow them to see that the item is spam, and
# add the other spam-related display attributes
@@ -246,11 +227,12 @@ class Builder(object):
w.can_ban = True
if item._spam:
w.show_spam = True
w.moderator_banned = getattr(item,'moderator_banned', False)
w.autobanned, w.banner = ban_info.get(item._fullname,
(False, None))
ban_info = getattr(item, 'ban_info', {})
w.moderator_banned = ban_info.get('moderator_banned', False)
w.autobanned = ban_info.get('auto', False)
w.banner = ban_info.get('banner')
elif hasattr(item,'reported') and item.reported > 0:
elif getattr(item, 'reported', 0) > 0:
w.show_reports = True
# recache the user object: it may be None if user is not logged in,

2
r2/r2/models/link.py
View File

@@ -127,6 +127,8 @@ class Link(Thing, Printable):
ip = ip)
l._commit()
l.set_url_cache()
if author._spam:
admintools.spam(l, True, False, 'banned user')
return l
@classmethod

481
r2/r2/models/report.py
View File

@@ -19,25 +19,13 @@
# All portions of the code written by CondeNet are Copyright (c) 2006-2009
# CondeNet, Inc. All Rights Reserved.
################################################################################
import sqlalchemy as sa
from r2.lib.db import tdb_sql as tdb
from r2.lib.db.operators import desc
from r2.lib.db.thing import Thing, Relation, NotFound, MultiRelation,\
thing_prefix
from r2.lib.utils import tup, Storage
from link import Link, Comment, Message, Subreddit
from account import Account
from vote import score_changes
from r2.lib.db.thing import Thing, Relation, MultiRelation, thing_prefix
from r2.lib.utils import tup
from r2.lib.memoize import memoize
from r2.models import Link, Comment, Message, Subreddit, Account
from r2.models.vote import score_changes
from r2.config import cache
from r2.lib.cache import sgm
import datetime
import thing_changes as tc
from admintools import admintools
from pylons import g
class Report(MultiRelation('report',
Relation(Account, Link),
@@ -47,453 +35,78 @@ class Report(MultiRelation('report',
)):
_field = 'reported'
@property
def _user(self): return self._thing1
@property
def _thing(self): return self._thing2
@classmethod
def new(cls, user, thing):
from r2.lib.db import queries
# check if this report exists already!
rel = cls.rel(user, thing)
oldreport = list(rel._query(rel.c._thing1_id == user._id,
rel.c._thing2_id == thing._id,
data = True))
q = rel._fast_query(user, thing, ['-1', '0', '1'])
q = [ report for (tupl, report) in q.iteritems() if report ]
if q:
# stop if we've seen this before, so that we never get the
# same report from the same user twice
oldreport = q[0]
g.log.debug("Ignoring duplicate report %s" % oldreport)
return oldreport
# stop if we've seen this before, so that we never get the
# same report from the same user twice
if oldreport: return oldreport[0]
r = Report(user, thing, '0', amount = 0)
r = Report(user, thing, '0')
if not thing._loaded:
thing._load()
# mark item as reported
thing._incr(cls._field)
# mark author as reported
if hasattr(thing, 'author_id'):
aid = thing.author_id
author = Account._byID(aid)
author._incr(cls._field)
# mark user as having made a report
user._incr('report_made')
r._commit()
admintools.report(thing)
if hasattr(thing, 'author_id'):
author = Account._byID(thing.author_id, data=True)
author._incr('reported')
# update the reports queue if it exists
queries.new_report(r)
queries.new_report(thing)
# if the thing is already marked as spam, accept the report
if thing._spam:
cls.accept(r)
else:
# set the report amount to 0, updating the cache in the process
cls.set_amount(r, 0)
cls.accept(thing)
return r
@classmethod
def for_thing(cls, thing):
rel = cls.rel(Account, thing.__class__)
rels = rel._query(rel.c._thing2_id == thing._id)
return list(rels)
@classmethod
def set_amount(cls, r, amount):
old_amount = int(r._name)
if old_amount != amount:
r._name = str(amount)
r._commit()
#update the cache for the amount = 0 and amount = None cases
rel = cls.rels[(r._thing1.__class__, r._thing2.__class__)]
for a in set((old_amount, amount, None)):
# clear memoizing around this thing's author
if not r._thing2._loaded: r._thing2._load()
if hasattr(r._thing2, "author_id"):
cls._by_author(r._thing2, amount = a, _update = True)
for t in (r._thing1, r._thing2):
thing_key = cls._cache_prefix(rel, t.__class__,
amount = a) + str(t._id)
v = cache.get(thing_key)
if v is not None:
if a == old_amount and old_amount != amount and r._id in v:
v.remove(r._id)
elif r._id not in v:
v.append(r._id)
cache.set(thing_key, v)
@classmethod
def accept(cls, r, correct = True):
''' sets the various reporting fields, but does nothing to
the corresponding spam fields (handled by unreport)'''
def accept(cls, things, correct = True):
from r2.lib.db import queries
amount = 1 if correct else -1
oldamount = int(r._name)
things = tup(things)
# do nothing if nothing has changed
if amount == oldamount: return
things_by_cls = {}
for thing in things:
things_by_cls.setdefault(thing.__class__, []).append(thing)
up_change, down_change = score_changes(amount, oldamount)
# update the user who made the report
r._thing1._incr('report_correct', up_change)
r._thing1._incr('report_ignored', down_change)
to_clear = []
# update the amount
cls.set_amount(r, amount)
for thing_cls, cls_things in things_by_cls.iteritems():
# look up all of the reports for each thing
rel_cls = cls.rel(Account, thing_cls)
rels = rel_cls._query(rel_cls.c._thing2_id == [ x._id for x in cls_things ],
rel_cls.c._name == '0')
for r in rels:
r._name = '1' if correct else '-1'
r._commit()
queries.clear_report(r)
for thing in cls_things:
if thing.reported > 0:
thing.reported = 0
thing._commit()
to_clear.append(thing)
# update the thing's number of reports only if we made no
# decision prior to this
if oldamount == 0:
# update the author and thing field
if getattr(r._thing2, Report._field) > 0:
r._thing2._incr(Report._field, -1)
if hasattr(r._thing2, "author_id"):
aid = r._thing2.author_id
author = Account._byID(aid)
if getattr(author, Report._field) > 0:
author._incr(Report._field, -1)
if to_clear:
queries.clear_reports(to_clear)
admintools.report(r._thing2, -1)
@classmethod
@memoize('report._by_author')
def _by_author_cache(cls, author_id, amount = None):
res = {}
for types, rel in cls.rels.iteritems():
# grab the proper thing table
thing_type = types[0]
table, dtable = tdb.get_thing_table(thing_type._type_id)
# and the proper relationship table
tables = tdb.get_rel_table(rel._type_id)
rel_table, rel_dtable = tables[0], tables[3]
where = [dtable.c.key == 'author_id',
sa.func.substring(dtable.c.value, 1, 1000) == str(author_id),
dtable.c.thing_id == rel_table.c.thing2_id]
if amount is not None:
where.extend([rel_table.c.name == str(amount),
rel_table.c.rel_id == rel_dtable.c.thing_id])
s = sa.select([rel_table.c.rel_id],
sa.and_(*where))
rids = [x[0] for x in s.execute().fetchall()]
if rids: res[types] = rids
return res
@classmethod
def _by_author(cls, author, amount = None, _update = False):
res = []
rdict = cls._by_author_cache(author._id, amount = amount,
_update = _update)
for types, rids in rdict.iteritems():
res.extend(cls.rels[types]._byID(rids, data=True,
return_dict = False))
return res
@classmethod
def fastreported(cls, users, things, amount = None):
if amount is None:
amount = ('1', '0', '-1')
res = cls._fast_query(users, things, amount)
res = dict((tuple(k[:2]), v) for k, v in res.iteritems() if v)
return res
@classmethod
def reported(cls, users = None, things = None,
return_dict=True, amount = None):
# nothing given, nothing to give back
if not users and not things:
return {} if return_dict else []
if users: users = tup(users)
if things: things = tup(things)
# if both are given, we can use fast_query
if users and things:
return cls.fastreported(users, things)
# type_dict stores id keyed on (type, rel_key)
type_dict = {}
# if users, we have to search all the rel types on thing1_id
if users:
db_key = '_thing1_id'
uid = [t._id for t in users]
for key in cls.rels.keys():
type_dict[(Account, key)] = uid
# if things, we have to search only on types present in the list
if things:
db_key = '_thing2_id'
for t in things:
key = (t.__class__, (Account, t.__class__))
type_dict.setdefault(key, []).append(t._id)
def db_func(rel, db_key, amount):
def _db_func(ids):
q = rel._query(getattr(rel.c, db_key) == ids,
data = True)
if amount is not None:
q._filter(rel.c._name == str(amount))
r_ids = {}
# fill up the report listing from the query
for r in q:
key = getattr(r, db_key)
r_ids.setdefault(key, []).append(r._id)
# add blanks where no results were returned
for i in ids:
if i not in r_ids:
r_ids[i] = []
return r_ids
return _db_func
rval = []
for (thing_class, rel_key), ids in type_dict.iteritems():
rel = cls.rels[rel_key]
prefix = cls._cache_prefix(rel, thing_class, amount=amount)
# load from cache
res = sgm(cache, ids, db_func(rel, db_key, amount), prefix)
# append *objects* to end of list
res1 = []
for x in res.values(): res1.extend(x)
if res1:
rval.extend(rel._byID(res1, data=True, return_dict=False))
if return_dict:
return dict(((r._thing1, r._thing2, cls._field), r) for r in rval)
return rval
@classmethod
def _cache_prefix(cls, rel, t_class, amount = None):
# encode the amount keyword on the prefix
prefix = thing_prefix(rel.__name__) + '_' + \
thing_prefix(t_class.__name__)
if amount is not None:
prefix += ("_amount_%d" % amount)
return prefix
@classmethod
def get_reported_authors(cls, time = None, sort = None):
reports = {}
for t_cls in (Link, Comment, Message):
q = t_cls._query(t_cls.c._spam == False,
t_cls.c.reported != 0,
data = True)
q._sort = desc("_date")
if time:
q._filter(time)
reports.update(Report.reported(things = list(q), amount = 0))
# at this point, we have a full list of reports made on the interval specified
# build up an author to report list
authors = Account._byID([k[1].author_id
for k, v in reports.iteritems()],
data = True) if reports else []
# and build up a report on each author
author_rep = {}
for (tattler, thing, amount), r in reports.iteritems():
aid = thing.author_id
if not author_rep.get(aid):
author_rep[aid] = Storage(author = authors[aid])
author_rep[aid].num_reports = 1
author_rep[aid].acct_correct = tattler.report_correct
author_rep[aid].acct_wrong = tattler.report_ignored
author_rep[aid].most_recent = r._date
author_rep[aid].reporters = set([tattler])
else:
author_rep[aid].num_reports += 1
author_rep[aid].acct_correct += tattler.report_correct
author_rep[aid].acct_wrong += tattler.report_ignored
if author_rep[aid].most_recent < r._date:
author_rep[aid].most_recent = r._date
author_rep[aid].reporters.add(tattler)
authors = author_rep.values()
if sort == "hot":
def report_hotness(a):
return a.acct_correct / max(a.acct_wrong + a.acct_correct,1)
def better_reporter(a, b):
q = report_hotness(b) - report_hotness(a)
if q == 0:
return b.acct_correct - a.acct_correct
else:
return 1 if q > 0 else -1
authors.sort(better_reporter)
if sort == "top":
authors.sort(lambda x, y: y.num_reports - x.num_reports)
elif sort == "new":
def newer_reporter(a, b):
t = b.most_recent - a.most_recent
t0 = datetime.timedelta(0)
return 1 if t > t0 else -1 if t < t0 else 0
authors.sort(newer_reporter)
return authors
@classmethod
def get_reporters(cls, time = None, sort = None):
query = cls._query(cls.c._name == '0', eager_load = False,
data = False, thing_data = False)
if time:
query._filter(time)
query._sort = desc("_date")
account_dict = {}
min_report_time = {}
for r in query:
account_dict[r._thing1_id] = account_dict.get(r._thing1_id, 0) + 1
if min_report_time.get(r._thing1_id):
min_report_time[r._thing1_id] = min(min_report_time[r._thing1_id], r._date)
else:
min_report_time[r._thing1_id] = r._date
# grab users in chunks of 50
c_size = 50
accounts = account_dict.keys()
accounts = [Account._byID(accounts[i:i+c_size], return_dict = False, data = True)
for i in xrange(0, len(accounts), c_size)]
accts = []
for a in accounts:
accts.extend(a)
if sort == "hot" or sort == "top":
def report_hotness(a):
return a.report_correct / max(a.report_ignored + a.report_correct,1)
def better_reporter(a, b):
q = report_hotness(b) - report_hotness(a)
if q == 0:
return b.report_correct - a.report_correct
else:
return 1 if q > 0 else -1
accts.sort(better_reporter)
elif sort == "new":
def newer_reporter(a, b):
t = (min_report_time[b._id] - min_report_time[a._id])
t0 = datetime.timedelta(0)
return 1 if t > t0 else -1 if t < t0 else 0
accts.sort(newer_reporter)
return accts
def unreport(things, correct=False, auto = False, banned_by = ''):
from r2.lib.db import queries
things = tup(things)
# load authors (to set the spammer flag)
aids = set(t.author_id for t in things
if hasattr(t, 'author_id'))
authors = Account._byID(tuple(aids), data=True) if aids else {}
with_srs = [ t for t in things if hasattr(t, 'sr_id') ]
if with_srs:
# populate local cache in batch, because
# queries.{ban,unban,new_report,clear_report} will do a
# Subreddit._byID(link.sr_id) on each link individually. Not
# required for Messages or Subreddits, which don't have sr_ids
Subreddit._byID(set(t.sr_id for t in with_srs))
# load all reports (to set their amount to be +/-1)
reports = Report.reported(things=things, amount = 0)
# mark the reports as finalized:
for r in reports.values():
Report.accept(r, correct)
amount = 1 if correct else -1
spammer = {}
for t in things:
# clean up inconsistencies
if getattr(t, Report._field) != 0:
setattr(t, Report._field, 0)
t._commit()
# flag search indexer that something has changed
tc.changed(t)
# update the spam flag
if t._spam != correct and hasattr(t, 'author_id'):
# tally the spamminess of the author
spammer[t.author_id] = spammer.get(t.author_id,0) + amount
if correct:
queries.ban(t, auto = auto)
elif not correct and t._spam:
queries.unban(t)
#will be empty if the items didn't have authors
for s, v in spammer.iteritems():
if authors[s].spammer + v >= 0:
authors[s]._incr('spammer', v)
# mark all as spam
admintools.spam(things, amount = amount, auto = auto, banned_by = banned_by)
def unreport_account(user, correct = True, types = (Link, Comment, Message),
auto = False, banned_by = ''):
for typ in types:
table, dtable = tdb.get_thing_table(typ._type_id)
by_user_query = sa.and_(table.c.thing_id == dtable.c.thing_id,
dtable.c.key == 'author_id',
sa.func.substring(dtable.c.value, 1, 1000) == str(user._id))
s = sa.select(["count(*)"],
sa.and_(by_user_query, table.c.spam == (not correct)))
# update the author's spamminess
count = s.execute().fetchone()[0] * (1 if correct else -1)
if user.spammer + count >= 0:
user._incr('spammer', count)
things= list(typ._query(typ.c.author_id == user._id,
typ.c._spam == (not correct),
data = False, limit=300))
admintools.spam(things, amount = 1 if correct else -1,
mark_as_spam = False,
auto = auto, banned_by = banned_by)
u = """UPDATE %(table)s SET spam='%(spam)s' FROM %(dtable)s
WHERE %(table)s.thing_id = %(dtable)s.thing_id
AND %(dtable)s.key = 'author_id'
AND substring(%(dtable)s.value, 1, 1000) = '%(author_id)s'"""
u = u % dict(spam = 't' if correct else 'f',
table = table.name,
dtable = dtable.name,
author_id = user._id)
table.bind.execute(u)
# grab a list of all the things we just blew away and update the cache
s = sa.select([table.c.thing_id], by_user_query)
tids = [t[0] for t in s.execute().fetchall()]
keys = [thing_prefix(typ.__name__, i) for i in tids]
cache.delete_multi(keys)
# mark the reports as finalized:
reports = Report._by_author(user, amount = 0)
for r in reports: Report.accept(r, correct)