Gracefully degrade when cassarel lookups fail.

The relation lookups currently do a multiget that will likely end up
hitting every node in the ring, until this can be fixed they are very
susceptible to load spikes. Rather than losing the whole request and
returning a 500/YBR to the user, if one of these lookups fail, we'll
just plow on.

r2/r2/lib/db/tdb_cassandra.py

@@ -28,6 +28,7 @@ from socket import gethostbyaddr
 from pylons import g
 
 from pycassa import ColumnFamily
+from pycassa.pool import MaximumRetryException
 from pycassa.cassandra.ttypes import ConsistencyLevel, NotFoundException
 from pycassa.system_manager import SystemManager, UTF8_TYPE, COUNTER_COLUMN_TYPE, TIME_UUID_TYPE
 from pycassa.types import DateType
@@ -54,6 +55,8 @@ db_create_tables = g.db_create_tables
 
 thing_types = {}
 
+TRANSIENT_EXCEPTIONS = (MaximumRetryException,)
+
 # The available consistency levels
 CL = Storage(ANY    = ConsistencyLevel.ANY,
              ONE    = ConsistencyLevel.ONE,

r2/r2/models/builder.py

@@ -35,7 +35,7 @@ from r2.lib.comment_tree import user_messages, subreddit_messages
 
 from r2.lib.wrapped import Wrapped
 from r2.lib import utils
-from r2.lib.db import operators
+from r2.lib.db import operators, tdb_cassandra
 from r2.lib.filters import _force_unicode
 from copy import deepcopy
 
@@ -90,7 +90,11 @@ class Builder(object):
                               if sr.can_ban(user))
 
         #get likes/dislikes
-        likes = queries.get_likes(user, items)
+        try:
+            likes = queries.get_likes(user, items)
+        except tdb_cassandra.TRANSIENT_EXCEPTIONS as e:
+            g.log.warning("Cassandra vote lookup failed: %r", e)
+            likes = {}
         uid = user._id if user else None
 
         types = {}

r2/r2/models/link.py

@@ -338,8 +338,13 @@ class Link(Thing, Printable):
         site = c.site
 
         if user_is_loggedin:
-            saved  = CassandraSave._fast_query(user, wrapped)
-            hidden = CassandraHide._fast_query(user, wrapped)
+            try:
+                saved = CassandraSave._fast_query(user, wrapped)
+                hidden = CassandraHide._fast_query(user, wrapped)
+            except tdb_cassandra.TRANSIENT_EXCEPTIONS as e:
+                g.log.warning("Cassandra save/hide lookup failed: %r", e)
+                saved = hidden = {}
+
             clicked = {}
         else:
             saved = hidden = clicked = {}