From 1f951aadc021dd2c3448fbbe70098ab5bfd9b80d Mon Sep 17 00:00:00 2001
From: Neil Williams <neil@reddit.com>
Date: Wed, 30 May 2012 22:44:13 -0700
Subject: [PATCH] Gracefully degrade when cassarel lookups fail.

The relation lookups currently do a multiget that will likely end up
hitting every node in the ring, until this can be fixed they are very
susceptible to load spikes. Rather than losing the whole request and
returning a 500/YBR to the user, if one of these lookups fail, we'll
just plow on.
---
 r2/r2/lib/db/tdb_cassandra.py | 3 +++
 r2/r2/models/builder.py       | 8 ++++++--
 r2/r2/models/link.py          | 9 +++++++--
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/r2/r2/lib/db/tdb_cassandra.py b/r2/r2/lib/db/tdb_cassandra.py
index 5492f18f2..4fa61fbe9 100644
--- a/r2/r2/lib/db/tdb_cassandra.py
+++ b/r2/r2/lib/db/tdb_cassandra.py
@@ -28,6 +28,7 @@ from socket import gethostbyaddr
 from pylons import g
 
 from pycassa import ColumnFamily
+from pycassa.pool import MaximumRetryException
 from pycassa.cassandra.ttypes import ConsistencyLevel, NotFoundException
 from pycassa.system_manager import SystemManager, UTF8_TYPE, COUNTER_COLUMN_TYPE, TIME_UUID_TYPE
 from pycassa.types import DateType
@@ -54,6 +55,8 @@ db_create_tables = g.db_create_tables
 
 thing_types = {}
 
+TRANSIENT_EXCEPTIONS = (MaximumRetryException,)
+
 # The available consistency levels
 CL = Storage(ANY    = ConsistencyLevel.ANY,
              ONE    = ConsistencyLevel.ONE,
diff --git a/r2/r2/models/builder.py b/r2/r2/models/builder.py
index 75e8b822a..bed77e4c1 100755
--- a/r2/r2/models/builder.py
+++ b/r2/r2/models/builder.py
@@ -35,7 +35,7 @@ from r2.lib.comment_tree import user_messages, subreddit_messages
 
 from r2.lib.wrapped import Wrapped
 from r2.lib import utils
-from r2.lib.db import operators
+from r2.lib.db import operators, tdb_cassandra
 from r2.lib.filters import _force_unicode
 from copy import deepcopy
 
@@ -90,7 +90,11 @@ class Builder(object):
                               if sr.can_ban(user))
 
         #get likes/dislikes
-        likes = queries.get_likes(user, items)
+        try:
+            likes = queries.get_likes(user, items)
+        except tdb_cassandra.TRANSIENT_EXCEPTIONS as e:
+            g.log.warning("Cassandra vote lookup failed: %r", e)
+            likes = {}
         uid = user._id if user else None
 
         types = {}
diff --git a/r2/r2/models/link.py b/r2/r2/models/link.py
index 3d91f75d8..10e4a2c84 100755
--- a/r2/r2/models/link.py
+++ b/r2/r2/models/link.py
@@ -338,8 +338,13 @@ class Link(Thing, Printable):
         site = c.site
 
         if user_is_loggedin:
-            saved  = CassandraSave._fast_query(user, wrapped)
-            hidden = CassandraHide._fast_query(user, wrapped)
+            try:
+                saved = CassandraSave._fast_query(user, wrapped)
+                hidden = CassandraHide._fast_query(user, wrapped)
+            except tdb_cassandra.TRANSIENT_EXCEPTIONS as e:
+                g.log.warning("Cassandra save/hide lookup failed: %r", e)
+                saved = hidden = {}
+
             clicked = {}
         else:
             saved = hidden = clicked = {}