Shard vote_link_qs by last digit of subreddit ID.

This should help with permacache contention.

r2/example.ini

@@ -337,6 +337,10 @@ spreadshirt_test_font =
 # Other magic settings
 ###
 
+# should we split link votes into separate queues based on subreddit id?
+# this helps with lock contention but isn't necessary on smaller sites
+shard_link_vote_queues = false
+
 # list of cnames allowed to render as reddit.com without a frame
 authorized_cnames = 
 

r2/r2/config/queues.py

@@ -71,7 +71,7 @@ class MessageQueue(object):
             self._bind(routing_key)
 
 
-def declare_queues():
+def declare_queues(g):
     queues = Queues({
         "scraper_q": MessageQueue(),
         "newcomments_q": MessageQueue(),
@@ -86,6 +86,12 @@ def declare_queues():
         "update_promos_q": MessageQueue(bind_to_self=True),
     })
 
+    if g.shard_link_vote_queues:
+        sharded_vote_queues = {"vote_link_%d_q" % i :
+                               MessageQueue(bind_to_self=True)
+                               for i in xrange(10)}
+        queues.declare(sharded_vote_queues)
+
     queues.cloudsearch_changes << "search_changes"
     queues.scraper_q << "new_link"
     queues.newcomments_q << "new_comment"

r2/r2/lib/app_globals.py

@@ -148,6 +148,7 @@ class Globals(object):
             'static_pre_gzipped',
             'static_secure_pre_gzipped',
             'trust_local_proxies',
+            'shard_link_vote_queues',
         ],
 
         ConfigValue.tuple: [
@@ -233,7 +234,7 @@ class Globals(object):
         self.config = ConfigValueParser(global_conf)
         self.config.add_spec(self.spec)
         self.plugins = PluginLoader(self.config.get("plugins", []))
-        self.queues = queues.declare_queues()
+        self.queues = queues.declare_queues(self)
 
         self.paths = paths
 

r2/r2/lib/db/queries.py

@@ -1371,7 +1371,10 @@ def queue_vote(user, thing, dir, ip, organic = False,
                 if thing._id36 in g.live_config["fastlane_links"]:
                     qname = vote_fastlane_q
                 else:
-                    qname = vote_link_q
+                    if g.shard_link_vote_queues:
+                        qname = "vote_link_%s_q" % str(thing.sr_id)[-1]
+                    else:
+                        qname = vote_link_q
 
             elif isinstance(thing, Comment):
                 if utils.to36(thing.link_id) in g.live_config["fastlane_links"]: