github/reddit
1
1
Fork 0
mirror of https://github.com/reddit-archive/reddit.git synced 2026-01-26 15:28:37 -05:00
Code Issues Packages Projects Releases Wiki Activity

Allow Cassandra models to choose a connection pool.

Browse Source 
Note: some CFs are explicitly marked for the "main" connection
pool and others are not. This is an artifact of the current
state of reddit.com's cassandra setup and is intentional. While
cassandra_default_pool remains "main", this will not cause an issue
for you.
This commit is contained in:
Neil Williams
2011-11-30 15:54:37 -08:00
parent 59fb2a9b89
commit 532be3fe2a
9 changed files with 46 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
r2/example.ini
View File

@@ -122,6 +122,8 @@ cassandra_seeds = 127.0.0.1:9160
# read/write consistency levels for Cassandra
cassandra_rcl = ONE
cassandra_wcl = ONE
# name of default connection pool to use when _connection_pool not specified
cassandra_default_pool = main
# -- url cache options --
url_caches = 127.0.0.1:11211

24
r2/r2/lib/app_globals.py
View File

@@ -217,18 +217,28 @@ class Globals(object):
if not self.cassandra_seeds:
raise ValueError("cassandra_seeds not set in the .ini")
self.cassandra = PycassaConnectionPool('reddit',
server_list = self.cassandra_seeds,
pool_size = len(self.cassandra_seeds),
# TODO: .ini setting
timeout=15, max_retries=3,
prefill=False)
keyspace = "reddit"
self.cassandra_pools = {
"main":
PycassaConnectionPool(
keyspace,
logging_name="main",
server_list=self.cassandra_seeds,
pool_size=len(self.cassandra_seeds),
timeout=2,
max_retries=3,
prefill=False
),
}
perma_memcache = (CMemcache(self.permacache_memcaches, num_clients = num_mc_clients)
if self.permacache_memcaches
else None)
self.permacache = CassandraCacheChain(localcache_cls(),
CassandraCache('permacache',
self.cassandra,
self.cassandra_pools[self.cassandra_default_pool],
read_consistency_level = self.cassandra_rcl,
write_consistency_level = self.cassandra_wcl),
memcache = perma_memcache,

19
r2/r2/lib/db/tdb_cassandra.py
View File

@@ -35,10 +35,11 @@ from uuid import uuid1
from itertools import chain
import cPickle as pickle
cassandra = g.cassandra
thing_cache = g.thing_cache
seeds = g.cassandra_seeds
connection_pools = g.cassandra_pools
default_connection_pool = g.cassandra_default_pool
keyspace = 'reddit'
thing_cache = g.thing_cache
disallow_db_writes = g.disallow_db_writes
tz = g.tz
log = g.log
@@ -93,7 +94,7 @@ def will_write(fn):
return fn(*a, **kw)
return _fn
def get_manager():
def get_manager(seeds):
# n.b. does not retry against multiple servers
server = seeds[0]
return SystemManager(server)
@@ -125,8 +126,12 @@ class ThingMeta(type):
if not getattr(cls, "_write_consistency_level", None):
cls._write_consistency_level = write_consistency_level
pool_name = getattr(cls, "_connection_pool", default_connection_pool)
connection_pool = connection_pools[pool_name]
cassandra_seeds = connection_pool.server_list
try:
cls._cf = ColumnFamily(cassandra,
cls._cf = ColumnFamily(connection_pool,
cf_name,
read_consistency_level = cls._read_consistency_level,
write_consistency_level = cls._write_consistency_level)
@@ -134,7 +139,7 @@ class ThingMeta(type):
if not db_create_tables:
raise
manager = get_manager()
manager = get_manager(cassandra_seeds)
log.warning("Creating Cassandra Column Family %s" % (cf_name,))
with make_lock('cassandra_schema'):
@@ -143,7 +148,7 @@ class ThingMeta(type):
log.warning("Created Cassandra Column Family %s" % (cf_name,))
# try again to look it up
cls._cf = ColumnFamily(cassandra,
cls._cf = ColumnFamily(connection_pool,
cf_name,
read_consistency_level = cls._read_consistency_level,
write_consistency_level = cls._write_consistency_level)

2
r2/r2/lib/subreddit_search.py
View File

@@ -8,7 +8,7 @@ from r2.lib.cache import CL_ONE
class SubredditsByPartialName(tdb_cassandra.View):
_use_db = True
_value_type = 'pickle'
_use_new_ring = True
_connection_pool = 'main'
_read_consistency_level = CL_ONE
def load_all_reddits():

4
r2/r2/models/flair.py
View File

@@ -83,7 +83,7 @@ class FlairTemplate(tdb_cassandra.Thing):
_bool_props = ('text_editable',)
_use_db = True
_use_new_ring = True
_connection_pool = 'main'
@classmethod
def _new(cls, text='', css_class='', text_editable=False):
@@ -137,7 +137,7 @@ class FlairTemplateBySubredditIndex(tdb_cassandra.Thing):
_int_props = ('sr_id',)
_use_db = True
_use_new_ring = True
_connection_pool = 'main'
_key_prefix = 'ft_'

5
r2/r2/models/link.py
View File

@@ -833,6 +833,7 @@ class CommentSortsCache(tdb_cassandra.View):
the candidate order"""
_use_db = True
_value_type = 'float'
_connection_pool = 'main'
_read_consistency_level = tdb_cassandra.CL.ONE
class StarkComment(Comment):
@@ -1175,9 +1176,11 @@ class SimpleRelation(tdb_cassandra.Relation):
except tdb_cassandra.NotFound:
pass
class CassandraSave(SimpleRelation):
_use_db = True
_cf_name = 'Save'
_connection_pool = 'main'
# thing1_cls = Account
# thing2_cls = Link
@@ -1211,6 +1214,7 @@ class CassandraHide(SimpleRelation):
_use_db = True
_cf_name = 'Hide'
_ttl = 7*24*60*60
_connection_pool = 'main'
@classmethod
def _hide(cls, *a, **kw):
@@ -1227,6 +1231,7 @@ class CassandraClick(SimpleRelation):
class SavesByAccount(tdb_cassandra.View):
_use_db = True
_cf_name = 'SavesByAccount'
_connection_pool = 'main'
class Inbox(MultiRelation('inbox',
Relation(Account, Comment),

6
r2/r2/models/oauth2.py
View File

@@ -69,7 +69,7 @@ class OAuth2Client(OAuth2Token):
redirect_uri="",
)
_use_db = True
_use_new_ring = True
_connection_pool = 'main'
@classmethod
def _new(cls, **kwargs):
@@ -90,7 +90,7 @@ class OAuth2AuthorizationCode(OAuth2Token):
_int_props = ("user_id",)
_warn_on_partial_ttl = False
_use_db = True
_use_new_ring = True
_connection_pool = 'main'
@classmethod
def _new(cls, client_id, redirect_uri, user_id, scope):
@@ -127,7 +127,7 @@ class OAuth2AccessToken(OAuth2Token):
)
_int_props = ("user_id",)
_use_db = True
_use_new_ring = True
_connection_pool = 'main'
@classmethod
def _new(cls, user_id, scope):

2
r2/r2/models/subreddit.py
View File

@@ -978,5 +978,5 @@ Subreddit.__bases__ += (UserRel('moderator', SRMember),
class SubredditPopularityByLanguage(tdb_cassandra.View):
_use_db = True
_value_type = 'pickle'
_use_new_ring = True
_connection_pool = 'main'
_read_consistency_level = CL_ONE

3
r2/r2/models/vote.py
View File

@@ -45,6 +45,7 @@ def score_changes(amount, old_amount):
class CassandraVote(tdb_cassandra.Relation):
_use_db = False
_connection_pool = 'main'
_bool_props = ('valid_user', 'valid_thing', 'organic')
_str_props = ('name', # one of '-1', '0', '1'
@@ -84,6 +85,7 @@ class CassandraVote(tdb_cassandra.Relation):
class VotesByLink(tdb_cassandra.View):
_use_db = True
_type_prefix = 'VotesByLink'
_connection_pool = 'main'
# _view_of = CassandraLinkVote
@@ -99,6 +101,7 @@ class VotesByLink(tdb_cassandra.View):
class VotesByDay(tdb_cassandra.View):
_use_db = True
_type_prefix = 'VotesByDay'
_connection_pool = 'main'
# _view_of = CassandraLinkVote