From 30885089bdac085882d04be416ecd7b140faec78 Mon Sep 17 00:00:00 2001
From: Neil Williams <neil@reddit.com>
Date: Mon, 20 Aug 2012 11:14:22 -0700
Subject: [PATCH] tdb_sql/Thing: Assume data props should be INSERTed on thing
 creation.

Previously, setting data props on a thing would result in a SELECT to
determine which data properties already existed, and then a sequence of
INSERTs and UPDATEs (with some batching) to get the database in the
right state.

The account master wasn't too happy with those extra selects, so we
made it so that the app would optimistically UPDATE and if no rows were
changed do INSERTs instead (with appropriate batching as above).

Unfortunately, this greatly increases the number of operations done when
adding new data props, such as at thing creation time, and on some
tables that's almost entirely all of the workload.

This patch takes advantage of the fact that we know no data attributes
will exist at thing creation time and therefore we can blindly INSERT
without doing UPDATEs or SELECTs first to try and reduce the number of
extraneous operations we do.
---
 r2/r2/lib/db/tdb_sql.py | 35 ++++++++++++++++++++++++++++++-----
 r2/r2/lib/db/thing.py   |  4 ++++
 2 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/r2/r2/lib/db/tdb_sql.py b/r2/r2/lib/db/tdb_sql.py
index 29c885784..0d9c397b7 100644
--- a/r2/r2/lib/db/tdb_sql.py
+++ b/r2/r2/lib/db/tdb_sql.py
@@ -489,7 +489,8 @@ def db2py(val, kind):
 
     return val
 
-def set_data(table, thing_id, **vals):
+
+def update_data(table, thing_id, **vals):
     transactions.add_engine(table.bind)
 
     u = table.update(sa.and_(table.c.thing_id == thing_id,
@@ -510,6 +511,22 @@ def set_data(table, thing_id, **vals):
 
     return len(inserts)
 
+
+def create_data(table, thing_id, **vals):
+    transactions.add_engine(table.bind)
+
+    inserts = []
+    for key, val in vals.iteritems():
+        val, kind = py2db(val, return_kind=True)
+        inserts.append(dict(key=key, value=val, kind=kind))
+
+    if inserts:
+        i = table.insert(values=dict(thing_id=thing_id))
+        i.execute(*inserts)
+
+    return 0
+
+
 def incr_data_prop(table, type_id, thing_id, prop, amount):
     t = table
     transactions.add_engine(t.bind)
@@ -554,9 +571,13 @@ def get_data(table, thing_id):
 
     return res
 
-def set_thing_data(type_id, thing_id, **vals):
+def set_thing_data(type_id, thing_id, brand_new_thing, **vals):
     table = get_thing_table(type_id, action = 'write')[1]
-    return set_data(table, thing_id, **vals)
+
+    if brand_new_thing:
+        return create_data(table, thing_id, **vals)
+    else:
+        return update_data(table, thing_id, **vals)
 
 def incr_thing_data(type_id, thing_id, prop, amount):
     table = get_thing_table(type_id, action = 'write')[1]
@@ -589,9 +610,13 @@ def get_thing(type_id, thing_id):
             res[row.thing_id] = stor
     return res
 
-def set_rel_data(rel_type_id, thing_id, **vals):
+def set_rel_data(rel_type_id, thing_id, brand_new_thing, **vals):
     table = get_rel_table(rel_type_id, action = 'write')[3]
-    return set_data(table, thing_id, **vals)
+
+    if brand_new_thing:
+        return create_data(table, thing_id, **vals)
+    else:
+        return update_data(table, thing_id, **vals)
 
 def incr_rel_data(rel_type_id, thing_id, prop, amount):
     table = get_rel_table(rel_type_id, action = 'write')[3]
diff --git a/r2/r2/lib/db/thing.py b/r2/r2/lib/db/thing.py
index ff74b747e..1a5a2b944 100644
--- a/r2/r2/lib/db/thing.py
+++ b/r2/r2/lib/db/thing.py
@@ -234,6 +234,9 @@ class DataThing(object):
     def _commit(self, keys=None):
         if not self._created:
             self._create()
+            just_created = True
+        else:
+            just_created = False
 
         with g.make_lock('commit_' + self._fullname):
             if not self._sync_latest():
@@ -259,6 +262,7 @@ class DataThing(object):
             if data_props:
                 useless_updates = self._set_data(self._type_id,
                                                  self._id,
+                                                 just_created,
                                                  **data_props)
             else:
                 useless_updates = 0