From 336608366358dd2cfe05650cb0528df62158cc7d Mon Sep 17 00:00:00 2001 From: Neil Williams Date: Fri, 15 Nov 2013 10:34:57 -0800 Subject: [PATCH] Create a vault for secret tokens and move some into it. This is intended to reduce the number of critical secrets stored in the INI file. An initial subset of secrets is moved into the vault to test things out. --- r2/example.ini | 17 +++--- r2/r2/lib/app_globals.py | 25 +++++++++ r2/r2/lib/validator/validator.py | 3 +- r2/r2/models/account.py | 9 +-- scripts/read_secrets | 73 +++++++++++++++++++++++++ scripts/write_secrets | 94 ++++++++++++++++++++++++++++++++ 6 files changed, 208 insertions(+), 13 deletions(-) create mode 100755 scripts/read_secrets create mode 100755 scripts/write_secrets diff --git a/r2/example.ini b/r2/example.ini index 3bdbde42d..fffa39103 100644 --- a/r2/example.ini +++ b/r2/example.ini @@ -7,6 +7,15 @@ # any name will do - e.g., 'foo.update' will create # 'foo.ini') +[secrets] +# the tokens in this section are base64 encoded +# general purpose secret +SECRET = YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXowMTIzNDU2Nzg5 +# secret for /prefs/feeds +FEEDSECRET = YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXowMTIzNDU2Nzg5 +# used for authenticating admin API calls w/o cookie +ADMINSECRET = YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXowMTIzNDU2Nzg5 + # # r2 - Pylons development environment configuration @@ -43,14 +52,6 @@ error_reporters = # the site's tagline, used in the title and description short_description = open source is awesome -# -- SECRETS! <-- update these first! -- -# global secret -SECRET = abcdefghijklmnopqrstuvwxyz0123456789 -# secret for /prefs/feeds -FEEDSECRET = abcdefghijklmnopqrstuvwxyz0123456789 -# used for authenticating admin API calls w/o cookie -ADMINSECRET = abcdefghijklmnopqrstuvwxyz0123456789 - CLOUDSEARCH_SEARCH_API = CLOUDSEARCH_DOC_API = CLOUDSEARCH_SUBREDDIT_SEARCH_API = diff --git a/r2/r2/lib/app_globals.py b/r2/r2/lib/app_globals.py index 62aa429da..65f573215 100755 --- a/r2/r2/lib/app_globals.py +++ b/r2/r2/lib/app_globals.py @@ -22,6 +22,8 @@ from datetime import datetime from urlparse import urlparse + +import base64 import ConfigParser import locale import json @@ -64,7 +66,9 @@ from r2.lib.stats import Stats, CacheStats, StatsCollectingConnectionPool from r2.lib.translation import get_active_langs, I18N_PATH from r2.lib.utils import config_gold_price, thread_dump + LIVE_CONFIG_NODE = "/config/live" +SECRETS_NODE = "/config/secrets" def extract_live_config(config, plugins): @@ -84,6 +88,24 @@ def extract_live_config(config, plugins): return parsed +def _decode_secrets(secrets): + return {key: base64.b64decode(value) for key, value in secrets.iteritems()} + + +def extract_secrets(config): + # similarly to the live_config one above, if we just did + # .options("secrets") we'd get back all the junk from DEFAULT too. bleh. + secrets = config._sections["secrets"].copy() + del secrets["__name__"] # magic value used by ConfigParser + return _decode_secrets(secrets) + + +def fetch_secrets(zk_client): + node_data = zk_client.get(SECRETS_NODE)[0] + secrets = json.loads(node_data) + return _decode_secrets(secrets) + + class Globals(object): spec = { @@ -434,14 +456,17 @@ class Globals(object): self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username, zk_password)) self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE) + self.secrets = fetch_secrets(self.zookeeper) self.throttles = LiveList(self.zookeeper, "/throttles", map_fn=ipaddress.ip_network, reduce_fn=ipaddress.collapse_addresses) else: self.zookeeper = None parser = ConfigParser.RawConfigParser() + parser.optionxform = str parser.read([self.config["__file__"]]) self.live_config = extract_live_config(parser, self.plugins) + self.secrets = extract_secrets(parser) self.throttles = tuple() # immutable since it's not real self.startup_timer.intermediate("zookeeper") diff --git a/r2/r2/lib/validator/validator.py b/r2/r2/lib/validator/validator.py index e2d8bb0bf..233e9130b 100644 --- a/r2/r2/lib/validator/validator.py +++ b/r2/r2/lib/validator/validator.py @@ -879,7 +879,8 @@ def make_or_admin_secret_cls(base_cls): def run(self, secret=None): '''If validation succeeds, return True if the secret was used, False otherwise''' - if secret and constant_time_compare(secret, g.ADMINSECRET): + if secret and constant_time_compare(secret, + g.secrets["ADMINSECRET"]): return True super(VOrAdminSecret, self).run() return False diff --git a/r2/r2/models/account.py b/r2/r2/models/account.py index ace7fbe1b..74c96c96b 100644 --- a/r2/r2/models/account.py +++ b/r2/r2/models/account.py @@ -239,7 +239,7 @@ class Account(Thing): self._load() timestr = timestr or time.strftime(COOKIE_TIMESTAMP_FORMAT) id_time = str(self._id) + ',' + timestr - to_hash = ','.join((id_time, self.password, g.SECRET)) + to_hash = ','.join((id_time, self.password, g.secrets["SECRET"])) return id_time + ',' + hashlib.sha1(to_hash).hexdigest() def make_admin_cookie(self, first_login=None, last_request=None): @@ -248,7 +248,7 @@ class Account(Thing): first_login = first_login or datetime.utcnow().strftime(COOKIE_TIMESTAMP_FORMAT) last_request = last_request or datetime.utcnow().strftime(COOKIE_TIMESTAMP_FORMAT) hashable = ','.join((first_login, last_request, request.ip, request.user_agent, self.password)) - mac = hmac.new(g.SECRET, hashable, hashlib.sha1).hexdigest() + mac = hmac.new(g.secrets["SECRET"], hashable, hashlib.sha1).hexdigest() return ','.join((first_login, last_request, mac)) def make_otp_cookie(self, timestamp=None): @@ -257,7 +257,7 @@ class Account(Thing): timestamp = timestamp or datetime.utcnow().strftime(COOKIE_TIMESTAMP_FORMAT) secrets = [request.user_agent, self.otp_secret, self.password] - signature = hmac.new(g.SECRET, ','.join([timestamp] + secrets), hashlib.sha1).hexdigest() + signature = hmac.new(g.secrets["SECRET"], ','.join([timestamp] + secrets), hashlib.sha1).hexdigest() return ",".join((timestamp, signature)) @@ -694,7 +694,8 @@ def valid_feed(name, feedhash, path): pass def make_feedhash(user, path): - return hashlib.sha1("".join([user.name, user.password, g.FEEDSECRET]) + return hashlib.sha1("".join([user.name, user.password, + g.secrets["FEEDSECRET"]]) ).hexdigest() def make_feedurl(user, path, ext = "rss"): diff --git a/scripts/read_secrets b/scripts/read_secrets new file mode 100755 index 000000000..adfa045f0 --- /dev/null +++ b/scripts/read_secrets @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# The contents of this file are subject to the Common Public Attribution +# License Version 1.0. (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# http://code.reddit.com/LICENSE. The License is based on the Mozilla Public +# License Version 1.1, but Sections 14 and 15 have been added to cover use of +# software over a computer network and provide for limited attribution for the +# Original Developer. In addition, Exhibit A has been modified to be consistent +# with Exhibit B. +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for +# the specific language governing rights and limitations under the License. +# +# The Original Code is reddit. +# +# The Original Developer is the Initial Developer. The Initial Developer of +# the Original Code is reddit Inc. +# +# All portions of the code written by reddit are Copyright (c) 2006-2013 reddit +# Inc. All Rights Reserved. +############################################################################### + +import ConfigParser +import base64 +import cStringIO +import os +import sys + +from r2.lib.utils import parse_ini_file +from r2.lib.zookeeper import connect_to_zookeeper +from r2.lib.app_globals import fetch_secrets + + +def read_secrets_from_zookeeper(config): + zk_hostlist = config.get("DEFAULT", "zookeeper_connection_string") + username = config.get("DEFAULT", "zookeeper_username") + password = config.get("DEFAULT", "zookeeper_password") + + client = connect_to_zookeeper(zk_hostlist, (username, password)) + secrets = fetch_secrets(client) + + ini = ConfigParser.RawConfigParser() + ini.add_section("secrets") + for name, secret in secrets.iteritems(): + ini.set("secrets", name, base64.b64encode(secret)) + + output = cStringIO.StringIO() + ini.write(output) + return output.getvalue() + + +def main(): + progname = os.path.basename(sys.argv[0]) + + try: + ini_file_name = sys.argv[1] + except IndexError: + print >> sys.stderr, "USAGE: %s INI" % progname + return 1 + + try: + with open(ini_file_name) as ini_file: + config = parse_ini_file(ini_file) + except (IOError, ConfigParser.Error), e: + print >> sys.stderr, "%s: %s: %s" % (progname, ini_file_name, e) + return 1 + + print read_secrets_from_zookeeper(config) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/write_secrets b/scripts/write_secrets new file mode 100755 index 000000000..8a6ac853d --- /dev/null +++ b/scripts/write_secrets @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# The contents of this file are subject to the Common Public Attribution +# License Version 1.0. (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# http://code.reddit.com/LICENSE. The License is based on the Mozilla Public +# License Version 1.1, but Sections 14 and 15 have been added to cover use of +# software over a computer network and provide for limited attribution for the +# Original Developer. In addition, Exhibit A has been modified to be consistent +# with Exhibit B. +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for +# the specific language governing rights and limitations under the License. +# +# The Original Code is reddit. +# +# The Original Developer is the Initial Developer. The Initial Developer of +# the Original Code is reddit Inc. +# +# All portions of the code written by reddit are Copyright (c) 2006-2013 reddit +# Inc. All Rights Reserved. +############################################################################### + +import base64 +import ConfigParser +import fileinput +import getpass +import json +import os +import sys + +import kazoo + +from kazoo.security import make_digest_acl + +from r2.lib.utils import parse_ini_file +from r2.lib.zookeeper import connect_to_zookeeper +from r2.lib.app_globals import SECRETS_NODE, extract_secrets + + +USERNAME = "live-config" + + +def _encode_secrets(secrets): + return json.dumps({key: base64.b64encode(secret) + for key, secret in secrets.iteritems()}) + + +def write_secrets_to_zookeeper(reddit_config, username, password, secrets): + # read the zk configuration from the app's config + zk_hostlist = reddit_config.get("DEFAULT", "zookeeper_connection_string") + app_username = reddit_config.get("DEFAULT", "zookeeper_username") + app_password = reddit_config.get("DEFAULT", "zookeeper_password") + + # connect to zk! + client = connect_to_zookeeper(zk_hostlist, (username, password)) + + # we're going to assume that any parent parts of the node path were + # already created by write_live_config. + json_data = _encode_secrets(secrets) + try: + client.create(SECRETS_NODE, json_data, acl=[ + make_digest_acl(username, password, read=True, write=True), + make_digest_acl(app_username, app_password, read=True), + ]) + except kazoo.exceptions.NodeExistsException: + client.set(SECRETS_NODE, json_data) + + +def main(): + progname = os.path.basename(sys.argv[0]) + + input = fileinput.input() + try: + config = parse_ini_file(input) + except (IOError, ConfigParser.Error), e: + print >> sys.stderr, "%s: %s" % (progname, e) + return 1 + + secrets = extract_secrets(config) + password = getpass.getpass("ZooKeeper Password: ") + + try: + write_secrets_to_zookeeper(config, USERNAME, password, secrets) + except kazoo.exceptions.NoAuthException: + print >> sys.stderr, "%s: incorrect password" % progname + return 1 + except Exception as e: + print >> sys.stderr, "%s: %s" % (progname, e) + return 1 + + +if __name__ == "__main__": + sys.exit(main())