From b40da70feb029f89904f5942993ad7627c87cca9 Mon Sep 17 00:00:00 2001 From: xiongchiamiov Date: Thu, 26 Mar 2015 14:44:30 -0700 Subject: [PATCH] Split uploaded media into separate buckets We've operated for a while with just one set of s3 buckets for storing static media (thumbnails, stylesheets, images uploaded for stylesheets). However, due to the additional-hop nature of image previews through imgix before reaching s3, we're finding it'd be convenient to have imgix-served images in a separate s3 bucket. The media providers now take a *category* on `put()`. We have a mapping from category to bucket names, which will (at least for the time being) remain `s3_media_buckets` for everything other than image previews. --- r2/example.ini | 1 + r2/r2/lib/media.py | 12 ++++++------ r2/r2/lib/providers/media/__init__.py | 1 - r2/r2/lib/providers/media/s3.py | 5 +++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/r2/example.ini b/r2/example.ini index 704d006ab..1431e200f 100644 --- a/r2/example.ini +++ b/r2/example.ini @@ -266,6 +266,7 @@ S3KEY_ID = S3SECRET_KEY = # May be one bucket, or many buckets seperated by commas s3_media_buckets = +s3_image_buckets = # Store direct urls for images, rather than buckets # For the bucket mybucket with the image helloworld.jpg the stored url would be: # true: http://{s3_media_domain}/mybucket/helloworld.jpg diff --git a/r2/r2/lib/media.py b/r2/r2/lib/media.py index f13fd98b9..7baffdf20 100644 --- a/r2/r2/lib/media.py +++ b/r2/r2/lib/media.py @@ -227,7 +227,7 @@ def _filename_from_content(contents): return base64.urlsafe_b64encode(hash_bytes).rstrip("=") -def upload_media(image, file_type='.jpg'): +def upload_media(image, file_type='.jpg', category='thumbs'): """Upload an image to the media provider.""" f = tempfile.NamedTemporaryFile(suffix=file_type, delete=False) try: @@ -258,7 +258,7 @@ def upload_media(image, file_type='.jpg'): optimize_jpeg(f.name) contents = open(f.name).read() file_name = _filename_from_content(contents) + file_type - return g.media_provider.put(file_name, contents) + return g.media_provider.put(category, file_name, contents) finally: os.unlink(f.name) return "" @@ -266,7 +266,7 @@ def upload_media(image, file_type='.jpg'): def upload_stylesheet(content): file_name = _filename_from_content(content) + ".css" - return g.media_provider.put(file_name, content) + return g.media_provider.put('stylesheets', file_name, content) def _scrape_media(url, autoplay=False, maxwidth=600, force=False, @@ -402,7 +402,7 @@ def upload_icon(image_data, size): image.thumbnail(size, Image.ANTIALIAS) icon_data = _image_to_str(image) file_name = _filename_from_content(icon_data) - return g.media_provider.put(file_name + ".png", icon_data) + return g.media_provider.put('icons', file_name + ".png", icon_data) def _make_custom_media_embed(media_object): @@ -515,7 +515,7 @@ class _ThumbnailOnlyScraper(Scraper): uid = _filename_from_content(image_data) image = str_to_image(image_data) - storage_url = upload_media(image) + storage_url = upload_media(image, category='previews') width, height = image.size preview_object = { 'uid': uid, @@ -657,7 +657,7 @@ class _EmbedlyScraper(Scraper): content_type, content = _fetch_url(thumbnail_url, referer=self.url) uid = _filename_from_content(content) image = str_to_image(content) - storage_url = upload_media(image) + storage_url = upload_media(image, category='previews') width, height = image.size preview_object = { 'uid': uid, diff --git a/r2/r2/lib/providers/media/__init__.py b/r2/r2/lib/providers/media/__init__.py index fd43afb95..545c90601 100644 --- a/r2/r2/lib/providers/media/__init__.py +++ b/r2/r2/lib/providers/media/__init__.py @@ -28,7 +28,6 @@ class MediaProvider(object): users to be able to view those objects over HTTP. """ - def make_inaccessible(self, url): """Make the content unavaiable, but do not remove. Content could be recovered at a later time. diff --git a/r2/r2/lib/providers/media/s3.py b/r2/r2/lib/providers/media/s3.py index 9f614aa06..de47dd624 100644 --- a/r2/r2/lib/providers/media/s3.py +++ b/r2/r2/lib/providers/media/s3.py @@ -64,6 +64,7 @@ class S3MediaProvider(MediaProvider): ], ConfigValue.tuple: [ "s3_media_buckets", + "s3_image_buckets", ], } @@ -114,8 +115,8 @@ class S3MediaProvider(MediaProvider): buckets = getattr(g, self.buckets[category]) # choose a bucket based on the filename name_without_extension = os.path.splitext(name)[0] - index = ord(name_without_extension[-1]) % len(g.s3_media_buckets) - bucket_name = g.s3_media_buckets[index] + index = ord(name_without_extension[-1]) % len(buckets) + bucket_name = buckets[index] # guess the mime type mime_type, encoding = mimetypes.guess_type(name)