robots.txt: Disallow /quarantine

Restrict crawlers from the quarantined content gate via robots.txt.

Also add robots=noindex,nofollow meta tag to /quarantine page.

r2/r2/controllers/post.py

@@ -89,10 +89,11 @@ class PostController(ApiController):
     )
     def GET_quarantine(self, dest):
         sr = UrlParser(dest).get_subreddit()
-        return BoringPage(_("opt in to potentially offensive content?"),
+        return BoringPage(
+            _("opt in to potentially offensive content?"),
             content=Quarantine(sr.name),
             show_sidebar=False,
-        ).render()
+            robots='noindex,nofollow').render()
 
     @validate(VModhash(fatal=False),
               over18 = nop('over18'),

r2/r2/templates/robots.txt

@@ -54,5 +54,6 @@ Disallow: /reddits/search
 Disallow: /search
 Disallow: /r/*/search
 Disallow: /over18
+Disallow: /quarantine
 Allow: /