adding support for robots.txt

r2/r2/config/rewrites.py

@@ -28,6 +28,7 @@ rewrites = (#these first two rules prevent the .embed rewrite from
             #This next rewrite makes it so that all the embed stuff works.
             ("^(.*)(?<!button)(\.js)$", "$1.embed"),
             ("^/favicon.ico$", "/static/favicon.ico"),
+            ("^/robots.txt$", "/static/robots.txt"),
             ("^/akamai-sureroute-test-object.html$", "/static/sureroute.html"),
             ("^/apple-touch-icon.png$", "/static/apple-touch-icon.png"))
 

r2/r2/public/static/robots.txt

@@ -0,0 +1,5 @@
+User-Agent: *
+Disallow: /goto
+Disallow: /*after=*
+Disallow: /*before=*
+Allow: /