aboutsummaryrefslogtreecommitdiffstats
path: root/public/robots.txt
diff options
context:
space:
mode:
Diffstat (limited to 'public/robots.txt')
-rw-r--r--public/robots.txt6
1 files changed, 5 insertions, 1 deletions
diff --git a/public/robots.txt b/public/robots.txt
index 80dfd8ef7..029ae0dbf 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -8,6 +8,10 @@
# Crawl-delay: 1
# http://www.bing.com/community/blogs/webmaster/archive/2009/08/10/crawl-delay-and-the-bing-crawler-msnbot.aspx
+# This file uses the non-standard extension characters * and $, which are supported by Google and Yahoo!
+# http://code.google.com/web/controlcrawlindex/docs/robots_txt.html
+# http://help.yahoo.com/l/us/yahoo/search/webcrawler/slurp-02.html
+
User-agent: *
Disallow: /annotate/
Disallow: /new/
@@ -19,4 +23,4 @@ Disallow: /user/contact/
Disallow: /feed/
Disallow: /profile/
Disallow: /signin
-
+Disallow: /body/*/view_email$