diff options
| -rw-r--r-- | public/robots.txt | 23 |
1 files changed, 12 insertions, 11 deletions
diff --git a/public/robots.txt b/public/robots.txt index 6a8628c93..a187d0775 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -13,17 +13,18 @@ # http://help.yahoo.com/l/us/yahoo/search/webcrawler/slurp-02.html User-agent: * -Disallow: /annotate/ -Disallow: /new/ -Disallow: /search/ -Disallow: /similar/ -Disallow: /track/ -Disallow: /upload/ -Disallow: /user/contact/ -Disallow: /feed/ -Disallow: /profile/ -Disallow: /signin -Disallow: /body/*/view_email$ +Disallow: */annotate/ +Disallow: */new/ +Disallow: */search/ +Disallow: */similar/ +Disallow: */track/ +Disallow: */upload/ +Disallow: */user/contact/ +Disallow: */feed/ +Disallow: */profile/ +Disallow: */signin +Disallow: */request/*/response/ +Disallow: */body/*/view_email$ # The following adding Jan 2012 to stop robots crawling pages # generated in error (see |