diff options
| -rw-r--r-- | app/controllers/general_controller.rb | 5 | ||||
| -rw-r--r-- | app/controllers/request_controller.rb | 7 | ||||
| -rw-r--r-- | app/views/layouts/default.rhtml | 5 | ||||
| -rw-r--r-- | todo.txt | 2 |
4 files changed, 16 insertions, 3 deletions
diff --git a/app/controllers/general_controller.rb b/app/controllers/general_controller.rb index 6c5424bee..6bd2ed2fb 100644 --- a/app/controllers/general_controller.rb +++ b/app/controllers/general_controller.rb @@ -5,7 +5,7 @@ # Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: general_controller.rb,v 1.34 2008-09-03 00:56:40 francis Exp $ +# $Id: general_controller.rb,v 1.35 2008-09-03 09:03:57 francis Exp $ class GeneralController < ApplicationController @@ -88,6 +88,9 @@ class GeneralController < ApplicationController @track_thing = TrackThing.create_track_for_search_query(query) @feed_autodetect = [ { :url => do_track_url(@track_thing, 'feed'), :title => @track_thing.params[:title_in_rss] } ] + + # No point bots crawling all the pages of search results. + @no_crawl = true end # For debugging diff --git a/app/controllers/request_controller.rb b/app/controllers/request_controller.rb index dd12a91b5..7614aee73 100644 --- a/app/controllers/request_controller.rb +++ b/app/controllers/request_controller.rb @@ -4,7 +4,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: request_controller.rb,v 1.105 2008-09-02 17:59:39 francis Exp $ +# $Id: request_controller.rb,v 1.106 2008-09-03 09:03:57 francis Exp $ class RequestController < ApplicationController @@ -48,6 +48,11 @@ class RequestController < ApplicationController @info_request = InfoRequest.find_by_url_title(params[:url_title]) @xapian_object = ::ActsAsXapian::Similar.new([InfoRequestEvent], @info_request.info_request_events, :offset => (@page - 1) * @per_page, :limit => @per_page, :collapse_by_prefix => 'request_collapse') + + # Stop robots crawling similar request lists. There is no point them + # doing so. Google bot was going dozens of pages in, and they are slow + # pages to generate, having an impact on server load. + @no_crawl = true end def list diff --git a/app/views/layouts/default.rhtml b/app/views/layouts/default.rhtml index 91d0f0d6b..aafc8e57a 100644 --- a/app/views/layouts/default.rhtml +++ b/app/views/layouts/default.rhtml @@ -19,6 +19,11 @@ <link rel="alternate" type="application/atom+xml" title="<%=h feed[:title] %>" href="<%=h feed[:url]%>"> <% end %> <% end %> + + <% if @no_crawl %> + <meta name="robots" content="noindex, nofollow"> + <% end %> + </head> <body> <div id="banner"> @@ -7,7 +7,6 @@ Look for pass by value efficiencies Site move: Install PostgresSQL 8.3 Move database - Move email foibackup to a new server Next ==== @@ -244,6 +243,7 @@ Quoting fixing TODO: http://www.whatdotheyknow.com/request/functions_council_43#incoming-4509 http://www.whatdotheyknow.com/request/york_road_tube_re_opening_feasib#incoming-3509 http://www.whatdotheyknow.com/request/controlled_drinking_zones_5#incoming-4210 + http://www.whatdotheyknow.com/request/road_and_junction_specifications#incoming-3598 http://www.whatdotheyknow.com/request/public_inspection_periods_for_lo_2#outgoing-1707 # square bracket in link |