diff options
| -rw-r--r-- | app/controllers/general_controller.rb | 10 | ||||
| -rw-r--r-- | app/controllers/request_controller.rb | 12 | ||||
| -rw-r--r-- | todo.txt | 7 |
3 files changed, 7 insertions, 22 deletions
diff --git a/app/controllers/general_controller.rb b/app/controllers/general_controller.rb index 6d2151861..9041fd12e 100644 --- a/app/controllers/general_controller.rb +++ b/app/controllers/general_controller.rb @@ -5,7 +5,7 @@ # Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: general_controller.rb,v 1.53 2009-09-04 13:44:39 francis Exp $ +# $Id: general_controller.rb,v 1.54 2009-09-09 00:03:09 francis Exp $ class GeneralController < ApplicationController @@ -114,14 +114,6 @@ class GeneralController < ApplicationController @track_thing = TrackThing.create_track_for_search_query(@query) @feed_autodetect = [ { :url => do_track_url(@track_thing, 'feed'), :title => @track_thing.params[:title_in_rss] } ] - - # No point bots crawling all the pages of search results. - @no_crawl = true - - # If we came from the front page (@bodies is true) and found no bodies - #if @bodies && @xapian_bodies.results.size == 0 - # flash[:notice] = 'No authorities found with that name. <a href="/body/list/other">Browse all</a> or <a href="/help/about#missing_body">ask us to add one</a>.' - #end end # For debugging diff --git a/app/controllers/request_controller.rb b/app/controllers/request_controller.rb index 6fa61002b..5d17e4f5c 100644 --- a/app/controllers/request_controller.rb +++ b/app/controllers/request_controller.rb @@ -4,7 +4,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: request_controller.rb,v 1.177 2009-09-08 22:48:55 francis Exp $ +# $Id: request_controller.rb,v 1.178 2009-09-09 00:03:10 francis Exp $ class RequestController < ApplicationController @@ -73,11 +73,6 @@ class RequestController < ApplicationController @xapian_object = ::ActsAsXapian::Similar.new([InfoRequestEvent], @info_request.info_request_events, :offset => (@page - 1) * @per_page, :limit => @per_page, :collapse_by_prefix => 'request_collapse') - # Stop robots crawling similar request lists. There is no point them - # doing so. Google bot was going dozens of pages in, and they are slow - # pages to generate, having an impact on server load. - @no_crawl = true - if (@page > 1) @page_desc = " (page " + @page.to_s + ")" else @@ -111,6 +106,11 @@ class RequestController < ApplicationController @feed_autodetect = [ { :url => do_track_url(@track_thing, 'feed'), :title => @track_thing.params[:title_in_rss] } ] + # Don't let robots go more than 20 pages in + if @page > 20 + @no_crawl = true + end + cache_in_squid end @@ -5,18 +5,11 @@ And email admins Things to make bots not crawl: /request/13683/response?internal_review=1 /request/febrile_neutropenia_154?unfold=1 - /list/recent?page=671 Next (things that will reduce admin time mainly) ==== -defunct/not_apply stuff -* When you do reply / follow up to authority that is defunct/not_apply mention it -http://www.whatdotheyknow.com/request/14960/response/39838#followup -* Fix up text when you send follow up and the authority no longer exists -http://www.whatdotheyknow.com/request/14960/response#followup - Have proper house rules / site terms and conditions page. Perhaps borrow from these house rules: http://www.theyworkforyou.com/houserules/ |