diff options
| -rw-r--r-- | app/controllers/general_controller.rb | 20 | ||||
| -rw-r--r-- | app/models/incoming_message.rb | 7 | ||||
| -rw-r--r-- | app/models/info_request.rb | 6 | ||||
| -rw-r--r-- | app/models/outgoing_message.rb | 7 | ||||
| -rw-r--r-- | app/models/public_body.rb | 8 | ||||
| -rw-r--r-- | app/models/user.rb | 7 | ||||
| -rw-r--r-- | app/views/general/search.rhtml | 15 | ||||
| -rw-r--r-- | config/routes.rb | 4 | ||||
| -rwxr-xr-x | script/rebuild-solr-index | 16 | ||||
| -rwxr-xr-x | script/update-solr-index | 6 | ||||
| -rw-r--r-- | todo.txt | 52 | ||||
| -rw-r--r-- | vendor/plugins/acts_as_solr/lib/parser_methods.rb | 8 |
12 files changed, 93 insertions, 63 deletions
diff --git a/app/controllers/general_controller.rb b/app/controllers/general_controller.rb index 95e228957..0920b2eb6 100644 --- a/app/controllers/general_controller.rb +++ b/app/controllers/general_controller.rb @@ -5,7 +5,7 @@ # Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: general_controller.rb,v 1.8 2008-03-10 13:06:53 francis Exp $ +# $Id: general_controller.rb,v 1.9 2008-03-13 11:29:46 francis Exp $ class GeneralController < ApplicationController @@ -50,7 +50,18 @@ class GeneralController < ApplicationController # Actual search def search @per_page = 20 - @query = params[:query].join("/") + @query = params[:query] + @sortby = params[:sortby] + @page = (params[:page] || "1").to_i + + # Work out sorting method + if @sortby.nil? + order = nil + elsif @sortby == 'newest' + order = 'created_at desc' + else + raise "Unknown sort order " + @sortby + end # Used for simpler word highlighting view code for users and public bodies query_nopunc = @query.gsub(/[^a-z0-9]/i, " ") @@ -58,7 +69,7 @@ class GeneralController < ApplicationController @highlight_words = query_nopunc.split(" ") @solr_object = InfoRequest.multi_solr_search(@query, :models => [ OutgoingMessage, IncomingMessage, PublicBody, User ], - :limit => @per_page, :offset => ((params[:page]||"1").to_i-1) * @per_page, + :limit => @per_page, :offset => (@page - 1) * @per_page, :highlight => { :prefix => '<span class="highlight">', :suffix => '</span>', @@ -68,9 +79,10 @@ class GeneralController < ApplicationController "get_text_for_indexing", # IncomingMessage "name", "short_name", # PublicBody "name" # User - ]} + ]}, :order => order ) @search_results = @solr_object.results + @search_hits = @solr_object.total_hits # Extract better Solr highlighting for info request related results @highlighting = @solr_object.highlights diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb index a44cddcfe..f098660bb 100644 --- a/app/models/incoming_message.rb +++ b/app/models/incoming_message.rb @@ -18,7 +18,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: incoming_message.rb,v 1.57 2008-03-12 16:07:13 francis Exp $ +# $Id: incoming_message.rb,v 1.58 2008-03-13 11:29:47 francis Exp $ # TODO @@ -50,7 +50,10 @@ class IncomingMessage < ActiveRecord::Base has_many :outgoing_message_followups, :class_name => OutgoingMessage - acts_as_solr :fields => [ :get_text_for_indexing ], :if => "$do_solr_index" + acts_as_solr :fields => [ + :get_text_for_indexing, + { :created_at => :date } + ], :if => "$do_solr_index" # Return the structured TMail::Mail object # Documentation at http://i.loveruby.net/en/projects/tmail/doc/ diff --git a/app/models/info_request.rb b/app/models/info_request.rb index 793506dfe..3f8a56b1a 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -22,7 +22,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: info_request.rb,v 1.61 2008-03-12 16:07:13 francis Exp $ +# $Id: info_request.rb,v 1.62 2008-03-13 11:29:47 francis Exp $ require 'digest/sha1' @@ -66,8 +66,8 @@ class InfoRequest < ActiveRecord::Base acts_as_solr :fields => [ :title, :initial_request_text, - { :status => :string } -# { :created_at => :date } + { :status => :string }, + { :created_at => :date } ], :if => "$do_solr_index" def status # for name in Solr queries calculate_status diff --git a/app/models/outgoing_message.rb b/app/models/outgoing_message.rb index 3e21a8af7..e12d91e65 100644 --- a/app/models/outgoing_message.rb +++ b/app/models/outgoing_message.rb @@ -21,7 +21,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: outgoing_message.rb,v 1.34 2008-03-12 16:07:13 francis Exp $ +# $Id: outgoing_message.rb,v 1.35 2008-03-13 11:29:47 francis Exp $ class OutgoingMessage < ActiveRecord::Base belongs_to :info_request @@ -32,7 +32,10 @@ class OutgoingMessage < ActiveRecord::Base belongs_to :incoming_message_followup, :foreign_key => 'incoming_message_followup_id', :class_name => 'IncomingMessage' - acts_as_solr :fields => [ :body ], :if => "$do_solr_index" + acts_as_solr :fields => [ + :body, + { :created_at => :date } + ], :if => "$do_solr_index" # How the default letter starts and ends def get_salutation diff --git a/app/models/public_body.rb b/app/models/public_body.rb index 031c1e476..9c5fe6927 100644 --- a/app/models/public_body.rb +++ b/app/models/public_body.rb @@ -22,7 +22,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: public_body.rb,v 1.35 2008-03-12 16:07:13 francis Exp $ +# $Id: public_body.rb,v 1.36 2008-03-13 11:29:47 francis Exp $ require 'csv' require 'set' @@ -59,7 +59,11 @@ class PublicBody < ActiveRecord::Base acts_as_versioned self.non_versioned_columns << 'created_at' << 'updated_at' - acts_as_solr :fields => [{:name => { :boost => 10.0 }}, {:short_name => { :boost => 10.0 }} ] + acts_as_solr :fields => [ + {:name => { :boost => 10.0 }}, + {:short_name => { :boost => 10.0 }}, + { :created_at => :date } + ] # When name or short name is changed, also change the url name def short_name=(short_name) diff --git a/app/models/user.rb b/app/models/user.rb index 14a1d63e3..6c426c810 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -20,7 +20,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: user.rb,v 1.35 2008-03-12 16:07:13 francis Exp $ +# $Id: user.rb,v 1.36 2008-03-13 11:29:47 francis Exp $ require 'digest/sha1' @@ -39,7 +39,10 @@ class User < ActiveRecord::Base attr_accessor :password_confirmation validates_confirmation_of :password, :message =>"^Please enter the same password twice" - acts_as_solr :fields => [ {:name => { :boost => 5.0 }} ] + acts_as_solr :fields => [ + {:name => { :boost => 5.0 }}, + { :created_at => :date } + ] def validate errors.add(:email, "doesn't look like a valid address") unless MySociety::Validate.is_valid_email(self.email) diff --git a/app/views/general/search.rhtml b/app/views/general/search.rhtml index 761e7e22c..e91aef2e5 100644 --- a/app/views/general/search.rhtml +++ b/app/views/general/search.rhtml @@ -1,8 +1,11 @@ <% if @query.nil? %> - <% @title = "Search requests, public bodies and users" %> + <% @title = "Search Freedom of Information requests, public bodies and users" %> <h1><%=@title%></h1> +<% elsif @search_hits == 0 %> + <% @title = "Nothing found for '" + h(@query) + "'" %> + <% else %> - <% @title = "Search for '" + h(@query) + "'" %> + <% @title = "Results " + ((@page-1)*@per_page+1).to_s + "-" + [@page*@per_page, @search_hits].min.to_s + " of " + @search_hits.to_s + " for '" + h(@query) + "'" %> <% end%> <% form_tag({:action => "search_redirect"}, {:id => "search_form"}) do %> @@ -16,11 +19,13 @@ <% end %> <% if not @search_results.nil? %> + <%=link_to_unless @sortby.nil?, "Show most relevant results first", { :sortby => nil } %> + | + <%=link_to_unless @sortby == 'newest', "Newest results first", { :sortby => "newest" } %> <h1><%=@title%></h1> <% if @search_results.empty? %> - None found. <% else %> <% for search_result in @search_results %> <% if search_result.class.to_s == 'InfoRequest' %> @@ -39,13 +44,13 @@ <% end %> <% end %> - <%= will_paginate WillPaginate::Collection.new(params[:page]||1, @per_page, @solr_object.total_hits) %> + <%= will_paginate WillPaginate::Collection.new(@page, @per_page, @search_hits) %> <% end %> <% if @search_results.nil? or @search_results.empty? %> <h2>Search tips</h2> <ul> - <li>Enter words that you want to find, e.g. climbing lane</li> + <li>Enter words that you want to find separated by spaces, e.g. climbing lane</li> <li>Use OR (in capital letters) where you don't mind which word, e.g. commons OR lords <li>Use quotes when you want to find an exact phrase, e.g. "Liverpool City Council" <li>Type status: to select based on the status of the request. diff --git a/config/routes.rb b/config/routes.rb index dc428fa6c..5d2fa4948 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -4,7 +4,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: routes.rb,v 1.45 2008-03-12 13:01:05 francis Exp $ +# $Id: routes.rb,v 1.46 2008-03-13 11:29:47 francis Exp $ ActionController::Routing::Routes.draw do |map| # The priority is based upon order of creation: first created -> highest priority. @@ -18,7 +18,7 @@ ActionController::Routing::Routes.draw do |map| general.auto_complete_for_public_body_query 'auto_complete_for_public_body_query', :action => 'auto_complete_for_public_body_query' general.search_redirect '/search', :action => 'search_redirect' - general.search '/search/*query', :action => 'search' + general.search '/search/:query/:sortby', :action => 'search', :sortby => nil general.fai_test '/test', :action => 'fai_test' end diff --git a/script/rebuild-solr-index b/script/rebuild-solr-index new file mode 100755 index 000000000..5d21b999b --- /dev/null +++ b/script/rebuild-solr-index @@ -0,0 +1,16 @@ +#!/bin/bash + +# Reindex everything in Solr, not just changed things + +LOC=`dirname $0` + +# Info requests are updated incrementally by update-solr-index, so first set +# them all to not up to date, then rebuild. +$LOC/runner "ActiveRecord::Base.connection.execute(\"update info_requests set solr_up_to_date = 'f'\")" +$LOC/runner 'InfoRequest.update_solr_index()' + +# These are updated in real time when the model is changed - so they aren't +# called in update-solr-index. So for this rebuild script, just rebuild them. +$LOC/runner 'PublicBody.rebuild_solr_index()' +$LOC/runner 'User.rebuild_solr_index()' + diff --git a/script/update-solr-index b/script/update-solr-index index 33415aa9c..75e4304eb 100755 --- a/script/update-solr-index +++ b/script/update-solr-index @@ -2,6 +2,12 @@ LOC=`dirname $0` +# Info request (and hence incoming and outgoing message) indexing is updated +# incrementally by this separate batch job. This is so when an incoming +# message arrives it doesn't have to call Solr, so errors in the indexing won't +# go back to public body. $LOC/runner 'InfoRequest.update_solr_index()' +# Public bodies and users are updated by the model, immediately when it is saved. +# So nothing to do for them. @@ -1,13 +1,14 @@ Contact: - Check messages look nice - + Check user contact messages look nice on live site (think about subject?) Search: -Date ranges http://lucene.apache.org/java/docs/queryparsersyntax.html +Date sorting + - add public bodies and users + - make updated at use last event date -Search by type of public body +Search by tags on public bodies Hide backpaged things from search http://www.whatdotheyknow.com/search/status:successful%20OR%20status:partially_successful @@ -62,8 +63,6 @@ Search and replace text "FOI" and "Freedom of Information" out the way more - but put it in the title tag "public body" --> "public authority"? -hidden_field_tags are all wrong - they store value1 x - Adam's woes: http://foi.mysociety.org/request/18/response/31 - No doubt leaks email address via download.bin :( @@ -77,12 +76,6 @@ Sort the requests by when something last happened to them (this needs thought as sort orders we need) Sort by due date -Fix the fastcgi errors -[Mon Jan 21 10:38:45 2008] [error] [client 81.107.40.81] FastCGI: incomplete headers (0 bytes) rec -eived from server "/data/vhost/foi.mysociety.org/docs/dispatch.fcgi" - -"Government" in about page - Send email to remind people to classify Send email to tell admins something isn't classified Send email to remind people to clarify @@ -90,6 +83,8 @@ Send email to remind people to clarify Later ===== +Search date ranges http://lucene.apache.org/java/docs/queryparsersyntax.html (at Louise's mail) + Preview when sending followups - especially people need to see quoting/subject when sending "my response is late" @@ -121,10 +116,6 @@ Get deploying of Lucene search working Consider on staging sites making follow ups go to dummy address also -Make the email address for responses - - something bland for domain name - - perhaps put name of user in the part before the @ ? - Synthesise these tips into our handful of snappy snappy bullet points http://community.foe.co.uk/tools/right_to_know/tips.html http://www.justice.gov.uk/requestinginformation.htm @@ -140,9 +131,6 @@ Generic alerting/tracking system including Requests with related content Blog posts / Wikipedia articles about this request -Finish about page. Use these: - http://www.ico.gov.uk/for_the_public/access_to_official_information.aspx - Remember me box This can't possible be the best way, it is too depressing: http://onrails.org/articles/2006/02/18/auto-login @@ -150,6 +138,7 @@ Remember me box Screen scrape this and add link to it on the public body page http://www.ico.gov.uk/Home/tools_and_resources/decision_notices.aspx + (10:32:14) richard: you just need to count the number of rows of text and compare it to the number of rows in the textbox (10:32:29) richard: then increase the height of the textbox by 1em-ish (10:32:52) Matthew: their function is called autogrow_textarea() by the way, if you just want to look at it... @@ -161,15 +150,6 @@ http://www.mysociety.org/moin.cgi/FreedomOfInformation And comments on proposal http://www.mysociety.org/2006/04/04/freedom-of-information-archive/ -Check FOE site lots -http://community.foe.co.uk/tools/right_to_know/request_generator.html - -Look at this basic US site -http://www.rcfp.org/foi_letter/generate.php - -This is interesting -http://www.liverpool.gov.uk/Council_government_and_democracy/About_your_council/Data_protection_and_FOI/FOI/Suppliers_and_Contractors/index.asp - For grey hints in input fields http://pauldowman.com/projects/fieldhints/ @@ -178,18 +158,10 @@ Hyperlink Section 1(3) to the act and to guidance notes http://www.ico.gov.uk/what_we_cover/freedom_of_information/guidance.aspx -Way of contacting other users - "We will not reveal your email address to anybody" - are there circumstances - (e.g. somebody messaging creator of a request and it bouncing) where we - may reasonably do this, and should say it may happen? - -- I think this is fine EXCEPT for the facility to send messages. Do we have - fake hash addresses two way for each user for the message-to-user sending - thing? Would make sense. - Properly escape all name_and_email functions Give hotline to call body for extra help -Advice and assistance in making FoI requests to the House of Lords is available +e.g. Advice and assistance in making FoI requests to the House of Lords is available from the Freedom of Information Officer, Parliamentary Archives, House of Lords. London SW1A 0PW. Telephone 020 7219 0100. E-mail: foilords@parliament.uk http://www.parliament.uk/parliamentary_publications_and_archives/freedom_of_information_in_the_house_of_lords/lords__foi___how_to_obtain_information.cfm @@ -198,8 +170,6 @@ Including 'Request for information' in the subject line of your e-mail will assi http://www.parliament.uk/parliamentary_publications_and_archives/freedom_of_information_in_the_house_of_lords/lords__foi___how_to_obtain_information.cfm Medway also hint at this with subject= on mailto link -Consider putting the requesting user's name in the request's magic email address - Ask what they learnt from request, and other things? and reward by putting their request on front page @@ -217,6 +187,7 @@ e.g. Environmental Information Regulations apply for some bodies (e.g. environment agency) do automatically tag such bodies eir? +http://community.foe.co.uk/tools/right_to_know/faq.html#why_specify Add geographical location to council import @@ -245,7 +216,8 @@ Schools list: www.edubase.gov.uk Heather has emailed me NHS Trusts providing Mental Health Services FOE site has email addresses of lots of bodies, e.g. -http://community.foe.co.uk/app/tc + http://community.foe.co.uk/app/tc +Has handy links to publication schemes that are now mainly broken Be sure to add: - all in schedule 1 diff --git a/vendor/plugins/acts_as_solr/lib/parser_methods.rb b/vendor/plugins/acts_as_solr/lib/parser_methods.rb index 5ba181c02..0f1288039 100644 --- a/vendor/plugins/acts_as_solr/lib/parser_methods.rb +++ b/vendor/plugins/acts_as_solr/lib/parser_methods.rb @@ -37,7 +37,13 @@ module ActsAsSolr #:nodoc: end query_options[:field_list] = [field_list, 'score'] - query = "(#{query.gsub(/ *: */,"_t:")}) #{models}" + if query.strip.empty? # make it work with empty queries + models.sub!("AND ", "") + query = models + else + query = "(#{query.gsub(/ *: */,"_t:")})" + query = query + " #{models}" + end order = options[:order].split(/\s*,\s*/).collect{|e| e.gsub(/\s+/,'_t ').gsub(/\bscore_t\b/, 'score') }.join(',') if options[:order] query_options[:query] = replace_types([query])[0] # TODO adjust replace_types to work with String or Array |