Index first message text with info requests themselves.

Fix routing for search queries with full stops in.
Public body search indexing.
User search indexing.
Link to specific users out of ones with same name.

17 files changed, 172 insertions, 75 deletions

diff --git a/app/controllers/general_controller.rb b/app/controllers/general_controller.rb
index 342355c75..986579874 100644
--- a/app/controllers/general_controller.rb
+++ b/app/controllers/general_controller.rb
@@ -5,7 +5,7 @@
 # Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: general_controller.rb,v 1.3 2008-03-06 21:49:33 francis Exp $
+# $Id: general_controller.rb,v 1.4 2008-03-07 23:13:38 francis Exp $
 
 class GeneralController < ApplicationController
 
@@ -39,20 +39,24 @@ class GeneralController < ApplicationController
 
     # Just does a redirect from ?query= search to /query
     def search_redirect
-        query = params[:query]
-        redirect_to search_url(:query => query)
+        @query = params[:query]
+        if @query.nil?
+            render :action => "search"
+        else
+            redirect_to search_url(:query => @query)
+        end
     end
 
     # Actual search
     def search
         @per_page = 20
-        query = params[:query]
+        @query = params[:query].join("/")
 
-        query_nopunc = query.gsub(/[^a-z0-9]/i, " ")
+        query_nopunc = @query.gsub(/[^a-z0-9]/i, " ")
         query_nopunc = query_nopunc.gsub(/\s+/, " ")
         @highlight_words = query_nopunc.split(" ")
 
-        @solr_object = InfoRequest.multi_solr_search(query, :models => [ OutgoingMessage, IncomingMessage ],
+        @solr_object = InfoRequest.multi_solr_search(@query, :models => [ OutgoingMessage, IncomingMessage, PublicBody, User ],
             :limit => @per_page, :offset => ((params[:page]||"1").to_i-1) * @per_page)
         @search_results = @solr_object.results
     end
diff --git a/app/helpers/link_to_helper.rb b/app/helpers/link_to_helper.rb
index 3e6ee7a40..e05aebc0b 100644
--- a/app/helpers/link_to_helper.rb
+++ b/app/helpers/link_to_helper.rb
@@ -5,7 +5,7 @@
 # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: link_to_helper.rb,v 1.21 2008-03-05 22:03:15 francis Exp $
+# $Id: link_to_helper.rb,v 1.22 2008-03-07 23:13:38 francis Exp $
 
 module LinkToHelper
 
@@ -47,7 +47,7 @@ module LinkToHelper
 
     # Users
     def user_url(user)
-        return show_user_url(:url_name => user.url_name, :only_path => true)
+        return show_user_url(:url_name => user.url_name, :only_path => true) + "#user-" + user.id.to_s
     end
     def user_link(user)
         link_to h(user.name), user_url(user)
diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb
index a22385347..661b3b659 100644
--- a/app/models/incoming_message.rb
+++ b/app/models/incoming_message.rb
@@ -18,7 +18,7 @@
 # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: incoming_message.rb,v 1.55 2008-03-07 10:13:57 francis Exp $
+# $Id: incoming_message.rb,v 1.56 2008-03-07 23:13:38 francis Exp $
 
 
 # TODO
@@ -320,13 +320,13 @@ class IncomingMessage < ActiveRecord::Base
         attachments = self.get_attachments_for_display
         for attachment in attachments
             if attachment.content_type == 'text/plain'
-                text += attachment.body
+                text += attachment.body + "\n\n"
             elsif attachment.content_type == 'application/msword'
                 tempfile = Tempfile.new('foipdf')
                 tempfile.print attachment.body
                 tempfile.flush
                 system("/usr/bin/wvText " + tempfile.path + " " + tempfile.path + ".txt")
-                text += File.read(tempfile.path + ".txt")
+                text += File.read(tempfile.path + ".txt") + "\n\n"
                 File.unlink(tempfile.path + ".txt")
                 tempfile.close
             elsif attachment.content_type == 'application/pdf'
@@ -334,7 +334,7 @@ class IncomingMessage < ActiveRecord::Base
                 tempfile.print attachment.body
                 tempfile.flush
                 IO.popen("/usr/bin/pdftotext " + tempfile.path + " -", "r") do |child|
-                    text += child.read()
+                    text += child.read() + "\n\n"
                 end
                 tempfile.close
             end
@@ -344,7 +344,7 @@ class IncomingMessage < ActiveRecord::Base
 
     # Returns text for indexing
     def get_text_for_indexing
-        return get_body_for_quoting + get_attachment_text
+        return get_body_for_quoting + "\n\n" + get_attachment_text
     end
 
     # Returns the name of the person the incoming message is from, or nil if there isn't one
diff --git a/app/models/info_request.rb b/app/models/info_request.rb
index 9bff40a60..b6da611d9 100644
--- a/app/models/info_request.rb
+++ b/app/models/info_request.rb
@@ -21,7 +21,7 @@
 # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: info_request.rb,v 1.57 2008-03-06 20:10:29 francis Exp $
+# $Id: info_request.rb,v 1.58 2008-03-07 23:13:38 francis Exp $
 
 require 'digest/sha1'
 
@@ -64,7 +64,16 @@ class InfoRequest < ActiveRecord::Base
     end
 
     # Full text search indexing
-    acts_as_solr :fields => [ :title ], :if => "$do_solr_index"
+    acts_as_solr :fields => [ 
+        :title, 
+        :initial_request_text, 
+        { :status => :string }
+#        { :created_at => :date } 
+    ], :if => "$do_solr_index"
+    def status # for name in Solr queries
+        calculate_status
+    end
+
     $do_solr_index = false
     def self.update_solr_index
         $do_solr_index = true
@@ -79,7 +88,13 @@ class InfoRequest < ActiveRecord::Base
                     raise "failed to solr_save"
                 end
                 for outgoing_message in info_request.outgoing_messages
-                    outgoing_message.solr_save
+                    # Initial request text is indexed for InfoRequest models -
+                    # see :initial_request_text in acts_as_solr entry above
+                    if outgoing_message.message_type != 'initial_request' 
+                        outgoing_message.solr_save
+                    else
+                        outgoing_message.solr_destroy
+                    end
                 end
                 for incoming_message in info_request.incoming_messages
                     incoming_message.solr_save
@@ -210,6 +225,7 @@ public
     end
 
     # Work out what the situation of the request is
+    #   waiting_classification
     #   waiting_response
     #   waiting_response_overdue  # XXX calculated, should be cached for display?
     #   waiting_clarification
@@ -217,6 +233,10 @@ public
     #   successful
     #   partially_successful
     def calculate_status
+        if self.awaiting_description
+            return 'waiting_classification'
+        end
+
         # See if response would be overdue 
         date_today = Time.now.strftime("%Y-%m-%d")
         date_response = date_response_required_by.strftime("%Y-%m-%d")
@@ -405,7 +425,7 @@ public
     # Display version of status
     def display_status
         status = self.calculate_status
-        if self.awaiting_description
+        if status == 'waiting_classification'
             "Awaiting classification."
         elsif status == 'waiting_response'
             "Awaiting response."
diff --git a/app/models/public_body.rb b/app/models/public_body.rb
index c45ec75a6..82eaa5822 100644
--- a/app/models/public_body.rb
+++ b/app/models/public_body.rb
@@ -22,14 +22,12 @@
 # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: public_body.rb,v 1.33 2008-03-05 18:14:03 francis Exp $
+# $Id: public_body.rb,v 1.34 2008-03-07 23:13:38 francis Exp $
 
 require 'csv'
 require 'set'
 
 class PublicBody < ActiveRecord::Base
-    #acts_as_solr :fields => [:name, :short_name]
-    
     validates_presence_of :name
     validates_presence_of :url_name
     validates_presence_of :request_email
@@ -61,6 +59,8 @@ class PublicBody < ActiveRecord::Base
     acts_as_versioned
     self.non_versioned_columns << 'created_at' << 'updated_at'
 
+    acts_as_solr :fields => [{:name => { :boost => 10.0 }}, {:short_name => { :boost => 10.0 }} ]
+
     # When name or short name is changed, also change the url name
     def short_name=(short_name)
         write_attribute(:short_name, short_name)
diff --git a/app/models/user.rb b/app/models/user.rb
index bc1028ae1..547ae4225 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -20,7 +20,7 @@
 # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: user.rb,v 1.33 2008-02-28 16:25:30 francis Exp $
+# $Id: user.rb,v 1.34 2008-03-07 23:13:38 francis Exp $
 
 require 'digest/sha1'
 
@@ -39,6 +39,8 @@ class User < ActiveRecord::Base
     attr_accessor :password_confirmation
     validates_confirmation_of :password, :message =>"^Please enter the same password twice"
 
+    acts_as_solr :fields => [ {:name => { :boost => 5.0 }} ]
+
     def validate
         errors.add(:email, "doesn't look like a valid address") unless MySociety::Validate.is_valid_email(self.email)
     end
diff --git a/app/views/body/_body_listing.rhtml b/app/views/body/_body_listing.rhtml
index 0287997c8..7d673a552 100644
--- a/app/views/body/_body_listing.rhtml
+++ b/app/views/body/_body_listing.rhtml
@@ -2,19 +2,6 @@
     None found.
 <% else %>
     <% for public_body in public_bodies %>
-        <p class="body_listing">
-            <%= public_body_link(public_body) %> 
-            
-            <br>
-            <% if not public_body.short_name.empty? %>
-                Also called <%=h public_body.short_name %>.
-                <br>
-            <% end %>
-
-            <span class="request_listing_bottomline">
-                <%= pluralize(public_body.info_requests.size, "request") %> made.
-                Added on <%= simple_date(public_body.created_at) %>.
-            </span>
-        </p>
+        <%= render :partial => 'body/body_listing_single', :locals => { :public_body => public_body } %>
     <% end %>
 <% end %>
diff --git a/app/views/body/_body_listing_single.rhtml b/app/views/body/_body_listing_single.rhtml
new file mode 100644
index 000000000..c060ccdb3
--- /dev/null
+++ b/app/views/body/_body_listing_single.rhtml
@@ -0,0 +1,19 @@
+<% if @highlight_words.nil? 
+      @highlight_words = []
+   end %>
+
+<p class="body_listing">
+    <%= link_to highlight_words(public_body.name, @highlight_words), public_body_url(public_body) %>
+    
+    <br>
+    <% if not public_body.short_name.empty? %>
+        Also called <%=highlight_words(public_body.short_name, @highlight_words) %>.
+        <br>
+    <% end %>
+
+    <span class="body_listing_bottomline">
+        <%= pluralize(public_body.info_requests.size, "request") %> made.
+        Added on <%= simple_date(public_body.created_at) %>.
+    </span>
+</p>
+
diff --git a/app/views/general/search.rhtml b/app/views/general/search.rhtml
index 56d5e44b3..27f36fba8 100644
--- a/app/views/general/search.rhtml
+++ b/app/views/general/search.rhtml
@@ -1,28 +1,41 @@
-<% @title = "Freedom of Information requests matching '" + h(params[:query]) + "'" %>
+<% if @query.nil? %>
+    <% @title = "Search requests, public bodies and users" %>
+    <h1><%=@title%></h1>
+<% else %>
+    <% @title = "Search for '" + h(@query) + "'" %>
+<% end%>
 
 <% form_tag({:action => "search_redirect"}, {:id => "search_form"}) do %>
    <p>
-       <%= text_field_tag 'query', params[:query], { :size => 40 } %>
+       <%= text_field_tag 'query', @query, { :size => 40 } %>
        <%= submit_tag "Search" %>
    </p>
 <% end %>
 
-<h1><%=@title%></h1>
+<% if not @search_results.nil? %>
 
-<% if @search_results.empty? %>
-    None found.
-<% else %>
-    <% for search_result in @search_results %>
-        <% if search_result.class.to_s == 'InfoRequest' %>
-            <%= render :partial => 'request/request_listing_single', :locals => { :info_request => search_result } %>
-        <% elsif search_result.class.to_s == 'OutgoingMessage' %>
-            <%= render :partial => 'request/request_listing_via_outgoing', :locals => { :info_request => search_result.info_request, :outgoing_message => search_result } %>
-        <% elsif search_result.class.to_s == 'IncomingMessage' %>
-            <%= render :partial => 'request/request_listing_via_incoming', :locals => { :info_request => search_result.info_request, :incoming_message => search_result } %>
-        <% else %>
-            <p><strong>Unknown search result type <%=search_result.class.to_s%></strong></p>
+    <h1><%=@title%></h1>
+
+    <% if @search_results.empty? %>
+        None found.
+    <% else %>
+        <% for search_result in @search_results %>
+            <% if search_result.class.to_s == 'InfoRequest' %>
+                <%= render :partial => 'request/request_listing_single', :locals => { :info_request => search_result } %>
+            <% elsif search_result.class.to_s == 'OutgoingMessage' %>
+                <%= render :partial => 'request/request_listing_via_outgoing', :locals => { :info_request => search_result.info_request, :outgoing_message => search_result } %>
+            <% elsif search_result.class.to_s == 'IncomingMessage' %>
+                <%= render :partial => 'request/request_listing_via_incoming', :locals => { :info_request => search_result.info_request, :incoming_message => search_result } %>
+            <% elsif search_result.class.to_s == 'PublicBody' %>
+                <%= render :partial => 'body/body_listing_single', :locals => { :public_body => search_result } %>
+            <% elsif search_result.class.to_s == 'User' %>
+                <%= render :partial => 'user/user_listing_single', :locals => { :display_user => search_result } %>
+            <% else %>
+                <p><strong>Unknown search result type <%=search_result.class.to_s%></strong></p>
+            <% end %>
         <% end %>
     <% end %>
-<% end %>
 
-<%= will_paginate WillPaginate::Collection.new(params[:page]||1, @per_page, @solr_object.total_hits) %>
+    <%= will_paginate WillPaginate::Collection.new(params[:page]||1, @per_page, @solr_object.total_hits) %>
+
+<% end %>
diff --git a/app/views/request/_request_listing_via_incoming.rhtml b/app/views/request/_request_listing_via_incoming.rhtml
index b2b3d7091..80efb1eb3 100644
--- a/app/views/request/_request_listing_via_incoming.rhtml
+++ b/app/views/request/_request_listing_via_incoming.rhtml
@@ -1,5 +1,5 @@
 <p class="request_listing">
-    <%= link_to "Response to '" + highlight_words(info_request.title, @highlight_words) + "'", request_url(info_request)+"#incoming-"+incoming_message.id.to_s %>
+    <%= link_to "Response to '" + h(info_request.title) + "'", request_url(info_request)+"#incoming-"+incoming_message.id.to_s %>
 
     <br>
     <%= excerpt_and_highlight(incoming_message.get_text_for_indexing, @highlight_words) %>
@@ -10,7 +10,7 @@
         <%= info_request.display_status %>
         </strong>
 
-        Response from <%= public_body_link(info_request.public_body) %>
+        Response by <%= public_body_link(info_request.public_body) %>
         to <%= user_link(info_request.user) %>
         on <%= simple_date(incoming_message.sent_at) %>.
     </span>
diff --git a/app/views/request/_request_listing_via_outgoing.rhtml b/app/views/request/_request_listing_via_outgoing.rhtml
index ce20231b6..02e236dad 100644
--- a/app/views/request/_request_listing_via_outgoing.rhtml
+++ b/app/views/request/_request_listing_via_outgoing.rhtml
@@ -1,8 +1,8 @@
 <p class="request_listing">
     <% if outgoing_message.message_type == 'initial_request' %>
-        <%= link_to "Initial request for '" + highlight_words(info_request.title, @highlight_words) + "'", request_url(info_request)+"#outgoing-"+outgoing_message.id.to_s %>
+        <% raise "initial_request shouldn't be indexed for search, text is indexed in InfoRequest model" %>
     <% elsif outgoing_message.message_type == 'followup' %>
-        <%= link_to "Initial request for '" + highlight_words(info_request.title, @highlight_words) + "'", request_url(info_request)+"#outgoing-"+outgoing_message.id.to_s %>
+        <%= link_to "Follow up message for '" + h(info_request.title) + "'", request_url(info_request)+"#outgoing-"+outgoing_message.id.to_s %>
     <% else %>
         <% raise "unknown outgoing message type" %>
     <% end %>
@@ -16,9 +16,9 @@
         <%= info_request.display_status %>
         </strong>
 
-        Requested from <%= public_body_link(info_request.public_body) %>
+        Follow up sent to <%= public_body_link(info_request.public_body) %>
         by <%= user_link(info_request.user) %>
-        on <%= simple_date(info_request.created_at) %>.
+        on <%= simple_date(outgoing_message.created_at) %>.
     </span>
 </p>
 
diff --git a/app/views/user/_user_listing_single.rhtml b/app/views/user/_user_listing_single.rhtml
new file mode 100644
index 000000000..7df5b38ab
--- /dev/null
+++ b/app/views/user/_user_listing_single.rhtml
@@ -0,0 +1,16 @@
+<% if @highlight_words.nil? 
+      @highlight_words = []
+   end %>
+
+<p class="user_listing">
+    <%= link_to highlight_words(display_user.name, @highlight_words), user_url(display_user) %>
+    
+    <br>
+    <!--<br>-->
+
+    <span class="user_listing_bottomline">
+        <%= pluralize(display_user.info_requests.size, "request") %> made.
+        Joined on <%= simple_date(display_user.created_at) %>.
+    </span>
+</p>
+
diff --git a/app/views/user/show.rhtml b/app/views/user/show.rhtml
index 41e0ec645..582e79703 100644
--- a/app/views/user/show.rhtml
+++ b/app/views/user/show.rhtml
@@ -5,20 +5,21 @@
 <% end%>
 
 <% for display_user in @display_users %>
-    <h1><%=@title%></h1>
-    <p class="subtitle">Joined on <%= simple_date(display_user.created_at) %></p>
+    <div class="single_user" id="user-<%=display_user.id.to_s%>">
+        <h1><%=@title%></h1>
+        <p class="subtitle">Joined on <%= simple_date(display_user.created_at) %></p>
 
-    <% if display_user.info_requests.empty? %>
-        <p><%= display_user == @user ? 'You have' : 'This person has' %>
-        made no Freedom of Information requests using this site.</p>
+        <% if display_user.info_requests.empty? %>
+            <p><%= display_user == @user ? 'You have' : 'This person has' %>
+            made no Freedom of Information requests using this site.</p>
 
-    <% else %>
-        <p><%= display_user == @user ? 'You have' : 'This person has' %>
-        made <%=pluralize(display_user.info_requests.size, "Freedom of Information request") %>
-        using this site.</p>
-
-        <%= render :partial => 'request/request_listing', :locals => { :info_requests => display_user.info_requests.sort { |a,b| b.created_at <=> a.created_at } } %>
-    <% end %>
+        <% else %>
+            <p><%= display_user == @user ? 'You have' : 'This person has' %>
+            made <%=pluralize(display_user.info_requests.size, "Freedom of Information request") %>
+            using this site.</p>
 
+            <%= render :partial => 'request/request_listing', :locals => { :info_requests => display_user.info_requests.sort { |a,b| b.created_at <=> a.created_at } } %>
+        <% end %>
+    </div>
 <% end %>
 
diff --git a/config/routes.rb b/config/routes.rb
index b30a2f2fd..07b1c570f 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -4,7 +4,7 @@
 # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: routes.rb,v 1.42 2008-03-06 01:23:39 francis Exp $
+# $Id: routes.rb,v 1.43 2008-03-07 23:13:39 francis Exp $
 
 ActionController::Routing::Routes.draw do |map|
     # The priority is based upon order of creation: first created -> highest priority.
@@ -18,7 +18,7 @@ ActionController::Routing::Routes.draw do |map|
       general.auto_complete_for_public_body_query 'auto_complete_for_public_body_query', :action => 'auto_complete_for_public_body_query'
 
       general.search_redirect '/search',      :action => 'search_redirect'
-      general.search '/search/:query',      :action => 'search'
+      general.search '/search/*query',      :action => 'search'
     end
 
     map.with_options :controller => 'request' do |request|
diff --git a/public/stylesheets/main.css b/public/stylesheets/main.css
index 1d27faafa..a6c768003 100644
--- a/public/stylesheets/main.css
+++ b/public/stylesheets/main.css
@@ -360,10 +360,26 @@ table#list_requests .odd {
 /* /search - searching */
 
 .highlight {
-    background-color: #eeee00;
+    background-color: #dddd00;
     color: #000000;
 }
 
+.user_listing {
+}
+.user_listing_bottomline {
+    color: #7aa600;
+}
+.user_listing_bottomline a:link {
+    color: #7777cc;
+}
+.user_listing_bottomline a:visited {
+    color: #551a8b;
+}
+.user_listing_bottomline a:active {
+    color: #ff0000;
+}
+
+
 /* /body - listing bodies */
 
 .body_listing {
diff --git a/todo.txt b/todo.txt
index 6b1253d58..ec96de12d 100644
--- a/todo.txt
+++ b/todo.txt
@@ -1,6 +1,13 @@
+CVS commit message:
+Index first message text with info requests themselves.
+Fix routing for search queries with full stops in.
+Public body search indexing.
+User search indexing.
+Link to specific users out of ones with same name.
+
+
 Search:
 
-Don't show same request so many times
 Status and stuff
 Date ranges 
 
@@ -9,11 +16,23 @@ Search for public bodies
 
 Rubbish highlighting:
 http://www.whatdotheyknow.com/search/%22MS%20Office%22
+Seems to be returning text anyway, so may as well highlight
+http://localhost:8999/solr/select/?q=house&version=2.2&start=0&rows=10&indent=on
 
 One of the PDFs on live site has:
 Error: PDF version 1.6 -- xpdf supports version 1.5 (continuing anyway)
 Need to upgrade to poppler-utils?
 
+
+Should we index by individual piece of correspondence, or by whole info requests?
+    Advantages of individual:
+        Ordinary search results link more directly to object
+        Easy to alert when a new piece of correspondence occurs
+    Advantages of whole info requests:
+        Easier to alert to new correspondence on request containing keyword
+        Shorter search results
+        Syntax of filtering requests by type is clearer
+
 FOI requests to use to test it
 ==============================
 
diff --git a/vendor/plugins/acts_as_solr/lib/parser_methods.rb b/vendor/plugins/acts_as_solr/lib/parser_methods.rb
index fc92d8851..f0c6f7a9d 100644
--- a/vendor/plugins/acts_as_solr/lib/parser_methods.rb
+++ b/vendor/plugins/acts_as_solr/lib/parser_methods.rb
@@ -124,4 +124,4 @@ module ActsAsSolr #:nodoc:
     end
   end
 
-end
\ No newline at end of file
+end