First pass at indexing info requests with solr

16 files changed, 203 insertions, 71 deletions

diff --git a/app/controllers/general_controller.rb b/app/controllers/general_controller.rb
new file mode 100644
index 000000000..b4f7c6eac
--- /dev/null
+++ b/app/controllers/general_controller.rb
@@ -0,0 +1,66 @@
+# app/controllers/general_controller.rb:
+# For pages like front page, general search, that aren't specific to a
+# particular model.
+#
+# Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved.
+# Email: francis@mysociety.org; WWW: http://www.mysociety.org/
+#
+# $Id: general_controller.rb,v 1.1 2008-03-06 01:23:38 francis Exp $
+
+class GeneralController < ApplicationController
+
+    # Fancy javascript smancy for auto complete search on front page
+    def auto_complete_for_public_body_query
+        @public_bodies = public_body_query(params[:public_body][:query])
+
+        render :partial => "public_body_query"
+    end
+
+    # Actual front page
+    def frontpage
+        # Public body search on the left
+        @public_bodies = []
+        @query_made = false
+        if params[:public_body] and params[:public_body][:query]
+            # Try and do exact match - redirect if it is made
+            @public_body = PublicBody.find_by_name(params[:public_body][:query])
+            if not @public_body.nil?
+                redirect_to new_request_to_body_url(:public_body_id => @public_body.id.to_s)
+            end
+            # Otherwise use search engine to find public body
+            @public_bodies = public_body_query(params[:public_body][:query])
+            @query_made = true
+        end
+
+        # Get all successful requests for display on the right  
+        @info_requests = InfoRequest.find :all, :order => "created_at desc", :conditions => "prominence = 'normal' and described_state in ('successful', 'partially_successful')", :limit => 3
+    end
+
+
+    # Just does a redirect from ?query= search to /query
+    def search_redirect
+        query = params[:query]
+        redirect_to search_url(:query => query)
+    end
+
+    # Actual search
+    def search
+        query = params[:query]
+        @solr_object = InfoRequest.multi_solr_search(query, :models => [ OutgoingMessage, IncomingMessage ])
+        @search_results = @solr_object.results
+    end
+ 
+    private
+
+    # Used in front page search for public body
+    def public_body_query(query)
+        # @public_bodies = PublicBody.find_by_solr(query).results
+
+        criteria = '%' + query + '%'
+        @public_bodies = PublicBody.find(:all, 
+                    :conditions => ["name ilike ? or short_name ilike ?", criteria, criteria],
+                    :order => 'name', :limit=>10)  
+        return @public_bodies
+    end
+end
+ 
diff --git a/app/controllers/request_controller.rb b/app/controllers/request_controller.rb
index 76cf70ee1..977e4cd32 100644
--- a/app/controllers/request_controller.rb
+++ b/app/controllers/request_controller.rb
@@ -4,7 +4,7 @@
 # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: request_controller.rb,v 1.64 2008-02-29 13:24:22 francis Exp $
+# $Id: request_controller.rb,v 1.65 2008-03-06 01:23:38 francis Exp $
 
 class RequestController < ApplicationController
     
@@ -39,31 +39,6 @@ class RequestController < ApplicationController
     def list
         @info_requests = InfoRequest.paginate :order => "created_at desc", :page => params[:page], :per_page => 25, :conditions => "prominence = 'normal'"
     end
-    
-    # Fancy javascript smancy for auto complete search
-    def auto_complete_for_public_body_query
-        @public_bodies = public_body_query(params[:public_body][:query])
-
-        render :partial => "public_body_query"
-    end
-    def frontpage
-        # Public body search on the left
-        @public_bodies = []
-        @query_made = false
-        if params[:public_body] and params[:public_body][:query]
-            # Try and do exact match - redirect if it is made
-            @public_body = PublicBody.find_by_name(params[:public_body][:query])
-            if not @public_body.nil?
-                redirect_to new_request_to_body_url(:public_body_id => @public_body.id.to_s)
-            end
-            # Otherwise use search engine to find public body
-            @public_bodies = public_body_query(params[:public_body][:query])
-            @query_made = true
-        end
-
-        # Get all successful requests for display on the right  
-        @info_requests = InfoRequest.find :all, :order => "created_at desc", :conditions => "prominence = 'normal' and described_state in ('successful', 'partially_successful')", :limit => 3
-    end
 
     # Page new form posts to
     def new
@@ -284,16 +259,5 @@ class RequestController < ApplicationController
         render :text => @attachment.body
     end
 
-    private
-
-    # Used in front page search for public body
-    def public_body_query(query)
-        # @public_bodies = PublicBody.find_by_solr(query).results
-
-        criteria = '%' + query + '%'
-        @public_bodies = PublicBody.find(:all, 
-                    :conditions => ["name ilike ? or short_name ilike ?", criteria, criteria],
-                    :order => 'name', :limit=>10)  
-        return @public_bodies
-    end
 end
+
diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb
index b3560e006..02256805e 100644
--- a/app/models/incoming_message.rb
+++ b/app/models/incoming_message.rb
@@ -18,7 +18,7 @@
 # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: incoming_message.rb,v 1.48 2008-02-28 14:25:51 francis Exp $
+# $Id: incoming_message.rb,v 1.49 2008-03-06 01:23:38 francis Exp $
 
 
 # TODO
@@ -50,6 +50,8 @@ class IncomingMessage < ActiveRecord::Base
 
     has_many :outgoing_message_followups, :class_name => OutgoingMessage
 
+    acts_as_solr :fields => [ :get_main_body_text ], :if => "$do_solr_index"
+
     # Return the structured TMail::Mail object
     # Documentation at http://i.loveruby.net/en/projects/tmail/doc/
     def mail
diff --git a/app/models/info_request.rb b/app/models/info_request.rb
index df2e32c8a..53f349535 100644
--- a/app/models/info_request.rb
+++ b/app/models/info_request.rb
@@ -21,7 +21,7 @@
 # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: info_request.rb,v 1.53 2008-02-29 16:01:32 francis Exp $
+# $Id: info_request.rb,v 1.54 2008-03-06 01:23:38 francis Exp $
 
 require 'digest/sha1'
 
@@ -63,6 +63,28 @@ class InfoRequest < ActiveRecord::Base
         end
     end
 
+    # Full text search indexing
+    acts_as_solr :fields => [ :title ], :if => "$do_solr_index"
+    $do_solr_index = false
+    def self.update_solr_index
+        $do_solr_index = true
+        InfoRequest.rebuild_solr_index(0) do |ar, options| 
+            ar.find(:all, :conditions => ["not solr_up_to_date"])
+        end
+        OutgoingMessage.rebuild_solr_index(0) do |ar, options| 
+            ar.find(:all, :conditions => ["not (select solr_up_to_date from info_requests where id = outgoing_messages.info_request_id)"])
+        end
+        IncomingMessage.rebuild_solr_index(0)  do |ar, options| 
+            ar.find(:all, :conditions => ["not (select solr_up_to_date from info_requests where id = incoming_messages.info_request_id)"])
+        end
+        InfoRequest.update_all("solr_up_to_date = 't'")
+        $do_solr_index = false
+    end
+    def before_update
+        self.solr_up_to_date = false
+        true
+    end
+
 public
     # When name is changed, also change the url name
     def title=(title)
@@ -110,7 +132,9 @@ public
 
     # Return info request corresponding to an incoming email address, or nil if
     # none found. Checks the hash to ensure the email came from the public body -
-    # only they are sent the email address with the has in it.
+    # only they are sent the email address with the has in it. (We don't check
+    # the prefix and domain, as sometimes those change, or might be elided by
+    # copying an email, and that doesn't matter)
     def self.find_by_incoming_email(incoming_email)
         incoming_email =~ /request-(\d+)-([a-z0-9]+)/
         id = $1.to_i
diff --git a/app/models/outgoing_message.rb b/app/models/outgoing_message.rb
index 03b3e5fef..bbbf1c7bb 100644
--- a/app/models/outgoing_message.rb
+++ b/app/models/outgoing_message.rb
@@ -21,7 +21,7 @@
 # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: outgoing_message.rb,v 1.32 2008-02-27 14:01:30 francis Exp $
+# $Id: outgoing_message.rb,v 1.33 2008-03-06 01:23:38 francis Exp $
 
 class OutgoingMessage < ActiveRecord::Base
     belongs_to :info_request
@@ -32,6 +32,8 @@ class OutgoingMessage < ActiveRecord::Base
 
     belongs_to :incoming_message_followup, :foreign_key => 'incoming_message_followup_id', :class_name => 'IncomingMessage'
 
+    acts_as_solr :fields => [ :body ], :if => "$do_solr_index"
+
     # How the default letter starts and ends
     def get_salutation
         ret = "Dear "
diff --git a/app/views/request/_public_body_query.rhtml b/app/views/general/_public_body_query.rhtml
index 77efdfcc1..77efdfcc1 100644
--- a/app/views/request/_public_body_query.rhtml
+++ b/app/views/general/_public_body_query.rhtml
diff --git a/app/views/request/frontpage.rhtml b/app/views/general/frontpage.rhtml
index 508f5105c..e3fb54ca8 100644
--- a/app/views/request/frontpage.rhtml
+++ b/app/views/general/frontpage.rhtml
@@ -58,6 +58,12 @@
 
 <div id="find_information">
     <h1>Explore information that others requested</h1>
-    <%= render :partial => 'request_listing', :locals => { :info_requests => @info_requests } %>
+    <% form_tag({:action => "search_redirect"}, {:id => "search_form"}) do %>
+       <p>
+           <%= text_field_tag 'query', params[:query], { :size => 20 } %>
+           <%= submit_tag "Search" %>
+       </p>
+    <% end %>
+    <%= render :partial => 'request/request_listing', :locals => { :info_requests => @info_requests } %>
     <p><a href="/list">View all information</a></p>
 </div>
diff --git a/app/views/general/search.rhtml b/app/views/general/search.rhtml
new file mode 100644
index 000000000..5d848c66a
--- /dev/null
+++ b/app/views/general/search.rhtml
@@ -0,0 +1,30 @@
+<% @title = "Freedom of Information requests matching '" + h(params[:query]) + "'" %>
+
+<% form_tag({:action => "search_redirect"}, {:id => "search_form"}) do %>
+   <p>
+       <%= text_field_tag 'query', params[:query], { :size => 40 } %>
+       <%= submit_tag "Search" %>
+   </p>
+<% end %>
+
+<h1><%=@title%></h1>
+
+<% if @search_results.empty? %>
+    None found.
+<% else %>
+    <% for search_result in @search_results %>
+        <% if search_result.class.to_s == 'InfoRequest' %>
+            <%= render :partial => 'request/request_listing_single', :locals => { :info_request => search_result } %>
+        <% elsif search_result.class.to_s == 'OutgoingMessage' %>
+            <%= render :partial => 'request/request_listing_single', :locals => { :info_request => search_result.info_request } %>
+        <% elsif search_result.class.to_s == 'IncomingMessage' %>
+            <%= render :partial => 'request/request_listing_single', :locals => { :info_request => search_result.info_request } %>
+        <% else %>
+            <p><strong>Unknown search result type <%=search_result.class.to_s%></strong></p>
+        <% end %>
+    <% end %>
+<% end %>
+
+<%
+#= will_paginate(@info_requests) 
+%>
diff --git a/app/views/request/_request_listing.rhtml b/app/views/request/_request_listing.rhtml
index 4c31f470a..ebfbc60a4 100644
--- a/app/views/request/_request_listing.rhtml
+++ b/app/views/request/_request_listing.rhtml
@@ -1,19 +1,8 @@
-<% for info_request in info_requests %>
-    <p class="request_listing">
-        <%= request_link(info_request) %>
-
-        <br>
-        <%=h excerpt(info_request.initial_request_text, "", 300) %>
-        <br>
-
-        <span class="request_listing_bottomline">
-            <strong>
-            <%= info_request.display_status %>
-            </strong>
-
-            Requested from <%= public_body_link(info_request.public_body) %>
-            by <%= user_link(info_request.user) %>
-            on <%= simple_date(info_request.created_at) %>.
-        </span>
-    </p>
+<% if info_requests.empty? %>
+    None found.
+<% else %>
+    <% for info_request in info_requests %>
+        <%= render :partial => 'request/request_listing_single', :locals => { :info_request => info_request } %>
+    <% end %>
 <% end %>
+
diff --git a/app/views/request/_request_listing_single.rhtml b/app/views/request/_request_listing_single.rhtml
new file mode 100644
index 000000000..b91d9c640
--- /dev/null
+++ b/app/views/request/_request_listing_single.rhtml
@@ -0,0 +1,18 @@
+<p class="request_listing">
+    <%= request_link(info_request) %>
+
+    <br>
+    <%=h excerpt(info_request.initial_request_text, "", 300) %>
+    <br>
+
+    <span class="request_listing_bottomline">
+        <strong>
+        <%= info_request.display_status %>
+        </strong>
+
+        Requested from <%= public_body_link(info_request.public_body) %>
+        by <%= user_link(info_request.user) %>
+        on <%= simple_date(info_request.created_at) %>.
+    </span>
+</p>
+
diff --git a/config/routes.rb b/config/routes.rb
index 81cb9c0ab..b30a2f2fd 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -4,7 +4,7 @@
 # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: routes.rb,v 1.41 2008-03-03 00:43:52 francis Exp $
+# $Id: routes.rb,v 1.42 2008-03-06 01:23:39 francis Exp $
 
 ActionController::Routing::Routes.draw do |map|
     # The priority is based upon order of creation: first created -> highest priority.
@@ -13,13 +13,21 @@ ActionController::Routing::Routes.draw do |map|
     # map.connect 'products/:id', :controller => 'catalog', :action => 'view'
     # Keep in mind you can assign values other than :controller and :action
     
+    map.with_options :controller => 'general' do |general|
+      general.frontpage           '/',            :action => 'frontpage'
+      general.auto_complete_for_public_body_query 'auto_complete_for_public_body_query', :action => 'auto_complete_for_public_body_query'
+
+      general.search_redirect '/search',      :action => 'search_redirect'
+      general.search '/search/:query',      :action => 'search'
+    end
+
     map.with_options :controller => 'request' do |request|
-      request.frontpage           '/',            :action => 'frontpage'
-      request.auto_complete_for_public_body_query 'auto_complete_for_public_body_query', :action => 'auto_complete_for_public_body_query'
 
       request.request_list   '/list',        :action => 'list'
+
       request.new_request    '/new',         :action => 'new'
       request.new_request_to_body    '/new/:public_body_id',         :action => 'new'
+
       request.show_request     '/request/:url_title', :action => 'show'
       request.describe_state   '/request/:id/describe', :action => 'describe_state'
       request.show_response_no_followup    '/request/:id/response', :action => 'show_response'
diff --git a/db/migrate/041_index_requests_with_solr.rb b/db/migrate/041_index_requests_with_solr.rb
new file mode 100644
index 000000000..6c6f161c9
--- /dev/null
+++ b/db/migrate/041_index_requests_with_solr.rb
@@ -0,0 +1,9 @@
+class IndexRequestsWithSolr < ActiveRecord::Migration
+    def self.up
+        add_column :info_requests, :solr_up_to_date, :boolean, :default => false, :null => false
+    end
+
+    def self.down
+        remove_column :info_requests, :solr_up_to_date
+    end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 58ddb696d..2965f7090 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -9,7 +9,7 @@
 #
 # It's strongly recommended to check this file into your version control system.
 
-ActiveRecord::Schema.define(:version => 40) do
+ActiveRecord::Schema.define(:version => 41) do
 
   create_table "incoming_messages", :force => true do |t|
     t.integer  "info_request_id",                    :null => false
@@ -37,6 +37,7 @@ ActiveRecord::Schema.define(:version => 40) do
     t.boolean  "awaiting_description", :default => false,    :null => false
     t.string   "prominence",           :default => "normal", :null => false
     t.text     "url_title",                                  :null => false
+    t.boolean  "solr_up_to_date",      :default => false,    :null => false
   end
 
   add_index "info_requests", ["created_at"], :name => "index_info_requests_on_created_at"
diff --git a/public/stylesheets/main.css b/public/stylesheets/main.css
index 7244e4d10..f49aa54c9 100644
--- a/public/stylesheets/main.css
+++ b/public/stylesheets/main.css
@@ -226,25 +226,25 @@ div.fieldWithErrors { display:inline; }
     width: 64%;
 }
 
-#request_frontpage h1 {
+#general_frontpage h1 {
     margin-bottom: 1em;
 }
-#request_frontpage #make_requests {
+#general_frontpage #make_requests {
     text-align: center;
     width: 45%;
     float: left;
     margin-bottom: 1em;
 }
-#request_frontpage #public_body_form {
+#general_frontpage #public_body_form {
 }
 
-#request_frontpage #find_information {
+#general_frontpage #find_information {
     width: 45%;
     float: right;
     margin-bottom: 1em;
     text-align: center;
 }
-#request_frontpage .request_listing {
+#general_frontpage .request_listing {
     text-align: left;
 }
 
diff --git a/todo.txt b/todo.txt
index e2074f2ff..2e4c858c0 100644
--- a/todo.txt
+++ b/todo.txt
@@ -1,3 +1,12 @@
+Search:
+
+Go to correct id
+Add cron job
+Add indexing of PDFs and DOCs etc.
+Pagination - http://www.quarkruby.com/2007/8/12/acts_as_solr-for-search-and-faceting
+Date ranges and types and stuff
+Highlighting - http://www.quarkruby.com/2007/9/14/advanced-acts_as_solr
+
 FOI requests to use to test it
 ==============================
 
@@ -13,6 +22,8 @@ BAILII - relationship with law courts, robots.txt ?
 Next
 ====
 
+Need something to mark contact as bad, e.g. for university of huddersfield
+
 This is knackered:
 http://foi.mysociety.org/request/14/response/44/attach/3/Marie's%20letter.tif
 
@@ -168,6 +179,8 @@ Add geographical location to council import
 Sources of public bodies
 ========================
 
+Call university of huddersfield
+
 Three lists of departments
     http://www.number10.gov.uk/output/Page30.asp
     http://en.wikipedia.org/wiki/Departments_of_the_United_Kingdom_Government
diff --git a/vendor/plugins/acts_as_solr/lib/acts_methods.rb b/vendor/plugins/acts_as_solr/lib/acts_methods.rb
index fba00cf02..47ae046ae 100644
--- a/vendor/plugins/acts_as_solr/lib/acts_methods.rb
+++ b/vendor/plugins/acts_as_solr/lib/acts_methods.rb
@@ -169,4 +169,4 @@ module ActsAsSolr #:nodoc:
     end
     
   end
-end
\ No newline at end of file
+end