Change from acts_as_solr to acts_as_xapian

16 files changed, 136 insertions, 170 deletions

diff --git a/app/controllers/application.rb b/app/controllers/application.rb
index 2f27f2872..79521e240 100644
--- a/app/controllers/application.rb
+++ b/app/controllers/application.rb
@@ -6,7 +6,7 @@
 # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: application.rb,v 1.39 2008-04-17 23:19:55 francis Exp $
+# $Id: application.rb,v 1.40 2008-04-24 23:52:59 francis Exp $
 
 
 class ApplicationController < ActionController::Base
@@ -130,11 +130,11 @@ class ApplicationController < ActionController::Base
     # Convert URL name for sort by order, to Lucene query 
     def order_to_sort_by(sortby)
         if sortby.nil?
-            return nil
+            return [nil, true]
         elsif sortby == 'newest'
-            return 'created_at desc'
+            return ['created_at', false]
         elsif sortby == 'described'
-            return 'rss_at desc' # use this for RSS
+            return ['rss_at', false] # use this for RSS
         else
             raise "Unknown sort order " + @sortby
         end
@@ -146,7 +146,9 @@ class ApplicationController < ActionController::Base
         @sortby = sortby
 
         # Work out sorting method
-        order = order_to_sort_by(@sortby)
+        order_pair = order_to_sort_by(@sortby)
+        order = order_pair[0]
+        ascending = order_pair[1]
 
         # Peform the search
         @per_page = per_page
@@ -155,17 +157,15 @@ class ApplicationController < ActionController::Base
         else
             @page = this_page
         end
-        solr_object = InfoRequest.full_search(@query, order, @per_page, @page, true) 
-        @search_results = solr_object.results
-        @search_hits = solr_object.total_hits
+        xapian_object = InfoRequest.full_search(@query, order, ascending, @per_page, @page, true) 
+        @search_results = xapian_object.results
+        @search_hits = xapian_object.matches_estimated
+        @search_spelling = xapian_object.spelling_correction
 
         # Calculate simple word highlighting view code for users and public bodies
         query_nopunc = @query.gsub(/[^a-z0-9]/i, " ")
         query_nopunc = query_nopunc.gsub(/\s+/, " ")
         @highlight_words = query_nopunc.split(" ")
-
-        # Extract better Solr highlighting for info request related results
-        @highlighting = solr_object.highlights
     end
 
     # URL generating functions are needed by all controllers (for redirects),
diff --git a/app/controllers/general_controller.rb b/app/controllers/general_controller.rb
index 7dbfa6b53..9e2dccc9a 100644
--- a/app/controllers/general_controller.rb
+++ b/app/controllers/general_controller.rb
@@ -5,7 +5,7 @@
 # Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: general_controller.rb,v 1.18 2008-04-15 23:53:10 francis Exp $
+# $Id: general_controller.rb,v 1.19 2008-04-24 23:52:59 francis Exp $
 
 class GeneralController < ApplicationController
 
@@ -70,7 +70,7 @@ class GeneralController < ApplicationController
 
     # Used in front page search for public body
     def public_body_query(query)
-        # @public_bodies = PublicBody.find_by_solr(query).results
+        # XXX try using search now we have spell correction?
 
         criteria = '%' + query + '%'
         @public_bodies = PublicBody.find(:all, 
diff --git a/app/controllers/track_controller.rb b/app/controllers/track_controller.rb
index 2a4ed9364..f3d83f776 100644
--- a/app/controllers/track_controller.rb
+++ b/app/controllers/track_controller.rb
@@ -5,7 +5,7 @@
 # Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: track_controller.rb,v 1.7 2008-04-18 01:57:43 francis Exp $
+# $Id: track_controller.rb,v 1.8 2008-04-24 23:52:59 francis Exp $
 
 class TrackController < ApplicationController
 
@@ -63,8 +63,7 @@ class TrackController < ApplicationController
         respond_to :atom
     end
 
-    # Delete a track
-#    def delete
+    # Change or delete a track
     def update
         track_thing = TrackThing.find(params[:track_id].to_i)
 
diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb
index f655209ef..c2a279c4a 100644
--- a/app/helpers/application_helper.rb
+++ b/app/helpers/application_helper.rb
@@ -5,7 +5,7 @@
 # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: application_helper.rb,v 1.19 2008-04-03 15:29:51 francis Exp $
+# $Id: application_helper.rb,v 1.20 2008-04-24 23:52:59 francis Exp $
 
 module ApplicationHelper
     # URL generating functions are needed by all controllers (for redirects),
@@ -44,9 +44,22 @@ module ApplicationHelper
     end
 
     # Highlight words, also escapes HTML (other than spans that we add)
-    def highlight_words(t, words)
+    def highlight_words(t, words, html = true)
         t = h(t)
-        t = highlight(t, words, '<span class="highlight">\1</span>')
+        if html
+            t = highlight(t, words, '<span class="highlight">\1</span>')
+        else
+            t = highlight(t, words, '*\1*')
+        end
+        return t
+    end
+    def highlight_and_excerpt(t, words, excount, html = true)
+        newt = excerpt(t, words[0], excount)
+        if not newt
+            newt = excerpt(t, '', excount)
+        end
+        t = newt
+        t = highlight_words(t, words, html)
         return t
     end
 end
diff --git a/app/models/info_request.rb b/app/models/info_request.rb
index 49245ab1b..22ef40883 100644
--- a/app/models/info_request.rb
+++ b/app/models/info_request.rb
@@ -13,7 +13,6 @@
 #  awaiting_description :boolean         default(false), not null
 #  prominence           :string(255)     default("normal"), not null
 #  url_title            :text            not null
-#  solr_up_to_date      :boolean         default(false), not null
 #
 
 # models/info_request.rb:
@@ -22,9 +21,10 @@
 # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: info_request.rb,v 1.95 2008-04-21 16:44:06 francis Exp $
+# $Id: info_request.rb,v 1.96 2008-04-24 23:52:59 francis Exp $
 
 require 'digest/sha1'
+require 'vendor/plugins/acts_as_xapian/lib/acts_as_xapian'
 
 class InfoRequest < ActiveRecord::Base
     validates_presence_of :title, :message => "^Please enter a summary of your request"
@@ -64,67 +64,19 @@ class InfoRequest < ActiveRecord::Base
         end
     end
 
-    # Full text search indexing
-    $do_solr_index = false
-    $do_solr_index_marking = false
-    def InfoRequest.update_solr_index
-        #STDERR.puts "self.update_solr_index"
-        $do_solr_index = true
-
-        # Index each item separately in a transaction, so solr_up_to_date is right 
-        ids_to_refresh = InfoRequest.find(:all, :conditions => ["not solr_up_to_date"]).map() { |i| i.id }
-        for id in ids_to_refresh
-            #STDERR.puts "updating id " + id.to_s
-            ActiveRecord::Base.transaction do
-                info_request = InfoRequest.find(id, :lock =>true)
-                do_index = (info_request.prominence != 'backpage')
-
-                info_request.calculate_event_states
-
-                # index all the events
-                for event in info_request.info_request_events
-                    if do_index and event.indexed_by_solr
-                        event.solr_save
-                    else
-                        event.solr_destroy
-                    end
-                end
-
-                $do_solr_index = false # disable indexing again while we save it, or else destroyed things get put back
-                $do_solr_index_marking = true # but record that we want to set solr_up_to_date to be true, so before_update doesn't clobber it
-                info_request.solr_up_to_date = true
-                #STDERR.puts "saving " + info_request.solr_up_to_date.to_s
-                info_request.save!
-                $do_solr_index_marking = false
-                $do_solr_index = true
-            end
-        end
-        InfoRequestEvent.solr_optimize
-        $do_solr_index = false
-    end
-    def before_update
-        # If we're not mid index, then mark we need to index later
-        if not $do_solr_index_marking
-            self.solr_up_to_date = false
-        end
-        true
-    end
-
     # Central function to do all searches
-    def InfoRequest.full_search(query, order, per_page, page, html_highlight)
+    # (Not really the right place to put it, but everything can get it here, and it
+    # does *mainly* find info requests, via their events, so hey)
+    def InfoRequest.full_search(query, order, ascending, per_page, page, html_highlight)
+        # XXX handle order better
+        # XXX html_highlight
         offset = (page - 1) * per_page
-        return InfoRequestEvent.multi_solr_search(query, :models => [ PublicBody, User ],
-            :limit => per_page, :offset => offset, 
-            :highlight => { 
-                :prefix => html_highlight ? '<span class="highlight">' : "*",
-                :suffix => html_highlight ? '</span>' : "*",
-                :fragsize => 250,
-                :fields => ["solr_text_main", "title", # InfoRequestEvent
-                           "name", "short_name", # PublicBody
-                           "name" # User
-            ]}, 
-            :order => order,
-            :include => { :InfoRequestEvent => [ { :incoming_message => { :info_request => :public_body }}, :outgoing_message, { :info_request => [ :user, :public_body ] } ] } 
+        return ::ActsAsXapian::Search.new(
+            [InfoRequestEvent, PublicBody, User], query,
+            :offset => offset, :limit => per_page,
+            :sort_by_prefix => order,
+            :sort_by_ascending => ascending, 
+            :collapse_by_prefix => "request_collapse"
         )
     end
 
@@ -135,7 +87,7 @@ class InfoRequest < ActiveRecord::Base
             t = Time.now.usec - t
             secs = t / 1000000.0
             STDOUT.write secs.to_s + " query " + i.to_s + "\n"
-            results = InfoRequest.full_search(query, "created_at desc", 25, 1, false).results
+            results = InfoRequest.full_search(query, "created_at", false, 25, 1, false).results
         end
     end
 
@@ -499,9 +451,6 @@ public
         self.incoming_messages.each { |a| a.destroy }
         self.outgoing_messages.each { |a| a.destroy }
         self.user_info_request_sent_alerts.each { |a| a.destroy }
-        for event in self.info_request_events
-            event.solr_destroy
-        end
         self.info_request_events.each { |a| a.destroy }
         self.destroy
     end
diff --git a/app/models/info_request_event.rb b/app/models/info_request_event.rb
index e8e1ecaa5..a89e98cad 100644
--- a/app/models/info_request_event.rb
+++ b/app/models/info_request_event.rb
@@ -20,7 +20,7 @@
 # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: info_request_event.rb,v 1.39 2008-04-24 22:50:03 angie Exp $
+# $Id: info_request_event.rb,v 1.40 2008-04-24 23:52:59 francis Exp $
 
 class InfoRequestEvent < ActiveRecord::Base
     belongs_to :info_request
@@ -53,20 +53,19 @@ class InfoRequestEvent < ActiveRecord::Base
     ]
 
     # Full text search indexing
-    acts_as_solr :fields => [ 
-        { :solr_text_main => :text },
-        { :title => :text },
-        { :status => :string },
-        { :requested_by => :string },
-        { :requested_from => :string },
-        { :request => :string },
-        { :created_at => :date },
-        { :rss_at => :date },
-        { :variety => :string }
-    ], :if => "$do_solr_index"
-    def status # for name in Solr queries
-        self.calculated_state
-    end
+    acts_as_xapian :texts => [ :search_text_main, :title ],
+        :values => [ [ :created_at, 0, "created_at", :date ],
+                     [ :rss_at, 1, "rss_at", :date ],
+                     [ :request, 2, "request_collapse", :string ]
+                   ],
+        :terms => [ [ :calculated_state, 'S', "status" ],
+                [ :requested_by, 'B', "requested_by" ],
+                [ :requested_from, 'F', "requested_from" ],
+                [ :request, 'R', "request" ],
+                [ :variety, 'V', "variety" ]
+        ],
+        :if => :indexed_by_search,
+        :eager_load => [ { :incoming_message => { :info_request => :public_body }}, :outgoing_message, { :info_request => [ :user, :public_body ] } ]
     def requested_by
         self.info_request.user.url_name
     end
@@ -83,7 +82,7 @@ class InfoRequestEvent < ActiveRecord::Base
         # types, just use the create at date.
         self.last_described_at || self.created_at
     end
-    def solr_text_main
+    def search_text_main
         text = ''
         if self.event_type == 'sent' 
             text = text + self.outgoing_message.body_without_salutation + "\n\n"
@@ -102,8 +101,11 @@ class InfoRequestEvent < ActiveRecord::Base
         end
         return ''
     end
-    def indexed_by_solr
+    def indexed_by_search
         if ['sent', 'followup_sent', 'response'].include?(self.event_type)
+            if info_request.prominence == 'backpage'
+                return false
+            end
             return true
         else
             return false
diff --git a/app/models/public_body.rb b/app/models/public_body.rb
index ab06209e3..4d073b4a2 100644
--- a/app/models/public_body.rb
+++ b/app/models/public_body.rb
@@ -21,7 +21,7 @@
 # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: public_body.rb,v 1.66 2008-04-22 00:54:07 francis Exp $
+# $Id: public_body.rb,v 1.67 2008-04-24 23:52:59 francis Exp $
 
 require 'csv'
 require 'set'
@@ -79,13 +79,9 @@ class PublicBody < ActiveRecord::Base
         attr_accessor :created_at
     end
 
-
-    acts_as_solr :fields => [
-        {:name => { :boost => 10.0 }}, 
-        {:short_name => { :boost => 10.0 }},
-        { :created_at => :date },
-        { :variety => :string }
-    ]
+    acts_as_xapian :texts => [ :name, :short_name ],
+        :values => [ [ :created_at, 0, "created_at", :date ] ],
+        :terms => [ [ :variety, 'V', "variety" ] ]
     def variety
         "authority"
     end
diff --git a/app/models/track_mailer.rb b/app/models/track_mailer.rb
index c5d2ef35f..5bd41a715 100644
--- a/app/models/track_mailer.rb
+++ b/app/models/track_mailer.rb
@@ -4,7 +4,7 @@
 # Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: track_mailer.rb,v 1.6 2008-04-14 14:46:48 francis Exp $
+# $Id: track_mailer.rb,v 1.7 2008-04-24 23:52:59 francis Exp $
 
 class TrackMailer < ApplicationMailer
     def event_digest(user, email_about_things)
@@ -41,13 +41,13 @@ class TrackMailer < ApplicationMailer
                 end
 
                 # Query for things in this track
-                solr_object = InfoRequest.full_search(track_thing.track_query, 'created_at desc', 100, 1, false) 
+                xapian_object = InfoRequest.full_search(track_thing.track_query, 'created_at', false, 100, 1, false) 
 
                 # Go through looking for unalerted things
                 alert_results = []
-                for result in solr_object.results
-                    if result.class.to_s == "InfoRequestEvent"
-                        if not done_info_request_events.include?(result.id) and track_thing.created_at < result.created_at
+                for result in xapian_object.results
+                    if result[:model].class.to_s == "InfoRequestEvent"
+                        if not done_info_request_events.include?(result[:model].id) and track_thing.created_at < result[:model].created_at
                             # OK alert this one
                             alert_results.push(result)
                         end
@@ -68,7 +68,7 @@ class TrackMailer < ApplicationMailer
                 for track_thing, alert_results in email_about_things
                     STDERR.puts "  tracking " + track_thing.track_query
                     for result in alert_results.reverse
-                        STDERR.puts "    result " + result.class.to_s + " id " + result.id.to_s
+                        STDERR.puts "    result " + result[:model].class.to_s + " id " + result[:model].id.to_s
                     end
                 end
 
@@ -81,8 +81,8 @@ class TrackMailer < ApplicationMailer
                 for result in alert_results
                     track_things_sent_email = TrackThingsSentEmail.new
                     track_things_sent_email.track_thing_id = track_thing.id
-                    if result.class.to_s == "InfoRequestEvent"
-                        track_things_sent_email.info_request_event_id = result.id
+                    if result[:model].class.to_s == "InfoRequestEvent"
+                        track_things_sent_email.info_request_event_id = result[:model].id
                     else
                         raise "need to add other types to TrackMailer.alert_tracks (mark alerted)"
                     end
diff --git a/app/models/user.rb b/app/models/user.rb
index 76b63c310..9042fda26 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -21,7 +21,7 @@
 # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: user.rb,v 1.51 2008-04-21 16:44:06 francis Exp $
+# $Id: user.rb,v 1.52 2008-04-24 23:52:59 francis Exp $
 
 require 'digest/sha1'
 
@@ -41,11 +41,9 @@ class User < ActiveRecord::Base
     attr_accessor :password_confirmation
     validates_confirmation_of :password, :message =>"^Please enter the same password twice"
 
-    acts_as_solr :fields => [ 
-        {:name => { :boost => 5.0 }},
-        { :created_at => :date },
-        { :variety => :string }
-    ]
+    acts_as_xapian :texts => [ :name ],
+        :values => [ [ :created_at, 0, "created_at", :date ] ],
+        :terms => [ [ :variety, 'V', "variety" ] ]
     def variety
         "user"
     end
diff --git a/app/views/general/frontpage.rhtml b/app/views/general/frontpage.rhtml
index ddbe20ea1..9584b1608 100644
--- a/app/views/general/frontpage.rhtml
+++ b/app/views/general/frontpage.rhtml
@@ -62,10 +62,10 @@
         <p>None yet.</p>
     <% else %>
         <% for search_result in @search_results %>
-            <% if search_result.class.to_s == 'InfoRequestEvent' %>
-                <%= render :partial => 'request/request_listing_via_event', :locals => { :event => search_result, :info_request => search_result.info_request } %>
+            <% if search_result[:model].class.to_s == 'InfoRequestEvent' %>
+                <%= render :partial => 'request/request_listing_via_event', :locals => { :event => search_result[:model], :info_request => search_result[:model].info_request } %>
             <% else %>
-                <p><strong>Unexpected search result type <%=search_result.class.to_s%></strong></p>
+                <p><strong>Unexpected search result type <%=search_result[:model].class.to_s%></strong></p>
             <% end %>
         <% end %>
     <% end %>
diff --git a/app/views/general/search.rhtml b/app/views/general/search.rhtml
index d94272922..a919bff45 100644
--- a/app/views/general/search.rhtml
+++ b/app/views/general/search.rhtml
@@ -26,17 +26,21 @@
 
     <h1><%=@title%></h1>
 
+    <% if @search_spelling %>
+        <p id="did_you_mean">Did you mean: <%= link_to @search_spelling, search_url(:query => @search_spelling) %></p>
+    <% end %>
+
     <% if @search_results.empty? %>
     <% else %>
         <% for search_result in @search_results %>
-            <% if search_result.class.to_s == 'PublicBody' %>
-                <%= render :partial => 'body/body_listing_single', :locals => { :public_body => search_result } %>
-            <% elsif search_result.class.to_s == 'User' %>
-                <%= render :partial => 'user/user_listing_single', :locals => { :display_user => search_result } %>
-            <% elsif search_result.class.to_s == 'InfoRequestEvent' %>
-                <%= render :partial => 'request/request_listing_via_event', :locals => { :event => search_result, :info_request => search_result.info_request } %>
+            <% if search_result[:model].class.to_s == 'PublicBody' %>
+                <%= render :partial => 'body/body_listing_single', :locals => { :public_body => search_result[:model] } %>
+            <% elsif search_result[:model].class.to_s == 'User' %>
+                <%= render :partial => 'user/user_listing_single', :locals => { :display_user => search_result[:model] } %>
+            <% elsif search_result[:model].class.to_s == 'InfoRequestEvent' %>
+                <%= render :partial => 'request/request_listing_via_event', :locals => { :event => search_result[:model], :info_request => search_result[:model].info_request } %>
             <% else %>
-                <p><strong>Unknown search result type <%=search_result.class.to_s%></strong></p>
+                <p><strong>Unknown search result type <%=search_result[:model].class.to_s%></strong></p>
             <% end %>
         <% end %>
     <% end %>
diff --git a/app/views/request/_request_listing_via_event.rhtml b/app/views/request/_request_listing_via_event.rhtml
index 8fe1d23ca..c292f8122 100644
--- a/app/views/request/_request_listing_via_event.rhtml
+++ b/app/views/request/_request_listing_via_event.rhtml
@@ -1,26 +1,29 @@
+<% if @highlight_words.nil?
+      @highlight_words = []
+end %>
+
 <div class="request_listing">
 	<span class="head">
-		<% if not @highlighting.nil? and @highlighting['InfoRequestEvent'][event.id].include?('title') %>
-			<%= link_to @highlighting['InfoRequestEvent'][event.id]["title"], request_url(info_request) %>
-			<% @highlighting['InfoRequestEvent'][event.id].delete("title") %>
-		<% elsif not event.incoming_message.nil? %>
-			<%= link_to "Response to '" + h(info_request.title) + "'", incoming_message_url(event.incoming_message) %>
-		<% elsif not event.outgoing_message.nil? and event.event_type == 'followup_sent' %>
-			<%= link_to "Follow up message for '" + h(info_request.title) + "'", outgoing_message_url(event.outgoing_message) %>
-		<% else %>
-			<%= link_to h(info_request.title), request_url(info_request) %>
-		<% end %>
+        <% if not event.incoming_message.nil? %>
+            <%= link_to "Response to '" + h(info_request.title) + "'", incoming_message_url(event.incoming_message) %>
+        <% elsif not event.outgoing_message.nil? and event.event_type == 'followup_sent' %>
+            <%= link_to "Follow up message for '" + h(info_request.title) + "'", outgoing_message_url(event.outgoing_message) %>
+        <% else %>
+            <%= link_to highlight_words(info_request.title, @highlight_words), request_url(info_request) %>
+        <% end %>
 	</span>
     <span class="desc">	
-		<% if not @highlighting.nil? and @highlighting['InfoRequestEvent'][event.id].size > 0 %>
-			<%= @highlighting['InfoRequestEvent'][event.id].values.join(" ") %>
-		<% elsif not event.outgoing_message.nil? %>
-			<%= excerpt(event.outgoing_message.body_without_salutation, "", 150) %>
-		<% elsif not event.incoming_message.nil? %>
-			<%= excerpt(event.incoming_message.get_body_for_quoting, "", 150) %>
-		<% else %>
-			<%= excerpt(info_request.initial_request_text, "", 150) %>
-		<% end %>
+        <%
+            # this will work, but will spawn catdoc and be slow!
+            #highlight_and_excerpt(event.search_text_main, @highlight_words, 150)
+        %>
+        <% if not event.outgoing_message.nil? %>
+            <%= highlight_and_excerpt(event.outgoing_message.body_without_salutation, @highlight_words, 150) %>
+        <% elsif not event.incoming_message.nil? %>
+            <%= highlight_and_excerpt(event.incoming_message.get_body_for_quoting, @highlight_words, 150) %>
+        <% else %>
+            <%= highlight_and_excerpt(info_request.initial_request_text, @highlight_words, 150) %>
+        <% end %>
 	</span>
 
     <span class="bottomline icon_<%= info_request.calculate_status %>">
diff --git a/app/views/request/list.rhtml b/app/views/request/list.rhtml
index f35f217d1..3b024b764 100644
--- a/app/views/request/list.rhtml
+++ b/app/views/request/list.rhtml
@@ -4,10 +4,10 @@
     <p>No requests made yet.</p>
 <% else %>
     <% for search_result in @search_results %>
-        <% if search_result.class.to_s == 'InfoRequestEvent' %>
-            <%= render :partial => 'request/request_listing_via_event', :locals => { :event => search_result, :info_request => search_result.info_request } %>
+        <% if search_result[:model].class.to_s == 'InfoRequestEvent' %>
+            <%= render :partial => 'request/request_listing_via_event', :locals => { :event => search_result[:model], :info_request => search_result[:model].info_request } %>
         <% else %>
-            <p><strong>Unexpected search result type <%=search_result.class.to_s%></strong></p>
+            <p><strong>Unexpected search result type <%=search_result[:model].class.to_s%></strong></p>
         <% end %>
     <% end %>
 <% end %>
diff --git a/app/views/track/atom_feed.atom.builder b/app/views/track/atom_feed.atom.builder
index 627c53d11..bc7a48de2 100644
--- a/app/views/track/atom_feed.atom.builder
+++ b/app/views/track/atom_feed.atom.builder
@@ -2,16 +2,16 @@ atom_feed do |feed|
     feed.title(@track_thing.params[:title_in_rss])
 
     for search_result in @search_results
-        feed.entry(search_result) do |entry|
+        feed.entry(search_result[:model]) do |entry|
             # Get the HTML content from the same partial template as website search does
             content = ''
-            if search_result.class.to_s == 'InfoRequestEvent'
-                content += render :partial => 'request/request_listing_via_event', :locals => { :event => search_result, :info_request => search_result.info_request }
+            if search_result[:model].class.to_s == 'InfoRequestEvent'
+                content += render :partial => 'request/request_listing_via_event', :locals => { :event => search_result[:model], :info_request => search_result[:model].info_request }
             else
-                content = "<p><strong>Unknown search result type " + search_result.class.to_s + "</strong></p>"
+                content = "<p><strong>Unknown search result type " + search_result[:model].class.to_s + "</strong></p>"
             end
             # Pull out the heading as separate item, from the partial template
-            content.match(/(<a href="[^>]*">(.*)<\/a>\s+<br>)/)
+            content.match(/(<span class="head">\s+<a href="[^>]*">(.*)<\/a>\s+<\/span>)/)
             heading = $1
             heading_text = $2
             content.sub!(heading, "")
diff --git a/app/views/track_mailer/event_digest.rhtml b/app/views/track_mailer/event_digest.rhtml
index ee702cb77..1eb33a892 100644
--- a/app/views/track_mailer/event_digest.rhtml
+++ b/app/views/track_mailer/event_digest.rhtml
@@ -1,3 +1,7 @@
+<% if @highlight_words.nil? 
+      @highlight_words = []
+end %>
+
 <%
         # Construct the main text of the mail
         main_text = ''
@@ -5,17 +9,15 @@
             main_text += track_thing.params[:title_in_email] + "\n"
             main_text += ("=" * track_thing.params[:title_in_email].size) + "\n\n"
             for result in alert_results.reverse
-                if result.class.to_s == "InfoRequestEvent"
-                    event = result
+                if result[:model].class.to_s == "InfoRequestEvent"
+                    event = result[:model]
 
-                    if not @highlighting.nil? and @highlighting['InfoRequestEvent'][event.id].size > 0
-                        extract = @highlighting['InfoRequestEvent'][event.id].values.join(" ")
-                    elsif not event.outgoing_message.nil?
-                        extract = excerpt(event.outgoing_message.body_without_salutation, "", 150)
+                    if not event.outgoing_message.nil?
+                        extract = highlight_and_excerpt(event.outgoing_message.body_without_salutation, @highlight_words, 150, false)
                     elsif not event.incoming_message.nil?
-                        extract = excerpt(event.incoming_message.get_body_for_quoting, "", 150)
+                        extract = highlight_and_excerpt(event.incoming_message.get_body_for_quoting, @highlight_words, 150, false)
                     else
-                        extract = excerpt(info_request.initial_request_text, "", 150)
+                        extract = highlight_and_excerpt(info_request.initial_request_text, @highlight_words, 150, false)
                     end
                     extract.gsub!(/\s+/, ' ')
 
@@ -44,7 +46,7 @@
             main_text += "\n"
         end
 
-        #STDERR.puts main_text
+        STDERR.puts main_text
         #STDERR.puts @unsubscribe_url
 %><%=main_text%>Alter your subscription
 =======================
diff --git a/app/views/user/_user_listing_single.rhtml b/app/views/user/_user_listing_single.rhtml
index b8517c671..3e5368b54 100644
--- a/app/views/user/_user_listing_single.rhtml
+++ b/app/views/user/_user_listing_single.rhtml
@@ -1,6 +1,6 @@
 <% if @highlight_words.nil? 
       @highlight_words = []
-   end %>
+end %>
 
 <p class="user_listing">
     <%= link_to highlight_words(display_user.name, @highlight_words), user_url(display_user) %>