Don't grab all attachment text if there are no highlight words.

4 files changed, 31 insertions, 8 deletions

diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb
index dbc0f6213..17a8b6ba1 100644
--- a/app/models/incoming_message.rb
+++ b/app/models/incoming_message.rb
@@ -19,7 +19,7 @@
 # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: incoming_message.rb,v 1.222 2009-09-15 18:26:23 francis Exp $
+# $Id: incoming_message.rb,v 1.223 2009-09-15 21:30:39 francis Exp $
 
 # TODO
 # Move some of the (e.g. quoting) functions here into rblib, as they feel
@@ -926,6 +926,7 @@ class IncomingMessage < ActiveRecord::Base
     end
 
     # Returns all attachments for use in display code
+    # XXX is this called multiple times and should be cached?
     def get_attachments_for_display
         ensure_parts_counted
 
@@ -935,7 +936,13 @@ class IncomingMessage < ActiveRecord::Base
         for leaf in leaves
             if leaf != main_part
                 attachment = FOIAttachment.new
+
                 attachment.body = leaf.body
+                # As leaf.body causes MIME decoding which uses lots of RAM, do garbage collection here
+                # to prevent excess memory use. XXX not really sure if this helps reduce
+                # peak RAM use overall. Anyway, maybe there is something better to do than this.
+                GC.start 
+
                 attachment.filename = _get_censored_part_file_name(leaf)
                 if leaf.within_rfc822_attachment
                     attachment.within_rfc822_subject = leaf.within_rfc822_attachment.subject
@@ -1036,13 +1043,19 @@ class IncomingMessage < ActiveRecord::Base
 
     # Returns text version of attachment text
     def get_attachment_text
+        #STDOUT.puts 'start ' + MySociety::DebugHelpers::allocated_string_size_around_gc
         if self.cached_attachment_text.nil?
             attachment_text = self.get_attachment_text_internal
+            #STDOUT.puts 'after get_attachment_text_internal ' + MySociety::DebugHelpers::allocated_string_size_around_gc
             self.cached_attachment_text = attachment_text
+            #STDOUT.puts 'after assign to cached_attachment_text ' + MySociety::DebugHelpers::allocated_string_size_around_gc
             self.save!
+            #STDOUT.puts 'after save!' + MySociety::DebugHelpers::allocated_string_size_around_gc
         end
+        #STDOUT.puts 'after cache ' + MySociety::DebugHelpers::allocated_string_size_around_gc
 
         # Remove any privacy things
+        #STDOUT.puts 'before dup ' + MySociety::DebugHelpers::allocated_string_size_around_gc
         text = self.cached_attachment_text.dup
         #STDOUT.puts 'before mask_special_emails ' + MySociety::DebugHelpers::allocated_string_size_around_gc
         self.mask_special_emails!(text)
@@ -1153,6 +1166,10 @@ class IncomingMessage < ActiveRecord::Base
     def get_text_for_indexing
         return get_body_for_quoting + "\n\n" + get_attachment_text
     end
+    # Used when there are no highlight words, so full attachment text not required
+    def get_text_for_indexing_quick
+        return get_body_for_quoting
+    end
 
     # Returns the name of the person the incoming message is from, or nil if
     # there isn't one or if there is only an email address. XXX can probably
diff --git a/app/models/info_request_event.rb b/app/models/info_request_event.rb
index b803fe45d..0f20dbc42 100644
--- a/app/models/info_request_event.rb
+++ b/app/models/info_request_event.rb
@@ -21,7 +21,7 @@
 # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
 # Email: francis@mysociety.org; WWW: http://www.mysociety.org/
 #
-# $Id: info_request_event.rb,v 1.89 2009-09-09 00:30:28 francis Exp $
+# $Id: info_request_event.rb,v 1.90 2009-09-15 21:30:39 francis Exp $
 
 class InfoRequestEvent < ActiveRecord::Base
     belongs_to :info_request
@@ -126,14 +126,18 @@ class InfoRequestEvent < ActiveRecord::Base
         # format it here as no datetime support in Xapian's value ranges
         return self.created_at.strftime("%Y%m%d%H%M%S") 
     end
-    def search_text_main
+    def search_text_main(quick = false)
         text = ''
         if self.event_type == 'sent' 
             text = text + self.outgoing_message.get_text_for_indexing + "\n\n"
         elsif self.event_type == 'followup_sent'
             text = text + self.outgoing_message.get_text_for_indexing + "\n\n"
         elsif self.event_type == 'response'
-            text = text + self.incoming_message.get_text_for_indexing + "\n\n"
+            if quick
+                text = text + self.incoming_message.get_text_for_indexing_quick + "\n\n"
+            else
+                text = text + self.incoming_message.get_text_for_indexing + "\n\n"
+            end
         elsif self.event_type == 'comment'
             text = text + self.comment.body + "\n\n"
         else
diff --git a/app/views/request/_request_listing_via_event.rhtml b/app/views/request/_request_listing_via_event.rhtml
index 4faa2b0e5..839d52cfd 100644
--- a/app/views/request/_request_listing_via_event.rhtml
+++ b/app/views/request/_request_listing_via_event.rhtml
@@ -15,7 +15,11 @@ end %>
         <% end %>
 	</span>
     <span class="desc">	
-        <%= highlight_and_excerpt(event.search_text_main, @highlight_words, 150) %>
+        <% if @highlight_words == [] %>
+            <%= highlight_and_excerpt(event.search_text_main(quick = true), @highlight_words, 150) %>
+        <% else %>
+            <%= highlight_and_excerpt(event.search_text_main, @highlight_words, 150) %>
+        <% end %>
 	</span>
 
     <span class="bottomline icon_<%= info_request.calculate_status %>">
diff --git a/todo.txt b/todo.txt
index 5ca63d317..f82d90777 100644
--- a/todo.txt
+++ b/todo.txt
@@ -1,6 +1,4 @@
-mask_special_emails
-html_mask_stuff
-
+get_attachments_for_display multiple calls
 
 Next (things that will reduce admin time mainly)
 ====