diff options
| -rw-r--r-- | app/models/incoming_message.rb | 33 | ||||
| -rw-r--r-- | app/models/info_request_event.rb | 5 | ||||
| -rw-r--r-- | db/migrate/060_add_cached_main_text.rb | 9 | ||||
| -rw-r--r-- | db/schema.rb | 3 | ||||
| -rw-r--r-- | todo.txt | 2 | ||||
| -rw-r--r-- | vendor/plugins/attr_lazy/init.rb | 9 | ||||
| -rw-r--r-- | vendor/plugins/attr_lazy/lib/attr_lazy.rb | 94 |
7 files changed, 145 insertions, 10 deletions
diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb index 385657704..653097bab 100644 --- a/app/models/incoming_message.rb +++ b/app/models/incoming_message.rb @@ -18,7 +18,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: incoming_message.rb,v 1.120 2008-07-16 23:45:41 francis Exp $ +# $Id: incoming_message.rb,v 1.121 2008-07-17 03:30:54 francis Exp $ # TODO # Move some of the (e.g. quoting) functions here into rblib, as they feel @@ -110,6 +110,11 @@ class IncomingMessage < ActiveRecord::Base has_many :outgoing_message_followups, :foreign_key => 'incoming_message_followup_id', :class_name => 'OutgoingMessage' + # Some emails are large (10Mb), making things like search results and the + # front page list of requests slow to display as the data is transferred from + # the database. + attr_lazy :raw_data + # Return the structured TMail::Mail object # Documentation at http://i.loveruby.net/en/projects/tmail/doc/ def mail @@ -121,12 +126,21 @@ class IncomingMessage < ActiveRecord::Base end # Number the attachments in depth first tree order, for use in URLs. + # XXX This fills in part.rfc822_attachment and part.url_part_number within + # all the parts of the email (see TMail monkeypatch above for how these + # attributes are added). ensure_parts_counted must be called before using + # the attributes. This calculation is done only when required to avoid + # having to load and parse the email unnecessarily. def after_initialize - if !self.mail.nil? + @parts_counted = false + end + def ensure_parts_counted + if not @parts_counted @count_parts_count = 0 count_parts_recursive(self.mail) # we carry on using these numeric ids for attachments uudecoded from within text parts @count_first_uudecode_count = @count_parts_count + @parts_counted = true end end def count_parts_recursive(part) @@ -147,8 +161,8 @@ class IncomingMessage < ActiveRecord::Base end end # And look up by URL part number to get an attachment + # XXX relies on get_attachments_for_display calling ensure_parts_counted def self.get_attachment_by_url_part_number(attachments, found_url_part_number) - @count_parts_count = 0 attachments.each do |a| if a.url_part_number == found_url_part_number return a @@ -378,7 +392,7 @@ class IncomingMessage < ActiveRecord::Base end # If the part is an attachment of email in text form if curr_mail.content_type == 'message/rfc822' - # This has been expanded from text to an email in count_parts_recursive above + ensure_parts_counted # fills in rfc822_attachment variable leaves_found += get_attachment_leaves_recursive(curr_mail.rfc822_attachment) else # Store leaf @@ -390,7 +404,14 @@ class IncomingMessage < ActiveRecord::Base # Returns body text from main text part of email, converted to UTF-8, with uudecode removed def get_main_body_text - text = get_main_body_text_internal + # Cached as loading raw_data can be quite huge, and need this for just + # search results + if self.cached_main_body_text.nil? + text = self.get_main_body_text_internal + self.cached_main_body_text = text + self.save! + end + text = self.cached_main_body_text # Strip the uudecode parts from main text text = text.split(/^begin.+^`\n^end\n/sm).join(" ") @@ -530,6 +551,8 @@ class IncomingMessage < ActiveRecord::Base # Returns all attachments for use in display code def get_attachments_for_display + ensure_parts_counted + main_part = get_main_body_text_part leaves = get_attachment_leaves attachments = [] diff --git a/app/models/info_request_event.rb b/app/models/info_request_event.rb index a5eb308c6..7a76649ba 100644 --- a/app/models/info_request_event.rb +++ b/app/models/info_request_event.rb @@ -20,7 +20,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: info_request_event.rb,v 1.47 2008-07-16 23:45:41 francis Exp $ +# $Id: info_request_event.rb,v 1.48 2008-07-17 03:30:54 francis Exp $ class InfoRequestEvent < ActiveRecord::Base belongs_to :info_request @@ -69,7 +69,8 @@ class InfoRequestEvent < ActiveRecord::Base [ :variety, 'V', "variety" ] ], :if => :indexed_by_search, - :eager_load => [ { :incoming_message => { :info_request => :public_body }}, :outgoing_message, { :info_request => [ :user, :public_body ] } ] + :eager_load => [ :incoming_message, :outgoing_message, { :info_request => [ :user, :public_body ] } ] + def requested_by self.info_request.user.url_name end diff --git a/db/migrate/060_add_cached_main_text.rb b/db/migrate/060_add_cached_main_text.rb new file mode 100644 index 000000000..c372b471c --- /dev/null +++ b/db/migrate/060_add_cached_main_text.rb @@ -0,0 +1,9 @@ +class AddCachedMainText < ActiveRecord::Migration + def self.up + add_column :incoming_messages, :cached_main_body_text, :text + end + + def self.down + remove_column :incoming_messages, :cached_main_body_text + end +end diff --git a/db/schema.rb b/db/schema.rb index 939a07c2b..3212913c8 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -9,7 +9,7 @@ # # It's strongly recommended to check this file into your version control system. -ActiveRecord::Schema.define(:version => 59) do +ActiveRecord::Schema.define(:version => 60) do create_table "acts_as_xapian_jobs", :force => true do |t| t.string "model", :null => false @@ -25,6 +25,7 @@ ActiveRecord::Schema.define(:version => 59) do t.datetime "created_at", :null => false t.datetime "updated_at", :null => false t.text "cached_attachment_text" + t.text "cached_main_body_text" end create_table "info_request_events", :force => true do |t| @@ -55,7 +55,6 @@ Maybe remove public tracking completely Comments interleaved with body Antispam on contact form -Incoming email to unknown FOI is a bit rubbish Do something about shared spreadsheet @@ -220,7 +219,6 @@ http://www.ico.gov.uk/Home/tools_and_resources/decision_notices.aspx Description for each body as to what info it holds Link to: - Website itself (for general search) XXX can get this from domain of email and some scanning, with override needed Aliases (not just short name, but multiple real names e.g. for museums) Disclosure logs Publication schemes (http://www.ico.gov.uk/what_we_cover/freedom_of_information/publication_schemes.aspx) diff --git a/vendor/plugins/attr_lazy/init.rb b/vendor/plugins/attr_lazy/init.rb new file mode 100644 index 000000000..7bd84b894 --- /dev/null +++ b/vendor/plugins/attr_lazy/init.rb @@ -0,0 +1,9 @@ +# attr_lazy/init.rb: +# +# Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved. +# Email: francis@mysociety.org; WWW: http://www.mysociety.org/ +# +# $Id: init.rb,v 1.1 2008-07-17 03:30:55 francis Exp $ + +require 'attr_lazy' + diff --git a/vendor/plugins/attr_lazy/lib/attr_lazy.rb b/vendor/plugins/attr_lazy/lib/attr_lazy.rb new file mode 100644 index 000000000..eaa700ff9 --- /dev/null +++ b/vendor/plugins/attr_lazy/lib/attr_lazy.rb @@ -0,0 +1,94 @@ +# Taken from here, initial code at top (the refactoring doesn't work). +# http://refactormycode.com/codes/219-activerecord-lazy-attribute-loading-plugin-for-rails + +module AttrLazy + + def self.included(base_class) + base_class.extend(ClassMethods) + end + + module ClassMethods + def attr_lazy(*parameters) + columns = parameters + cattr_accessor :attr_lazy_configuration + self.attr_lazy_configuration = { + :columns => columns + } + + # prevent barfing if the plugin is reloaded/loaded twice + unless self.respond_to? :column_names_for_join_base_without_attr_lazy + self.extend AttrLazy::SingletonMethods + class << self + alias_method_chain :construct_finder_sql, :attr_lazy + alias_method_chain :column_names_for_join_base, :attr_lazy + end + end + + include AttrLazy::InstanceMethods + columns.each do |col| + class_eval("def #{col}; read_lazy_attribute :#{col}; end", __FILE__, __LINE__) + end + + end + + def column_names_for_join_base + column_names + end + + def column_names_for_join_base_with_attr_lazy + @column_names_for_join_base ||= columns.collect{|c| + c.name unless attr_lazy_configuration[:columns].include?(c.name.to_sym) + }.compact + end + end + + module SingletonMethods + + def construct_finder_sql_with_attr_lazy(options) + options = {:select => unlazy_column_list}.merge(options) + construct_finder_sql_without_attr_lazy(options) + end + + def unlazy_column_list + @unlazy ||= columns.collect do |c| + "#{quoted_table_name}.#{connection.quote_column_name(c.name)}" unless attr_lazy_columns.include?(c.name.to_sym) + end.compact.join ',' + end + + def attr_lazy_columns + attr_lazy_configuration[:columns] + end + + end + + module InstanceMethods + + def read_lazy_attribute(att) + @lazy_attribute_values ||= {} + if attribute_names.include?(att.to_s) + read_attribute att + else + @lazy_attribute_values[att] ||= self.class.find(id, :select => att)[att] + end + end + + end + +end + +class ActiveRecord::Associations::ClassMethods::JoinDependency::JoinBase + def column_names_with_alias + unless @column_names_with_alias + @column_names_with_alias = [] + ([active_record.primary_key] + (active_record.column_names_for_join_base - [active_record.primary_key])).each_with_index do |column_name, i| + @column_names_with_alias << [column_name, "#{ aliased_prefix }_r#{ i }"] + end + end + return @column_names_with_alias + end +end + +ActiveRecord::Base.send :include, AttrLazy + + + |