Merge remote-tracking branch 'origin/feature/isolate-mail-handling' into develop

25 files changed, 284 insertions, 171 deletions

diff --git a/Gemfile b/Gemfile
index 1da116ae2..61eae74f8 100644
--- a/Gemfile
+++ b/Gemfile
@@ -16,6 +16,7 @@ gem 'gettext_i18n_rails', '>= 0.7.1'
 gem 'gettext', '~> 2.3.3'
 gem 'json', '~> 1.5.1'
 gem 'mahoro'
+gem 'mail', :platforms => :ruby_19
 gem 'memcache-client', :require => 'memcache'
 gem 'locale', '>= 2.0.5'
 gem 'net-http-local'
diff --git a/Gemfile.lock b/Gemfile.lock
index f2cdd9d40..c43136893 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -21,6 +21,7 @@ GEM
       net-ssh (>= 2.0.14)
       net-ssh-gateway (>= 1.1.0)
     chunky_png (1.2.6)
+    columnize (0.3.6)
     compass (0.12.2)
       chunky_png (~> 1.2)
       fssm (>= 0.2.7)
@@ -156,6 +157,7 @@ DEPENDENCIES
   json (~> 1.5.1)
   locale (>= 2.0.5)
   mahoro
+  mail
   mailcatcher
   memcache-client
   net-http-local
diff --git a/app/controllers/admin_request_controller.rb b/app/controllers/admin_request_controller.rb
index 6b6a8525e..1de63be59 100644
--- a/app/controllers/admin_request_controller.rb
+++ b/app/controllers/admin_request_controller.rb
@@ -207,8 +207,7 @@ class AdminRequestController < AdminController
                 end
 
                 raw_email_data = incoming_message.raw_email.data
-                mail = TMail::Mail.parse(raw_email_data)
-                mail.base64_decode
+                mail = MailHandler.mail_from_raw_email(raw_email_data)
                 destination_request.receive(mail, raw_email_data, true)
 
                 incoming_message_id = incoming_message.id
diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb
index 5205d0a2d..123319125 100644
--- a/app/models/incoming_message.rb
+++ b/app/models/incoming_message.rb
@@ -38,14 +38,6 @@ require 'zip/zip'
 require 'mapi/msg'
 require 'mapi/convert'
 
-# Monkeypatch! Adding some extra members to store extra info in.
-module TMail
-    class Mail
-        attr_accessor :url_part_number
-        attr_accessor :rfc822_attachment # when a whole email message is attached as text
-        attr_accessor :within_rfc822_attachment # for parts within a message attached as text (for getting subject mainly)
-    end
-end
 
 class IncomingMessage < ActiveRecord::Base
     belongs_to :info_request
@@ -70,17 +62,10 @@ class IncomingMessage < ActiveRecord::Base
         'application/zip' => 1,
     }
 
-    # Return the structured TMail::Mail object
-    # Documentation at http://i.loveruby.net/en/projects/tmail/doc/
+    # Return a cached structured mail object
     def mail(force = nil)
         if (!force.nil? || @mail.nil?) && !self.raw_email.nil?
-            # Hack round bug in TMail's MIME decoding.
-            # Report of TMail bug:
-            # http://rubyforge.org/tracker/index.php?func=detail&aid=21810&group_id=4512&atid=17370
-            copy_of_raw_data = self.raw_email.data.gsub(/; boundary=\s+"/im,'; boundary="')
-
-            @mail = TMail::Mail.parse(copy_of_raw_data)
-            @mail.base64_decode
+            @mail = MailHandler.mail_from_raw_email(self.raw_email.data)
         end
         @mail
     end
@@ -207,9 +192,9 @@ class IncomingMessage < ActiveRecord::Base
 
     # Number the attachments in depth first tree order, for use in URLs.
     # XXX This fills in part.rfc822_attachment and part.url_part_number within
-    # all the parts of the email (see TMail monkeypatch above for how these
-    # attributes are added). ensure_parts_counted must be called before using
-    # the attributes.
+    # all the parts of the email (see monkeypatches in lib/mail_handler/tmail_extensions and
+    # lib/mail_handler/mail_extensions for how these attributes are added). ensure_parts_counted
+    # must be called before using the attributes.
     def ensure_parts_counted
         @count_parts_count = 0
         _count_parts_recursive(self.mail)
@@ -222,20 +207,20 @@ class IncomingMessage < ActiveRecord::Base
                 _count_parts_recursive(p)
             end
         else
-            part_filename = TMail::Mail.get_part_file_name(part)
+            part_filename = MailHandler.get_part_file_name(part)
             begin
                 if part.content_type == 'message/rfc822'
                     # An email attached as text
                     # e.g. http://www.whatdotheyknow.com/request/64/response/102
-                    part.rfc822_attachment = TMail::Mail.parse(part.body)
+                    part.rfc822_attachment = MailHandler.mail_from_raw_email(part.body, decode=false)
                 elsif part.content_type == 'application/vnd.ms-outlook' || part_filename && AlaveteliFileTypes.filename_to_mimetype(part_filename) == 'application/vnd.ms-outlook'
                     # An email attached as an Outlook file
                     # e.g. http://www.whatdotheyknow.com/request/chinese_names_for_british_politi
                     msg = Mapi::Msg.open(StringIO.new(part.body))
-                    part.rfc822_attachment = TMail::Mail.parse(msg.to_mime.to_s)
+                    part.rfc822_attachment = MailHandler.mail_from_raw_email(msg.to_mime.to_s, decode=false)
                 elsif part.content_type == 'application/ms-tnef'
                     # A set of attachments in a TNEF file
-                    part.rfc822_attachment = TNEF.as_tmail(part.body)
+                    part.rfc822_attachment = MailHandler.mail_from_tnef(part.body)
                 end
             rescue
                 # If attached mail doesn't parse, treat it as text part
@@ -473,16 +458,6 @@ class IncomingMessage < ActiveRecord::Base
         return text
     end
 
-    # Internal function
-    def _get_part_file_name(mail)
-        part_file_name = TMail::Mail.get_part_file_name(mail)
-        if part_file_name.nil?
-            return nil
-        end
-        part_file_name = part_file_name.dup
-        return part_file_name
-    end
-
     # (This risks losing info if the unchosen alternative is the only one to contain
     # useful info, but let's worry about that another time)
     def get_attachment_leaves
@@ -534,7 +509,7 @@ class IncomingMessage < ActiveRecord::Base
             end
             # PDFs often come with this mime type, fix it up for view code
             if curr_mail.content_type == 'application/octet-stream'
-                part_file_name = self._get_part_file_name(curr_mail)
+                part_file_name = MailHandler.get_part_file_name(curr_mail)
                 calc_mime = AlaveteliFileTypes.filename_and_content_to_mimetype(part_file_name, curr_mail.body)
                 if calc_mime
                     curr_mail.content_type = calc_mime
@@ -814,7 +789,7 @@ class IncomingMessage < ActiveRecord::Base
             attachment = self.foi_attachments.find_or_create_by_hexdigest(:hexdigest => hexdigest)
             attachment.update_attributes(:url_part_number => leaf.url_part_number,
                                          :content_type => leaf.content_type,
-                                         :filename => _get_part_file_name(leaf),
+                                         :filename => MailHandler.get_part_file_name(leaf),
                                          :charset => leaf.charset,
                                          :within_rfc822_subject => within_rfc822_subject,
                                          :body => body)
diff --git a/app/models/info_request.rb b/app/models/info_request.rb
index e9335228a..194f8e105 100644
--- a/app/models/info_request.rb
+++ b/app/models/info_request.rb
@@ -275,7 +275,7 @@ public
         return self.magic_email("request-")
     end
     def incoming_name_and_email
-        return TMail::Address.address_from_name_and_email(self.user_name, self.incoming_email).to_s
+        return MailHandler.address_from_name_and_email(self.user_name, self.incoming_email)
     end
 
     # Subject lines for emails about the request
@@ -707,11 +707,11 @@ public
         return self.public_body.is_followupable?
     end
     def recipient_name_and_email
-        return TMail::Address.address_from_name_and_email(
+        return MailHandler.address_from_name_and_email(
             _("{{law_used}} requests at {{public_body}}",
                 :law_used => self.law_used_short,
                 :public_body => self.public_body.short_or_long_name),
-            self.recipient_email).to_s
+            self.recipient_email)
     end
 
     # History of some things that have happened
diff --git a/app/models/info_request_event.rb b/app/models/info_request_event.rb
index 5a8e3416f..09eba31ab 100644
--- a/app/models/info_request_event.rb
+++ b/app/models/info_request_event.rb
@@ -384,7 +384,7 @@ class InfoRequestEvent < ActiveRecord::Base
         if prev_addr.nil? || curr_addr.nil?
             return false
         end
-        return TMail::Address.parse(prev_addr).address == TMail::Address.parse(curr_addr).address
+        return MailHandler.address_from_string(prev_addr) == MailHandler.address_from_string(curr_addr)
     end
 
     def json_for_api(deep, snippet_highlight_proc = nil)
diff --git a/app/models/outgoing_mailer.rb b/app/models/outgoing_mailer.rb
index 277794c69..503166b8a 100644
--- a/app/models/outgoing_mailer.rb
+++ b/app/models/outgoing_mailer.rb
@@ -47,7 +47,8 @@ class OutgoingMailer < ApplicationMailer
             return info_request.recipient_name_and_email
         else
             # calling safe_mail_from from so censor rules are run
-            return TMail::Address.address_from_name_and_email(incoming_message_followup.safe_mail_from, incoming_message_followup.from_email).to_s
+            return MailHandler.address_from_name_and_email(incoming_message_followup.safe_mail_from,
+                                                           incoming_message_followup.from_email)
         end
     end
     # Used in the preview of followup
diff --git a/app/models/request_mailer.rb b/app/models/request_mailer.rb
index 90c4c6b53..493d6961c 100644
--- a/app/models/request_mailer.rb
+++ b/app/models/request_mailer.rb
@@ -204,15 +204,14 @@ class RequestMailer < ApplicationMailer
     #
     # That is because we want to be sure we properly record the actual message
     # received in its raw form - so any information won't be lost in a round
-    # trip via TMail, or by bugs in it, and so we can use something other than
-    # TMail at a later date. And so we can offer an option to download the
+    # trip via the mail handler, or by bugs in it, and so we can use something
+    # other than TMail at a later date. And so we can offer an option to download the
     # actual original mail sent by the authority in the admin interface (so
     # can check that attachment decoding failures are problems in the message,
     # not in our code). ]
     def self.receive(raw_email)
         logger.info "Received mail:\n #{raw_email}" unless logger.nil?
-        mail = TMail::Mail.parse(raw_email)
-        mail.base64_decode
+        mail = MailHandler.mail_from_raw_email(raw_email)
         new.receive(mail, raw_email)
     end
 
diff --git a/app/models/user.rb b/app/models/user.rb
index 70386f7e4..6e1e21481 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -203,7 +203,7 @@ class User < ActiveRecord::Base
 
 # For use in to/from in email messages
     def name_and_email
-        return TMail::Address.address_from_name_and_email(self.name, self.email).to_s
+        return MailHandler.address_from_name_and_email(self.name, self.email)
     end
 
     # The "internal admin" is a special user for internal use.
diff --git a/config/environment.rb b/config/environment.rb
index 492446a43..e79efdcfa 100644
--- a/config/environment.rb
+++ b/config/environment.rb
@@ -92,6 +92,8 @@ Rails::Initializer.run do |config|
      require 'routing_filters.rb'
   end
 
+  config.autoload_paths << "#{RAILS_ROOT}/lib/mail_handler"
+
   # See Rails::Configuration for more options
   ENV['RECAPTCHA_PUBLIC_KEY'] = Configuration::recaptcha_public_key
   ENV['RECAPTCHA_PRIVATE_KEY'] = Configuration::recaptcha_private_key
@@ -140,7 +142,6 @@ WillPaginate::ViewHelpers.pagination_options[:renderer] = 'WillPaginateExtension
 
 # Load monkey patches and other things from lib/
 require 'ruby19.rb'
-require 'tmail_extensions.rb'
 require 'activesupport_cache_extensions.rb'
 require 'timezone_fixes.rb'
 require 'use_spans_for_errors.rb'
@@ -148,12 +149,12 @@ require 'make_html_4_compliant.rb'
 require 'activerecord_errors_extensions.rb'
 require 'willpaginate_extension.rb'
 require 'sendmail_return_path.rb'
-require 'tnef.rb'
 require 'i18n_fixes.rb'
 require 'rack_quote_monkeypatch.rb'
 require 'world_foi_websites.rb'
 require 'alaveteli_external_command.rb'
 require 'quiet_opener.rb'
+require 'mail_handler'
 
 if !Configuration.exception_notifications_from.blank? && !Configuration.exception_notifications_to.blank?
   ExceptionNotification::Notifier.sender_address = Configuration::exception_notifications_from
diff --git a/lib/mail_handler/backends/mail_backend.rb b/lib/mail_handler/backends/mail_backend.rb
new file mode 100644
index 000000000..0e198adf0
--- /dev/null
+++ b/lib/mail_handler/backends/mail_backend.rb
@@ -0,0 +1,51 @@
+require 'mail'
+
+module MailHandler
+    module Backends
+        module MailBackend
+
+            def backend()
+                'Mail'
+            end
+
+            # Note that the decode flag is not yet used
+            def mail_from_raw_email(data, decode=true)
+                Mail.new(data)
+            end
+
+             # Extracts all attachments from the given TNEF file as a Mail object
+            def mail_from_tnef(content)
+                main = Mail.new
+                tnef_attachments(content).each do |attachment|
+                    main.add_file(attachment)
+                end
+                main.ready_to_send!
+                main
+            end
+
+            # Return a copy of the file name for the mail part
+            def get_part_file_name(mail_part)
+                part_file_name = mail_part.filename
+                part_file_name.nil? ? nil : part_file_name.dup
+            end
+
+            # Format
+            def address_from_name_and_email(name, email)
+                if !MySociety::Validate.is_valid_email(email)
+                    raise "invalid email " + email + " passed to address_from_name_and_email"
+                end
+                if name.nil?
+                    return Mail::Address.new(email)
+                end
+                address = Mail::Address.new
+                address.display_name = name
+                address.address = email
+                address.to_s
+            end
+
+            def address_from_string(string)
+                Mail::Address.new(string).address
+            end
+        end
+    end
+end
\ No newline at end of file
diff --git a/lib/mail_handler/backends/mail_extensions.rb b/lib/mail_handler/backends/mail_extensions.rb
new file mode 100644
index 000000000..cbe0491ed
--- /dev/null
+++ b/lib/mail_handler/backends/mail_extensions.rb
@@ -0,0 +1,7 @@
+module Mail
+    class Message
+        attr_accessor :url_part_number
+        attr_accessor :rfc822_attachment # when a whole email message is attached as text
+        attr_accessor :within_rfc822_attachment # for parts within a message attached as text (for getting subject mainly)
+    end
+end
\ No newline at end of file
diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb
new file mode 100644
index 000000000..87aba73d7
--- /dev/null
+++ b/lib/mail_handler/backends/tmail_backend.rb
@@ -0,0 +1,62 @@
+module MailHandler
+    module Backends
+        module TmailBackend
+
+            def backend()
+                'TMail'
+            end
+
+            # Turn raw data into a structured TMail::Mail object
+            # Documentation at http://i.loveruby.net/en/projects/tmail/doc/
+            def mail_from_raw_email(data, decode=true)
+                # Hack round bug in TMail's MIME decoding.
+                # Report of TMail bug:
+                # http://rubyforge.org/tracker/index.php?func=detail&aid=21810&group_id=4512&atid=17370
+                copy_of_raw_data = data.gsub(/; boundary=\s+"/im,'; boundary="')
+                mail = TMail::Mail.parse(copy_of_raw_data)
+                mail.base64_decode if decode
+                mail
+            end
+
+            # Extracts all attachments from the given TNEF file as a TMail::Mail object
+            def mail_from_tnef(content)
+                main = TMail::Mail.new
+                main.set_content_type 'multipart', 'mixed', { 'boundary' => TMail.new_boundary }
+                tnef_attachments(content).each do |attachment|
+                    tmail_attachment = TMail::Mail.new
+                    tmail_attachment['content-location'] = attachment[:filename]
+                    tmail_attachment.body = attachment[:content]
+                    main.parts << tmail_attachment
+                end
+                main
+            end
+
+            # Return a copy of the file name for the mail part
+            def get_part_file_name(mail_part)
+                part_file_name = TMail::Mail.get_part_file_name(mail_part)
+                if part_file_name.nil?
+                    return nil
+                end
+                part_file_name = part_file_name.dup
+                return part_file_name
+            end
+
+            def address_from_name_and_email(name, email)
+                if !MySociety::Validate.is_valid_email(email)
+                    raise "invalid email " + email + " passed to address_from_name_and_email"
+                end
+                if name.nil?
+                    return TMail::Address.parse(email).to_s
+                end
+                # Botch an always quoted RFC address, then parse it
+                name = name.gsub(/(["\\])/, "\\\\\\1")
+                TMail::Address.parse('"' + name + '" <' + email + '>').to_s
+            end
+
+            def address_from_string(string)
+                TMail::Address.parse(string).address
+            end
+
+        end
+    end
+end
\ No newline at end of file
diff --git a/lib/tmail_extensions.rb b/lib/mail_handler/backends/tmail_extensions.rb
index 6a533e658..9359dfeea 100644
--- a/lib/tmail_extensions.rb
+++ b/lib/mail_handler/backends/tmail_extensions.rb
@@ -15,6 +15,12 @@ require 'tmail/interface'
 # These mainly used in app/models/incoming_message.rb
 module TMail
     class Mail
+        # Monkeypatch! Adding some extra members to store extra info in.
+
+        attr_accessor :url_part_number
+        attr_accessor :rfc822_attachment # when a whole email message is attached as text
+        attr_accessor :within_rfc822_attachment # for parts within a message attached as text (for getting subject mainly)
+
         # Monkeypatch! (check to see if this becomes a standard function in
         # TMail::Mail, then use that, whatever it is called)
         def Mail.get_part_file_name(part)
@@ -68,22 +74,6 @@ module TMail
 
     end
 
-    class Address
-        # Monkeypatch! Constructor which makes a TMail::Address given
-        # a name and an email
-        def Address.address_from_name_and_email(name, email)
-            if !MySociety::Validate.is_valid_email(email)
-                raise "invalid email " + email + " passed to address_from_name_and_email"
-            end
-            if name.nil?
-                return TMail::Address.parse(email)
-            end
-            # Botch an always quoted RFC address, then parse it
-            name = name.gsub(/(["\\])/, "\\\\\\1")
-            return TMail::Address.parse('"' + name + '" <' + email + '>')
-        end
-    end
-
     module TextUtils
         # Monkeypatch! Much more aggressive list of characters to cause quoting
         # than in normal TMail. e.g. Have found real cases where @ needs quoting.
@@ -95,8 +85,8 @@ module TMail
     end
 end
 
-# Monkeypatch! TMail 1.2.7.1 will parse only one address out of a list of addresses with 
-# unquoted display parts https://github.com/mikel/tmail/issues#issue/9 - this monkeypatch 
+# Monkeypatch! TMail 1.2.7.1 will parse only one address out of a list of addresses with
+# unquoted display parts https://github.com/mikel/tmail/issues#issue/9 - this monkeypatch
 # fixes this issue.
 module TMail
 
diff --git a/lib/tnef.rb b/lib/mail_handler/mail_handler.rb
index 1c941f8b0..24d14b5c8 100644
--- a/lib/tnef.rb
+++ b/lib/mail_handler/mail_handler.rb
@@ -1,13 +1,23 @@
+# Handles the parsing of email
 require 'tmpdir'
 
-class TNEF
+module MailHandler
 
-    # Extracts all attachments from the given TNEF file as a TMail::Mail object
-    # The TNEF file also contains the message body, but in general this is the
+    if RUBY_VERSION.to_f >= 1.9
+        require 'backends/mail_extensions'
+        require 'backends/mail_backend'
+        include Backends::MailBackend
+    else
+        require 'backends/tmail_extensions'
+        require 'backends/tmail_backend'
+        include Backends::TmailBackend
+    end
+
+    # Returns a set of attachments from the given TNEF contents
+    # The TNEF contents also contains the message body, but in general this is the
     # same as the message body in the message proper.
-    def self.as_tmail(content)
-        main = TMail::Mail.new
-        main.set_content_type 'multipart', 'mixed', { 'boundary' => TMail.new_boundary }
+    def tnef_attachments(content)
+        attachments = []
         Dir.mktmpdir do |dir|
             IO.popen("#{`which tnef`.chomp} -K -C #{dir}", "w") do |f|
                 f.write(content)
@@ -23,10 +33,8 @@ class TNEF
             Dir.new(dir).sort.each do |file| # sort for deterministic behaviour
                 if file != "." && file != ".."
                     file_content = File.open("#{dir}/#{file}", "r").read
-                    attachment = TMail::Mail.new
-                    attachment['content-location'] = file
-                    attachment.body = file_content
-                    main.parts << attachment
+                    attachments << { :content => file_content,
+                                     :filename => file }
                     found += 1
                 end
             end
@@ -34,7 +42,11 @@ class TNEF
                 raise IOError, "tnef produced no attachments"
             end
         end
-        main
+        attachments
     end
 
+    # Turn instance methods into class methods
+    extend self
+
 end
+
diff --git a/lib/tasks/translation.rake b/lib/tasks/translation.rake
index f6611cc80..273c12bfa 100644
--- a/lib/tasks/translation.rake
+++ b/lib/tasks/translation.rake
@@ -4,7 +4,7 @@ namespace :translation do
     include Usage
 
     def write_email(email, email_description, output_file)
-        mail_object = TMail::Mail.parse(email.to_s)
+        mail_object =  MailHandler.mail_from_raw_email(email.to_s, decode=false)
         output_file.write("\n")
         output_file.write("Description of email: #{email_description}\n")
         output_file.write("Subject line: #{mail_object.subject}\n")
@@ -86,7 +86,7 @@ namespace :translation do
                                       'fixtures',
                                       'files',
                                       'incoming-request-plain.email'))
-        response_mail = TMail::Mail.parse(content)
+        response_mail =  MailHandler.mail_from_raw_email(content, decode=false)
 
         response_mail.from = "authority@example.com"
         stopped_responses_email = RequestMailer.create_stopped_responses(info_request,
diff --git a/script/handle-mail-replies.rb b/script/handle-mail-replies.rb
index f4ffb61f8..c05dca453 100755
--- a/script/handle-mail-replies.rb
+++ b/script/handle-mail-replies.rb
@@ -16,17 +16,19 @@ $alaveteli_dir = File.expand_path(File.join(File.dirname(__FILE__), '..'))
 $:.push(File.join($alaveteli_dir, "commonlib", "rblib"))
 load "config.rb"
 $:.push(File.join($alaveteli_dir, "lib"))
+$:.push(File.join($alaveteli_dir, "lib", "mail_handler"))
 require "configuration"
 MySociety::Config.set_file(File.join($alaveteli_dir, 'config', 'general'), true)
 MySociety::Config.load_default
 
 require 'action_mailer'
+require 'mail_handler'
 
 def main(in_test_mode)
     Dir.chdir($alaveteli_dir) do
         raw_message = $stdin.read
         begin
-            message = TMail::Mail.parse(raw_message)
+            message = MailHandler.mail_from_raw_email(raw_message, decode=false)
         rescue
             # Error parsing message. Just pass it on, to be on the safe side.
             forward_on(raw_message) unless in_test_mode
diff --git a/spec/lib/mail_handler/mail_handler_spec.rb b/spec/lib/mail_handler/mail_handler_spec.rb
new file mode 100644
index 000000000..a3fba0698
--- /dev/null
+++ b/spec/lib/mail_handler/mail_handler_spec.rb
@@ -0,0 +1,23 @@
+# coding: utf-8
+require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper')
+
+describe 'when creating a mail object from raw data' do
+
+    it 'should correctly parse a multipart email with a linebreak in the boundary' do
+        mail = get_fixture_mail('space-boundary.email')
+        mail.parts.size.should == 2
+        mail.multipart?.should == true
+    end
+
+    it 'should parse multiple to addresses with unqoted display names' do
+        mail = get_fixture_mail('multiple-unquoted-display-names.email')
+        mail.to.should == ["request-66666-caa77777@whatdotheyknow.com", "foi@example.com"]
+    end
+
+    it 'should convert an iso8859 email to utf8' do
+        mail = get_fixture_mail('iso8859_2_raw_email.email')
+        mail.subject.should have_text(/gjatë/u)
+        mail.body.is_utf8?.should == true
+    end
+
+end
diff --git a/spec/lib/tmail_extensions_spec.rb b/spec/lib/tmail_extensions_spec.rb
deleted file mode 100644
index bd89e6a84..000000000
--- a/spec/lib/tmail_extensions_spec.rb
+++ /dev/null
@@ -1,45 +0,0 @@
-# coding: utf-8
-# This is a test of the set_content_type monkey patch in
-# lib/tmail_extensions.rb
-
-require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
-
-describe "when using TMail" do
-
-    before(:each) do
-        ActionMailer::Base.deliveries.clear
-    end
-
-    it "should load an email with funny MIME settings" do
-        # just send it to the holding pen
-        InfoRequest.holding_pen_request.incoming_messages.size.should == 0
-        receive_incoming_mail("humberside-police-odd-mime-type.email", 'dummy')
-        InfoRequest.holding_pen_request.incoming_messages.size.should == 1
-
-        # clear the notification of new message in holding pen
-        deliveries = ActionMailer::Base.deliveries
-        deliveries.size.should == 1
-        deliveries.clear
-
-        incoming_message = InfoRequest.holding_pen_request.incoming_messages[0]
-
-        # This will raise an error if the bug in TMail hasn't been fixed
-        incoming_message.get_body_for_html_display()
-    end
-
-    it 'should parse multiple to addresses with unqoted display names' do
-        mail = TMail::Mail.parse(load_file_fixture('multiple-unquoted-display-names.email'))
-        mail.to.should == ["request-66666-caa77777@whatdotheyknow.com", "foi@example.com"]
-    end
-
-    it 'should convert to utf8' do
-        # NB this isn't actually a TMail extension, but is core TMail;
-        # this was just a convenient place to assert the UTF8
-        # conversion is working
-        mail = TMail::Mail.parse(load_file_fixture('iso8859_2_raw_email.email'))
-        mail.subject.should have_text(/gjatë/u)
-        mail.body.is_utf8?.should == true
-    end
-
-end
-
diff --git a/spec/models/incoming_message_spec.rb b/spec/models/incoming_message_spec.rb
index b038c43d9..fdbcd1e23 100644
--- a/spec/models/incoming_message_spec.rb
+++ b/spec/models/incoming_message_spec.rb
@@ -85,6 +85,26 @@ describe IncomingMessage, " when dealing with incoming mail" do
       end
     end
 
+
+    it "should load an email with funny MIME settings" do
+        ActionMailer::Base.deliveries.clear
+        # just send it to the holding pen
+        InfoRequest.holding_pen_request.incoming_messages.size.should == 0
+        receive_incoming_mail("humberside-police-odd-mime-type.email", 'dummy')
+        InfoRequest.holding_pen_request.incoming_messages.size.should == 1
+
+        # clear the notification of new message in holding pen
+        deliveries = ActionMailer::Base.deliveries
+        deliveries.size.should == 1
+        deliveries.clear
+
+        incoming_message = InfoRequest.holding_pen_request.incoming_messages[0]
+
+        # This will raise an error if the bug in TMail hasn't been fixed
+        incoming_message.get_body_for_html_display()
+    end
+
+
 end
 
 describe IncomingMessage, "when parsing HTML mail" do
@@ -399,14 +419,8 @@ end
 
 describe IncomingMessage, " when uudecoding bad messages" do
 
-    before(:each) do
-        load_raw_emails_data
-    end
-
     it "should be able to do it at all" do
-        mail_body = load_file_fixture('incoming-request-bad-uuencoding.email')
-        mail = TMail::Mail.parse(mail_body)
-        mail.base64_decode
+        mail = get_fixture_mail('incoming-request-bad-uuencoding.email')
         im = incoming_messages(:useless_incoming_message)
         im.stub!(:mail).and_return(mail)
         im.extract_attachments!
@@ -418,9 +432,7 @@ describe IncomingMessage, " when uudecoding bad messages" do
     end
 
     it "should apply censor rules" do
-        mail_body = load_file_fixture('incoming-request-bad-uuencoding.email')
-        mail = TMail::Mail.parse(mail_body)
-        mail.base64_decode
+        mail = get_fixture_mail('incoming-request-bad-uuencoding.email')
 
         im = incoming_messages(:useless_incoming_message)
         im.stub!(:mail).and_return(mail)
@@ -443,14 +455,8 @@ end
 
 describe IncomingMessage, "when messages are attached to messages" do
 
-    before(:each) do
-        load_raw_emails_data
-    end
-
     it "should flatten all the attachments out" do
-        mail_body = load_file_fixture('incoming-request-attach-attachments.email')
-        mail = TMail::Mail.parse(mail_body)
-        mail.base64_decode
+        mail = get_fixture_mail('incoming-request-attach-attachments.email')
 
         im = incoming_messages(:useless_incoming_message)
         im.stub!(:mail).and_return(mail)
@@ -468,14 +474,8 @@ end
 
 describe IncomingMessage, "when Outlook messages are attached to messages" do
 
-    before(:each) do
-        load_raw_emails_data
-    end
-
     it "should flatten all the attachments out" do
-        mail_body = load_file_fixture('incoming-request-oft-attachments.email')
-        mail = TMail::Mail.parse(mail_body)
-        mail.base64_decode
+        mail = get_fixture_mail('incoming-request-oft-attachments.email')
 
         im = incoming_messages(:useless_incoming_message)
         im.stub!(:mail).and_return(mail)
@@ -490,14 +490,8 @@ end
 
 describe IncomingMessage, "when TNEF attachments are attached to messages" do
 
-    before(:each) do
-        load_raw_emails_data
-    end
-
     it "should flatten all the attachments out" do
-        mail_body = load_file_fixture('incoming-request-tnef-attachments.email')
-        mail = TMail::Mail.parse(mail_body)
-        mail.base64_decode
+        mail = get_fixture_mail('incoming-request-tnef-attachments.email')
 
         im = incoming_messages(:useless_incoming_message)
         im.stub!(:mail).and_return(mail)
diff --git a/spec/models/info_request_event_spec.rb b/spec/models/info_request_event_spec.rb
index 7352f3be0..796f8b840 100644
--- a/spec/models/info_request_event_spec.rb
+++ b/spec/models/info_request_event_spec.rb
@@ -54,36 +54,71 @@ describe InfoRequestEvent do
 
     end
 
-    describe "doing search/index stuff" do 
+    describe "doing search/index stuff" do
 
         before(:each) do
             load_raw_emails_data
             parse_all_incoming_messages
         end
 
-        it 'should get search text for outgoing messages' do 
+        it 'should get search text for outgoing messages' do
             event = info_request_events(:useless_outgoing_message_event)
             message = outgoing_messages(:useless_outgoing_message).body
             event.search_text_main.should == message + "\n\n"
         end
 
-        it 'should get search text for incoming messages' do 
+        it 'should get search text for incoming messages' do
             event = info_request_events(:useless_incoming_message_event)
             event.search_text_main.strip.should == "No way! I'm not going to tell you that in a month of Thursdays.\n\nThe Geraldine Quango"
         end
 
-        it 'should get clipped text for incoming messages, and cache it too' do 
+        it 'should get clipped text for incoming messages, and cache it too' do
             event = info_request_events(:useless_incoming_message_event)
-            
+
             event.incoming_message_selective_columns("cached_main_body_text_folded").cached_main_body_text_folded = nil
             event.search_text_main(true).strip.should == "No way! I'm not going to tell you that in a month of Thursdays.\n\nThe Geraldine Quango"
             event.incoming_message_selective_columns("cached_main_body_text_folded").cached_main_body_text_folded.should_not == nil
         end
 
-
     end
 
+    describe 'when asked if it has the same email as a previous send' do
+
+        before do
+            @info_request_event = InfoRequestEvent.new
+        end
+
+        it 'should return true if the email in its params and the previous email the request was sent to are both nil' do
+            @info_request_event.stub!(:params).and_return({})
+            @info_request_event.stub_chain(:info_request, :get_previous_email_sent_to).and_return(nil)
+            @info_request_event.same_email_as_previous_send?.should be_true
+        end
+
+        it 'should return false if one email address exists and the other does not' do
+            @info_request_event.stub!(:params).and_return(:email => 'test@example.com')
+            @info_request_event.stub_chain(:info_request, :get_previous_email_sent_to).and_return(nil)
+            @info_request_event.same_email_as_previous_send?.should be_false
+        end
 
+        it 'should return true if the addresses are identical' do
+            @info_request_event.stub!(:params).and_return(:email => 'test@example.com')
+            @info_request_event.stub_chain(:info_request, :get_previous_email_sent_to).and_return('test@example.com')
+            @info_request_event.same_email_as_previous_send?.should be_true
+        end
+
+        it 'should return false if the addresses are different' do
+            @info_request_event.stub!(:params).and_return(:email => 'test@example.com')
+            @info_request_event.stub_chain(:info_request, :get_previous_email_sent_to).and_return('different@example.com')
+            @info_request_event.same_email_as_previous_send?.should be_false
+        end
+
+        it 'should return true if the addresses have different formats' do
+            @info_request_event.stub!(:params).and_return(:email => 'A Test <test@example.com>')
+            @info_request_event.stub_chain(:info_request, :get_previous_email_sent_to).and_return('test@example.com')
+            @info_request_event.same_email_as_previous_send?.should be_true
+        end
+
+    end
 
 end
 
diff --git a/spec/models/request_mailer_spec.rb b/spec/models/request_mailer_spec.rb
index 906756784..0f09e6926 100644
--- a/spec/models/request_mailer_spec.rb
+++ b/spec/models/request_mailer_spec.rb
@@ -98,7 +98,7 @@ describe RequestMailer, " when receiving incoming mail" do
         mail.multipart?.should == true
         mail.parts.size.should == 2
         message_part = mail.parts[0].to_s
-        bounced_mail = TMail::Mail.parse(mail.parts[1].body)
+        bounced_mail = MailHandler.mail_from_raw_email(mail.parts[1].body, decode=false)
         bounced_mail.to.should == [ ir.incoming_email ]
         bounced_mail.from.should == [ 'geraldinequango@localhost' ]
         bounced_mail.body.include?("That's so totally a rubbish question").should be_true
diff --git a/spec/models/track_mailer_spec.rb b/spec/models/track_mailer_spec.rb
index 1bf77dab5..9bf03c3d0 100644
--- a/spec/models/track_mailer_spec.rb
+++ b/spec/models/track_mailer_spec.rb
@@ -169,7 +169,7 @@ describe TrackMailer do
 
         it 'should deliver one email, with right headers' do
             @user = mock_model(User,
-                    :name_and_email => TMail::Address.address_from_name_and_email('Tippy Test', 'tippy@localhost'),
+                    :name_and_email => MailHandler.address_from_name_and_email('Tippy Test', 'tippy@localhost'),
                     :url_name => 'tippy_test'
             )
 
diff --git a/spec/script/handle-mail-replies_spec.rb b/spec/script/handle-mail-replies_spec.rb
index 406af9ee3..90a8de27c 100644
--- a/spec/script/handle-mail-replies_spec.rb
+++ b/spec/script/handle-mail-replies_spec.rb
@@ -5,7 +5,7 @@ def mail_reply_test(email_filename)
     Dir.chdir Rails.root do
         xc = ExternalCommand.new("script/handle-mail-replies", "--test")
         xc.run(load_file_fixture(email_filename))
-        
+
         xc.err.should == ""
         return xc
     end
@@ -14,7 +14,7 @@ end
 describe "When filtering" do
     it "should not fail when not in test mode" do
         xc = ExternalCommand.new("script/handle-mail-replies")
-        xc.run(load_file_fixture("track-response-exim-bounce.email"))        
+        xc.run(load_file_fixture("track-response-exim-bounce.email"))
         xc.err.should == ""
     end
 
@@ -23,19 +23,19 @@ describe "When filtering" do
         r.status.should == 1
         r.out.should == "user@example.com\n"
     end
-    
+
     it "should detect a WebShield delivery error message" do
         r = mail_reply_test("track-response-webshield-bounce.email")
         r.status.should == 1
         r.out.should == "failed.user@example.co.uk\n"
     end
-    
+
     it "should detect a MS Exchange non-permanent delivery error message" do
         r = mail_reply_test("track-response-ms-bounce.email")
         r.status.should == 1
         r.out.should == ""
     end
-    
+
     it "should pass on a non-bounce message" do
         r = mail_reply_test("incoming-request-bad-uuencoding.email")
         r.status.should == 0
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 248dff70e..e6d81540b 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -208,6 +208,10 @@ def load_raw_emails_data
     end
 end
 
+def get_fixture_mail(filename)
+    MailHandler.mail_from_raw_email(load_file_fixture(filename))
+end
+
 def parse_all_incoming_messages
     IncomingMessage.find(:all).each{|x| x.parse_raw_email!}
 end