Merge branch 'feature/rework-mail-handling' into develop

1 files changed, 204 insertions, 5 deletions

diff --git a/lib/mail_handler/backends/mail_backend.rb b/lib/mail_handler/backends/mail_backend.rb
index 8dd2e6b48..b75e6ed63 100644
--- a/lib/mail_handler/backends/mail_backend.rb
+++ b/lib/mail_handler/backends/mail_backend.rb
@@ -23,15 +23,23 @@ module MailHandler
                 main
             end
 
+            # Returns an outlook message as a Mail object
+            def mail_from_outlook(content)
+                msg = Mapi::Msg.open(StringIO.new(content))
+                mail = mail_from_raw_email(msg.to_mime.to_s)
+                mail.ready_to_send!
+                mail
+            end
+
             # Return a copy of the file name for the mail part
-            def get_part_file_name(mail_part)
-                part_file_name = mail_part.filename
+            def get_part_file_name(part)
+                part_file_name = part.filename
                 part_file_name.nil? ? nil : part_file_name.dup
             end
 
             # Get the body of a mail part
-            def get_part_body(mail_part)
-                mail_part.body.decoded
+            def get_part_body(part)
+                part.body.decoded
             end
 
             # Return the first from field if any
@@ -102,13 +110,204 @@ module MailHandler
                 mail.header[header] ? mail.header[header].to_s : nil
             end
 
+            # Detects whether a mail part is an Outlook email
+            def is_outlook?(part)
+                filename = get_part_file_name(part)
+                return true if get_content_type(part) == 'application/vnd.ms-outlook'
+                if filename && AlaveteliFileTypes.filename_to_mimetype(filename) == 'application/vnd.ms-outlook'
+                    return true
+                end
+                return false
+            end
+
+            # Convert a mail part which is an attached mail in one of
+            # several formats into a mail object and set it as the
+            # rfc822_attachment on the part. If the mail part can't be
+            # converted, the content type on the part is updated to
+            # 'text/plain' for an RFC822 attachment, and 'application/octet-stream'
+            # for other types
+            def decode_attached_part(part, parent_mail)
+                if get_content_type(part) == 'message/rfc822'
+                    # An email attached as text
+                    part.rfc822_attachment = mail_from_raw_email(part.body)
+                    if part.rfc822_attachment.nil?
+                        # Attached mail didn't parse, so treat as text
+                        part.content_type = 'text/plain'
+                    end
+                elsif is_outlook?(part)
+                    part.rfc822_attachment = mail_from_outlook(part.body.decoded)
+                    if part.rfc822_attachment.nil?
+                         # Attached mail didn't parse, so treat as binary
+                         part.content_type = 'application/octet-stream'
+                    end
+                elsif get_content_type(part) == 'application/ms-tnef'
+                    # A set of attachments in a TNEF file
+                    part.rfc822_attachment = mail_from_tnef(part.body.decoded)
+                    if part.rfc822_attachment.nil?
+                        # Attached mail didn't parse, so treat as binary
+                        part.content_type = 'application/octet-stream'
+                    end
+                end
+                if part.rfc822_attachment
+                    expand_and_normalize_parts(part.rfc822_attachment, parent_mail)
+                end
+            end
+
+            # Expand and normalize a mail part recursively. Decodes attached messages into
+            # Mail objects wherever possible. Sets a default content type if none is
+            # set. Tries to set a more specific content type for binary content types.
+            def expand_and_normalize_parts(part, parent_mail)
+                if part.multipart?
+                  part.parts.each{ |sub_part| expand_and_normalize_parts(sub_part, parent_mail) }
+                else
+                  part_filename = get_part_file_name(part)
+                  charset = part.charset # save this, because overwriting content_type also resets charset
+
+                  # Don't allow nil content_types
+                  if get_content_type(part).nil?
+                      part.content_type = 'application/octet-stream'
+                  end
+
+                  # PDFs often come with this mime type, fix it up for view code
+                  if get_content_type(part) == 'application/octet-stream'
+                      part_body = get_part_body(part)
+                      calc_mime = AlaveteliFileTypes.filename_and_content_to_mimetype(part_filename,
+                                                                                      part_body)
+                      if calc_mime
+                          part.content_type = calc_mime
+                      end
+                  end
+
+                  # Use standard content types for Word documents etc.
+                  part.content_type = normalise_content_type(get_content_type(part))
+                  decode_attached_part(part, parent_mail)
+                  part.charset = charset
+                end
+            end
+
+            # Count the parts in a mail part recursively, including any attached messages.
+            # Set the count on the parent mail, and set a url_part_number on the part itself.
+            # Set the count for the first uudecoded part on the parent mail also.
+            def count_parts(part, parent_mail)
+                if part.multipart?
+                    part.parts.each { |p| count_parts(p, parent_mail) }
+                else
+                    if part.rfc822_attachment
+                        count_parts(part.rfc822_attachment, parent_mail)
+                    else
+                        parent_mail.count_parts_count += 1
+                        part.url_part_number = parent_mail.count_parts_count
+                    end
+                end
+                parent_mail.count_first_uudecode_count = parent_mail.count_parts_count
+            end
+
+            # Choose the best part from alternatives
+            def choose_best_alternative(mail)
+                if mail.html_part
+                    return mail.html_part
+                elsif mail.text_part
+                    return mail.text_part
+                else
+                    return mail.parts.first
+                end
+            end
+
+            # Expand and normalize the parts of a mail, select the best part
+            # wherever there is an alternative, and then count the returned
+            # leaves and assign url_part values to them
+            def get_attachment_leaves(mail)
+                expand_and_normalize_parts(mail, mail)
+                leaves = _get_attachment_leaves_recursive(mail, nil, mail)
+                mail.count_parts_count = 0
+                count_parts(mail, mail)
+                return leaves
+            end
+
+            # Recurse through a mail part, selecting the best part wherever there is
+            # an alternative
+            def _get_attachment_leaves_recursive(part, within_rfc822_attachment, parent_mail)
+                leaves_found = []
+                if part.multipart?
+                    raise "no parts on multipart mail" if part.parts.size == 0
+                    if part.sub_type == 'alternative'
+                        best_part = choose_best_alternative(part)
+                        leaves_found += _get_attachment_leaves_recursive(best_part,
+                                                                         within_rfc822_attachment,
+                                                                         parent_mail)
+                    else
+                        # Add all parts
+                        part.parts.each do |sub_part|
+                            leaves_found += _get_attachment_leaves_recursive(sub_part,
+                                                                             within_rfc822_attachment,
+                                                                             parent_mail)
+                        end
+                    end
+                else
+                    # Add all the parts of a decoded attached message
+                    if part.rfc822_attachment
+                        leaves_found += _get_attachment_leaves_recursive(part.rfc822_attachment,
+                                                                         part.rfc822_attachment,
+                                                                         parent_mail)
+                    else
+                        # Store leaf
+                        part.within_rfc822_attachment = within_rfc822_attachment
+                        leaves_found += [part]
+                    end
+                end
+                return leaves_found
+            end
+
+            # Add selected useful headers from an attached message to its body
+            def extract_attached_message_headers(leaf)
+                body = get_part_body(leaf)
+                # Test to see if we are in the first part of the attached
+                # RFC822 message and it is text, if so add headers.
+                if leaf.within_rfc822_attachment == leaf && get_content_type(leaf) == 'text/plain'
+                    headers = ""
+                    [ 'Date', 'Subject', 'From', 'To', 'Cc' ].each do |header|
+                        if header_value = get_header_string(header, leaf.within_rfc822_attachment)
+                            if !header_value.blank?
+                                headers = headers + header + ": " + header_value.to_s + "\n"
+                            end
+                        end
+                    end
+                    # XXX call _convert_part_body_to_text here, but need to get charset somehow
+                    # e.g. http://www.whatdotheyknow.com/request/1593/response/3088/attach/4/Freedom%20of%20Information%20request%20-%20car%20oval%20sticker:%20Article%2020,%20Convention%20on%20Road%20Traffic%201949.txt
+                    body = headers + "\n" + body
+                end
+                body
+            end
+
+            # Generate a hash of the attributes associated with each significant part of a Mail object
+            def get_attachment_attributes(mail)
+                leaves = get_attachment_leaves(mail)
+                attachments = []
+                for leaf in leaves
+                    body = get_part_body(leaf)
+                    if leaf.within_rfc822_attachment
+                        within_rfc822_subject = leaf.within_rfc822_attachment.subject
+                        body = extract_attached_message_headers(leaf)
+                    end
+                    leaf_attributes = { :url_part_number => leaf.url_part_number,
+                                        :content_type => get_content_type(leaf),
+                                        :filename => get_part_file_name(leaf),
+                                        :charset => leaf.charset,
+                                        :within_rfc822_subject => within_rfc822_subject,
+                                        :body => body,
+                                        :hexdigest => Digest::MD5.hexdigest(body) }
+                    attachments << leaf_attributes
+                end
+                return attachments
+            end
+
             # Format
             def address_from_name_and_email(name, email)
                 if !MySociety::Validate.is_valid_email(email)
                     raise "invalid email " + email + " passed to address_from_name_and_email"
                 end
                 if name.nil?
-                    return Mail::Address.new(email)
+                    return Mail::Address.new(email).to_s
                 end
                 address = Mail::Address.new
                 address.display_name = name