diff options
Diffstat (limited to 'lib/mail_handler')
| -rw-r--r-- | lib/mail_handler/backends/mail_backend.rb | 323 | ||||
| -rw-r--r-- | lib/mail_handler/backends/mail_extensions.rb | 67 | ||||
| -rw-r--r-- | lib/mail_handler/backends/tmail_backend.rb | 290 | ||||
| -rw-r--r-- | lib/mail_handler/backends/tmail_extensions.rb | 138 | ||||
| -rw-r--r-- | lib/mail_handler/mail_handler.rb | 179 |
5 files changed, 997 insertions, 0 deletions
diff --git a/lib/mail_handler/backends/mail_backend.rb b/lib/mail_handler/backends/mail_backend.rb new file mode 100644 index 000000000..b75e6ed63 --- /dev/null +++ b/lib/mail_handler/backends/mail_backend.rb @@ -0,0 +1,323 @@ +require 'mail' + +module MailHandler + module Backends + module MailBackend + + def backend() + 'Mail' + end + + # Note that the decode flag is not yet used + def mail_from_raw_email(data, decode=true) + Mail.new(data) + end + + # Extracts all attachments from the given TNEF file as a Mail object + def mail_from_tnef(content) + main = Mail.new + tnef_attachments(content).each do |attachment| + main.add_file(attachment) + end + main.ready_to_send! + main + end + + # Returns an outlook message as a Mail object + def mail_from_outlook(content) + msg = Mapi::Msg.open(StringIO.new(content)) + mail = mail_from_raw_email(msg.to_mime.to_s) + mail.ready_to_send! + mail + end + + # Return a copy of the file name for the mail part + def get_part_file_name(part) + part_file_name = part.filename + part_file_name.nil? ? nil : part_file_name.dup + end + + # Get the body of a mail part + def get_part_body(part) + part.body.decoded + end + + # Return the first from field if any + def first_from(mail) + if mail[:from] + begin + mail[:from].addrs[0] + mail[:from].decoded + return mail[:from].addrs[0] + rescue + return mail[:from].value + end + else + nil + end + end + + # Return the first from address if any + def get_from_address(mail) + first_from = first_from(mail) + if first_from + if first_from.is_a?(String) + return nil + else + return first_from.address + end + else + return nil + end + end + + # Return the first from name if any + def get_from_name(mail) + first_from = first_from(mail) + if first_from + if first_from.is_a?(String) + return nil + else + return first_from.display_name ? eval(%Q{"#{first_from.display_name}"}) : nil + end + else + return nil + end + end + + def get_all_addresses(mail) + envelope_to = mail['envelope-to'] ? [mail['envelope-to'].value] : [] + ((mail.to || []) + + (mail.cc || []) + + (envelope_to || [])).uniq + end + + def empty_return_path?(mail) + return false if mail['return-path'].nil? + return true if mail['return-path'].value.blank? + return false + end + + def get_auto_submitted(mail) + mail['auto-submitted'] ? mail['auto-submitted'].value : nil + end + + def get_content_type(part) + part.content_type ? part.content_type.split(';')[0] : nil + end + + def get_header_string(header, mail) + mail.header[header] ? mail.header[header].to_s : nil + end + + # Detects whether a mail part is an Outlook email + def is_outlook?(part) + filename = get_part_file_name(part) + return true if get_content_type(part) == 'application/vnd.ms-outlook' + if filename && AlaveteliFileTypes.filename_to_mimetype(filename) == 'application/vnd.ms-outlook' + return true + end + return false + end + + # Convert a mail part which is an attached mail in one of + # several formats into a mail object and set it as the + # rfc822_attachment on the part. If the mail part can't be + # converted, the content type on the part is updated to + # 'text/plain' for an RFC822 attachment, and 'application/octet-stream' + # for other types + def decode_attached_part(part, parent_mail) + if get_content_type(part) == 'message/rfc822' + # An email attached as text + part.rfc822_attachment = mail_from_raw_email(part.body) + if part.rfc822_attachment.nil? + # Attached mail didn't parse, so treat as text + part.content_type = 'text/plain' + end + elsif is_outlook?(part) + part.rfc822_attachment = mail_from_outlook(part.body.decoded) + if part.rfc822_attachment.nil? + # Attached mail didn't parse, so treat as binary + part.content_type = 'application/octet-stream' + end + elsif get_content_type(part) == 'application/ms-tnef' + # A set of attachments in a TNEF file + part.rfc822_attachment = mail_from_tnef(part.body.decoded) + if part.rfc822_attachment.nil? + # Attached mail didn't parse, so treat as binary + part.content_type = 'application/octet-stream' + end + end + if part.rfc822_attachment + expand_and_normalize_parts(part.rfc822_attachment, parent_mail) + end + end + + # Expand and normalize a mail part recursively. Decodes attached messages into + # Mail objects wherever possible. Sets a default content type if none is + # set. Tries to set a more specific content type for binary content types. + def expand_and_normalize_parts(part, parent_mail) + if part.multipart? + part.parts.each{ |sub_part| expand_and_normalize_parts(sub_part, parent_mail) } + else + part_filename = get_part_file_name(part) + charset = part.charset # save this, because overwriting content_type also resets charset + + # Don't allow nil content_types + if get_content_type(part).nil? + part.content_type = 'application/octet-stream' + end + + # PDFs often come with this mime type, fix it up for view code + if get_content_type(part) == 'application/octet-stream' + part_body = get_part_body(part) + calc_mime = AlaveteliFileTypes.filename_and_content_to_mimetype(part_filename, + part_body) + if calc_mime + part.content_type = calc_mime + end + end + + # Use standard content types for Word documents etc. + part.content_type = normalise_content_type(get_content_type(part)) + decode_attached_part(part, parent_mail) + part.charset = charset + end + end + + # Count the parts in a mail part recursively, including any attached messages. + # Set the count on the parent mail, and set a url_part_number on the part itself. + # Set the count for the first uudecoded part on the parent mail also. + def count_parts(part, parent_mail) + if part.multipart? + part.parts.each { |p| count_parts(p, parent_mail) } + else + if part.rfc822_attachment + count_parts(part.rfc822_attachment, parent_mail) + else + parent_mail.count_parts_count += 1 + part.url_part_number = parent_mail.count_parts_count + end + end + parent_mail.count_first_uudecode_count = parent_mail.count_parts_count + end + + # Choose the best part from alternatives + def choose_best_alternative(mail) + if mail.html_part + return mail.html_part + elsif mail.text_part + return mail.text_part + else + return mail.parts.first + end + end + + # Expand and normalize the parts of a mail, select the best part + # wherever there is an alternative, and then count the returned + # leaves and assign url_part values to them + def get_attachment_leaves(mail) + expand_and_normalize_parts(mail, mail) + leaves = _get_attachment_leaves_recursive(mail, nil, mail) + mail.count_parts_count = 0 + count_parts(mail, mail) + return leaves + end + + # Recurse through a mail part, selecting the best part wherever there is + # an alternative + def _get_attachment_leaves_recursive(part, within_rfc822_attachment, parent_mail) + leaves_found = [] + if part.multipart? + raise "no parts on multipart mail" if part.parts.size == 0 + if part.sub_type == 'alternative' + best_part = choose_best_alternative(part) + leaves_found += _get_attachment_leaves_recursive(best_part, + within_rfc822_attachment, + parent_mail) + else + # Add all parts + part.parts.each do |sub_part| + leaves_found += _get_attachment_leaves_recursive(sub_part, + within_rfc822_attachment, + parent_mail) + end + end + else + # Add all the parts of a decoded attached message + if part.rfc822_attachment + leaves_found += _get_attachment_leaves_recursive(part.rfc822_attachment, + part.rfc822_attachment, + parent_mail) + else + # Store leaf + part.within_rfc822_attachment = within_rfc822_attachment + leaves_found += [part] + end + end + return leaves_found + end + + # Add selected useful headers from an attached message to its body + def extract_attached_message_headers(leaf) + body = get_part_body(leaf) + # Test to see if we are in the first part of the attached + # RFC822 message and it is text, if so add headers. + if leaf.within_rfc822_attachment == leaf && get_content_type(leaf) == 'text/plain' + headers = "" + [ 'Date', 'Subject', 'From', 'To', 'Cc' ].each do |header| + if header_value = get_header_string(header, leaf.within_rfc822_attachment) + if !header_value.blank? + headers = headers + header + ": " + header_value.to_s + "\n" + end + end + end + # XXX call _convert_part_body_to_text here, but need to get charset somehow + # e.g. http://www.whatdotheyknow.com/request/1593/response/3088/attach/4/Freedom%20of%20Information%20request%20-%20car%20oval%20sticker:%20Article%2020,%20Convention%20on%20Road%20Traffic%201949.txt + body = headers + "\n" + body + end + body + end + + # Generate a hash of the attributes associated with each significant part of a Mail object + def get_attachment_attributes(mail) + leaves = get_attachment_leaves(mail) + attachments = [] + for leaf in leaves + body = get_part_body(leaf) + if leaf.within_rfc822_attachment + within_rfc822_subject = leaf.within_rfc822_attachment.subject + body = extract_attached_message_headers(leaf) + end + leaf_attributes = { :url_part_number => leaf.url_part_number, + :content_type => get_content_type(leaf), + :filename => get_part_file_name(leaf), + :charset => leaf.charset, + :within_rfc822_subject => within_rfc822_subject, + :body => body, + :hexdigest => Digest::MD5.hexdigest(body) } + attachments << leaf_attributes + end + return attachments + end + + # Format + def address_from_name_and_email(name, email) + if !MySociety::Validate.is_valid_email(email) + raise "invalid email " + email + " passed to address_from_name_and_email" + end + if name.nil? + return Mail::Address.new(email).to_s + end + address = Mail::Address.new + address.display_name = name + address.address = email + address.to_s + end + + def address_from_string(string) + Mail::Address.new(string).address + end + end + end +end
\ No newline at end of file diff --git a/lib/mail_handler/backends/mail_extensions.rb b/lib/mail_handler/backends/mail_extensions.rb new file mode 100644 index 000000000..f756abd1a --- /dev/null +++ b/lib/mail_handler/backends/mail_extensions.rb @@ -0,0 +1,67 @@ +require 'mail/message' +require 'mail/fields/common/parameter_hash' +module Mail + class Message + attr_accessor :url_part_number + attr_accessor :rfc822_attachment # when a whole email message is attached as text + attr_accessor :within_rfc822_attachment # for parts within a message attached as text (for getting subject mainly) + attr_accessor :count_parts_count + attr_accessor :count_first_uudecode_count + + # A patched version of the message initializer to work around a bug where stripping the original + # input removes meaningful spaces - e.g. in the case of uuencoded bodies. + def initialize(*args, &block) + @body = nil + @body_raw = nil + @separate_parts = false + @text_part = nil + @html_part = nil + @errors = nil + @header = nil + @charset = 'UTF-8' + @defaulted_charset = true + + @perform_deliveries = true + @raise_delivery_errors = true + + @delivery_handler = nil + + @delivery_method = Mail.delivery_method.dup + + @transport_encoding = Mail::Encodings.get_encoding('7bit') + + @mark_for_delete = false + + if args.flatten.first.respond_to?(:each_pair) + init_with_hash(args.flatten.first) + else + # The replacement of this commented out line is the change. + # init_with_string(args.flatten[0].to_s.strip) + init_with_string(args.flatten[0].to_s) + end + + if block_given? + instance_eval(&block) + end + + self + end + end + + # A patched version of the parameter hash that handles nil values without throwing + # an error. + class ParameterHash < IndifferentHash + + def encoded + map.sort { |a,b| a.first.to_s <=> b.first.to_s }.map do |key_name, value| + # The replacement of this commented out line is the change + # unless value.ascii_only? + unless value.nil? || value.ascii_only? + value = Mail::Encodings.param_encode(value) + key_name = "#{key_name}*" + end + %Q{#{key_name}=#{quote_token(value)}} + end.join(";\r\n\s") + end + end +end
\ No newline at end of file diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb new file mode 100644 index 000000000..02124cdb1 --- /dev/null +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -0,0 +1,290 @@ +module MailHandler + module Backends + module TmailBackend + + def backend() + 'TMail' + end + + # Turn raw data into a structured TMail::Mail object + # Documentation at http://i.loveruby.net/en/projects/tmail/doc/ + def mail_from_raw_email(data, decode=true) + # Hack round bug in TMail's MIME decoding. + # Report of TMail bug: + # http://rubyforge.org/tracker/index.php?func=detail&aid=21810&group_id=4512&atid=17370 + copy_of_raw_data = data.gsub(/; boundary=\s+"/im,'; boundary="') + mail = TMail::Mail.parse(copy_of_raw_data) + mail.base64_decode if decode + mail + end + + # Extracts all attachments from the given TNEF file as a TMail::Mail object + def mail_from_tnef(content) + main = TMail::Mail.new + main.set_content_type 'multipart', 'mixed', { 'boundary' => TMail.new_boundary } + tnef_attachments(content).each do |attachment| + tmail_attachment = TMail::Mail.new + tmail_attachment['content-location'] = attachment[:filename] + tmail_attachment.body = attachment[:content] + main.parts << tmail_attachment + end + main + end + + # Return a copy of the file name for the mail part + def get_part_file_name(mail_part) + part_file_name = TMail::Mail.get_part_file_name(mail_part) + if part_file_name.nil? + return nil + end + part_file_name = part_file_name.dup + return part_file_name + end + + # Get the body of a mail part + def get_part_body(mail_part) + mail_part.body + end + + # Return the first from address if any + def get_from_address(mail) + if mail.from_addrs.nil? || mail.from_addrs.size == 0 + return nil + end + mail.from_addrs[0].spec + end + + # Return the first from name if any + def get_from_name(mail) + mail.from_name_if_present + end + + def get_all_addresses(mail) + ((mail.to || []) + + (mail.cc || []) + + (mail.envelope_to || [])).uniq + end + + def empty_return_path?(mail) + return false if mail['return-path'].nil? + return true if mail['return-path'].addr.to_s == '<>' + return false + end + + def get_auto_submitted(mail) + mail['auto-submitted'] ? mail['auto-submitted'].body : nil + end + + def get_content_type(part) + part.content_type + end + + def get_header_string(header, mail) + mail.header_string(header) + end + + # Number the attachments in depth first tree order, for use in URLs. + # XXX This fills in part.rfc822_attachment and part.url_part_number within + # all the parts of the email (see monkeypatches in lib/mail_handler/tmail_extensions and + # lib/mail_handler/mail_extensions for how these attributes are added). ensure_parts_counted + # must be called before using the attributes. + def ensure_parts_counted(mail) + mail.count_parts_count = 0 + _count_parts_recursive(mail, mail) + # we carry on using these numeric ids for attachments uudecoded from within text parts + mail.count_first_uudecode_count = mail.count_parts_count + end + def _count_parts_recursive(part, mail) + if part.multipart? + part.parts.each do |p| + _count_parts_recursive(p, mail) + end + else + part_filename = get_part_file_name(part) + begin + if part.content_type == 'message/rfc822' + # An email attached as text + # e.g. http://www.whatdotheyknow.com/request/64/response/102 + part.rfc822_attachment = mail_from_raw_email(part.body, decode=false) + elsif part.content_type == 'application/vnd.ms-outlook' || part_filename && AlaveteliFileTypes.filename_to_mimetype(part_filename) == 'application/vnd.ms-outlook' + # An email attached as an Outlook file + # e.g. http://www.whatdotheyknow.com/request/chinese_names_for_british_politi + msg = Mapi::Msg.open(StringIO.new(part.body)) + part.rfc822_attachment = mail_from_raw_email(msg.to_mime.to_s, decode=false) + elsif part.content_type == 'application/ms-tnef' + # A set of attachments in a TNEF file + part.rfc822_attachment = mail_from_tnef(part.body) + end + rescue + # If attached mail doesn't parse, treat it as text part + part.rfc822_attachment = nil + else + unless part.rfc822_attachment.nil? + _count_parts_recursive(part.rfc822_attachment, mail) + end + end + if part.rfc822_attachment.nil? + mail.count_parts_count += 1 + part.url_part_number = mail.count_parts_count + end + end + end + + def get_attachment_attributes(mail) + leaves = get_attachment_leaves(mail) + # XXX we have to call ensure_parts_counted after get_attachment_leaves + # which is really messy. + ensure_parts_counted(mail) + attachment_attributes = [] + for leaf in leaves + body = get_part_body(leaf) + # As leaf.body causes MIME decoding which uses lots of RAM, do garbage collection here + # to prevent excess memory use. XXX not really sure if this helps reduce + # peak RAM use overall. Anyway, maybe there is something better to do than this. + GC.start + if leaf.within_rfc822_attachment + within_rfc822_subject = leaf.within_rfc822_attachment.subject + # Test to see if we are in the first part of the attached + # RFC822 message and it is text, if so add headers. + # XXX should probably use hunting algorithm to find main text part, rather than + # just expect it to be first. This will do for now though. + if leaf.within_rfc822_attachment == leaf && leaf.content_type == 'text/plain' + headers = "" + for header in [ 'Date', 'Subject', 'From', 'To', 'Cc' ] + if leaf.within_rfc822_attachment.header.include?(header.downcase) + header_value = leaf.within_rfc822_attachment.header[header.downcase] + if !header_value.blank? + headers = headers + header + ": " + header_value.to_s + "\n" + end + end + end + # XXX call _convert_part_body_to_text here, but need to get charset somehow + # e.g. http://www.whatdotheyknow.com/request/1593/response/3088/attach/4/Freedom%20of%20Information%20request%20-%20car%20oval%20sticker:%20Article%2020,%20Convention%20on%20Road%20Traffic%201949.txt + body = headers + "\n" + body + + # This is quick way of getting all headers, but instead we only add some a) to + # make it more usable, b) as at least one authority accidentally leaked security + # information into a header. + #attachment.body = leaf.within_rfc822_attachment.port.to_s + end + end + attachment_attributes << {:url_part_number => leaf.url_part_number, + :content_type => get_content_type(leaf), + :filename => get_part_file_name(leaf), + :charset => leaf.charset, + :within_rfc822_subject => within_rfc822_subject, + :body => body, + :hexdigest => Digest::MD5.hexdigest(body) } + end + attachment_attributes + end + + # (This risks losing info if the unchosen alternative is the only one to contain + # useful info, but let's worry about that another time) + def get_attachment_leaves(mail) + return _get_attachment_leaves_recursive(mail, mail) + end + def _get_attachment_leaves_recursive(curr_mail, parent_mail, within_rfc822_attachment = nil) + leaves_found = [] + if curr_mail.multipart? + if curr_mail.parts.size == 0 + raise "no parts on multipart mail" + end + + if curr_mail.sub_type == 'alternative' + # Choose best part from alternatives + best_part = nil + # Take the last text/plain one, or else the first one + curr_mail.parts.each do |m| + if not best_part + best_part = m + elsif m.content_type == 'text/plain' + best_part = m + end + end + # Take an HTML one as even higher priority. (They tend + # to render better than text/plain, e.g. don't wrap links here: + # http://www.whatdotheyknow.com/request/amount_and_cost_of_freedom_of_in#incoming-72238 ) + curr_mail.parts.each do |m| + if m.content_type == 'text/html' + best_part = m + end + end + leaves_found += _get_attachment_leaves_recursive(best_part, parent_mail, within_rfc822_attachment) + else + # Add all parts + curr_mail.parts.each do |m| + leaves_found += _get_attachment_leaves_recursive(m, parent_mail, within_rfc822_attachment) + end + end + else + # XXX Yuck. this section alters various content_types. That puts + # it into conflict with ensure_parts_counted which it has to be + # called both before and after. It will fail with cases of + # attachments of attachments etc. + charset = curr_mail.charset # save this, because overwriting content_type also resets charset + # Don't allow nil content_types + if curr_mail.content_type.nil? + curr_mail.content_type = 'application/octet-stream' + end + # PDFs often come with this mime type, fix it up for view code + if curr_mail.content_type == 'application/octet-stream' + part_file_name = get_part_file_name(curr_mail) + part_body = get_part_body(curr_mail) + calc_mime = AlaveteliFileTypes.filename_and_content_to_mimetype(part_file_name, part_body) + if calc_mime + curr_mail.content_type = calc_mime + end + end + + # Use standard content types for Word documents etc. + curr_mail.content_type = normalise_content_type(curr_mail.content_type) + if curr_mail.content_type == 'message/rfc822' + ensure_parts_counted(parent_mail) # fills in rfc822_attachment variable + if curr_mail.rfc822_attachment.nil? + # Attached mail didn't parse, so treat as text + curr_mail.content_type = 'text/plain' + end + end + if curr_mail.content_type == 'application/vnd.ms-outlook' || curr_mail.content_type == 'application/ms-tnef' + ensure_parts_counted(parent_mail) # fills in rfc822_attachment variable + if curr_mail.rfc822_attachment.nil? + # Attached mail didn't parse, so treat as binary + curr_mail.content_type = 'application/octet-stream' + end + end + # If the part is an attachment of email + if curr_mail.content_type == 'message/rfc822' || curr_mail.content_type == 'application/vnd.ms-outlook' || curr_mail.content_type == 'application/ms-tnef' + ensure_parts_counted(parent_mail) # fills in rfc822_attachment variable + leaves_found += _get_attachment_leaves_recursive(curr_mail.rfc822_attachment, parent_mail, curr_mail.rfc822_attachment) + else + # Store leaf + curr_mail.within_rfc822_attachment = within_rfc822_attachment + leaves_found += [curr_mail] + end + # restore original charset + curr_mail.charset = charset + end + return leaves_found + end + + + def address_from_name_and_email(name, email) + if !MySociety::Validate.is_valid_email(email) + raise "invalid email " + email + " passed to address_from_name_and_email" + end + if name.nil? + return TMail::Address.parse(email).to_s + end + # Botch an always quoted RFC address, then parse it + name = name.gsub(/(["\\])/, "\\\\\\1") + TMail::Address.parse('"' + name + '" <' + email + '>').to_s + end + + def address_from_string(string) + TMail::Address.parse(string).address + end + + end + end +end
\ No newline at end of file diff --git a/lib/mail_handler/backends/tmail_extensions.rb b/lib/mail_handler/backends/tmail_extensions.rb new file mode 100644 index 000000000..3576a8eca --- /dev/null +++ b/lib/mail_handler/backends/tmail_extensions.rb @@ -0,0 +1,138 @@ +# lib/tmail_extensions.rb: +# Extensions / fixes to TMail. +# +# Copyright (c) 2009 UK Citizens Online Democracy. All rights reserved. +# Email: francis@mysociety.org; WWW: http://www.mysociety.org/ + +require 'racc/parser' +require 'tmail' +require 'tmail/scanner' +require 'tmail/utils' +require 'tmail/interface' + +# Monkeypatch! + +# These mainly used in app/models/incoming_message.rb +module TMail + class Mail + # Monkeypatch! Adding some extra members to store extra info in. + + attr_accessor :url_part_number + attr_accessor :rfc822_attachment # when a whole email message is attached as text + attr_accessor :within_rfc822_attachment # for parts within a message attached as text (for getting subject mainly) + attr_accessor :count_parts_count + attr_accessor :count_first_uudecode_count + + # Monkeypatch! (check to see if this becomes a standard function in + # TMail::Mail, then use that, whatever it is called) + def Mail.get_part_file_name(part) + file_name = (part['content-location'] && + part['content-location'].body) || + part.sub_header("content-type", "name") || + part.sub_header("content-disposition", "filename") + file_name = file_name.strip if file_name + file_name + end + + # Monkeypatch! Return the name part of from address, or nil if there isn't one + def from_name_if_present + if self.from && self.from_addrs[0].name + return TMail::Unquoter.unquote_and_convert_to(self.from_addrs[0].name, "utf-8") + else + return nil + end + end + + # Monkeypatch! Generalisation of To:, Cc: + def envelope_to(default = nil) + # XXX assumes only one envelope-to, and no parsing needed + val = self.header_string('envelope-to') + return val ? [val,] : [] + end + + # Monkeypatch! + # Bug fix to this function - is for message in humberside-police-odd-mime-type.email + # Which was originally: https://secure.mysociety.org/admin/foi/request/show_raw_email/11209 + # See test in spec/lib/tmail_extensions.rb + def set_content_type( str, sub = nil, param = nil ) + if sub + main, sub = str, sub + else + main, sub = str.split(%r</>, 2) + raise ArgumentError, "sub type missing: #{str.inspect}" unless sub + end + if h = @header['content-type'] + h.main_type = main + h.sub_type = sub + h.params.clear if !h.params.nil? # XXX this if statement is the fix # XXX disabled until works with test + else + store 'Content-Type', "#{main}/#{sub}" + end + @header['content-type'].params.replace param if param + str + end + # Need to make sure this alias calls the Monkeypatch too + alias content_type= set_content_type + + end + + module TextUtils + # Monkeypatch! Much more aggressive list of characters to cause quoting + # than in normal TMail. e.g. Have found real cases where @ needs quoting. + # We list characters to allow, rather than characters not to allow. + NEW_PHRASE_UNSAFE=/[^A-Za-z0-9!#\$%&'*+\-\/=?^_`{|}~ ]/n + def quote_phrase( str ) + (NEW_PHRASE_UNSAFE === str) ? dquote(str) : str + end + end +end + +# Monkeypatch! TMail 1.2.7.1 will parse only one address out of a list of addresses with +# unquoted display parts https://github.com/mikel/tmail/issues#issue/9 - this monkeypatch +# fixes this issue. +module TMail + + class Parser < Racc::Parser + +module_eval <<'..end lib/tmail/parser.y modeval..id2dd1c7d21d', 'lib/tmail/parser.y', 340 + + def self.special_quote_address(str) #:nodoc: + # Takes a string which is an address and adds quotation marks to special + # edge case methods that the RACC parser can not handle. + # + # Right now just handles two edge cases: + # + # Full stop as the last character of the display name: + # Mikel L. <mikel@me.com> + # Returns: + # "Mikel L." <mikel@me.com> + # + # Unquoted @ symbol in the display name: + # mikel@me.com <mikel@me.com> + # Returns: + # "mikel@me.com" <mikel@me.com> + # + # Any other address not matching these patterns just gets returned as is. + case + # This handles the missing "" in an older version of Apple Mail.app + # around the display name when the display name contains a '@' + # like 'mikel@me.com <mikel@me.com>' + # Just quotes it to: '"mikel@me.com" <mikel@me.com>' + when str =~ /\A([^"][^<]+@[^>]+[^"])\s(<.*?>)\Z/ + return "\"#{$1}\" #{$2}" + # This handles cases where 'Mikel A. <mikel@me.com>' which is a trailing + # full stop before the address section. Just quotes it to + # '"Mikel A." <mikel@me.com>' + when str =~ /\A(.*?\.)\s(<.*?>)\s*\Z/ + return "\"#{$1}\" #{$2}" + else + str + end + end + +..end lib/tmail/parser.y modeval..id2dd1c7d21d + end # class Parser + +end # module TMail + + diff --git a/lib/mail_handler/mail_handler.rb b/lib/mail_handler/mail_handler.rb new file mode 100644 index 000000000..8b227b9ca --- /dev/null +++ b/lib/mail_handler/mail_handler.rb @@ -0,0 +1,179 @@ +# Handles the parsing of email +require 'tmpdir' + +module MailHandler + + if RUBY_VERSION.to_f >= 1.9 + require 'mail' + require 'backends/mail_extensions' + require 'backends/mail_backend' + include Backends::MailBackend + else + require 'action_mailer' + require 'backends/tmail_extensions' + require 'backends/tmail_backend' + include Backends::TmailBackend + end + + # Returns a set of attachments from the given TNEF contents + # The TNEF contents also contains the message body, but in general this is the + # same as the message body in the message proper. + def tnef_attachments(content) + attachments = [] + Dir.mktmpdir do |dir| + IO.popen("#{`which tnef`.chomp} -K -C #{dir}", "wb") do |f| + f.write(content) + f.close + if $?.signaled? + raise IOError, "tnef exited with signal #{$?.termsig}" + end + if $?.exited? && $?.exitstatus != 0 + raise IOError, "tnef exited with status #{$?.exitstatus}" + end + end + found = 0 + Dir.new(dir).sort.each do |file| # sort for deterministic behaviour + if file != "." && file != ".." + file_content = File.open("#{dir}/#{file}", "rb").read + attachments << { :content => file_content, + :filename => file } + found += 1 + end + end + if found == 0 + raise IOError, "tnef produced no attachments" + end + end + attachments + end + + def normalise_content_type(content_type) + # e.g. http://www.whatdotheyknow.com/request/93/response/250 + if content_type == 'application/excel' or content_type == 'application/msexcel' or content_type == 'application/x-ms-excel' + content_type = 'application/vnd.ms-excel' + end + if content_type == 'application/mspowerpoint' or content_type == 'application/x-ms-powerpoint' + content_type = 'application/vnd.ms-powerpoint' + end + if content_type == 'application/msword' or content_type == 'application/x-ms-word' + content_type = 'application/vnd.ms-word' + end + if content_type == 'application/x-zip-compressed' + content_type = 'application/zip' + end + + # e.g. http://www.whatdotheyknow.com/request/copy_of_current_swessex_scr_opt#incoming-9928 + if content_type == 'application/acrobat' + content_type = 'application/pdf' + end + + return content_type + end + + def get_attachment_text_one_file(content_type, body, charset = 'utf-8') + # note re. charset: TMail always tries to convert email bodies + # to UTF8 by default, so normally it should already be that. + text = '' + # XXX - tell all these command line tools to return utf-8 + if content_type == 'text/plain' + text += body + "\n\n" + else + tempfile = Tempfile.new('foiextract') + tempfile.binmode + tempfile.print body + tempfile.flush + default_params = { :append_to => text, :binary_output => false } + if content_type == 'application/vnd.ms-word' + AlaveteliExternalCommand.run("wvText", tempfile.path, tempfile.path + ".txt") + # Try catdoc if we get into trouble (e.g. for InfoRequestEvent 2701) + if not File.exists?(tempfile.path + ".txt") + AlaveteliExternalCommand.run("catdoc", tempfile.path, default_params) + else + text += File.read(tempfile.path + ".txt") + "\n\n" + File.unlink(tempfile.path + ".txt") + end + elsif content_type == 'application/rtf' + # catdoc on RTF prodcues less comments and extra bumf than --text option to unrtf + AlaveteliExternalCommand.run("catdoc", tempfile.path, default_params) + elsif content_type == 'text/html' + # lynx wordwraps links in its output, which then don't + # get formatted properly by Alaveteli. We use elinks + # instead, which doesn't do that. + AlaveteliExternalCommand.run("elinks", "-eval", "set document.codepage.assume = \"#{charset}\"", + "-eval", "set document.codepage.force_assumed = 1", + "-dump-charset", "utf-8", + "-force-html", "-dump", + tempfile.path, + default_params.merge(:env => {"LANG" => "C"})) + elsif content_type == 'application/vnd.ms-excel' + # Bit crazy using /usr/bin/strings - but xls2csv, xlhtml and + # py_xls2txt only extract text from cells, not from floating + # notes. catdoc may be fooled by weird character sets, but will + # probably do for UK FOI requests. + AlaveteliExternalCommand.run("/usr/bin/strings", tempfile.path, default_params) + elsif content_type == 'application/vnd.ms-powerpoint' + # ppthtml seems to catch more text, but only outputs HTML when + # we want text, so just use catppt for now + AlaveteliExternalCommand.run("catppt", tempfile.path, default_params) + elsif content_type == 'application/pdf' + AlaveteliExternalCommand.run("pdftotext", tempfile.path, "-", default_params) + elsif content_type == 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + # This is Microsoft's XML office document format. + # Just pull out the main XML file, and strip it of text. + xml = AlaveteliExternalCommand.run("/usr/bin/unzip", "-qq", + "-c", + tempfile.path, + "word/document.xml", + {:binary_output => false}) + if !xml.nil? + doc = REXML::Document.new(xml) + text += doc.each_element( './/text()' ){}.join(" ") + end + elsif content_type == 'application/zip' + # recurse into zip files + begin + zip_file = Zip::ZipFile.open(tempfile.path) + text += get_attachment_text_from_zip_file(zip_file) + zip_file.close() + rescue + $stderr.puts("Error processing zip file: #{$!.inspect}") + end + end + tempfile.close + end + + return text + end + def get_attachment_text_from_zip_file(zip_file) + + text = "" + for entry in zip_file + if entry.file? + filename = entry.to_s + begin + body = entry.get_input_stream.read + rescue + # move to next attachment silently if there were problems + # XXX really should reduce this to specific exceptions? + # e.g. password protected + next + end + calc_mime = AlaveteliFileTypes.filename_to_mimetype(filename) + if calc_mime + content_type = calc_mime + else + content_type = 'application/octet-stream' + end + + text += get_attachment_text_one_file(content_type, body) + + end + end + return text + end + + # Turn instance methods into class methods + extend self + +end + |