From be5e69a7dccaa6c76408f9b7883980bd79bdba28 Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Thu, 15 Nov 2012 11:00:36 +0000 Subject: First skeletal version of separate mail handling library. --- lib/mail_handler/backends/mail_backend.rb | 12 ++++++++++++ lib/mail_handler/backends/tmail_backend.rb | 11 +++++++++++ 2 files changed, 23 insertions(+) create mode 100644 lib/mail_handler/backends/mail_backend.rb create mode 100644 lib/mail_handler/backends/tmail_backend.rb (limited to 'lib/mail_handler/backends') diff --git a/lib/mail_handler/backends/mail_backend.rb b/lib/mail_handler/backends/mail_backend.rb new file mode 100644 index 000000000..ad1322923 --- /dev/null +++ b/lib/mail_handler/backends/mail_backend.rb @@ -0,0 +1,12 @@ +require 'mail' + +module MailHandler + module Backends + module MailBackend + + def backend() + 'Mail' + end + end + end +end \ No newline at end of file diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb new file mode 100644 index 000000000..b2d36d65f --- /dev/null +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -0,0 +1,11 @@ +module MailHandler + module Backends + module TmailBackend + + def backend() + 'TMail' + end + + end + end +end \ No newline at end of file -- cgit v1.2.3 From ca7bb3d9f7f7e38ca670dee4352a6c81e2b9d19a Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Thu, 15 Nov 2012 11:07:48 +0000 Subject: Move TMail monkey patch to MailHandler Tmail backend. --- lib/mail_handler/backends/tmail_backend.rb | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'lib/mail_handler/backends') diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb index b2d36d65f..e5bfa3f49 100644 --- a/lib/mail_handler/backends/tmail_backend.rb +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -1,3 +1,12 @@ +# Monkeypatch! Adding some extra members to store extra info in. +module TMail + class Mail + attr_accessor :url_part_number + attr_accessor :rfc822_attachment # when a whole email message is attached as text + attr_accessor :within_rfc822_attachment # for parts within a message attached as text (for getting subject mainly) + end +end + module MailHandler module Backends module TmailBackend -- cgit v1.2.3 From faa38d57e2eb87c288741e9cba5550e9532282ca Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Thu, 15 Nov 2012 11:38:32 +0000 Subject: Add methods for both backends to generate a mail object from raw data. --- lib/mail_handler/backends/mail_backend.rb | 5 +++++ lib/mail_handler/backends/tmail_backend.rb | 12 ++++++++++++ 2 files changed, 17 insertions(+) (limited to 'lib/mail_handler/backends') diff --git a/lib/mail_handler/backends/mail_backend.rb b/lib/mail_handler/backends/mail_backend.rb index ad1322923..f7dbb7624 100644 --- a/lib/mail_handler/backends/mail_backend.rb +++ b/lib/mail_handler/backends/mail_backend.rb @@ -7,6 +7,11 @@ module MailHandler def backend() 'Mail' end + + def mail_from_raw_email(data) + Mail.new(data) + end + end end end \ No newline at end of file diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb index e5bfa3f49..ff00d92df 100644 --- a/lib/mail_handler/backends/tmail_backend.rb +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -15,6 +15,18 @@ module MailHandler 'TMail' end + # Turn raw data into a structured TMail::Mail object + # Documentation at http://i.loveruby.net/en/projects/tmail/doc/ + def mail_from_raw_email(data) + # Hack round bug in TMail's MIME decoding. + # Report of TMail bug: + # http://rubyforge.org/tracker/index.php?func=detail&aid=21810&group_id=4512&atid=17370 + copy_of_raw_data = data.gsub(/; boundary=\s+"/im,'; boundary="') + mail = TMail::Mail.parse(copy_of_raw_data) + mail.base64_decode + mail + end + end end end \ No newline at end of file -- cgit v1.2.3 From 8834f67db8cd94a0285dd1bb4702db834e08e995 Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Thu, 15 Nov 2012 12:48:16 +0000 Subject: Use mail handler for making mail objects of attached emails. Add a flag to use base64 decoding or not. Note that currently the Mail-based backend doesn't use it - I think that eventually we'll want to have the mail handler wrap the mail object with it's own interface. --- lib/mail_handler/backends/mail_backend.rb | 3 ++- lib/mail_handler/backends/tmail_backend.rb | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'lib/mail_handler/backends') diff --git a/lib/mail_handler/backends/mail_backend.rb b/lib/mail_handler/backends/mail_backend.rb index f7dbb7624..6a5fff13f 100644 --- a/lib/mail_handler/backends/mail_backend.rb +++ b/lib/mail_handler/backends/mail_backend.rb @@ -8,7 +8,8 @@ module MailHandler 'Mail' end - def mail_from_raw_email(data) + # Note that the decode flag is not yet used + def mail_from_raw_email(data, decode=true) Mail.new(data) end diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb index ff00d92df..2f59b1161 100644 --- a/lib/mail_handler/backends/tmail_backend.rb +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -17,13 +17,13 @@ module MailHandler # Turn raw data into a structured TMail::Mail object # Documentation at http://i.loveruby.net/en/projects/tmail/doc/ - def mail_from_raw_email(data) + def mail_from_raw_email(data, decode=true) # Hack round bug in TMail's MIME decoding. # Report of TMail bug: # http://rubyforge.org/tracker/index.php?func=detail&aid=21810&group_id=4512&atid=17370 copy_of_raw_data = data.gsub(/; boundary=\s+"/im,'; boundary="') mail = TMail::Mail.parse(copy_of_raw_data) - mail.base64_decode + mail.base64_decode if decode mail end -- cgit v1.2.3 From 4bdab94e9d4f0a64647e5f8534c1fea8b4ba2809 Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Thu, 15 Nov 2012 14:04:55 +0000 Subject: Move TMail extensions to mail handler. --- lib/mail_handler/backends/tmail_backend.rb | 9 -- lib/mail_handler/backends/tmail_extensions.rb | 152 ++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 9 deletions(-) create mode 100644 lib/mail_handler/backends/tmail_extensions.rb (limited to 'lib/mail_handler/backends') diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb index 2f59b1161..fc675c1ed 100644 --- a/lib/mail_handler/backends/tmail_backend.rb +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -1,12 +1,3 @@ -# Monkeypatch! Adding some extra members to store extra info in. -module TMail - class Mail - attr_accessor :url_part_number - attr_accessor :rfc822_attachment # when a whole email message is attached as text - attr_accessor :within_rfc822_attachment # for parts within a message attached as text (for getting subject mainly) - end -end - module MailHandler module Backends module TmailBackend diff --git a/lib/mail_handler/backends/tmail_extensions.rb b/lib/mail_handler/backends/tmail_extensions.rb new file mode 100644 index 000000000..bc994b9f3 --- /dev/null +++ b/lib/mail_handler/backends/tmail_extensions.rb @@ -0,0 +1,152 @@ +# lib/tmail_extensions.rb: +# Extensions / fixes to TMail. +# +# Copyright (c) 2009 UK Citizens Online Democracy. All rights reserved. +# Email: francis@mysociety.org; WWW: http://www.mysociety.org/ + +require 'racc/parser' +require 'tmail' +require 'tmail/scanner' +require 'tmail/utils' +require 'tmail/interface' + +# Monkeypatch! + +# These mainly used in app/models/incoming_message.rb +module TMail + class Mail + # Monkeypatch! Adding some extra members to store extra info in. + + attr_accessor :url_part_number + attr_accessor :rfc822_attachment # when a whole email message is attached as text + attr_accessor :within_rfc822_attachment # for parts within a message attached as text (for getting subject mainly) + + # Monkeypatch! (check to see if this becomes a standard function in + # TMail::Mail, then use that, whatever it is called) + def Mail.get_part_file_name(part) + file_name = (part['content-location'] && + part['content-location'].body) || + part.sub_header("content-type", "name") || + part.sub_header("content-disposition", "filename") + file_name = file_name.strip if file_name + file_name + end + + # Monkeypatch! Return the name part of from address, or nil if there isn't one + def from_name_if_present + if self.from && self.from_addrs[0].name + return TMail::Unquoter.unquote_and_convert_to(self.from_addrs[0].name, "utf-8") + else + return nil + end + end + + # Monkeypatch! Generalisation of To:, Cc: + def envelope_to(default = nil) + # XXX assumes only one envelope-to, and no parsing needed + val = self.header_string('envelope-to') + return val ? [val,] : [] + end + + # Monkeypatch! + # Bug fix to this function - is for message in humberside-police-odd-mime-type.email + # Which was originally: https://secure.mysociety.org/admin/foi/request/show_raw_email/11209 + # See test in spec/lib/tmail_extensions.rb + def set_content_type( str, sub = nil, param = nil ) + if sub + main, sub = str, sub + else + main, sub = str.split(%r, 2) + raise ArgumentError, "sub type missing: #{str.inspect}" unless sub + end + if h = @header['content-type'] + h.main_type = main + h.sub_type = sub + h.params.clear if !h.params.nil? # XXX this if statement is the fix # XXX disabled until works with test + else + store 'Content-Type', "#{main}/#{sub}" + end + @header['content-type'].params.replace param if param + str + end + # Need to make sure this alias calls the Monkeypatch too + alias content_type= set_content_type + + end + + class Address + # Monkeypatch! Constructor which makes a TMail::Address given + # a name and an email + def Address.address_from_name_and_email(name, email) + if !MySociety::Validate.is_valid_email(email) + raise "invalid email " + email + " passed to address_from_name_and_email" + end + if name.nil? + return TMail::Address.parse(email) + end + # Botch an always quoted RFC address, then parse it + name = name.gsub(/(["\\])/, "\\\\\\1") + return TMail::Address.parse('"' + name + '" <' + email + '>') + end + end + + module TextUtils + # Monkeypatch! Much more aggressive list of characters to cause quoting + # than in normal TMail. e.g. Have found real cases where @ needs quoting. + # We list characters to allow, rather than characters not to allow. + NEW_PHRASE_UNSAFE=/[^A-Za-z0-9!#\$%&'*+\-\/=?^_`{|}~ ]/n + def quote_phrase( str ) + (NEW_PHRASE_UNSAFE === str) ? dquote(str) : str + end + end +end + +# Monkeypatch! TMail 1.2.7.1 will parse only one address out of a list of addresses with +# unquoted display parts https://github.com/mikel/tmail/issues#issue/9 - this monkeypatch +# fixes this issue. +module TMail + + class Parser < Racc::Parser + +module_eval <<'..end lib/tmail/parser.y modeval..id2dd1c7d21d', 'lib/tmail/parser.y', 340 + + def self.special_quote_address(str) #:nodoc: + # Takes a string which is an address and adds quotation marks to special + # edge case methods that the RACC parser can not handle. + # + # Right now just handles two edge cases: + # + # Full stop as the last character of the display name: + # Mikel L. + # Returns: + # "Mikel L." + # + # Unquoted @ symbol in the display name: + # mikel@me.com + # Returns: + # "mikel@me.com" + # + # Any other address not matching these patterns just gets returned as is. + case + # This handles the missing "" in an older version of Apple Mail.app + # around the display name when the display name contains a '@' + # like 'mikel@me.com ' + # Just quotes it to: '"mikel@me.com" ' + when str =~ /\A([^"][^<]+@[^>]+[^"])\s(<.*?>)\Z/ + return "\"#{$1}\" #{$2}" + # This handles cases where 'Mikel A. ' which is a trailing + # full stop before the address section. Just quotes it to + # '"Mikel A." ' + when str =~ /\A(.*?\.)\s(<.*?>)\s*\Z/ + return "\"#{$1}\" #{$2}" + else + str + end + end + +..end lib/tmail/parser.y modeval..id2dd1c7d21d + end # class Parser + +end # module TMail + + -- cgit v1.2.3 From 388c75bfbd18fcaf273d95c21dc132ad19f0cefe Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Thu, 15 Nov 2012 16:12:23 +0000 Subject: Move handling of TNEF mail attachments to mail handler --- lib/mail_handler/backends/mail_backend.rb | 9 +++++++++ lib/mail_handler/backends/tmail_backend.rb | 13 +++++++++++++ 2 files changed, 22 insertions(+) (limited to 'lib/mail_handler/backends') diff --git a/lib/mail_handler/backends/mail_backend.rb b/lib/mail_handler/backends/mail_backend.rb index 6a5fff13f..a6f2a6a44 100644 --- a/lib/mail_handler/backends/mail_backend.rb +++ b/lib/mail_handler/backends/mail_backend.rb @@ -13,6 +13,15 @@ module MailHandler Mail.new(data) end + # Extracts all attachments from the given TNEF file as a Mail object + def mail_from_tnef(content) + main = Mail.new + tnef_attachments(content).each do |attachment| + main.add_file(attachment) + end + main.ready_to_send! + main + end end end end \ No newline at end of file diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb index fc675c1ed..4daa5469f 100644 --- a/lib/mail_handler/backends/tmail_backend.rb +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -18,6 +18,19 @@ module MailHandler mail end + # Extracts all attachments from the given TNEF file as a TMail::Mail object + def mail_from_tnef(content) + main = TMail::Mail.new + main.set_content_type 'multipart', 'mixed', { 'boundary' => TMail.new_boundary } + tnef_attachments(content).each do |attachment| + tmail_attachment = TMail::Mail.new + tmail_attachment['content-location'] = attachment[:filename] + tmail_attachment.body = attachment[:content] + main.parts << tmail_attachment + end + main + end + end end end \ No newline at end of file -- cgit v1.2.3 From 9ef3f43fca535ffb52d2420bcfd8f18e5213b943 Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Thu, 15 Nov 2012 16:21:38 +0000 Subject: Add some extra accessors to Mail::Message for now --- lib/mail_handler/backends/mail_extensions.rb | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 lib/mail_handler/backends/mail_extensions.rb (limited to 'lib/mail_handler/backends') diff --git a/lib/mail_handler/backends/mail_extensions.rb b/lib/mail_handler/backends/mail_extensions.rb new file mode 100644 index 000000000..cbe0491ed --- /dev/null +++ b/lib/mail_handler/backends/mail_extensions.rb @@ -0,0 +1,7 @@ +module Mail + class Message + attr_accessor :url_part_number + attr_accessor :rfc822_attachment # when a whole email message is attached as text + attr_accessor :within_rfc822_attachment # for parts within a message attached as text (for getting subject mainly) + end +end \ No newline at end of file -- cgit v1.2.3 From daf42a44bf019128374f3cc636d95b8308a01f2e Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Thu, 15 Nov 2012 16:39:46 +0000 Subject: Move _get_file_part_name to mail handler. --- lib/mail_handler/backends/mail_backend.rb | 7 +++++++ lib/mail_handler/backends/tmail_backend.rb | 10 ++++++++++ 2 files changed, 17 insertions(+) (limited to 'lib/mail_handler/backends') diff --git a/lib/mail_handler/backends/mail_backend.rb b/lib/mail_handler/backends/mail_backend.rb index a6f2a6a44..fd2fa627f 100644 --- a/lib/mail_handler/backends/mail_backend.rb +++ b/lib/mail_handler/backends/mail_backend.rb @@ -22,6 +22,13 @@ module MailHandler main.ready_to_send! main end + + # Return a copy of the file name for the mail part + def get_part_file_name(mail_part) + part_file_name = mail_part.filename + part_file_name.nil? ? nil : part_file_name.dup + end + end end end \ No newline at end of file diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb index 4daa5469f..0fa90a657 100644 --- a/lib/mail_handler/backends/tmail_backend.rb +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -31,6 +31,16 @@ module MailHandler main end + # Return a copy of the file name for the mail part + def get_part_file_name(mail_part) + part_file_name = TMail::Mail.get_part_file_name(mail_part) + if part_file_name.nil? + return nil + end + part_file_name = part_file_name.dup + return part_file_name + end + end end end \ No newline at end of file -- cgit v1.2.3 From f5ced2133cd1a66e18b225208fa96f4f36a20889 Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Thu, 15 Nov 2012 16:56:06 +0000 Subject: Move address_from_name_and_email to mail handler. --- lib/mail_handler/backends/mail_backend.rb | 13 +++++++++++++ lib/mail_handler/backends/tmail_backend.rb | 11 +++++++++++ lib/mail_handler/backends/tmail_extensions.rb | 20 ++------------------ 3 files changed, 26 insertions(+), 18 deletions(-) (limited to 'lib/mail_handler/backends') diff --git a/lib/mail_handler/backends/mail_backend.rb b/lib/mail_handler/backends/mail_backend.rb index fd2fa627f..bc4bf564c 100644 --- a/lib/mail_handler/backends/mail_backend.rb +++ b/lib/mail_handler/backends/mail_backend.rb @@ -29,6 +29,19 @@ module MailHandler part_file_name.nil? ? nil : part_file_name.dup end + # Format + def address_from_name_and_email(name, email) + if !MySociety::Validate.is_valid_email(email) + raise "invalid email " + email + " passed to address_from_name_and_email" + end + if name.nil? + return Mail::Address.new(email) + end + address = Mail::Address.new + address.display_name = name + address.address = email + address.to_s + end end end end \ No newline at end of file diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb index 0fa90a657..1c5489145 100644 --- a/lib/mail_handler/backends/tmail_backend.rb +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -41,6 +41,17 @@ module MailHandler return part_file_name end + def address_from_name_and_email(name, email) + if !MySociety::Validate.is_valid_email(email) + raise "invalid email " + email + " passed to address_from_name_and_email" + end + if name.nil? + return TMail::Address.parse(email) + end + # Botch an always quoted RFC address, then parse it + name = name.gsub(/(["\\])/, "\\\\\\1") + TMail::Address.parse('"' + name + '" <' + email + '>').to_s + end end end end \ No newline at end of file diff --git a/lib/mail_handler/backends/tmail_extensions.rb b/lib/mail_handler/backends/tmail_extensions.rb index bc994b9f3..9359dfeea 100644 --- a/lib/mail_handler/backends/tmail_extensions.rb +++ b/lib/mail_handler/backends/tmail_extensions.rb @@ -74,22 +74,6 @@ module TMail end - class Address - # Monkeypatch! Constructor which makes a TMail::Address given - # a name and an email - def Address.address_from_name_and_email(name, email) - if !MySociety::Validate.is_valid_email(email) - raise "invalid email " + email + " passed to address_from_name_and_email" - end - if name.nil? - return TMail::Address.parse(email) - end - # Botch an always quoted RFC address, then parse it - name = name.gsub(/(["\\])/, "\\\\\\1") - return TMail::Address.parse('"' + name + '" <' + email + '>') - end - end - module TextUtils # Monkeypatch! Much more aggressive list of characters to cause quoting # than in normal TMail. e.g. Have found real cases where @ needs quoting. @@ -101,8 +85,8 @@ module TMail end end -# Monkeypatch! TMail 1.2.7.1 will parse only one address out of a list of addresses with -# unquoted display parts https://github.com/mikel/tmail/issues#issue/9 - this monkeypatch +# Monkeypatch! TMail 1.2.7.1 will parse only one address out of a list of addresses with +# unquoted display parts https://github.com/mikel/tmail/issues#issue/9 - this monkeypatch # fixes this issue. module TMail -- cgit v1.2.3 From 125ca970ad4e2b5e424265c632ae31c6dde62da7 Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Thu, 15 Nov 2012 17:25:11 +0000 Subject: Wrap address parsing in a address_from_string method in the mail handler. --- lib/mail_handler/backends/mail_backend.rb | 4 ++++ lib/mail_handler/backends/tmail_backend.rb | 5 +++++ 2 files changed, 9 insertions(+) (limited to 'lib/mail_handler/backends') diff --git a/lib/mail_handler/backends/mail_backend.rb b/lib/mail_handler/backends/mail_backend.rb index bc4bf564c..0e198adf0 100644 --- a/lib/mail_handler/backends/mail_backend.rb +++ b/lib/mail_handler/backends/mail_backend.rb @@ -42,6 +42,10 @@ module MailHandler address.address = email address.to_s end + + def address_from_string(string) + Mail::Address.new(string).address + end end end end \ No newline at end of file diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb index 1c5489145..0a1236e77 100644 --- a/lib/mail_handler/backends/tmail_backend.rb +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -52,6 +52,11 @@ module MailHandler name = name.gsub(/(["\\])/, "\\\\\\1") TMail::Address.parse('"' + name + '" <' + email + '>').to_s end + + def address_from_string(string) + TMail::Address.parse(string).address + end + end end end \ No newline at end of file -- cgit v1.2.3 From 24c3ceb2315734ab6e43ae4f75673e251b98a96e Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Thu, 15 Nov 2012 17:44:08 +0000 Subject: Bugfix - need to convert to string in the just email address case. --- lib/mail_handler/backends/tmail_backend.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/mail_handler/backends') diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb index 0a1236e77..87aba73d7 100644 --- a/lib/mail_handler/backends/tmail_backend.rb +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -46,7 +46,7 @@ module MailHandler raise "invalid email " + email + " passed to address_from_name_and_email" end if name.nil? - return TMail::Address.parse(email) + return TMail::Address.parse(email).to_s end # Botch an always quoted RFC address, then parse it name = name.gsub(/(["\\])/, "\\\\\\1") -- cgit v1.2.3