From 6ceaadf8954c03d7d2723e639f7449de93fec6fa Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Wed, 5 Dec 2012 16:23:50 +0000 Subject: Move part counting to the mail handler. --- lib/mail_handler/backends/tmail_backend.rb | 48 ++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) (limited to 'lib/mail_handler/backends/tmail_backend.rb') diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb index 4b7291d00..3f77f9f8b 100644 --- a/lib/mail_handler/backends/tmail_backend.rb +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -83,6 +83,54 @@ module MailHandler mail.header_string(header) end + # Number the attachments in depth first tree order, for use in URLs. + # XXX This fills in part.rfc822_attachment and part.url_part_number within + # all the parts of the email (see monkeypatches in lib/mail_handler/tmail_extensions and + # lib/mail_handler/mail_extensions for how these attributes are added). ensure_parts_counted + # must be called before using the attributes. + def ensure_parts_counted(mail) + mail.count_parts_count = 0 + _count_parts_recursive(mail, mail) + # we carry on using these numeric ids for attachments uudecoded from within text parts + mail.count_first_uudecode_count = mail.count_parts_count + end + def _count_parts_recursive(part, mail) + if part.multipart? + part.parts.each do |p| + _count_parts_recursive(p, mail) + end + else + part_filename = MailHandler.get_part_file_name(part) + begin + if part.content_type == 'message/rfc822' + # An email attached as text + # e.g. http://www.whatdotheyknow.com/request/64/response/102 + part.rfc822_attachment = MailHandler.mail_from_raw_email(part.body, decode=false) + elsif part.content_type == 'application/vnd.ms-outlook' || part_filename && AlaveteliFileTypes.filename_to_mimetype(part_filename) == 'application/vnd.ms-outlook' + # An email attached as an Outlook file + # e.g. http://www.whatdotheyknow.com/request/chinese_names_for_british_politi + msg = Mapi::Msg.open(StringIO.new(part.body)) + part.rfc822_attachment = MailHandler.mail_from_raw_email(msg.to_mime.to_s, decode=false) + elsif part.content_type == 'application/ms-tnef' + # A set of attachments in a TNEF file + part.rfc822_attachment = MailHandler.mail_from_tnef(part.body) + end + rescue + # If attached mail doesn't parse, treat it as text part + part.rfc822_attachment = nil + else + unless part.rfc822_attachment.nil? + _count_parts_recursive(part.rfc822_attachment, mail) + end + end + if part.rfc822_attachment.nil? + mail.count_parts_count += 1 + part.url_part_number = mail.count_parts_count + end + end + end + + def address_from_name_and_email(name, email) if !MySociety::Validate.is_valid_email(email) raise "invalid email " + email + " passed to address_from_name_and_email" -- cgit v1.2.3 From 1b1527b30b6b10493eafd4b63d318bc14bd0d07f Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Wed, 5 Dec 2012 16:27:53 +0000 Subject: Move the getting of attachment leaves to the mail handler. --- lib/mail_handler/backends/tmail_backend.rb | 88 ++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) (limited to 'lib/mail_handler/backends/tmail_backend.rb') diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb index 3f77f9f8b..4df4780a3 100644 --- a/lib/mail_handler/backends/tmail_backend.rb +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -130,6 +130,94 @@ module MailHandler end end + # (This risks losing info if the unchosen alternative is the only one to contain + # useful info, but let's worry about that another time) + def get_attachment_leaves(mail) + return _get_attachment_leaves_recursive(mail, mail) + end + def _get_attachment_leaves_recursive(curr_mail, parent_mail, within_rfc822_attachment = nil) + leaves_found = [] + if curr_mail.multipart? + if curr_mail.parts.size == 0 + raise "no parts on multipart mail" + end + + if curr_mail.sub_type == 'alternative' + # Choose best part from alternatives + best_part = nil + # Take the last text/plain one, or else the first one + curr_mail.parts.each do |m| + if not best_part + best_part = m + elsif m.content_type == 'text/plain' + best_part = m + end + end + # Take an HTML one as even higher priority. (They tend + # to render better than text/plain, e.g. don't wrap links here: + # http://www.whatdotheyknow.com/request/amount_and_cost_of_freedom_of_in#incoming-72238 ) + curr_mail.parts.each do |m| + if m.content_type == 'text/html' + best_part = m + end + end + leaves_found += _get_attachment_leaves_recursive(best_part, parent_mail, within_rfc822_attachment) + else + # Add all parts + curr_mail.parts.each do |m| + leaves_found += _get_attachment_leaves_recursive(m, parent_mail, within_rfc822_attachment) + end + end + else + # XXX Yuck. this section alters various content_type's. That puts + # it into conflict with MailHandler.ensure_parts_counted which it has to be + # called both before and after. It will fail with cases of + # attachments of attachments etc. + charset = curr_mail.charset # save this, because overwriting content_type also resets charset + # Don't allow nil content_types + if curr_mail.content_type.nil? + curr_mail.content_type = 'application/octet-stream' + end + # PDFs often come with this mime type, fix it up for view code + if curr_mail.content_type == 'application/octet-stream' + part_file_name = MailHandler.get_part_file_name(curr_mail) + part_body = MailHandler.get_part_body(curr_mail) + calc_mime = AlaveteliFileTypes.filename_and_content_to_mimetype(part_file_name, part_body) + if calc_mime + curr_mail.content_type = calc_mime + end + end + + # Use standard content types for Word documents etc. + curr_mail.content_type = MailHandler.normalise_content_type(curr_mail.content_type) + if curr_mail.content_type == 'message/rfc822' + MailHandler.ensure_parts_counted(parent_mail) # fills in rfc822_attachment variable + if curr_mail.rfc822_attachment.nil? + # Attached mail didn't parse, so treat as text + curr_mail.content_type = 'text/plain' + end + end + if curr_mail.content_type == 'application/vnd.ms-outlook' || curr_mail.content_type == 'application/ms-tnef' + MailHandler.ensure_parts_counted(parent_mail) # fills in rfc822_attachment variable + if curr_mail.rfc822_attachment.nil? + # Attached mail didn't parse, so treat as binary + curr_mail.content_type = 'application/octet-stream' + end + end + # If the part is an attachment of email + if curr_mail.content_type == 'message/rfc822' || curr_mail.content_type == 'application/vnd.ms-outlook' || curr_mail.content_type == 'application/ms-tnef' + MailHandler.ensure_parts_counted(parent_mail) # fills in rfc822_attachment variable + leaves_found += _get_attachment_leaves_recursive(curr_mail.rfc822_attachment, parent_mail, curr_mail.rfc822_attachment) + else + # Store leaf + curr_mail.within_rfc822_attachment = within_rfc822_attachment + leaves_found += [curr_mail] + end + # restore original charset + curr_mail.charset = charset + end + return leaves_found + end def address_from_name_and_email(name, email) if !MySociety::Validate.is_valid_email(email) -- cgit v1.2.3 From 53faf19864cc8bbf872de889e6f59574fc92950c Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Wed, 5 Dec 2012 16:34:12 +0000 Subject: Remove redundant references to MailHandler. --- lib/mail_handler/backends/tmail_backend.rb | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'lib/mail_handler/backends/tmail_backend.rb') diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb index 4df4780a3..4e9cf6628 100644 --- a/lib/mail_handler/backends/tmail_backend.rb +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -100,20 +100,20 @@ module MailHandler _count_parts_recursive(p, mail) end else - part_filename = MailHandler.get_part_file_name(part) + part_filename = get_part_file_name(part) begin if part.content_type == 'message/rfc822' # An email attached as text # e.g. http://www.whatdotheyknow.com/request/64/response/102 - part.rfc822_attachment = MailHandler.mail_from_raw_email(part.body, decode=false) + part.rfc822_attachment = mail_from_raw_email(part.body, decode=false) elsif part.content_type == 'application/vnd.ms-outlook' || part_filename && AlaveteliFileTypes.filename_to_mimetype(part_filename) == 'application/vnd.ms-outlook' # An email attached as an Outlook file # e.g. http://www.whatdotheyknow.com/request/chinese_names_for_british_politi msg = Mapi::Msg.open(StringIO.new(part.body)) - part.rfc822_attachment = MailHandler.mail_from_raw_email(msg.to_mime.to_s, decode=false) + part.rfc822_attachment = mail_from_raw_email(msg.to_mime.to_s, decode=false) elsif part.content_type == 'application/ms-tnef' # A set of attachments in a TNEF file - part.rfc822_attachment = MailHandler.mail_from_tnef(part.body) + part.rfc822_attachment = mail_from_tnef(part.body) end rescue # If attached mail doesn't parse, treat it as text part @@ -170,7 +170,7 @@ module MailHandler end else # XXX Yuck. this section alters various content_type's. That puts - # it into conflict with MailHandler.ensure_parts_counted which it has to be + # it into conflict with ensure_parts_counted which it has to be # called both before and after. It will fail with cases of # attachments of attachments etc. charset = curr_mail.charset # save this, because overwriting content_type also resets charset @@ -180,8 +180,8 @@ module MailHandler end # PDFs often come with this mime type, fix it up for view code if curr_mail.content_type == 'application/octet-stream' - part_file_name = MailHandler.get_part_file_name(curr_mail) - part_body = MailHandler.get_part_body(curr_mail) + part_file_name = get_part_file_name(curr_mail) + part_body = get_part_body(curr_mail) calc_mime = AlaveteliFileTypes.filename_and_content_to_mimetype(part_file_name, part_body) if calc_mime curr_mail.content_type = calc_mime @@ -189,16 +189,16 @@ module MailHandler end # Use standard content types for Word documents etc. - curr_mail.content_type = MailHandler.normalise_content_type(curr_mail.content_type) + curr_mail.content_type = normalise_content_type(curr_mail.content_type) if curr_mail.content_type == 'message/rfc822' - MailHandler.ensure_parts_counted(parent_mail) # fills in rfc822_attachment variable + ensure_parts_counted(parent_mail) # fills in rfc822_attachment variable if curr_mail.rfc822_attachment.nil? # Attached mail didn't parse, so treat as text curr_mail.content_type = 'text/plain' end end if curr_mail.content_type == 'application/vnd.ms-outlook' || curr_mail.content_type == 'application/ms-tnef' - MailHandler.ensure_parts_counted(parent_mail) # fills in rfc822_attachment variable + ensure_parts_counted(parent_mail) # fills in rfc822_attachment variable if curr_mail.rfc822_attachment.nil? # Attached mail didn't parse, so treat as binary curr_mail.content_type = 'application/octet-stream' @@ -206,7 +206,7 @@ module MailHandler end # If the part is an attachment of email if curr_mail.content_type == 'message/rfc822' || curr_mail.content_type == 'application/vnd.ms-outlook' || curr_mail.content_type == 'application/ms-tnef' - MailHandler.ensure_parts_counted(parent_mail) # fills in rfc822_attachment variable + ensure_parts_counted(parent_mail) # fills in rfc822_attachment variable leaves_found += _get_attachment_leaves_recursive(curr_mail.rfc822_attachment, parent_mail, curr_mail.rfc822_attachment) else # Store leaf -- cgit v1.2.3 From 8d7a02933e9d867bd2d3ce2209df2c80a316fe34 Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Wed, 5 Dec 2012 16:35:35 +0000 Subject: Fix typo. --- lib/mail_handler/backends/tmail_backend.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/mail_handler/backends/tmail_backend.rb') diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb index 4e9cf6628..92d478541 100644 --- a/lib/mail_handler/backends/tmail_backend.rb +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -169,7 +169,7 @@ module MailHandler end end else - # XXX Yuck. this section alters various content_type's. That puts + # XXX Yuck. this section alters various content_types. That puts # it into conflict with ensure_parts_counted which it has to be # called both before and after. It will fail with cases of # attachments of attachments etc. -- cgit v1.2.3 From 0375214ca0c295f0316ec80be8aebdbd1d1c1b8a Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Wed, 5 Dec 2012 16:41:43 +0000 Subject: Add a wrapper method for get_attachment_leaves. --- lib/mail_handler/backends/tmail_backend.rb | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'lib/mail_handler/backends/tmail_backend.rb') diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb index 92d478541..9bcc2ab1f 100644 --- a/lib/mail_handler/backends/tmail_backend.rb +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -130,6 +130,14 @@ module MailHandler end end + def get_attachment_attributes(mail) + leaves = get_attachment_leaves(mail) + # XXX we have to call ensure_parts_counted after get_attachment_leaves + # which is really messy. + ensure_parts_counted(mail) + leaves + end + # (This risks losing info if the unchosen alternative is the only one to contain # useful info, but let's worry about that another time) def get_attachment_leaves(mail) -- cgit v1.2.3 From 7f79f32bd1021fde6a4c026072febcfabc6d0c72 Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Wed, 5 Dec 2012 16:57:38 +0000 Subject: Move mail-specific stuff to mail handler. --- lib/mail_handler/backends/tmail_backend.rb | 47 +++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) (limited to 'lib/mail_handler/backends/tmail_backend.rb') diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb index 9bcc2ab1f..f28eaad79 100644 --- a/lib/mail_handler/backends/tmail_backend.rb +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -135,7 +135,52 @@ module MailHandler # XXX we have to call ensure_parts_counted after get_attachment_leaves # which is really messy. ensure_parts_counted(mail) - leaves + attachment_attributes = [] + for leaf in leaves + body = MailHandler.get_part_body(leaf) + # As leaf.body causes MIME decoding which uses lots of RAM, do garbage collection here + # to prevent excess memory use. XXX not really sure if this helps reduce + # peak RAM use overall. Anyway, maybe there is something better to do than this. + GC.start + if leaf.within_rfc822_attachment + within_rfc822_subject = leaf.within_rfc822_attachment.subject + # Test to see if we are in the first part of the attached + # RFC822 message and it is text, if so add headers. + # XXX should probably use hunting algorithm to find main text part, rather than + # just expect it to be first. This will do for now though. + # Example request that needs this: + # http://www.whatdotheyknow.com/request/2923/response/7013/attach/2/Cycle%20Path%20Bank.txt + if leaf.within_rfc822_attachment == leaf && leaf.content_type == 'text/plain' + headers = "" + for header in [ 'Date', 'Subject', 'From', 'To', 'Cc' ] + if leaf.within_rfc822_attachment.header.include?(header.downcase) + header_value = leaf.within_rfc822_attachment.header[header.downcase] + # Example message which has a blank Date header: + # http://www.whatdotheyknow.com/request/30747/response/80253/attach/html/17/Common%20Purpose%20Advisory%20Group%20Meeting%20Tuesday%202nd%20March.txt.html + if !header_value.blank? + headers = headers + header + ": " + header_value.to_s + "\n" + end + end + end + # XXX call _convert_part_body_to_text here, but need to get charset somehow + # e.g. http://www.whatdotheyknow.com/request/1593/response/3088/attach/4/Freedom%20of%20Information%20request%20-%20car%20oval%20sticker:%20Article%2020,%20Convention%20on%20Road%20Traffic%201949.txt + body = headers + "\n" + body + + # This is quick way of getting all headers, but instead we only add some a) to + # make it more usable, b) as at least one authority accidentally leaked security + # information into a header. + #attachment.body = leaf.within_rfc822_attachment.port.to_s + end + end + attachment_attributes << {:url_part_number => leaf.url_part_number, + :content_type => MailHandler.get_content_type(leaf), + :filename => MailHandler.get_part_file_name(leaf), + :charset => leaf.charset, + :within_rfc822_subject => within_rfc822_subject, + :body => body, + :hexdigest => Digest::MD5.hexdigest(body) } + end + attachment_attributes end # (This risks losing info if the unchosen alternative is the only one to contain -- cgit v1.2.3 From 3a55a3eb5601bbb3ae83d32bc92ffdd9a27961c4 Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Wed, 5 Dec 2012 17:00:18 +0000 Subject: Remove redundant references to MailHandler --- lib/mail_handler/backends/tmail_backend.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib/mail_handler/backends/tmail_backend.rb') diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb index f28eaad79..3ce82a50c 100644 --- a/lib/mail_handler/backends/tmail_backend.rb +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -137,7 +137,7 @@ module MailHandler ensure_parts_counted(mail) attachment_attributes = [] for leaf in leaves - body = MailHandler.get_part_body(leaf) + body = get_part_body(leaf) # As leaf.body causes MIME decoding which uses lots of RAM, do garbage collection here # to prevent excess memory use. XXX not really sure if this helps reduce # peak RAM use overall. Anyway, maybe there is something better to do than this. @@ -173,8 +173,8 @@ module MailHandler end end attachment_attributes << {:url_part_number => leaf.url_part_number, - :content_type => MailHandler.get_content_type(leaf), - :filename => MailHandler.get_part_file_name(leaf), + :content_type => get_content_type(leaf), + :filename => get_part_file_name(leaf), :charset => leaf.charset, :within_rfc822_subject => within_rfc822_subject, :body => body, -- cgit v1.2.3 From f026dbc4c23ca25dd31dc6d3d132d6f1668728de Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Thu, 6 Dec 2012 13:46:05 +0000 Subject: Convert example URL to spec. --- lib/mail_handler/backends/tmail_backend.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib/mail_handler/backends/tmail_backend.rb') diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb index 3ce82a50c..e28765d54 100644 --- a/lib/mail_handler/backends/tmail_backend.rb +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -155,9 +155,7 @@ module MailHandler for header in [ 'Date', 'Subject', 'From', 'To', 'Cc' ] if leaf.within_rfc822_attachment.header.include?(header.downcase) header_value = leaf.within_rfc822_attachment.header[header.downcase] - # Example message which has a blank Date header: - # http://www.whatdotheyknow.com/request/30747/response/80253/attach/html/17/Common%20Purpose%20Advisory%20Group%20Meeting%20Tuesday%202nd%20March.txt.html - if !header_value.blank? + if !header_value.blank? headers = headers + header + ": " + header_value.to_s + "\n" end end -- cgit v1.2.3 From 36c82a1fce563c0f23720589e1ac2741576bd4dd Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Thu, 6 Dec 2012 13:58:18 +0000 Subject: Remove url in comment - has been converted to spec. --- lib/mail_handler/backends/tmail_backend.rb | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib/mail_handler/backends/tmail_backend.rb') diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb index e28765d54..f5da3049e 100644 --- a/lib/mail_handler/backends/tmail_backend.rb +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -148,8 +148,6 @@ module MailHandler # RFC822 message and it is text, if so add headers. # XXX should probably use hunting algorithm to find main text part, rather than # just expect it to be first. This will do for now though. - # Example request that needs this: - # http://www.whatdotheyknow.com/request/2923/response/7013/attach/2/Cycle%20Path%20Bank.txt if leaf.within_rfc822_attachment == leaf && leaf.content_type == 'text/plain' headers = "" for header in [ 'Date', 'Subject', 'From', 'To', 'Cc' ] -- cgit v1.2.3 From ac1c1329eeebefec8c9952cbae372fe8c4255307 Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Thu, 6 Dec 2012 16:34:52 +0000 Subject: Convert url in comment to spec. Conflicts: lib/mail_handler/backends/tmail_backend.rb --- lib/mail_handler/backends/tmail_backend.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib/mail_handler/backends/tmail_backend.rb') diff --git a/lib/mail_handler/backends/tmail_backend.rb b/lib/mail_handler/backends/tmail_backend.rb index f5da3049e..02124cdb1 100644 --- a/lib/mail_handler/backends/tmail_backend.rb +++ b/lib/mail_handler/backends/tmail_backend.rb @@ -268,6 +268,7 @@ module MailHandler return leaves_found end + def address_from_name_and_email(name, email) if !MySociety::Validate.is_valid_email(email) raise "invalid email " + email + " passed to address_from_name_and_email" -- cgit v1.2.3