From b9a2d996408c1ef703aa78311f8c2a0f6e0d7afc Mon Sep 17 00:00:00 2001 From: Henare Degan Date: Sun, 10 Feb 2013 16:51:30 +1100 Subject: Update this to match the older version of the mail gem we're using --- lib/mail_handler/backends/mail_extensions.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'lib/mail_handler/backends/mail_extensions.rb') diff --git a/lib/mail_handler/backends/mail_extensions.rb b/lib/mail_handler/backends/mail_extensions.rb index f756abd1a..83dce5733 100644 --- a/lib/mail_handler/backends/mail_extensions.rb +++ b/lib/mail_handler/backends/mail_extensions.rb @@ -50,8 +50,7 @@ module Mail # A patched version of the parameter hash that handles nil values without throwing # an error. - class ParameterHash < IndifferentHash - + class ParameterHash < HashWithIndifferentAccess def encoded map.sort { |a,b| a.first.to_s <=> b.first.to_s }.map do |key_name, value| # The replacement of this commented out line is the change @@ -64,4 +63,4 @@ module Mail end.join(";\r\n\s") end end -end \ No newline at end of file +end -- cgit v1.2.3 From a74855165779821ba531fd3f9c3767fc3d10ac60 Mon Sep 17 00:00:00 2001 From: Henare Degan Date: Sat, 2 Mar 2013 15:38:46 +1100 Subject: Backport newer Mail code to fix decoding problems. #850 --- lib/mail_handler/backends/mail_extensions.rb | 38 ++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) (limited to 'lib/mail_handler/backends/mail_extensions.rb') diff --git a/lib/mail_handler/backends/mail_extensions.rb b/lib/mail_handler/backends/mail_extensions.rb index 83dce5733..29ae32542 100644 --- a/lib/mail_handler/backends/mail_extensions.rb +++ b/lib/mail_handler/backends/mail_extensions.rb @@ -63,4 +63,42 @@ module Mail end.join(";\r\n\s") end end + + # HACK: Backport encoding fixes for Ruby 1.8 from Mail 2.5 + # Can be removed when we no longer support Ruby 1.8 + class Ruby18 + def Ruby18.b_value_decode(str) + match = str.match(/\=\?(.+)?\?[Bb]\?(.+)?\?\=/m) + if match + encoding = match[1] + str = Ruby18.decode_base64(match[2]) + str = Iconv.conv('UTF-8//IGNORE', fix_encoding(encoding), str) + end + str + end + + def Ruby18.q_value_decode(str) + match = str.match(/\=\?(.+)?\?[Qq]\?(.+)?\?\=/m) + if match + encoding = match[1] + string = match[2].gsub(/_/, '=20') + # Remove trailing = if it exists in a Q encoding + string = string.sub(/\=$/, '') + str = Encodings::QuotedPrintable.decode(string) + str = Iconv.conv('UTF-8//IGNORE', fix_encoding(encoding), str) + end + str + end + + private + + def Ruby18.fix_encoding(encoding) + case encoding.upcase + when 'UTF8' + 'UTF-8' + else + encoding + end + end + end end -- cgit v1.2.3 From 9564ca12234e27e2292b5382596664634d4994e0 Mon Sep 17 00:00:00 2001 From: Henare Degan Date: Sat, 2 Mar 2013 18:07:30 +1100 Subject: Backport Mail 2.5's to_yaml to prevent an exception with Ruby 1.8 --- lib/mail_handler/backends/mail_extensions.rb | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'lib/mail_handler/backends/mail_extensions.rb') diff --git a/lib/mail_handler/backends/mail_extensions.rb b/lib/mail_handler/backends/mail_extensions.rb index 29ae32542..c9f16e361 100644 --- a/lib/mail_handler/backends/mail_extensions.rb +++ b/lib/mail_handler/backends/mail_extensions.rb @@ -46,6 +46,28 @@ module Mail self end + + # HACK: Backported from Mail 2.5 for Ruby 1.8 support + # Can be removed when we no longer support Ruby 1.8 + def to_yaml(opts = {}) + hash = {} + hash['headers'] = {} + header.fields.each do |field| + hash['headers'][field.name] = field.value + end + hash['delivery_handler'] = delivery_handler.to_s if delivery_handler + hash['transport_encoding'] = transport_encoding.to_s + special_variables = [:@header, :@delivery_handler, :@transport_encoding] + if multipart? + hash['multipart_body'] = [] + body.parts.map { |part| hash['multipart_body'] << part.to_yaml } + special_variables.push(:@body, :@text_part, :@html_part) + end + (instance_variables.map(&:to_sym) - special_variables).each do |var| + hash[var.to_s] = instance_variable_get(var) + end + hash.to_yaml(opts) + end end # A patched version of the parameter hash that handles nil values without throwing -- cgit v1.2.3 From 930619d4cf3a779abaf28e4cfa9d287e552bdc05 Mon Sep 17 00:00:00 2001 From: Henare Degan Date: Tue, 26 Feb 2013 16:07:46 +1100 Subject: Revert "Update this to match the older version of the mail gem we're using" This reverts commit b9a2d996408c1ef703aa78311f8c2a0f6e0d7afc. --- lib/mail_handler/backends/mail_extensions.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib/mail_handler/backends/mail_extensions.rb') diff --git a/lib/mail_handler/backends/mail_extensions.rb b/lib/mail_handler/backends/mail_extensions.rb index c9f16e361..34ced7319 100644 --- a/lib/mail_handler/backends/mail_extensions.rb +++ b/lib/mail_handler/backends/mail_extensions.rb @@ -72,7 +72,8 @@ module Mail # A patched version of the parameter hash that handles nil values without throwing # an error. - class ParameterHash < HashWithIndifferentAccess + class ParameterHash < IndifferentHash + def encoded map.sort { |a,b| a.first.to_s <=> b.first.to_s }.map do |key_name, value| # The replacement of this commented out line is the change -- cgit v1.2.3 From 2f6af09899e2822121d060db55fdd78b15f099db Mon Sep 17 00:00:00 2001 From: Henare Degan Date: Thu, 7 Mar 2013 18:16:16 +1100 Subject: Backport Mail's encoding code from 2.5 for Ruby 1.9 Decoding messages in Ruby 1.9 was screwing up but not dying like 1.8. Backporting this fixes the problem. --- lib/mail_handler/backends/mail_extensions.rb | 45 ++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) (limited to 'lib/mail_handler/backends/mail_extensions.rb') diff --git a/lib/mail_handler/backends/mail_extensions.rb b/lib/mail_handler/backends/mail_extensions.rb index 34ced7319..611b44c4c 100644 --- a/lib/mail_handler/backends/mail_extensions.rb +++ b/lib/mail_handler/backends/mail_extensions.rb @@ -124,4 +124,49 @@ module Mail end end end + + # HACK: Backport encoding fixes for Ruby 1.9 from Mail 2.5 + # Can be removed when Rails relies on Mail > 2.5 + class Ruby19 + def Ruby19.b_value_decode(str) + match = str.match(/\=\?(.+)?\?[Bb]\?(.+)?\?\=/m) + if match + encoding = match[1] + str = Ruby19.decode_base64(match[2]) + str.force_encoding(fix_encoding(encoding)) + end + decoded = str.encode("utf-8", :invalid => :replace, :replace => "") + decoded.valid_encoding? ? decoded : decoded.encode("utf-16le", :invalid => :replace, :replace => "").encode("utf-8") + end + + def Ruby19.q_value_decode(str) + match = str.match(/\=\?(.+)?\?[Qq]\?(.+)?\?\=/m) + if match + encoding = match[1] + str = Encodings::QuotedPrintable.decode(match[2]) + str.force_encoding(fix_encoding(encoding)) + end + decoded = str.encode("utf-8", :invalid => :replace, :replace => "") + decoded.valid_encoding? ? decoded : decoded.encode("utf-16le", :invalid => :replace, :replace => "").encode("utf-8") + end + + # mails somtimes includes invalid encodings like iso885915 or utf8 so we transform them to iso885915 or utf8 + # TODO: add this as a test somewhere + # Encoding.list.map{|e| [e.to_s.upcase==fix_encoding(e.to_s.downcase.gsub("-", "")), e.to_s] }.select {|a,b| !b} + # Encoding.list.map{|e| [e.to_s==fix_encoding(e.to_s), e.to_s] }.select {|a,b| !b} + def Ruby19.fix_encoding(encoding) + case encoding + # ISO-8859-15, ISO-2022-JP and alike + when /iso-?(\d{4})-?(\w{1,2})/i then return "ISO-#{$1}-#{$2}" + # "ISO-2022-JP-KDDI" and alike + when /iso-?(\d{4})-?(\w{1,2})-?(\w*)/i then return "ISO-#{$1}-#{$2}-#{$3}" + # UTF-8, UTF-32BE and alike + when /utf-?(\d{1,2})?(\w{1,2})/i then return "UTF-#{$1}#{$2}" + # Windows-1252 and alike + when /Windows-?(.*)/i then return "Windows-#{$1}" + #more aliases to be added if needed + else return encoding + end + end + end end -- cgit v1.2.3 From 5738367d0f38627ba0227758be0433af23faefbc Mon Sep 17 00:00:00 2001 From: Matthew Landauer Date: Tue, 19 Mar 2013 11:41:19 +1100 Subject: Rails security update The rails upgrade also forces a mail gem upgrade. To make things work again we need to remove part of the monkeypatched backporting of encoding fixes. --- lib/mail_handler/backends/mail_extensions.rb | 67 ---------------------------- 1 file changed, 67 deletions(-) (limited to 'lib/mail_handler/backends/mail_extensions.rb') diff --git a/lib/mail_handler/backends/mail_extensions.rb b/lib/mail_handler/backends/mail_extensions.rb index 611b44c4c..d25012e39 100644 --- a/lib/mail_handler/backends/mail_extensions.rb +++ b/lib/mail_handler/backends/mail_extensions.rb @@ -46,28 +46,6 @@ module Mail self end - - # HACK: Backported from Mail 2.5 for Ruby 1.8 support - # Can be removed when we no longer support Ruby 1.8 - def to_yaml(opts = {}) - hash = {} - hash['headers'] = {} - header.fields.each do |field| - hash['headers'][field.name] = field.value - end - hash['delivery_handler'] = delivery_handler.to_s if delivery_handler - hash['transport_encoding'] = transport_encoding.to_s - special_variables = [:@header, :@delivery_handler, :@transport_encoding] - if multipart? - hash['multipart_body'] = [] - body.parts.map { |part| hash['multipart_body'] << part.to_yaml } - special_variables.push(:@body, :@text_part, :@html_part) - end - (instance_variables.map(&:to_sym) - special_variables).each do |var| - hash[var.to_s] = instance_variable_get(var) - end - hash.to_yaml(opts) - end end # A patched version of the parameter hash that handles nil values without throwing @@ -124,49 +102,4 @@ module Mail end end end - - # HACK: Backport encoding fixes for Ruby 1.9 from Mail 2.5 - # Can be removed when Rails relies on Mail > 2.5 - class Ruby19 - def Ruby19.b_value_decode(str) - match = str.match(/\=\?(.+)?\?[Bb]\?(.+)?\?\=/m) - if match - encoding = match[1] - str = Ruby19.decode_base64(match[2]) - str.force_encoding(fix_encoding(encoding)) - end - decoded = str.encode("utf-8", :invalid => :replace, :replace => "") - decoded.valid_encoding? ? decoded : decoded.encode("utf-16le", :invalid => :replace, :replace => "").encode("utf-8") - end - - def Ruby19.q_value_decode(str) - match = str.match(/\=\?(.+)?\?[Qq]\?(.+)?\?\=/m) - if match - encoding = match[1] - str = Encodings::QuotedPrintable.decode(match[2]) - str.force_encoding(fix_encoding(encoding)) - end - decoded = str.encode("utf-8", :invalid => :replace, :replace => "") - decoded.valid_encoding? ? decoded : decoded.encode("utf-16le", :invalid => :replace, :replace => "").encode("utf-8") - end - - # mails somtimes includes invalid encodings like iso885915 or utf8 so we transform them to iso885915 or utf8 - # TODO: add this as a test somewhere - # Encoding.list.map{|e| [e.to_s.upcase==fix_encoding(e.to_s.downcase.gsub("-", "")), e.to_s] }.select {|a,b| !b} - # Encoding.list.map{|e| [e.to_s==fix_encoding(e.to_s), e.to_s] }.select {|a,b| !b} - def Ruby19.fix_encoding(encoding) - case encoding - # ISO-8859-15, ISO-2022-JP and alike - when /iso-?(\d{4})-?(\w{1,2})/i then return "ISO-#{$1}-#{$2}" - # "ISO-2022-JP-KDDI" and alike - when /iso-?(\d{4})-?(\w{1,2})-?(\w*)/i then return "ISO-#{$1}-#{$2}-#{$3}" - # UTF-8, UTF-32BE and alike - when /utf-?(\d{1,2})?(\w{1,2})/i then return "UTF-#{$1}#{$2}" - # Windows-1252 and alike - when /Windows-?(.*)/i then return "Windows-#{$1}" - #more aliases to be added if needed - else return encoding - end - end - end end -- cgit v1.2.3 From d5725cac044cc46245edc209e7c61c717e0d23db Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 3 Jun 2013 15:11:05 +0100 Subject: Fix for subject lines with invalid UTF-8 as the last character This seems to be the bug mentioned here: http://po-ru.com/diary/fixing-invalid-utf-8-in-ruby-revisited/ That explains that some versions of Iconv don't ignore invalid characters when converting to UTF-8 even with //IGNORE if that invalid character happens to be at the end of the string. In fact, as Matthew Somerville pointed out, with some versions of iconv (e.g. 1.14 on Mac OS, apparently) it's necessary to add and remove more than one space at the end, in case the first character of the byte sequence indicates a long sequence. We add and remove 4 to be on the safe side. --- lib/mail_handler/backends/mail_extensions.rb | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'lib/mail_handler/backends/mail_extensions.rb') diff --git a/lib/mail_handler/backends/mail_extensions.rb b/lib/mail_handler/backends/mail_extensions.rb index d25012e39..54599639b 100644 --- a/lib/mail_handler/backends/mail_extensions.rb +++ b/lib/mail_handler/backends/mail_extensions.rb @@ -73,7 +73,12 @@ module Mail if match encoding = match[1] str = Ruby18.decode_base64(match[2]) - str = Iconv.conv('UTF-8//IGNORE', fix_encoding(encoding), str) + # Adding and removing trailing spaces is a workaround + # for Iconv.conv throwing an exception if it finds an + # invalid character at the end of the string, even + # with UTF-8//IGNORE: + # http://po-ru.com/diary/fixing-invalid-utf-8-in-ruby-revisited/ + str = Iconv.conv('UTF-8//IGNORE', fix_encoding(encoding), str + " ")[0...-4] end str end @@ -86,7 +91,12 @@ module Mail # Remove trailing = if it exists in a Q encoding string = string.sub(/\=$/, '') str = Encodings::QuotedPrintable.decode(string) - str = Iconv.conv('UTF-8//IGNORE', fix_encoding(encoding), str) + # Adding and removing trailing spaces is a workaround + # for Iconv.conv throwing an exception if it finds an + # invalid character at the end of the string, even + # with UTF-8//IGNORE: + # http://po-ru.com/diary/fixing-invalid-utf-8-in-ruby-revisited/ + str = Iconv.conv('UTF-8//IGNORE', fix_encoding(encoding), str + " ")[0...-4] end str end -- cgit v1.2.3 From a919141992a40599f99b32bd4a8312a0009f3f7a Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Tue, 4 Jun 2013 10:29:56 +0100 Subject: Backport ruby 1.9 fix for trailing = sign in message headers from mail 2.5 --- lib/mail_handler/backends/mail_extensions.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'lib/mail_handler/backends/mail_extensions.rb') diff --git a/lib/mail_handler/backends/mail_extensions.rb b/lib/mail_handler/backends/mail_extensions.rb index 54599639b..322c49bb5 100644 --- a/lib/mail_handler/backends/mail_extensions.rb +++ b/lib/mail_handler/backends/mail_extensions.rb @@ -112,4 +112,20 @@ module Mail end end end + class Ruby19 + + def Ruby19.q_value_decode(str) + match = str.match(/\=\?(.+)?\?[Qq]\?(.+)?\?\=/m) + if match + encoding = match[1] + str = Encodings::QuotedPrintable.decode(match[2].gsub(/_/, '=20')) + # Backport line from mail 2.5 to strip a trailing = character + # Remove trailing = if it exists in a Q encoding + str = str.sub(/\=$/, '') + str.force_encoding(fix_encoding(encoding)) + end + decoded = str.encode("utf-8", :invalid => :replace, :replace => "") + decoded.valid_encoding? ? decoded : decoded.encode("utf-16le", :invalid => :replace, :replace => "").encode("utf-8") + end + end end -- cgit v1.2.3