diff options
author | Louise Crow <louise.crow@gmail.com> | 2014-05-15 17:01:22 +0100 |
---|---|---|
committer | Louise Crow <louise.crow@gmail.com> | 2014-05-15 17:01:22 +0100 |
commit | bb5f95040d377d86629012347343fbf2c7dda016 (patch) | |
tree | be696ec2615b098db34449a982fcb09a68c9c49b /lib/normalize_string.rb | |
parent | 6d215fba5cc709c43f12f86da39a643e4be5922b (diff) | |
parent | 21027d0d1197e7ac447296ab68a25159860888b2 (diff) |
Merge remote-tracking branch 'origin/release/0.18'0.18
Diffstat (limited to 'lib/normalize_string.rb')
-rw-r--r-- | lib/normalize_string.rb | 12 |
1 files changed, 5 insertions, 7 deletions
diff --git a/lib/normalize_string.rb b/lib/normalize_string.rb index f02b18ee0..3b6116970 100644 --- a/lib/normalize_string.rb +++ b/lib/normalize_string.rb @@ -1,4 +1,4 @@ -require 'iconv' unless RUBY_VERSION.to_f >= 1.9 +require 'iconv' unless String.method_defined?(:encode) require 'charlock_holmes' class EncodingNormalizationError < StandardError @@ -23,17 +23,16 @@ def normalize_string_to_utf8(s, suggested_character_encoding=nil) to_try.push guessed_encoding to_try.each do |from_encoding| - if RUBY_VERSION.to_f >= 1.9 + if String.method_defined?(:encode) begin s.force_encoding from_encoding return s.encode('UTF-8') if s.valid_encoding? - rescue ArgumentError + rescue ArgumentError, Encoding::UndefinedConversionError # We get this is there are invalid bytes when # interpreted as from_encoding at the point of # the encode('UTF-8'); move onto the next one... end else - to_encoding = 'UTF-8' begin converted = Iconv.conv 'UTF-8', from_encoding, s return converted @@ -45,7 +44,6 @@ def normalize_string_to_utf8(s, suggested_character_encoding=nil) end end raise EncodingNormalizationError, "Couldn't find a valid character encoding for the string" - end def convert_string_to_utf8_or_binary(s, suggested_character_encoding=nil) @@ -69,13 +67,13 @@ def convert_string_to_utf8_or_binary(s, suggested_character_encoding=nil) result = normalize_string_to_utf8 s, suggested_character_encoding rescue EncodingNormalizationError result = s - s.force_encoding 'ASCII-8BIT' if RUBY_VERSION.to_f >= 1.9 + s.force_encoding 'ASCII-8BIT' if String.method_defined?(:encode) end result end def log_text_details(message, text) - if RUBY_VERSION.to_f >= 1.9 + if String.method_defined?(:encode) STDERR.puts "#{message}, we have text: #{text}, of class #{text.class} and encoding #{text.encoding}" else STDERR.puts "#{message}, we have text: #{text}, of class #{text.class}" |