From 203eea18feeaec3dc9a3e8e8af3b83de085b53ac Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Thu, 14 May 2015 15:52:10 +0100 Subject: Add method for forcing strings to valid utf-8 Try likely conversions but if that fails, just replace the characters that are invalid utf-8. --- lib/normalize_string.rb | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'lib/normalize_string.rb') diff --git a/lib/normalize_string.rb b/lib/normalize_string.rb index 3b6116970..de847cd16 100644 --- a/lib/normalize_string.rb +++ b/lib/normalize_string.rb @@ -72,6 +72,17 @@ def convert_string_to_utf8_or_binary(s, suggested_character_encoding=nil) result end +def convert_string_to_utf8(s, suggested_character_encoding=nil) + begin + result = normalize_string_to_utf8 s, suggested_character_encoding + rescue EncodingNormalizationError + result = s.force_encoding("utf-8").encode("utf-8", :invalid => :replace, + :undef => :replace, + :replace => "") if String.method_defined?(:encode) + end + result +end + def log_text_details(message, text) if String.method_defined?(:encode) STDERR.puts "#{message}, we have text: #{text}, of class #{text.class} and encoding #{text.encoding}" -- cgit v1.2.3