Add method for forcing strings to valid utf-8

Try likely conversions but if that fails, just replace the characters
that are invalid utf-8.

1 files changed, 11 insertions, 0 deletions

diff --git a/lib/normalize_string.rb b/lib/normalize_string.rb
index 3b6116970..de847cd16 100644
--- a/lib/normalize_string.rb
+++ b/lib/normalize_string.rb
@@ -72,6 +72,17 @@ def convert_string_to_utf8_or_binary(s, suggested_character_encoding=nil)
     result
 end
 
+def convert_string_to_utf8(s, suggested_character_encoding=nil)
+    begin
+        result = normalize_string_to_utf8 s, suggested_character_encoding
+    rescue EncodingNormalizationError
+        result = s.force_encoding("utf-8").encode("utf-8", :invalid => :replace,
+                                   :undef => :replace,
+                                   :replace => "") if String.method_defined?(:encode)
+    end
+    result
+end
+
 def log_text_details(message, text)
     if String.method_defined?(:encode)
         STDERR.puts "#{message}, we have text: #{text}, of class #{text.class} and encoding #{text.encoding}"