From 203eea18feeaec3dc9a3e8e8af3b83de085b53ac Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Thu, 14 May 2015 15:52:10 +0100 Subject: Add method for forcing strings to valid utf-8 Try likely conversions but if that fails, just replace the characters that are invalid utf-8. --- spec/lib/basic_encoding_spec.rb | 55 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) (limited to 'spec/lib/basic_encoding_spec.rb') diff --git a/spec/lib/basic_encoding_spec.rb b/spec/lib/basic_encoding_spec.rb index 43a65eab9..da5acf361 100644 --- a/spec/lib/basic_encoding_spec.rb +++ b/spec/lib/basic_encoding_spec.rb @@ -155,3 +155,58 @@ describe "convert_string_to_utf8_or_binary" do end end + +describe "convert_string_to_utf8" do + + describe "when passed uninterpretable character data" do + + it "should return it as a utf8 string" do + + converted = convert_string_to_utf8 random_string + converted.should == random_string + + if String.method_defined?(:encode) + converted.encoding.to_s.should == 'UTF-8' + end + + converted = convert_string_to_utf8 random_string,'UTF-8' + converted.should == random_string + + if String.method_defined?(:encode) + converted.encoding.to_s.should == 'UTF-8' + end + + end + end + + describe "when passed unlabelled Windows 1252 data" do + + it "should correctly convert it to UTF-8" do + + converted = convert_string_to_utf8 windows_1252_string + + converted.should == "DASH – DASH" + + if String.method_defined?(:encode) + converted.encoding.to_s.should == 'UTF-8' + end + end + + end + + describe "when passed GB 18030 data" do + + it "should correctly convert it to UTF-8 if unlabelled" do + + converted = convert_string_to_utf8 gb_18030_spam_string + + converted.should start_with("贵公司负责人") + + if String.method_defined?(:encode) + converted.encoding.to_s.should == 'UTF-8' + end + end + + end + +end \ No newline at end of file -- cgit v1.2.3