Prevent erroneous internal error messages with multibyte characters under ruby 1.8 by using mb_chars for length comparison - it counts multibyte characters.

2 files changed, 14 insertions, 3 deletions

diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb
index 4910d43f4..ae3c3b407 100644
--- a/app/models/incoming_message.rb
+++ b/app/models/incoming_message.rb
@@ -258,7 +258,7 @@ class IncomingMessage < ActiveRecord::Base
     # Used by binary_mask_stuff - replace text in place
     def _binary_mask_stuff_internal!(text)
         # Keep original size, so can check haven't resized it
-        orig_size = text.size
+        orig_size = text.mb_chars.size
 
         # Replace ASCII email addresses...
         text.gsub!(MySociety::Validate.email_find_regexp) do |email|
@@ -293,7 +293,7 @@ class IncomingMessage < ActiveRecord::Base
         # Replace censor items
         self.info_request.apply_censor_rules_to_binary!(text)
 
-        raise "internal error in binary_mask_stuff" if text.size != orig_size
+        raise "internal error in binary_mask_stuff" if text.mb_chars.size != orig_size
         return text
     end
 
diff --git a/spec/models/incoming_message_spec.rb b/spec/models/incoming_message_spec.rb
index 3c924dcb3..ff6a8e34e 100644
--- a/spec/models/incoming_message_spec.rb
+++ b/spec/models/incoming_message_spec.rb
@@ -328,7 +328,18 @@ describe IncomingMessage, " when censoring data" do
         data.should == "His email was x\000x\000x\000@\000x\000x\000x\000.\000x\000x\000x\000, indeed"
     end
 
-
+    it 'should handle multibyte characters correctly', :focus => true do
+        orig_data = 'á'
+        data = orig_data.dup
+        @regex_censor_rule = CensorRule.new()
+        @regex_censor_rule.text = 'á'
+        @regex_censor_rule.regexp = true
+        @regex_censor_rule.replacement = 'cat'
+        @regex_censor_rule.last_edit_editor = 'unknown'
+        @regex_censor_rule.last_edit_comment = 'none'
+        @im.info_request.censor_rules << @regex_censor_rule
+        lambda{ @im.binary_mask_stuff!(data, "text/plain") }.should_not raise_error
+    end
 
     def pdf_replacement_test(use_ghostscript_compression)
         config = MySociety::Config.load_default()