Handle the case where an incoming message is badly encoded and has no charset on the part we're using as a main part.

3 files changed, 49 insertions, 0 deletions

diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb
index 17066a375..fcb4671c5 100644
--- a/app/models/incoming_message.rb
+++ b/app/models/incoming_message.rb
@@ -644,6 +644,7 @@ class IncomingMessage < ActiveRecord::Base
             # Text looks like unlabelled nonsense,
             # strip out anything that isn't UTF-8
             begin
+                source_charset = 'utf-8' if source_charset.nil?
                 text = Iconv.conv('utf-8//IGNORE', source_charset, text) +
                     _("\n\n[ {{site_name}} note: The above text was badly encoded, and has had strange characters removed. ]",
                       :site_name => Configuration::site_name)
diff --git a/spec/fixtures/files/no-part-charset-bad-utf8.email b/spec/fixtures/files/no-part-charset-bad-utf8.email
new file mode 100644
index 000000000..7e67d0063
--- /dev/null
+++ b/spec/fixtures/files/no-part-charset-bad-utf8.email
@@ -0,0 +1,38 @@
+From xxxx@yahoo.cn Mon Oct 08 14:01:34 2012
+Return-path: <xxxx@yahoo.cn>
+Envelope-to: foi@atlas.ukcod.org.uk
+Delivery-date: Mon, 08 Oct 2012 14:01:34 +0100
+Received: (qmail 63864 invoked from network); 8 Oct 2012 13:01:12 -0000
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=yahoo.cn; s=s1024; t=1349701272; bh=T/mtlIYvhB/L5RO+CvTazeAdGf1n1zsGXBoA8EKGT9M=; h=Message-ID:X-Yahoo-Newman-Property:X-YMail-OSG:X-Yahoo-SMTP:Received:X-mailer:From:Subject:To:Content-Transfer-Encoding:Content-Type:Date; b=LYI/PXvA7DA746bmyprChUg7N8YDvN9XE/bhfTt5MW7siOmxHHzn1w+s5X33PvLI0x0UfJLo+MCkTnGPKnG5BYY38US8PkocJYyphrvF/eaUl3ALf8UvxHBOJX1iIi89Xp2NnfbS8lz9kZAWifb9GOnOA5/kLDcL5/WJXliit2k=
+Message-ID: <xxxx@xxxx.yahoo.com>
+X-Yahoo-Newman-Property: ymail-5
+X-YMail-OSG: nPs5jgsVM1myUoKjeEPTxxalz4BM6BZMEUYu.E8NPMPQyo_
+ Yej8T2WCTurn767NOwhuDIqNxC2QGZINqfjmKcdyW7a1P_Zxqr9GsjgxODci
+ ihwr7qYAGDDbcsrB.PX4epnJZHl3yAwoGW.1ReEZnXQANFcNep7.zNEbZ_2k
+ RU1IhI9aHYvxPxt5RWugwOoFRh9P8Ym35A88IMazNtVaBiBEXF6Vk8Aqr9XP
+ 3Vh9xOT9Pn6X8qOUjNXkdb3xB4S5AAIRSE9mqhL1KzHBwdVQs25IoM_2FV2b
+ gPsQGgL4_mwBH0WcEMhdj7Kn6Nfb44L.50E_V3DH.8P7KzDK8zNVXSbAqohX
+ Qi6MzUK2frr8IyZyYzHb.ekff7kAcJgUoHvhnyPar8tRYxhQT3_xsUTzsx8N
+ oWckVPh_i3OT7U4ObgekqgtteMoYqPH2eF1SZXamGBAs-
+X-Yahoo-SMTP: YUQHwRWswBDjbw_M.D6EP4KpT9khlJErDRBQi4ySZQ--
+X-mailer: MIME::Lite 3.027 (F2.74; T1.31; A2.07; B3.13; Q3.13)
+From: =?GB2312?B?zsJKaWFu?= Bing <xxxx@yahoo.cn>
+Subject: =?GB2312?B?yM7A1svJ?=
+To: FOI Person <EMAIL_TO>
+Content-Transfer-Encoding: base64
+Content-Type: text/plain
+Date: Tue, 9 Oct 2012 20:53:06 +0800
+
+DQogICAgICAgICAgufO5q8u+uLrU8MjLKL6twO0vssbO8SnE+rrDo7oNCiAgICAgICAgICAgILG+
+uavLvtTaMTk5N8Tqs8nBorn6vNK5pMnM16Ky4S7KtcGm0Nu68aGj09C2wMGiy7DO8Q0KICAgICAg
+ICAgINeo0rXIy9SxO9TayKu5+rj3s8fK0MnowaK31rmry76jqNXjva2hosnPuqOhornj1t2hor2t
+y9W1yA0KICAgICAgICAgILXYt72jqdLyvfjP7r3PtuDP1s3qs8myu8HLw7/Uws/6ytu27rbIoaPD
+v9TC09DSu7K/t9YNCiAgICAgICAgICDU9ta1tpCjqDYtNyXX89PSKbrNxtXGsaOoMC41JS0yJSDX
+89PSo6nTxbvdtPq/qrvyus/X96OsDQogICAgICAgICAgtePK/b3Ptc2ho7T6wO23ts6nyOfPwqO6
+DQogICAgICAgICAg1PbWtcuwOjEuMTcl16jTw9T21rWjuzI6xtXNqNT21rWjuzM6uqO52MBVv+6V
++CANCiAgICAgICAgICAgNC65+suwzajTw7v6tPI7IDUutdjLsM2o08O7+rTyDQogICAgICAgICAg
+ICAgPT09Pdaj1tiz0MW1PT09PSANCiAgICAgICAgICDGsb7dvvnOqraQhNW+1sHss/a78tPJxvPS
+tdaxvdO/qrP2o6zR6dakuvO4tr/uoaMNCiAgICAgICAgICAgICAgIMGqIMLnIDq7xr6twO0gICAg
+ICAgICAgICAgyMjP3zoxMzgtMjQzNi0wNTE1DQogICAgICAgICAgICDStc7xUVE6OTc4My05Njg5
+OCAgIEUtbWFpbDp3d3dheDg4QDEyNi5jb20NCg==
+
diff --git a/spec/models/incoming_message_spec.rb b/spec/models/incoming_message_spec.rb
index 03a3c8dc3..cf84b4595 100644
--- a/spec/models/incoming_message_spec.rb
+++ b/spec/models/incoming_message_spec.rb
@@ -56,6 +56,15 @@ describe IncomingMessage, " when dealing with incoming mail" do
         message.subject.should == "Câmara Responde:  Banco de ideias"
     end
 
+    it 'should not error on display of a message which has no charset set on the body part and
+        is not good utf-8' do
+        ir = info_requests(:fancy_dog_request)
+        receive_incoming_mail('no-part-charset-bad-utf8.email', ir.incoming_email)
+        puts ir.incoming_messages.inspect
+        message = ir.incoming_messages[1]
+        message.parse_raw_email!
+        message.get_main_body_text_internal.should include("The above text was badly encoded")
+    end
 
     it "should fold multiline sections" do
       {
@@ -77,6 +86,7 @@ describe IncomingMessage, "when parsing HTML mail" do
         plain_text = IncomingMessage._get_attachment_text_internal_one_file('text/html', html)
         plain_text.should match(/është/)
     end
+
 end
 
 describe IncomingMessage, "when getting the attachment text" do