Handle unparsed email contents as binary.

I think I was wrong in a83b379fd2d676172855825d0592937b234371e2 in
assuming that all email gets properly encoded for transfer. Looking
at the mail gem load method
https://github.com/mikel/mail/blob/b159e0a542962fdd5e292a48cfffa560d7cf412e/lib/mail/mail.rb#L175a,
it reads raw email content from a file in binary mode. So this commit
makes both reading and writing the raw_email a binary mode operation
and adds a data_as_text method for displaying the data in the admin
interface that coerces it to valid utf-8.

4 files changed, 62 insertions, 25 deletions

diff --git a/app/models/raw_email.rb b/app/models/raw_email.rb
index 3b466cb81..907d3c7a0 100644
--- a/app/models/raw_email.rb
+++ b/app/models/raw_email.rb
@@ -39,11 +39,26 @@ class RawEmail < ActiveRecord::Base
 
     def data=(d)
         FileUtils.mkdir_p(directory) unless File.exists?(directory)
-        File.atomic_write(filepath) { |file| file.write(d) }
+        File.atomic_write(filepath) do |file|
+            file.binmode
+            file.write(d)
+        end
     end
 
     def data
-        File.open(filepath, "r").read
+        File.open(filepath, "rb").read
+    end
+
+    def data_as_text
+        text = data
+        if text.respond_to?(:encoding)
+            text = text.encode("UTF-8", :invalid => :replace,
+                                        :undef => :replace,
+                                        :replace => "")
+        else
+            text = Iconv.conv('UTF-8//IGNORE', 'UTF-8', text)
+        end
+        text
     end
 
     def destroy_file_representation!
diff --git a/app/views/admin_request/show_raw_email.html.erb b/app/views/admin_request/show_raw_email.html.erb
index da22b6069..045989df6 100644
--- a/app/views/admin_request/show_raw_email.html.erb
+++ b/app/views/admin_request/show_raw_email.html.erb
@@ -59,5 +59,8 @@
 
 <p><%= link_to "Download", admin_request_download_raw_email_path(@raw_email) %></p>
 
-<pre><%=h(@raw_email.data).gsub(/\n/, '<br>').html_safe %></pre>
+<h2>Preview</h2>
+
+For an exact rendering of this email, use the "Download" link.
+<pre><%=h(@raw_email.data_as_text).gsub(/\n/, '<br>').html_safe %></pre>
 
diff --git a/spec/factories/raw_emails.rb b/spec/factories/raw_emails.rb
index 30fb24c37..b271515d2 100644
--- a/spec/factories/raw_emails.rb
+++ b/spec/factories/raw_emails.rb
@@ -1,5 +1,3 @@
 FactoryGirl.define do
-
     factory :raw_email
-
 end
diff --git a/spec/models/raw_email_spec.rb b/spec/models/raw_email_spec.rb
index aa82b0bc3..4f8327a79 100644
--- a/spec/models/raw_email_spec.rb
+++ b/spec/models/raw_email_spec.rb
@@ -1,3 +1,4 @@
+# -*- encoding : utf-8 -*-
 # == Schema Information
 #
 # Table name: raw_emails
@@ -16,28 +17,48 @@ describe User, "manipulating a raw email" do
         @raw_email.stub!(:incoming_message).and_return(incoming_message)
     end
 
-    it 'putting data in comes back out' do 
-        @raw_email.data = "Hello, world!"
-        @raw_email.save!
-        @raw_email.reload
-        @raw_email.data.should == "Hello, world!"
+    def roundtrip_data(raw_email, data)
+        raw_email.data = data
+        raw_email.save!
+        raw_email.reload
+        raw_email.data
     end
 
-    # TODO: this test fails, hopefully will be fixed in later Rails.
-    # Doesn't matter too much for us for storing raw_emails, it would seem,
-    # but keep an eye out.
-
-    # This is testing a bug in Rails PostgreSQL code
-    # http://blog.aradine.com/2009/09/rubys-marshal-and-activerecord-and.html
-    # https://rails.lighthouseapp.com/projects/8994/tickets/1063-binary-data-broken-with-postgresql-adapter
-#    it 'putting data in comes back out even if it has a backslash in it' do 
-#        @raw_email.data = "This \\ that"
-#        @raw_email.save!
-#        @raw_email.reload
-#        $stderr.puts @raw_email.data
-#        $stderr.puts "This \\ that"
-#        @raw_email.data.should == "This \\ that"
-#    end
+    describe :data do
+
+        it 'roundtrips data unchanged' do
+            raw_email = FactoryGirl.create(:incoming_message).raw_email
+            data = roundtrip_data(raw_email, "Hello, world!")
+            data.should == "Hello, world!"
+        end
+
+        it 'returns an unchanged binary string with a valid encoding if the data is non-ascii and non-utf-8' do
+            raw_email = FactoryGirl.create(:incoming_message).raw_email
+            data = roundtrip_data(raw_email, "\xA0")
+
+            if data.respond_to?(:encoding)
+                data.encoding.to_s.should == 'ASCII-8BIT'
+                data.valid_encoding?.should be_true
+                data = data.force_encoding('UTF-8')
+            end
+            data.should == "\xA0"
+        end
+
+    end
+
+    describe :data_as_text do
+
+        it 'returns a utf-8 string with a valid encoding if the data is non-ascii and non-utf8' do
+            raw_email = FactoryGirl.create(:incoming_message).raw_email
+            roundtrip_data(raw_email, "\xA0ccc")
+            data_as_text = raw_email.data_as_text
+            data_as_text.should == "ccc"
+            if data_as_text.respond_to?(:encoding)
+                data_as_text.encoding.to_s.should == 'UTF-8'
+                data_as_text.valid_encoding?.should be_true
+            end
+        end
+    end
 
 end