Store raw_emails in the filesystem, not in the database. They don't need to be in the database (we never write to them, for example), and they bloat it unecessarily, making backups etc difficult.

NOTE: this migration could take a *very* long time.

2 files changed, 37 insertions, 4 deletions

diff --git a/app/models/info_request.rb b/app/models/info_request.rb
index dcef9e5b5..582e7aab9 100644
--- a/app/models/info_request.rb
+++ b/app/models/info_request.rb
@@ -435,11 +435,11 @@ public
 
         ActiveRecord::Base.transaction do
             raw_email = RawEmail.new
-            raw_email.data = raw_email_data
             incoming_message.raw_email = raw_email
             incoming_message.info_request = self
-            raw_email.save!
             incoming_message.save!
+            raw_email.data = raw_email_data
+            raw_email.save!
 
             self.awaiting_description = true
             self.log_event("response", { :incoming_message_id => incoming_message.id })
diff --git a/app/models/raw_email.rb b/app/models/raw_email.rb
index 0b70d1786..7a57399d5 100644
--- a/app/models/raw_email.rb
+++ b/app/models/raw_email.rb
@@ -6,7 +6,7 @@
 #  id          :integer         not null, primary key
 #  data_text   :text            
 #  data_binary :binary          
-#
+# - prepared to 277k.
 
 # models/raw_email.rb:
 # The fat part of models/incoming_message.rb
@@ -21,17 +21,50 @@ class RawEmail < ActiveRecord::Base
     
     has_one :incoming_message
 
+    before_destroy :destroy_file_representation!
 
     # We keep the old data_text field (which is of type text) for backwards
     # compatibility. We use the new data_binary field because only it works
     # properly in recent versions of PostgreSQL (get seg faults escaping
     # some binary strings).
 
+    def directory
+        request_id = self.incoming_message.info_request.id.to_s
+        File.join(MySociety::Config.get('RAW_EMAILS_LOCATION',
+                                        'files/raw_emails'), 
+                  request_id[0..2], request_id)
+    end
+
+    def filepath
+        File.join(self.directory, self.incoming_message.id.to_s)
+    end
+
     def data=(d)
-        write_attribute(:data_binary, d)
+        if !File.exists?(self.directory)
+            FileUtils.mkdir_p self.directory
+        end
+        File.open(self.filepath, "wb") { |file|
+            file.write d
+        }
     end
 
     def data
+        if !File.exists?(self.filepath)
+            dbdata
+        else
+            File.open(self.filepath, "rb" ).read
+        end
+    end
+
+    def destroy_file_representation!
+        File.delete(self.filepath)
+    end
+
+    def dbdata=(d)
+        write_attribute(:data_binary, d)
+    end
+
+    def dbdata
         d = read_attribute(:data_binary)
         if !d.nil?
             return d