Load all raw emails for testing

Previously there was just one raw email, and the test code relied
on that fact. Generalise it to handle multiple raw emails.

This change causes a number of tests to fail, because it exposes
failures that should have happened when the second raw email was
added but were masked by the fact that the text of this second
raw email was never loaded. These failures will be fixed in the
next commit.

21 files changed, 81 insertions, 64 deletions

diff --git a/script/load-sample-data b/script/load-sample-data
index 84b8a28eb..92846ce17 100755
--- a/script/load-sample-data
+++ b/script/load-sample-data
@@ -4,17 +4,16 @@
 # the fact that the fixtures aren't aware of the fact that RawEmails
 # have a filesystem representation of their contents
 
-LOC=`dirname $0`
+LOC=`dirname "$0"`
+
 rake --silent spec:db:fixtures:load
+
 "$LOC/runner" /dev/stdin <<END
 env = ENV["RAILS_ENV"]
-require "spec/spec_helper.rb" # Sets RAILS_ENV to 'test'
-ENV["RAILS_ENV"] = env
+require "spec/spec_helper.rb" # this sets RAILS_ENV to 'test'
+ENV["RAILS_ENV"] = env        # so restore to what it was before
 
-RawEmail.all().each do |email|
-    puts "Writing #{email.filepath}"
-    email.data = load_file_fixture("useless_raw_email.email")
-end
+load_raw_emails_data
 END
 
 echo "Loaded fixtures."
diff --git a/spec/controllers/admin_request_controller_spec.rb b/spec/controllers/admin_request_controller_spec.rb
index 6d3c955bb..5e6f6c2a5 100644
--- a/spec/controllers/admin_request_controller_spec.rb
+++ b/spec/controllers/admin_request_controller_spec.rb
@@ -6,7 +6,7 @@ describe AdminRequestController, "when administering requests" do
     before { basic_auth_login @request }
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
         @old_filters = ActionController::Routing::Routes.filters
         ActionController::Routing::Routes.filters = RoutingFilter::Chain.new
     end
@@ -53,7 +53,7 @@ describe AdminRequestController, "when administering the holding pen" do
     fixtures :users, :public_bodies, :public_body_translations, :public_body_versions, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
     before(:each) do
         basic_auth_login @request
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
         @old_filters = ActionController::Routing::Routes.filters
         ActionController::Routing::Routes.filters = RoutingFilter::Chain.new
     end
diff --git a/spec/controllers/general_controller_spec.rb b/spec/controllers/general_controller_spec.rb
index b067707de..8eb445146 100644
--- a/spec/controllers/general_controller_spec.rb
+++ b/spec/controllers/general_controller_spec.rb
@@ -34,7 +34,7 @@ describe GeneralController, "when searching" do
     ]
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
         rebuild_xapian_index
     end
 
@@ -111,7 +111,7 @@ describe GeneralController, "when searching" do
     
     describe 'when constructing the list of recent requests' do
         before(:each) do
-          load_raw_emails_data(raw_emails)
+          load_raw_emails_data
           rebuild_xapian_index
         end
 
@@ -127,7 +127,7 @@ describe GeneralController, "when searching" do
 
       # rebuild xapian index after fixtures loaded
       before(:each) do
-          load_raw_emails_data(raw_emails)
+          load_raw_emails_data
           rebuild_xapian_index
       end
 
diff --git a/spec/controllers/public_body_controller_spec.rb b/spec/controllers/public_body_controller_spec.rb
index bd4e625ca..bf9888685 100644
--- a/spec/controllers/public_body_controller_spec.rb
+++ b/spec/controllers/public_body_controller_spec.rb
@@ -7,7 +7,7 @@ describe PublicBodyController, "when showing a body" do
     fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
         rebuild_xapian_index
     end
 
diff --git a/spec/controllers/request_controller_spec.rb b/spec/controllers/request_controller_spec.rb
index bae1f32e7..e43d8ea3d 100644
--- a/spec/controllers/request_controller_spec.rb
+++ b/spec/controllers/request_controller_spec.rb
@@ -7,7 +7,7 @@ describe RequestController, "when listing recent requests" do
     fixtures :users, :public_bodies, :public_body_translations, :public_body_versions, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
         rebuild_xapian_index
     end
     
@@ -125,7 +125,7 @@ describe RequestController, "when showing one request" do
     fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things # all needed as integrating views
     
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     it "should be successful" do
@@ -441,7 +441,7 @@ describe RequestController, "when changing prominence of a request" do
     fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :info_request_events, :track_things # all needed as integrating views
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     it "should not show hidden requests" do
@@ -755,7 +755,7 @@ describe RequestController, "when viewing an individual response for reply/follo
     fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things # all needed as integrating views
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     it "should ask for login if you are logged in as wrong person" do
@@ -805,7 +805,7 @@ describe RequestController, "when classifying an information request" do
         @dog_request = info_requests(:fancy_dog_request)
         @dog_request.stub!(:is_old_unclassified?).and_return(false)
         InfoRequest.stub!(:find).and_return(@dog_request)
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     def post_status(status)
@@ -1142,7 +1142,7 @@ describe RequestController, "when sending a followup message" do
     fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things # all needed as integrating views
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     it "should require login" do
@@ -1225,7 +1225,7 @@ describe RequestController, "sending overdue request alerts" do
     fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things # all needed as integrating views
     
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     it "should send an overdue alert mail to creators of overdue requests" do
@@ -1313,7 +1313,7 @@ describe RequestController, "sending unclassified new response reminder alerts"
     fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things # all needed as integrating views
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     it "should send an alert" do
@@ -1343,7 +1343,7 @@ describe RequestController, "clarification required alerts" do
     integrate_views
     fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things # all needed as integrating views
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     it "should send an alert" do
@@ -1397,7 +1397,7 @@ describe RequestController, "comment alerts" do
     integrate_views
     fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things # all needed as integrating views
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
  
     it "should send an alert (once and once only)" do
@@ -1472,7 +1472,7 @@ describe RequestController, "when viewing comments" do
     integrate_views
     fixtures :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     it "should link to the user who submitted it" do
@@ -1584,7 +1584,7 @@ describe RequestController, "when showing JSON version for API" do
     fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things 
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     it "should return data in JSON form" do
diff --git a/spec/controllers/request_game_controller_spec.rb b/spec/controllers/request_game_controller_spec.rb
index cc0808ef3..1383554a1 100644
--- a/spec/controllers/request_game_controller_spec.rb
+++ b/spec/controllers/request_game_controller_spec.rb
@@ -4,7 +4,7 @@ describe RequestGameController, "when playing the game" do
 
     fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things # all needed as integrating views
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     it "should show the game homepage" do
diff --git a/spec/controllers/track_controller_spec.rb b/spec/controllers/track_controller_spec.rb
index ad4d651cb..509aa6725 100644
--- a/spec/controllers/track_controller_spec.rb
+++ b/spec/controllers/track_controller_spec.rb
@@ -40,7 +40,7 @@ describe TrackController, "when sending alerts for a track" do
     include LinkToHelper # for main_url
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
         rebuild_xapian_index
     end
     
@@ -118,7 +118,7 @@ describe TrackController, "when viewing RSS feed for a track" do
     fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
         rebuild_xapian_index
     end
 
@@ -144,7 +144,7 @@ describe TrackController, "when viewing JSON version of a track feed" do
     fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
         rebuild_xapian_index
     end
 
diff --git a/spec/controllers/user_controller_spec.rb b/spec/controllers/user_controller_spec.rb
index 81d3ff2e5..101fb716e 100644
--- a/spec/controllers/user_controller_spec.rb
+++ b/spec/controllers/user_controller_spec.rb
@@ -10,7 +10,7 @@ describe UserController, "when showing a user" do
     integrate_views
     fixtures :users, :public_bodies, :public_body_translations, :public_body_versions, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
         rebuild_xapian_index
     end
    
diff --git a/spec/fixtures/files/useless_raw_email.email b/spec/fixtures/files/raw_emails/1.email
index 2e4585af7..2e4585af7 100644
--- a/spec/fixtures/files/useless_raw_email.email
+++ b/spec/fixtures/files/raw_emails/1.email
diff --git a/spec/fixtures/files/raw_emails/2.email b/spec/fixtures/files/raw_emails/2.email
new file mode 100644
index 000000000..eab0b0f8d
--- /dev/null
+++ b/spec/fixtures/files/raw_emails/2.email
@@ -0,0 +1,20 @@
+From: "FOI Person" <foiperson@localhost>
+To: "Bob Smith" <bob@localhost>
+Date: Tue, 13 Nov 2007 11:39:55 +0000
+Bcc: 
+Subject: Re: Your email
+Reply-To: 
+In-Reply-To: <471f1eae5d1cb_7347..fdbe67386164@cat.tmail>
+Content-Type: text/plain; charset=utf-8
+
+Dear “Bob”,
+
+In the financial year 2010–2011, this Department spent a
+total of nine hundred of your earth pounds on the purchase
+and repair of boring equipment.
+
+Yours most sincerely,
+
+Quentin Nobble-Boston,
+Permanent Under-Secretary,
+Department for Humpadinking
diff --git a/spec/integration/errors_spec.rb b/spec/integration/errors_spec.rb
index d03323445..ea9caaf12 100644
--- a/spec/integration/errors_spec.rb
+++ b/spec/integration/errors_spec.rb
@@ -17,7 +17,7 @@ describe "When rendering errors" do
     ]
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
         ActionController::Base.consider_all_requests_local = false
     end
 
diff --git a/spec/integration/search_request_spec.rb b/spec/integration/search_request_spec.rb
index d0a457923..317a3cab7 100644
--- a/spec/integration/search_request_spec.rb
+++ b/spec/integration/search_request_spec.rb
@@ -17,8 +17,7 @@ describe "When searching" do
     ]
 
     before(:each) do
-        emails = raw_emails.clone
-        load_raw_emails_data(emails)
+        load_raw_emails_data
         rebuild_xapian_index
     end
 
diff --git a/spec/integration/view_request_spec.rb b/spec/integration/view_request_spec.rb
index cf1e4ca6c..0787644ff 100644
--- a/spec/integration/view_request_spec.rb
+++ b/spec/integration/view_request_spec.rb
@@ -17,8 +17,7 @@ describe "When viewing requests" do
     ]
 
     before(:each) do
-        emails = raw_emails.clone
-        load_raw_emails_data(emails)
+        load_raw_emails_data
     end
 
     it "should not make endlessly recursive JSON <link>s" do
diff --git a/spec/models/foi_attachment_spec.rb b/spec/models/foi_attachment_spec.rb
index 05c4fc5fd..0d122aa1b 100644
--- a/spec/models/foi_attachment_spec.rb
+++ b/spec/models/foi_attachment_spec.rb
@@ -4,7 +4,7 @@ describe FoiAttachment, " when calculating due date" do
     fixtures :incoming_messages, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :users, :foi_attachments
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     it "sets the body" do
diff --git a/spec/models/incoming_message_spec.rb b/spec/models/incoming_message_spec.rb
index 7808ef24c..c096e61e0 100644
--- a/spec/models/incoming_message_spec.rb
+++ b/spec/models/incoming_message_spec.rb
@@ -6,7 +6,7 @@ describe IncomingMessage, " when dealing with incoming mail" do
 
     before(:each) do
         @im = incoming_messages(:useless_incoming_message)
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     after(:all) do
@@ -236,7 +236,7 @@ describe IncomingMessage, " when censoring data" do
         @censor_rule_2.last_edit_comment = "none"
         @im.info_request.censor_rules << @censor_rule_2
 
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     it "should do nothing to a JPEG" do
@@ -336,7 +336,7 @@ describe IncomingMessage, " when censoring whole users" do
         @censor_rule_1.last_edit_editor = "unknown"
         @censor_rule_1.last_edit_comment = "none"
         @im.info_request.user.censor_rules << @censor_rule_1
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     it "should apply censor rules to HTML files" do
@@ -357,7 +357,7 @@ describe IncomingMessage, " when uudecoding bad messages" do
     fixtures :incoming_messages, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :users, :foi_attachments
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     it "should be able to do it at all" do
@@ -401,7 +401,7 @@ describe IncomingMessage, "when messages are attached to messages" do
     fixtures :incoming_messages, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :users, :foi_attachments
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     it "should flatten all the attachments out" do
@@ -426,7 +426,7 @@ describe IncomingMessage, "when Outlook messages are attached to messages" do
     fixtures :incoming_messages, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :users, :foi_attachments
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     it "should flatten all the attachments out" do
@@ -449,7 +449,7 @@ describe IncomingMessage, "when TNEF attachments are attached to messages" do
     fixtures :incoming_messages, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :users, :foi_attachments
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     it "should flatten all the attachments out" do
diff --git a/spec/models/info_request_event_spec.rb b/spec/models/info_request_event_spec.rb
index a75f4b232..9a340c125 100644
--- a/spec/models/info_request_event_spec.rb
+++ b/spec/models/info_request_event_spec.rb
@@ -58,7 +58,7 @@ describe InfoRequestEvent do
         fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
         before(:each) do
-            load_raw_emails_data(raw_emails)
+            load_raw_emails_data
             parse_all_incoming_messages
         end
 
diff --git a/spec/models/info_request_spec.rb b/spec/models/info_request_spec.rb
index b1baa66a2..3c8860285 100644
--- a/spec/models/info_request_spec.rb
+++ b/spec/models/info_request_spec.rb
@@ -7,7 +7,7 @@ describe InfoRequest do
 
         before(:each) do
             @im = incoming_messages(:useless_incoming_message)
-            load_raw_emails_data(raw_emails)
+            load_raw_emails_data
         end
 
         it 'should compute a hash' do
diff --git a/spec/models/outgoing_mailer_spec.rb b/spec/models/outgoing_mailer_spec.rb
index c5fde93fc..9b8fb5f98 100644
--- a/spec/models/outgoing_mailer_spec.rb
+++ b/spec/models/outgoing_mailer_spec.rb
@@ -6,7 +6,7 @@ describe OutgoingMailer, " when working out follow up addresses" do
     # mocks. Put parts of the tests in spec/lib/tmail_extensions.rb
     fixtures :public_bodies, :public_body_translations, :public_body_versions, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     it "should parse them right" do
@@ -77,7 +77,7 @@ describe OutgoingMailer, "when working out follow up subjects" do
     fixtures :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
     end
 
     it "should prefix the title with 'Freedom of Information request -' for initial requests" do
diff --git a/spec/models/request_mailer_spec.rb b/spec/models/request_mailer_spec.rb
index 2888213d7..0dd0b749a 100644
--- a/spec/models/request_mailer_spec.rb
+++ b/spec/models/request_mailer_spec.rb
@@ -3,7 +3,7 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 describe RequestMailer, " when receiving incoming mail" do
     fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
         ActionMailer::Base.deliveries = []
     end
 
diff --git a/spec/models/xapian_spec.rb b/spec/models/xapian_spec.rb
index 6f4f49373..ba06f3eac 100644
--- a/spec/models/xapian_spec.rb
+++ b/spec/models/xapian_spec.rb
@@ -4,7 +4,7 @@ describe User, " when indexing users with Xapian" do
     fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
         rebuild_xapian_index
     end
     
@@ -39,7 +39,7 @@ end
 describe PublicBody, " when indexing public bodies with Xapian" do
     fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
         rebuild_xapian_index
     end
 
@@ -73,7 +73,7 @@ describe PublicBody, " when indexing requests by body they are to" do
     fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
         rebuild_xapian_index
     end
 
@@ -129,7 +129,7 @@ end
 describe User, " when indexing requests by user they are from" do
     fixtures :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things, :public_bodies, :public_body_versions, :public_body_translations
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
         rebuild_xapian_index
     end
 
@@ -208,7 +208,7 @@ end
 describe User, " when indexing comments by user they are by" do
     fixtures :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
         rebuild_xapian_index
     end
 
@@ -244,7 +244,7 @@ end
 describe InfoRequest, " when indexing requests by their title" do
     fixtures :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
         rebuild_xapian_index
     end
 
@@ -274,7 +274,7 @@ end
 describe InfoRequest, " when indexing requests by tag" do
     fixtures :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
         rebuild_xapian_index
     end
 
@@ -296,7 +296,7 @@ end
 describe PublicBody, " when indexing authorities by tag" do
     fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
         rebuild_xapian_index
     end
 
@@ -321,7 +321,7 @@ end
 describe PublicBody, " when only indexing selected things on a rebuild" do
     fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
     before(:each) do
-        load_raw_emails_data(raw_emails)
+        load_raw_emails_data
         rebuild_xapian_index
     end
 
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 33a28449e..065d9d080 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -107,7 +107,7 @@ def validate_html(html)
     File.open(tempfilename, "w+") do |f|
         f.puts html
     end
-    if not system($html_validation_script, tempfilename)
+    if not system($html_validation_script, *($html_validation_script_options +[tempfilename]))
         raise "HTML validation error in " + tempfilename + " HTTP status: " + @response.response_code.to_s
     end
     File.unlink(tempfilename)
@@ -131,6 +131,7 @@ utility_search_path = MySociety::Config.get("UTILITY_SEARCH_PATH", ["/usr/bin",
 $html_validation_script_found = false
 utility_search_path.each do |d|
     $html_validation_script = File.join(d, "validate")
+    $html_validation_script_options = ["--charset=utf-8"]
     if File.file? $html_validation_script and File.executable? $html_validation_script
         $html_validation_script_found = true
         break
@@ -178,13 +179,12 @@ def safe_mock_model(model, args = {})
   mock
 end
 
-def load_raw_emails_data(raw_emails)
-    raw_email = raw_emails(:useless_raw_email)
-    begin
-        raw_email.destroy_file_representation!
-    rescue Errno::ENOENT
+def load_raw_emails_data
+    raw_emails_yml = File.join(Spec::Runner.configuration.fixture_path, "raw_emails.yml")
+    for raw_email_id in YAML::load_file(raw_emails_yml).map{|k,v| v["id"]} do
+        raw_email = RawEmail.find(raw_email_id)
+        raw_email.data = load_file_fixture("raw_emails/%d.email" % [raw_email_id])
     end
-    raw_email.data = load_file_fixture("useless_raw_email.email")
 end
 
 def parse_all_incoming_messages