11 files changed, 253 insertions, 104 deletions
diff --git a/spec/models/has_tag_string_tag_spec.rb b/spec/models/has_tag_string_tag_spec.rb
index ba439acb8..1acd2e27d 100644
--- a/spec/models/has_tag_string_tag_spec.rb
+++ b/spec/models/has_tag_string_tag_spec.rb
@@ -1,7 +1,7 @@
 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
 describe HasTagString::HasTagStringTag, " when fiddling with tag strings " do
-    fixtures :public_bodies, :public_body_translations
+    fixtures :public_bodies, :public_body_translations, :public_body_versions, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     it "should be able to make a new tag and save it" do
         @tag = HasTagString::HasTagStringTag.new 
diff --git a/spec/models/incoming_message_spec.rb b/spec/models/incoming_message_spec.rb
index d6923da21..08d3d89f7 100644
--- a/spec/models/incoming_message_spec.rb
+++ b/spec/models/incoming_message_spec.rb
@@ -1,13 +1,18 @@
+# coding: utf-8
 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
 describe IncomingMessage, " when dealing with incoming mail" do
-    fixtures :incoming_messages, :raw_emails, :info_requests
+    fixtures :users, :raw_emails, :public_bodies, :public_body_translations, :public_body_versions, :info_requests, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     before(:each) do
         @im = incoming_messages(:useless_incoming_message)
         load_raw_emails_data(raw_emails)
     end
 
+    after(:all) do
+        ActionMailer::Base.deliveries.clear
+    end
+
     it "should return the mail Date header date for sent at" do
         @im.sent_at.should == @im.mail.date
     end
@@ -17,6 +22,45 @@ describe IncomingMessage, " when dealing with incoming mail" do
         TMail::Address.parse(em)
     end
 
+    it "should correctly fold various types of footer" do
+        Dir.glob(File.join(Spec::Runner.configuration.fixture_path, "files", "email-folding-example-*.txt")).each do |file|
+            message = File.read(file)
+            parsed = IncomingMessage.remove_quoted_sections(message)
+            expected = File.read("#{file}.expected")
+            parsed.should include(expected)
+        end
+    end
+
+    it "should correctly convert HTML even when there's a meta tag asserting that it is iso-8859-1 which would normally confuse elinks" do
+        ir = info_requests(:fancy_dog_request)
+        receive_incoming_mail('quoted-subject-iso8859-1.email', ir.incoming_email)
+        message = ir.incoming_messages[1]
+        message.parse_raw_email!
+        message.get_main_body_text_part.charset.should == "iso-8859-1"
+        message.get_main_body_text_internal.should include("política")
+    end
+
+    it "should unquote RFC 2047 headers" do
+        ir = info_requests(:fancy_dog_request)
+        receive_incoming_mail('quoted-subject-iso8859-1.email', ir.incoming_email)
+        message = ir.incoming_messages[1]
+        message.mail_from.should == "Coordenação de Relacionamento, Pesquisa e Informação/CEDI"
+        message.subject.should == "Câmara Responde:  Banco de ideias"
+    end
+
+
+    it "should fold multiline sections" do
+      {
+        "foo\n--------\nconfidential"                                       => "foo\nFOLDED_QUOTED_SECTION\n", # basic test
+        "foo\n--------\nbar - confidential"                                 => "foo\nFOLDED_QUOTED_SECTION\n", # allow scorechar inside folded section
+        "foo\n--------\nbar\n--------\nconfidential"                        => "foo\n--------\nbar\nFOLDED_QUOTED_SECTION\n", # don't assume that anything after a score is a folded section
+        "foo\n--------\nbar\n--------\nconfidential\n--------\nrest"        => "foo\n--------\nbar\nFOLDED_QUOTED_SECTION\nrest", # don't assume that a folded section continues to the end of the message
+        "foo\n--------\nbar\n- - - - - - - -\nconfidential\n--------\nrest" => "foo\n--------\nbar\nFOLDED_QUOTED_SECTION\nrest", # allow spaces in the score
+      }.each do |input,output|
+        IncomingMessage.remove_quoted_sections(input).should == output
+      end
+    end
+
 end
 
 describe IncomingMessage, "when parsing HTML mail" do 
@@ -42,7 +86,7 @@ end
 describe IncomingMessage, " display attachments" do
 
     it "should not show slashes in filenames" do
-        foi_attachment = FOIAttachment.new()
+        foi_attachment = FoiAttachment.new()
         # http://www.whatdotheyknow.com/request/post_commercial_manager_librarie#incoming-17233
         foi_attachment.filename = "FOI/09/066 RESPONSE TO FOI REQUEST RECEIVED 21st JANUARY 2009.txt"
         expected_display_filename = foi_attachment.filename.gsub(/\//, " ")
@@ -50,10 +94,11 @@ describe IncomingMessage, " display attachments" do
     end
 
     it "should not show slashes in subject generated filenames" do
-        foi_attachment = FOIAttachment.new()
+        foi_attachment = FoiAttachment.new()
         # http://www.whatdotheyknow.com/request/post_commercial_manager_librarie#incoming-17233
         foi_attachment.within_rfc822_subject = "FOI/09/066 RESPONSE TO FOI REQUEST RECEIVED 21st JANUARY 2009"
         foi_attachment.content_type = 'text/plain'
+        foi_attachment.ensure_filename!
         expected_display_filename = foi_attachment.within_rfc822_subject.gsub(/\//, " ") + ".txt"
         foi_attachment.display_filename.should == expected_display_filename
     end
@@ -79,16 +124,15 @@ describe IncomingMessage, " folding quoted parts of emails" do
 end
 
 describe IncomingMessage, " checking validity to reply to" do
-    def test_email(result, email, return_path, autosubmitted)
+    def test_email(result, email, return_path, autosubmitted = nil)
         @address = mock(TMail::Address)
         @address.stub!(:spec).and_return(email)
 
         @return_path = mock(TMail::ReturnPathHeader)
         @return_path.stub!(:addr).and_return(return_path)
-
-        @autosubmitted = mock(TMail::KeywordsHeader)
-        @autosubmitted.stub!(:keys).and_return(autosubmitted)
-
+        if !autosubmitted.nil?
+            @autosubmitted = TMail::UnstructuredHeader.new("auto-submitted", autosubmitted)
+        end
         @mail = mock(TMail::Mail)
         @mail.stub!(:from_addrs).and_return( [ @address ] )
         @mail.stub!(:[]).with("return-path").and_return(@return_path)
@@ -96,50 +140,49 @@ describe IncomingMessage, " checking validity to reply to" do
 
         @incoming_message = IncomingMessage.new()
         @incoming_message.stub!(:mail).and_return(@mail)
-
-        @incoming_message.valid_to_reply_to?.should == result
+        @incoming_message._calculate_valid_to_reply_to.should == result
     end
 
     it "says a valid email is fine" do
-        test_email(true, "team@mysociety.org", nil, [])
+        test_email(true, "team@mysociety.org", nil)
     end
 
     it "says postmaster email is bad" do
-        test_email(false, "postmaster@mysociety.org", nil, [])
+        test_email(false, "postmaster@mysociety.org", nil)
     end
 
     it "says Mailer-Daemon email is bad" do
-        test_email(false, "Mailer-Daemon@mysociety.org", nil, [])
+        test_email(false, "Mailer-Daemon@mysociety.org", nil)
     end
 
     it "says case mangled MaIler-DaemOn email is bad" do
-        test_email(false, "MaIler-DaemOn@mysociety.org", nil, [])
+        test_email(false, "MaIler-DaemOn@mysociety.org", nil)
     end
 
     it "says Auto_Reply email is bad" do
-        test_email(false, "Auto_Reply@mysociety.org", nil, [])
+        test_email(false, "Auto_Reply@mysociety.org", nil)
     end
 
     it "says DoNotReply email is bad" do
-        test_email(false, "DoNotReply@tube.tfl.gov.uk", nil, [])
+        test_email(false, "DoNotReply@tube.tfl.gov.uk", nil)
     end
 
     it "says a filled-out return-path is fine" do
-        test_email(true, "team@mysociety.org", "Return-path: <foo@baz.com>", [])
+        test_email(true, "team@mysociety.org", "Return-path: <foo@baz.com>")
     end
 
     it "says an empty return-path is bad" do
-        test_email(false, "team@mysociety.org", "<>", [])
+        test_email(false, "team@mysociety.org", "<>")
     end
 
     it "says an auto-submitted keyword is bad" do
-        test_email(false, "team@mysociety.org", nil, ["auto-replied"])
+        test_email(false, "team@mysociety.org", nil, "auto-replied")
     end
 
 end
 
 describe IncomingMessage, " checking validity to reply to with real emails" do
-    fixtures :incoming_messages, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :users
+    fixtures :users, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     after(:all) do
         ActionMailer::Base.deliveries.clear
@@ -163,7 +206,7 @@ describe IncomingMessage, " checking validity to reply to with real emails" do
 end
 
 describe IncomingMessage, " when censoring data" do
-    fixtures :incoming_messages, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :users
+    fixtures :users, :raw_emails, :public_bodies, :public_body_translations, :public_body_versions, :info_requests, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     before(:each) do
         @test_data = "There was a mouse called Stilton, he wished that he was blue."
@@ -262,10 +305,8 @@ describe IncomingMessage, " when censoring data" do
     end
 
     it "should apply censor rules to From: addresses" do
-        mock_mail = mock('Email object')
-        mock_mail.stub!(:from_name_if_present).and_return("Stilton Mouse")
-        @im.stub!(:mail).and_return(mock_mail)
-        
+        @im.stub!(:mail_from).and_return("Stilton Mouse")        
+        @im.stub!(:last_parsed).and_return(Time.now)        
         safe_mail_from = @im.safe_mail_from
         safe_mail_from.should == "Jarlsberg Mouse"
     end
@@ -273,7 +314,7 @@ describe IncomingMessage, " when censoring data" do
 end
 
 describe IncomingMessage, " when censoring whole users" do
-    fixtures :incoming_messages, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :users
+    fixtures :users, :raw_emails, :public_bodies, :public_body_translations, :public_body_versions, :info_requests, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     before(:each) do
         @test_data = "There was a mouse called Stilton, he wished that he was blue."
@@ -304,21 +345,23 @@ end
 
 
 describe IncomingMessage, " when uudecoding bad messages" do
+    fixtures :incoming_messages, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :users, :foi_attachments
+
+    before(:each) do
+        load_raw_emails_data(raw_emails)
+    end
+
     it "should be able to do it at all" do
         mail_body = load_file_fixture('incoming-request-bad-uuencoding.email')
         mail = TMail::Mail.parse(mail_body)
         mail.base64_decode
-
-        im = IncomingMessage.new
+        im = incoming_messages(:useless_incoming_message)
         im.stub!(:mail).and_return(mail)
-        ir = InfoRequest.new
-        im.info_request = ir
-        u = User.new
-        ir.user = u
-
-        attachments = im.get_main_body_text_uudecode_attachments
-        attachments.size.should == 1
-        attachments[0].filename.should == 'moo.txt'
+        im.extract_attachments!
+        attachments = im.foi_attachments
+        attachments.size.should == 2
+        attachments[1].filename.should == 'moo.txt'
+        im.get_attachments_for_display.size.should == 1
     end
 
     it "should apply censor rules" do
@@ -326,12 +369,9 @@ describe IncomingMessage, " when uudecoding bad messages" do
         mail = TMail::Mail.parse(mail_body)
         mail.base64_decode
 
-        im = IncomingMessage.new
+        im = incoming_messages(:useless_incoming_message)
         im.stub!(:mail).and_return(mail)
-        ir = InfoRequest.new
-        im.info_request = ir
-        u = User.new
-        ir.user = u
+        ir = info_requests(:fancy_dog_request)
 
         @censor_rule = CensorRule.new()
         @censor_rule.text = "moo"
@@ -339,26 +379,31 @@ describe IncomingMessage, " when uudecoding bad messages" do
         @censor_rule.last_edit_editor = "unknown"
         @censor_rule.last_edit_comment = "none"
         ir.censor_rules << @censor_rule
+        im.extract_attachments!
 
-        attachments = im.get_main_body_text_uudecode_attachments
+        attachments = im.get_attachments_for_display
         attachments.size.should == 1
-        attachments[0].filename.should == 'bah.txt'
+        attachments[0].display_filename.should == 'bah.txt'
     end
 
 end
 
 describe IncomingMessage, "when messages are attached to messages" do
+    fixtures :incoming_messages, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :users, :foi_attachments
+
+    before(:each) do
+        load_raw_emails_data(raw_emails)
+    end
+
     it "should flatten all the attachments out" do
         mail_body = load_file_fixture('incoming-request-attach-attachments.email')
         mail = TMail::Mail.parse(mail_body)
         mail.base64_decode
 
-        im = IncomingMessage.new
+        im = incoming_messages(:useless_incoming_message)
         im.stub!(:mail).and_return(mail)
-        ir = InfoRequest.new
-        im.info_request = ir
-        u = User.new
-        ir.user = u
+        
+        im.extract_attachments!
 
         attachments = im.get_attachments_for_display
         attachments.size.should == 3
@@ -369,17 +414,20 @@ describe IncomingMessage, "when messages are attached to messages" do
 end
 
 describe IncomingMessage, "when Outlook messages are attached to messages" do
+    fixtures :incoming_messages, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :users, :foi_attachments
+
+    before(:each) do
+        load_raw_emails_data(raw_emails)
+    end
+
     it "should flatten all the attachments out" do
         mail_body = load_file_fixture('incoming-request-oft-attachments.email')
         mail = TMail::Mail.parse(mail_body)
         mail.base64_decode
 
-        im = IncomingMessage.new
+        im = incoming_messages(:useless_incoming_message)
         im.stub!(:mail).and_return(mail)
-        ir = InfoRequest.new
-        im.info_request = ir
-        u = User.new
-        ir.user = u
+        im.extract_attachments!
 
         attachments = im.get_attachments_for_display
         attachments.size.should == 2
@@ -389,17 +437,20 @@ describe IncomingMessage, "when Outlook messages are attached to messages" do
 end
 
 describe IncomingMessage, "when TNEF attachments are attached to messages" do
+    fixtures :incoming_messages, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :users, :foi_attachments
+
+    before(:each) do
+        load_raw_emails_data(raw_emails)
+    end
+
     it "should flatten all the attachments out" do
         mail_body = load_file_fixture('incoming-request-tnef-attachments.email')
         mail = TMail::Mail.parse(mail_body)
         mail.base64_decode
 
-        im = IncomingMessage.new
+        im = incoming_messages(:useless_incoming_message)
         im.stub!(:mail).and_return(mail)
-        ir = InfoRequest.new
-        im.info_request = ir
-        u = User.new
-        ir.user = u
+        im.extract_attachments!
 
         attachments = im.get_attachments_for_display
         attachments.size.should == 2
diff --git a/spec/models/info_request_event_spec.rb b/spec/models/info_request_event_spec.rb
index 666f5cb1a..3229284cc 100644
--- a/spec/models/info_request_event_spec.rb
+++ b/spec/models/info_request_event_spec.rb
@@ -50,5 +50,30 @@ describe InfoRequestEvent do
         end
 
     end
+
+    describe "doing search/index stuff" do 
+        fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
+
+        before(:each) do
+            load_raw_emails_data(raw_emails)
+            parse_all_incoming_messages
+        end
+
+        it 'should get search text for outgoing messages' do 
+            event = info_request_events(:useless_outgoing_message_event)
+            message = outgoing_messages(:useless_outgoing_message).body
+            event.search_text_main.should == message + "\n\n"
+        end
+
+        it 'should get search text for incoming messages' do 
+            event = info_request_events(:useless_incoming_message_event)
+            event.search_text_main.strip.should == "No way! I'm not going to tell you that in a month of Thursdays.\n\nThe Geraldine Quango"
+        end
+
+
+    end
+
+
+
 end
 
diff --git a/spec/models/info_request_spec.rb b/spec/models/info_request_spec.rb
index b82052a0f..b1baa66a2 100644
--- a/spec/models/info_request_spec.rb
+++ b/spec/models/info_request_spec.rb
@@ -3,7 +3,7 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 describe InfoRequest do 
 
     describe "guessing a request from an email" do 
-        fixtures :info_requests, :public_bodies, :incoming_messages, :raw_emails
+        fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
         before(:each) do
             @im = incoming_messages(:useless_incoming_message)
@@ -74,7 +74,7 @@ describe InfoRequest do
     
     describe " when emailing" do
     
-        fixtures :info_requests, :info_request_events, :outgoing_messages, :public_bodies, :public_body_translations, :users, :comments
+        fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
         before do
             @info_request = info_requests(:fancy_dog_request)
@@ -143,8 +143,8 @@ describe InfoRequest do
         end
 
         it "should cope with indexing after item is deleted" do
+            IncomingMessage.find(:all).each{|x| x.parse_raw_email!}
             rebuild_xapian_index
-
             # delete event from underneath indexing; shouldn't cause error
             info_request_events(:useless_incoming_message_event).save!
             info_request_events(:useless_incoming_message_event).destroy
@@ -154,7 +154,7 @@ describe InfoRequest do
     end 
 
     describe "when calculating the status" do
-        fixtures :info_requests, :info_request_events, :holidays, :public_bodies, :public_body_translations, :outgoing_messages
+        fixtures :holidays, :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
         before do
             @ir = info_requests(:naughty_chicken_request)
@@ -196,7 +196,7 @@ describe InfoRequest do
 
     describe "when using a plugin and calculating the status" do
 
-        fixtures :info_requests
+        fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
         before do
             InfoRequest.send(:require, File.expand_path(File.dirname(__FILE__) + '/customstates'))
@@ -231,7 +231,7 @@ describe InfoRequest do
 
 
     describe "when calculating the status for a school" do
-        fixtures :info_requests, :info_request_events, :holidays, :public_bodies, :public_body_translations
+        fixtures :holidays, :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
         before do
             @ir = info_requests(:naughty_chicken_request)
@@ -380,8 +380,8 @@ describe InfoRequest do
         
         before do 
             Time.stub!(:now).and_return(Time.utc(2007, 11, 9, 23, 59))
-            @mock_comment_event = mock_model(InfoRequestEvent, :created_at => Time.now - 23.days, :event_type => 'comment')
-            @mock_response_event = mock_model(InfoRequestEvent, :created_at => Time.now - 22.days, :event_type => 'response')
+            @mock_comment_event = safe_mock_model(InfoRequestEvent, :created_at => Time.now - 23.days, :event_type => 'comment')
+            @mock_response_event = safe_mock_model(InfoRequestEvent, :created_at => Time.now - 22.days, :event_type => 'response')
             @info_request = InfoRequest.new(:prominence => 'normal', 
                                             :awaiting_description => true, 
                                             :info_request_events => [@mock_response_event, @mock_comment_event])
diff --git a/spec/models/outgoing_mailer_spec.rb b/spec/models/outgoing_mailer_spec.rb
index c96a3fb74..75c8053b4 100644
--- a/spec/models/outgoing_mailer_spec.rb
+++ b/spec/models/outgoing_mailer_spec.rb
@@ -4,7 +4,7 @@ describe OutgoingMailer, " when working out follow up addresses" do
     # This is done with fixtures as the code is a bit tangled with the way it
     # calls TMail.  XXX untangle it and make these tests spread out and using
     # mocks. Put parts of the tests in spec/lib/tmail_extensions.rb
-    fixtures :info_requests, :incoming_messages, :raw_emails, :public_bodies, :public_body_translations
+    fixtures :public_bodies, :public_body_translations, :public_body_versions, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
     before(:each) do
         load_raw_emails_data(raw_emails)
     end
@@ -70,7 +70,7 @@ describe OutgoingMailer, " when working out follow up addresses" do
 end
 
 describe OutgoingMailer, "when working out follow up subjects" do
-    fixtures :info_requests, :incoming_messages, :outgoing_messages, :raw_emails
+    fixtures :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     before(:each) do
         load_raw_emails_data(raw_emails)
diff --git a/spec/models/outgoing_message_spec.rb b/spec/models/outgoing_message_spec.rb
index 1956c4d73..58d9f398e 100644
--- a/spec/models/outgoing_message_spec.rb
+++ b/spec/models/outgoing_message_spec.rb
@@ -1,7 +1,7 @@
 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
 describe OutgoingMessage, " when making an outgoing message" do
-    fixtures :outgoing_messages, :info_requests, :incoming_messages, :public_bodies, :public_body_translations
+    fixtures :public_bodies, :public_body_translations, :public_body_versions, :info_requests, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     before do
         @om = outgoing_messages(:useless_outgoing_message)
@@ -38,7 +38,7 @@ end
 
 
 describe IncomingMessage, " when censoring data" do
-    fixtures :outgoing_messages, :info_requests
+    fixtures :info_requests, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     before do
         @om = outgoing_messages(:useless_outgoing_message)
diff --git a/spec/models/public_body_spec.rb b/spec/models/public_body_spec.rb
index 33ab8ffdb..07e8f291d 100644
--- a/spec/models/public_body_spec.rb
+++ b/spec/models/public_body_spec.rb
@@ -95,7 +95,7 @@ describe PublicBody, " using machine tags" do
 end
 
 describe PublicBody, "when finding_by_tags" do
-    fixtures :public_bodies, :public_body_translations
+    fixtures :public_bodies, :public_body_translations, :public_body_versions, :info_requests, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     before do
          @geraldine = public_bodies(:geraldine_public_body)
@@ -173,7 +173,7 @@ describe PublicBody, " when saving" do
 end
 
 describe PublicBody, "when searching" do
-    fixtures :public_bodies, :public_body_translations, :public_body_versions
+    fixtures :public_bodies, :public_body_translations, :public_body_versions, :info_requests, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     it "should find by existing url name" do
         body = PublicBody.find_by_url_name_with_historic('dfh')
@@ -226,6 +226,18 @@ describe PublicBody, "when searching" do
     end
 end
 
+describe PublicBody, " when dealing public body locales" do
+    it "shouldn't fail if it internal_admin_body was created in a locale other than the default" do
+        # first time, do it with the non-default locale
+        PublicBody.with_locale(:es) do
+            PublicBody.internal_admin_body
+        end
+
+        # second time
+        lambda {PublicBody.internal_admin_body }.should_not raise_error(ActiveRecord::RecordInvalid)
+    end
+end
+
 describe PublicBody, " when loading CSV files" do
     before(:each) do
         # InternalBody is created the first time it's accessed, which happens sometimes during imports,
@@ -302,8 +314,8 @@ describe PublicBody, " when loading CSV files" do
         notes.size.should == 4
         notes.should == [
             "line 2: creating new authority 'North West Fake Authority' (locale: en):\n\t\{\"name\":\"North West Fake Authority\",\"request_email\":\"north_west_foi@localhost\",\"home_page\":\"http://northwest.org\"\}", 
-            "line 3: creating new authority 'Scottish Fake Authority' (locale: en):\n\t\{\"tag_string\":\"scottish\",\"name\":\"Scottish Fake Authority\",\"request_email\":\"scottish_foi@localhost\",\"home_page\":\"http://scottish.org\"\}", 
-            "line 4: creating new authority 'Fake Authority of Northern Ireland' (locale: en):\n\t\{\"tag_string\":\"fake aTag\",\"name\":\"Fake Authority of Northern Ireland\",\"request_email\":\"ni_foi@localhost\"\}",
+            "line 3: creating new authority 'Scottish Fake Authority' (locale: en):\n\t\{\"name\":\"Scottish Fake Authority\",\"request_email\":\"scottish_foi@localhost\",\"home_page\":\"http://scottish.org\",\"tag_string\":\"scottish\"\}", 
+            "line 4: creating new authority 'Fake Authority of Northern Ireland' (locale: en):\n\t\{\"name\":\"Fake Authority of Northern Ireland\",\"request_email\":\"ni_foi@localhost\",\"tag_string\":\"fake aTag\"\}",
             "Notes: Some  bodies are in database, but not in CSV file:\n    Department for Humpadinking\n    Geraldine Quango\nYou may want to delete them manually.\n"
             ]
 
@@ -316,7 +328,7 @@ describe PublicBody, " when loading CSV files" do
 
         PublicBody.find_by_name('North West Fake Authority').tag_array_for_search.should == []
         PublicBody.find_by_name('Scottish Fake Authority').tag_array_for_search.should == ['scottish']
-        PublicBody.find_by_name('Fake Authority of Northern Ireland').tag_array_for_search.should == ['fake', 'aTag']
+        PublicBody.find_by_name('Fake Authority of Northern Ireland').tag_array_for_search.should == ['aTag', 'fake']
 
         # Import again to check the 'add' tag functionality works
         new_tags_file = load_file_fixture('fake-authority-add-tags.rb')
@@ -324,8 +336,8 @@ describe PublicBody, " when loading CSV files" do
 
         # Check tags were added successfully
         PublicBody.find_by_name('North West Fake Authority').tag_array_for_search.should == ['aTag']
-        PublicBody.find_by_name('Scottish Fake Authority').tag_array_for_search.should == ['scottish', 'aTag']
-        PublicBody.find_by_name('Fake Authority of Northern Ireland').tag_array_for_search.should == ['fake', 'aTag']
+        PublicBody.find_by_name('Scottish Fake Authority').tag_array_for_search.should == ['aTag', 'scottish']
+        PublicBody.find_by_name('Fake Authority of Northern Ireland').tag_array_for_search.should == ['aTag', 'fake']
     end
 
     it "should import tags successfully when the import tag is set" do
@@ -334,8 +346,8 @@ describe PublicBody, " when loading CSV files" do
 
         # Check new bodies were imported successfully
         PublicBody.find_by_name('North West Fake Authority').tag_array_for_search.should == ['fake']
-        PublicBody.find_by_name('Scottish Fake Authority').tag_array_for_search.should == ['scottish', 'fake']
-        PublicBody.find_by_name('Fake Authority of Northern Ireland').tag_array_for_search.should == ['fake', 'aTag']
+        PublicBody.find_by_name('Scottish Fake Authority').tag_array_for_search.should == ['fake', 'scottish']
+        PublicBody.find_by_name('Fake Authority of Northern Ireland').tag_array_for_search.should == ['aTag', 'fake']
         
         # Import again to check the 'replace' tag functionality works
         new_tags_file = load_file_fixture('fake-authority-add-tags.rb')
@@ -344,7 +356,7 @@ describe PublicBody, " when loading CSV files" do
         # Check tags were added successfully
         PublicBody.find_by_name('North West Fake Authority').tag_array_for_search.should == ['aTag']
         PublicBody.find_by_name('Scottish Fake Authority').tag_array_for_search.should == ['aTag']
-        PublicBody.find_by_name('Fake Authority of Northern Ireland').tag_array_for_search.should == ['fake', 'aTag']
+        PublicBody.find_by_name('Fake Authority of Northern Ireland').tag_array_for_search.should == ['aTag', 'fake']
     end
 
     it "should create bodies with names in multiple locales" do
@@ -357,9 +369,9 @@ describe PublicBody, " when loading CSV files" do
         notes.should == [
             "line 2: creating new authority 'North West Fake Authority' (locale: en):\n\t{\"name\":\"North West Fake Authority\",\"request_email\":\"north_west_foi@localhost\",\"home_page\":\"http://northwest.org\"}", 
             "line 2: creating new authority 'North West Fake Authority' (locale: es):\n\t{\"name\":\"Autoridad del Nordeste\"}", 
-            "line 3: creating new authority 'Scottish Fake Authority' (locale: en):\n\t{\"tag_string\":\"scottish\",\"name\":\"Scottish Fake Authority\",\"request_email\":\"scottish_foi@localhost\",\"home_page\":\"http://scottish.org\"}",
+            "line 3: creating new authority 'Scottish Fake Authority' (locale: en):\n\t{\"name\":\"Scottish Fake Authority\",\"request_email\":\"scottish_foi@localhost\",\"home_page\":\"http://scottish.org\",\"tag_string\":\"scottish\"}",
             "line 3: creating new authority 'Scottish Fake Authority' (locale: es):\n\t{\"name\":\"Autoridad Escocesa\"}",
-            "line 4: creating new authority 'Fake Authority of Northern Ireland' (locale: en):\n\t{\"tag_string\":\"fake aTag\",\"name\":\"Fake Authority of Northern Ireland\",\"request_email\":\"ni_foi@localhost\"}",
+            "line 4: creating new authority 'Fake Authority of Northern Ireland' (locale: en):\n\t{\"name\":\"Fake Authority of Northern Ireland\",\"request_email\":\"ni_foi@localhost\",\"tag_string\":\"fake aTag\"}",
             "line 4: creating new authority 'Fake Authority of Northern Ireland' (locale: es):\n\t{\"name\":\"Autoridad Irlandesa\"}",
             "Notes: Some  bodies are in database, but not in CSV file:\n    Department for Humpadinking\n    Geraldine Quango\nYou may want to delete them manually.\n"
             ]
@@ -385,8 +397,8 @@ describe PublicBody, " when loading CSV files" do
         notes.size.should == 4
         notes.should == [
             "line 2: creating new authority 'North West Fake Authority' (locale: en):\n\t{\"name\":\"North West Fake Authority\",\"request_email\":\"north_west_foi@localhost\",\"home_page\":\"http://northwest.org\"}",
-            "line 3: creating new authority 'Scottish Fake Authority' (locale: en):\n\t{\"tag_string\":\"scottish\",\"name\":\"Scottish Fake Authority\",\"request_email\":\"scottish_foi@localhost\",\"home_page\":\"http://scottish.org\"}", 
-            "line 4: creating new authority 'Fake Authority of Northern Ireland' (locale: en):\n\t{\"tag_string\":\"fake aTag\",\"name\":\"Fake Authority of Northern Ireland\",\"request_email\":\"ni_foi@localhost\"}",
+            "line 3: creating new authority 'Scottish Fake Authority' (locale: en):\n\t{\"name\":\"Scottish Fake Authority\",\"request_email\":\"scottish_foi@localhost\",\"home_page\":\"http://scottish.org\",\"tag_string\":\"scottish\"}", 
+            "line 4: creating new authority 'Fake Authority of Northern Ireland' (locale: en):\n\t{\"name\":\"Fake Authority of Northern Ireland\",\"request_email\":\"ni_foi@localhost\",\"tag_string\":\"fake aTag\"}",
             "Notes: Some  bodies are in database, but not in CSV file:\n    Department for Humpadinking\n    Geraldine Quango\nYou may want to delete them manually.\n"
             ]
 
diff --git a/spec/models/request_mailer_spec.rb b/spec/models/request_mailer_spec.rb
index fbe22c220..ef4ed8074 100644
--- a/spec/models/request_mailer_spec.rb
+++ b/spec/models/request_mailer_spec.rb
@@ -1,7 +1,7 @@
 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
 describe RequestMailer, " when receiving incoming mail" do
-    fixtures :info_requests, :incoming_messages, :raw_emails, :users, :public_bodies, :public_body_translations
+    fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
     before(:each) do
         load_raw_emails_data(raw_emails)
     end
diff --git a/spec/models/track_thing_spec.rb b/spec/models/track_thing_spec.rb
index 1a0324a78..4922a96c7 100644
--- a/spec/models/track_thing_spec.rb
+++ b/spec/models/track_thing_spec.rb
@@ -1,7 +1,7 @@
 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
 describe TrackThing, "when tracking changes" do
-    fixtures :track_things, :users
+    fixtures :users, :info_requests, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     before do
         @track_thing = track_things(:track_fancy_dog_search)
diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb
index 751a61060..e0a6c649e 100644
--- a/spec/models/user_spec.rb
+++ b/spec/models/user_spec.rb
@@ -152,10 +152,10 @@ end
 describe User, "when reindexing referencing models" do 
 
     before do 
-        @request_event = mock_model(InfoRequestEvent, :xapian_mark_needs_index => true)
-        @request = mock_model(InfoRequest, :info_request_events => [@request_event])
-        @comment_event = mock_model(InfoRequestEvent, :xapian_mark_needs_index => true)
-        @comment = mock_model(Comment, :info_request_events => [@comment_event])
+        @request_event = safe_mock_model(InfoRequestEvent, :xapian_mark_needs_index => true)
+        @request = safe_mock_model(InfoRequest, :info_request_events => [@request_event])
+        @comment_event = safe_mock_model(InfoRequestEvent, :xapian_mark_needs_index => true)
+        @comment = safe_mock_model(Comment, :info_request_events => [@comment_event])
         @user = User.new(:comments => [@comment], :info_requests => [@request])
     end
     
@@ -193,7 +193,7 @@ describe User, "when reindexing referencing models" do
 end
 
 describe User, "when checking abilities" do
-    fixtures :users
+    fixtures :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     before do
         @user = users(:bob_smith_user)
@@ -283,7 +283,7 @@ describe User, "when setting a profile photo" do
 end
 
 describe User, "when unconfirmed" do
-    fixtures :users
+    fixtures :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     before do
         @user = users(:unconfirmed_user)
@@ -295,7 +295,7 @@ describe User, "when unconfirmed" do
 end
 
 describe User, "when emails have bounced" do
-    fixtures :users
+    fixtures :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     it "should record bounces" do
         User.record_bounce_for_email("bob@localhost", "The reason we think the email bounced (e.g. a bounce message)")
diff --git a/spec/models/xapian_spec.rb b/spec/models/xapian_spec.rb
index 0c6fa6bb6..ec11c944b 100644
--- a/spec/models/xapian_spec.rb
+++ b/spec/models/xapian_spec.rb
@@ -1,9 +1,10 @@
 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
 describe User, " when indexing users with Xapian" do
-    fixtures :users
+    fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     it "should search by name" do
+        parse_all_incoming_messages
         rebuild_xapian_index
           # def InfoRequest.full_search(models, query, order, ascending, collapse, per_page, page)
         xapian_object = InfoRequest.full_search([User], "Silly", 'created_at', true, nil, 100, 1)
@@ -12,10 +13,10 @@ describe User, " when indexing users with Xapian" do
     end
 
     it "should search by 'about me' text" do
+        rebuild_xapian_index
         user = users(:bob_smith_user)
 
-        rebuild_xapian_index
-          # def InfoRequest.full_search(models, query, order, ascending, collapse, per_page, page)
+        # def InfoRequest.full_search(models, query, order, ascending, collapse, per_page, page)
         xapian_object = InfoRequest.full_search([User], "stuff", 'created_at', true, nil, 100, 1)
         xapian_object.results.size.should == 1
         xapian_object.results[0][:model].should == user
@@ -34,7 +35,7 @@ describe User, " when indexing users with Xapian" do
 end
 
 describe PublicBody, " when indexing public bodies with Xapian" do
-    fixtures :public_bodies, :public_body_translations, :incoming_messages, :outgoing_messages, :raw_emails, :comments, :info_requests
+    fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
     before(:each) do
         load_raw_emails_data(raw_emails)
     end
@@ -72,7 +73,7 @@ describe PublicBody, " when indexing public bodies with Xapian" do
 end
 
 describe PublicBody, " when indexing requests by body they are to" do
-    fixtures :public_bodies, :public_body_translations, :info_request_events, :info_requests, :raw_emails, :comments
+    fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
 
     before(:each) do
         load_raw_emails_data(raw_emails)
@@ -132,7 +133,7 @@ describe PublicBody, " when indexing requests by body they are to" do
 end
 
 describe User, " when indexing requests by user they are from" do
-    fixtures :users, :info_request_events, :info_requests, :incoming_messages, :outgoing_messages, :raw_emails, :comments
+    fixtures :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
     before(:each) do
         load_raw_emails_data(raw_emails)
     end
@@ -219,7 +220,7 @@ describe User, " when indexing requests by user they are from" do
 end
 
 describe User, " when indexing comments by user they are by" do
-    fixtures :users, :info_request_events, :info_requests, :comments, :incoming_messages, :outgoing_messages, :raw_emails, :comments
+    fixtures :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
     before(:each) do
         load_raw_emails_data(raw_emails)
     end
@@ -256,7 +257,7 @@ describe User, " when indexing comments by user they are by" do
 end
 
 describe InfoRequest, " when indexing requests by their title" do
-    fixtures :info_request_events, :info_requests, :incoming_messages, :raw_emails, :comments
+    fixtures :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
     before(:each) do
         load_raw_emails_data(raw_emails)
     end
@@ -287,7 +288,7 @@ describe InfoRequest, " when indexing requests by their title" do
 end
 
 describe InfoRequest, " when indexing requests by tag" do
-    fixtures :info_request_events, :info_requests, :incoming_messages, :raw_emails, :comments
+    fixtures :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
     before(:each) do
         load_raw_emails_data(raw_emails)
     end
@@ -309,7 +310,7 @@ describe InfoRequest, " when indexing requests by tag" do
 end
 
 describe PublicBody, " when indexing authorities by tag" do
-    fixtures :public_bodies, :public_body_translations, :incoming_messages, :outgoing_messages, :raw_emails, :comments
+    fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
     before(:each) do
         load_raw_emails_data(raw_emails)
     end
@@ -333,6 +334,66 @@ describe PublicBody, " when indexing authorities by tag" do
     end
 end
 
+describe PublicBody, " when only indexing selected things on a rebuild" do
+    fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things
+    before(:each) do
+        load_raw_emails_data(raw_emails)
+    end
+
+    it "should only index what we ask it to" do
+        rebuild_xapian_index
+        body = public_bodies(:geraldine_public_body)
+        body.tag_string = 'mice:3'
+        body.name = 'frobzn'
+        body.save!
+        # only reindex 'variety' term
+        dropfirst = true
+        terms = "V"
+        values = false
+        texts = false
+        rebuild_xapian_index(terms, values, texts, dropfirst)
+        xapian_object = InfoRequest.full_search([PublicBody], "tag:mice", 'created_at', true, nil, 100, 1)
+        xapian_object.results.size.should == 0
+        xapian_object = InfoRequest.full_search([PublicBody], "frobzn", 'created_at', true, nil, 100, 1)
+        xapian_object.results.size.should == 0
+        xapian_object = InfoRequest.full_search([PublicBody], "variety:authority", 'created_at', true, nil, 100, 1)
+        xapian_object.results.size.should == 2
+        # only reindex 'tag' and text
+        dropfirst = true
+        terms = "U"
+        values = false
+        texts = true
+        rebuild_xapian_index(terms, values, texts, dropfirst)
+        xapian_object = InfoRequest.full_search([PublicBody], "tag:mice", 'created_at', true, nil, 100, 1)
+        xapian_object.results.size.should == 1
+        xapian_object = InfoRequest.full_search([PublicBody], "frobzn", 'created_at', true, nil, 100, 1)
+        xapian_object.results.size.should == 1
+        xapian_object = InfoRequest.full_search([PublicBody], "variety:authority", 'created_at', true, nil, 100, 1)
+        xapian_object.results.size.should == 0
+        # only reindex 'variety' term, but keeping the existing data in-place
+        dropfirst = false
+        terms = "V"
+        texts = false
+        rebuild_xapian_index(terms, values, texts, dropfirst)
+        xapian_object = InfoRequest.full_search([PublicBody], "tag:mice", 'created_at', true, nil, 100, 1)
+        xapian_object.results.size.should == 1
+        xapian_object = InfoRequest.full_search([PublicBody], "frobzn", 'created_at', true, nil, 100, 1)
+        xapian_object.results.size.should == 1
+        xapian_object = InfoRequest.full_search([PublicBody], "variety:authority", 'created_at', true, nil, 100, 1)
+        xapian_object.results.size.should == 2
+        # only reindex 'variety' term, blowing away existing data
+        dropfirst = true
+        rebuild_xapian_index(terms, values, texts, dropfirst)
+        xapian_object = InfoRequest.full_search([PublicBody], "tag:mice", 'created_at', true, nil, 100, 1)
+        xapian_object.results.size.should == 0
+        xapian_object = InfoRequest.full_search([PublicBody], "frobzn", 'created_at', true, nil, 100, 1)
+        xapian_object.results.size.should == 0
+        xapian_object = InfoRequest.full_search([PublicBody], "variety:authority", 'created_at', true, nil, 100, 1)
+        xapian_object.results.size.should == 2
+    end
+end
+
+