diff options
38 files changed, 410 insertions, 127 deletions
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 5a295d3e7..1849f23f3 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -354,9 +354,7 @@ class ApplicationController < ActionController::Base @sortby = sortby # Work out sorting method - order_pair = order_to_sort_by(@sortby) - order = order_pair[0] - ascending = order_pair[1] + order, ascending = order_to_sort_by(@sortby) # Peform the search @per_page = per_page diff --git a/app/controllers/general_controller.rb b/app/controllers/general_controller.rb index 8ac41b05a..82b1b8629 100644 --- a/app/controllers/general_controller.rb +++ b/app/controllers/general_controller.rb @@ -45,20 +45,21 @@ class GeneralController < ApplicationController :joins => :translations) end end - # Get some successful requests # + # Get some successful requests begin query = 'variety:response (status:successful OR status:partially_successful)' - # query = 'variety:response' # XXX debug - sortby = "described" + sortby = "newest" max_count = 5 xapian_object = perform_search([InfoRequestEvent], query, sortby, 'request_title_collapse', max_count) @request_events = xapian_object.results.map { |r| r[:model] } - @request_events = @request_events.sort_by { |e| e.described_at }.reverse + + # If there are not yet enough successful requests, fill out the list with + # other requests if @request_events.count < max_count query = 'variety:sent' xapian_object = perform_search([InfoRequestEvent], query, sortby, 'request_title_collapse', max_count-@request_events.count) more_events = xapian_object.results.map { |r| r[:model] } - @request_events += more_events.sort_by { |e| e.described_at }.reverse + @request_events += more_events end rescue @request_events = [] diff --git a/app/models/foi_attachment.rb b/app/models/foi_attachment.rb index 74346227b..da92d1c2d 100644 --- a/app/models/foi_attachment.rb +++ b/app/models/foi_attachment.rb @@ -38,7 +38,11 @@ class FoiAttachment < ActiveRecord::Base BODY_MAX_DELAY = 5 def directory - base_dir = File.join(File.dirname(__FILE__), "../../cache", "attachments_#{ENV['RAILS_ENV']}") + rails_env = ENV['RAILS_ENV'] + if rails_env.nil? || rails_env.empty? + raise "$RAILS_ENV is not set" + end + base_dir = File.join(File.dirname(__FILE__), "../../cache", "attachments_#{rails_env}") return File.join(base_dir, self.hexdigest[0..2]) end diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb index 91f1cf7c0..131970ba6 100644 --- a/app/models/incoming_message.rb +++ b/app/models/incoming_message.rb @@ -127,6 +127,9 @@ class IncomingMessage < ActiveRecord::Base # The following fields may be absent; we treat them as cached # values in case we want to regenerate them (due to mail # parsing bugs, etc). + if self.raw_email.nil? + raise "Incoming message id=#{id} has no raw_email" + end if (!force.nil? || self.last_parsed.nil?) ActiveRecord::Base.transaction do self.extract_attachments! diff --git a/doc/INSTALL.md b/doc/INSTALL.md index b95534e4f..9dd05b93e 100644 --- a/doc/INSTALL.md +++ b/doc/INSTALL.md @@ -36,20 +36,24 @@ Some of the files also have a version number listed in config/packages - check that you have appropriate versions installed. Some also list "|" and offer a choice of packages. + You will also want to install mySociety's common ruby libraries and the Rails code. Run: - git submodule update --init + git submodule update --init to fetch the contents of the submodules. -Optionally, you may want to install -[wkhtmltopdf](http://code.google.com/p/wkhtmltopdf/downloads/list). +If you would like users to be able to download pretty PDFs as part of +the downloadable zipfile of their request history, you should also install +[wkhtmltopdf](http://code.google.com/p/wkhtmltopdf/downloads/list). We recommend downloading the latest, statically compiled version from the project website, as this allows running headless (i.e. without a graphical interface running) on Linux. If you do install `wkhtmltopdf`, you need to edit a setting in the config file to point -to it (see below). +to it (see below). If you don't install it, everything will still work, but +users will get ugly, plain text versions of their requests when they +download them. Version 1.44 of `pdftk` contains a bug which makes it to loop forever in certain edge conditions. Until it's incorporated into an official diff --git a/script/load-sample-data b/script/load-sample-data index 84b8a28eb..92846ce17 100755 --- a/script/load-sample-data +++ b/script/load-sample-data @@ -4,17 +4,16 @@ # the fact that the fixtures aren't aware of the fact that RawEmails # have a filesystem representation of their contents -LOC=`dirname $0` +LOC=`dirname "$0"` + rake --silent spec:db:fixtures:load + "$LOC/runner" /dev/stdin <<END env = ENV["RAILS_ENV"] -require "spec/spec_helper.rb" # Sets RAILS_ENV to 'test' -ENV["RAILS_ENV"] = env +require "spec/spec_helper.rb" # this sets RAILS_ENV to 'test' +ENV["RAILS_ENV"] = env # so restore to what it was before -RawEmail.all().each do |email| - puts "Writing #{email.filepath}" - email.data = load_file_fixture("useless_raw_email.email") -end +load_raw_emails_data END echo "Loaded fixtures." diff --git a/script/spec-all-pairs b/script/spec-all-pairs index 6f27e6b8e..338f56147 100755 --- a/script/spec-all-pairs +++ b/script/spec-all-pairs @@ -41,6 +41,7 @@ pairs_from_stdin() { do case "$line" in \*\ FAILED:\ *) + spec/*.rb spec/$.rb) line=${line#\* FAILED: } if ! test_pair $line then @@ -63,4 +64,4 @@ then else all_pairs fi -exit $?
\ No newline at end of file +exit $? diff --git a/spec/controllers/admin_request_controller_spec.rb b/spec/controllers/admin_request_controller_spec.rb index 6d3c955bb..5e6f6c2a5 100644 --- a/spec/controllers/admin_request_controller_spec.rb +++ b/spec/controllers/admin_request_controller_spec.rb @@ -6,7 +6,7 @@ describe AdminRequestController, "when administering requests" do before { basic_auth_login @request } before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data @old_filters = ActionController::Routing::Routes.filters ActionController::Routing::Routes.filters = RoutingFilter::Chain.new end @@ -53,7 +53,7 @@ describe AdminRequestController, "when administering the holding pen" do fixtures :users, :public_bodies, :public_body_translations, :public_body_versions, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do basic_auth_login @request - load_raw_emails_data(raw_emails) + load_raw_emails_data @old_filters = ActionController::Routing::Routes.filters ActionController::Routing::Routes.filters = RoutingFilter::Chain.new end diff --git a/spec/controllers/general_controller_spec.rb b/spec/controllers/general_controller_spec.rb index b067707de..9a5421a78 100644 --- a/spec/controllers/general_controller_spec.rb +++ b/spec/controllers/general_controller_spec.rb @@ -34,7 +34,7 @@ describe GeneralController, "when searching" do ] before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data rebuild_xapian_index end @@ -111,23 +111,33 @@ describe GeneralController, "when searching" do describe 'when constructing the list of recent requests' do before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data rebuild_xapian_index end it 'should list the newest successful request first' do - # Make sure the newest is listed first even if an older one has an even newer comment: + # Make sure the newest is listed first even if an older one + # has a newer comment or was reclassified more recently: # https://github.com/sebbacon/alaveteli/issues/370 + # + # This is a deliberate behaviour change, in that the + # previous behaviour (showing more-recently-reclassified + # requests first) was intentional. get :frontpage assigns[:request_events].first.info_request.should == info_requests(:another_boring_request) end + + it 'should coalesce duplicate requests' do + get :frontpage + assigns[:request_events].map(&:info_request).select{|x|x.url_title =~ /^spam/}.length.should == 1 + end end describe 'when using xapian search' do # rebuild xapian index after fixtures loaded before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data rebuild_xapian_index end @@ -158,21 +168,31 @@ describe GeneralController, "when searching" do it "should filter results based on end of URL being 'all'" do get :search, :combined => ['"bob"', "all"] - assigns[:xapian_requests].results.size.should == 2 - assigns[:xapian_users].results.size.should == 1 - assigns[:xapian_bodies].results.size.should == 0 + assigns[:xapian_requests].results.map{|x| x[:model]}.should =~ [ + info_request_events(:useless_outgoing_message_event), + info_request_events(:silly_outgoing_message_event), + info_request_events(:useful_incoming_message_event), + info_request_events(:another_useful_incoming_message_event), + ] + assigns[:xapian_users].results.map{|x| x[:model]}.should == [users(:bob_smith_user)] + assigns[:xapian_bodies].results.should == [] end it "should filter results based on end of URL being 'users'" do get :search, :combined => ['"bob"', "users"] assigns[:xapian_requests].should == nil - assigns[:xapian_users].results.size.should == 1 + assigns[:xapian_users].results.map{|x| x[:model]}.should == [users(:bob_smith_user)] assigns[:xapian_bodies].should == nil end it "should filter results based on end of URL being 'requests'" do get :search, :combined => ['"bob"', "requests"] - assigns[:xapian_requests].results.size.should == 2 + assigns[:xapian_requests].results.map{|x|x[:model]}.should =~ [ + info_request_events(:useless_outgoing_message_event), + info_request_events(:silly_outgoing_message_event), + info_request_events(:useful_incoming_message_event), + info_request_events(:another_useful_incoming_message_event), + ] assigns[:xapian_users].should == nil assigns[:xapian_bodies].should == nil end @@ -181,7 +201,7 @@ describe GeneralController, "when searching" do get :search, :combined => ['"quango"', "bodies"] assigns[:xapian_requests].should == nil assigns[:xapian_users].should == nil - assigns[:xapian_bodies].results.size.should == 1 + assigns[:xapian_bodies].results.map{|x|x[:model]}.should == [public_bodies(:geraldine_public_body)] end it "should show help when searching for nothing" do diff --git a/spec/controllers/public_body_controller_spec.rb b/spec/controllers/public_body_controller_spec.rb index bd4e625ca..4d27ea3f5 100644 --- a/spec/controllers/public_body_controller_spec.rb +++ b/spec/controllers/public_body_controller_spec.rb @@ -7,7 +7,7 @@ describe PublicBodyController, "when showing a body" do fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data rebuild_xapian_index end @@ -153,7 +153,6 @@ describe PublicBodyController, "when listing bodies" do get :list response.should render_template('list') assigns[:public_bodies].should =~ PublicBody.all(:conditions => "id <> #{PublicBody.internal_admin_body.id}") - end it "should list a machine tagged thing, should get it in both ways" do @@ -199,6 +198,11 @@ describe PublicBodyController, "when doing type ahead searches" do fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things integrate_views + + before(:each) do + load_raw_emails_data + rebuild_xapian_index + end it "should return nothing for the empty query string" do get :search_typeahead, :query => "" @@ -217,16 +221,16 @@ describe PublicBodyController, "when doing type ahead searches" do it "should return all requests matching any of the given keywords" do get :search_typeahead, :query => "Geraldine Humpadinking" response.should render_template('public_body/_search_ahead') - assigns[:xapian_requests].results.size.should == 2 - assigns[:xapian_requests].results[0][:model].name.should == public_bodies(:humpadink_public_body).name - assigns[:xapian_requests].results[1][:model].name.should == public_bodies(:geraldine_public_body).name + assigns[:xapian_requests].results.map{|x|x[:model]}.should =~ [ + public_bodies(:humpadink_public_body), + public_bodies(:geraldine_public_body), + ] end it "should return requests matching the given keywords in any of their locales" do get :search_typeahead, :query => "baguette" # part of the spanish notes response.should render_template('public_body/_search_ahead') - assigns[:xapian_requests].results.size.should == 1 - assigns[:xapian_requests].results[0][:model].name.should == public_bodies(:humpadink_public_body).name + assigns[:xapian_requests].results.map{|x|x[:model]}.should =~ [public_bodies(:humpadink_public_body)] end it "should not return matches for short words" do diff --git a/spec/controllers/request_controller_spec.rb b/spec/controllers/request_controller_spec.rb index bae1f32e7..77652b26d 100644 --- a/spec/controllers/request_controller_spec.rb +++ b/spec/controllers/request_controller_spec.rb @@ -7,7 +7,7 @@ describe RequestController, "when listing recent requests" do fixtures :users, :public_bodies, :public_body_translations, :public_body_versions, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data rebuild_xapian_index end @@ -125,7 +125,7 @@ describe RequestController, "when showing one request" do fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things # all needed as integrating views before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data end it "should be successful" do @@ -209,7 +209,8 @@ describe RequestController, "when showing one request" do it "should download attachments" do ir = info_requests(:fancy_dog_request) - ir.incoming_messages.each { |x| x.parse_raw_email! } + ir.incoming_messages.each { |x| x.parse_raw_email!(true) } + get :show, :url_title => 'why_do_you_have_such_a_fancy_dog' response.content_type.should == "text/html" size_before = assigns[:info_request_events].size @@ -221,11 +222,11 @@ describe RequestController, "when showing one request" do (assigns[:info_request_events].size - size_before).should == 1 ir.reload - get :get_attachment, :incoming_message_id => ir.incoming_messages[1].id, :id => ir.id, :part => 2, :file_name => ['hello.txt'] + get :get_attachment, :incoming_message_id => ir.incoming_messages[1].id, :id => ir.id, :part => 2, :file_name => ['hello.txt'], :skip_cache => 1 response.content_type.should == "text/plain" response.should have_text(/Second hello/) - get :get_attachment, :incoming_message_id => ir.incoming_messages[1].id, :id => ir.id, :part => 3, :file_name => ['hello.txt'] + get :get_attachment, :incoming_message_id => ir.incoming_messages[1].id, :id => ir.id, :part => 3, :file_name => ['hello.txt'], :skip_cache => 1 response.content_type.should == "text/plain" response.should have_text(/First hello/) end @@ -321,7 +322,7 @@ describe RequestController, "when showing one request" do receive_incoming_mail('incoming-request-two-same-name.email', ir.incoming_email) lambda { - get :get_attachment, :incoming_message_id => ir.incoming_messages[1].id, :id => ir.id, :part => 2, + get :get_attachment, :incoming_message_id => ir.incoming_messages[1].id, :id => ir.id, :part => 2, :file_name => ['http://trying.to.hack'] }.should raise_error(ActiveRecord::RecordNotFound) end @@ -335,12 +336,16 @@ describe RequestController, "when showing one request" do censor_rule.last_edit_editor = "unknown" censor_rule.last_edit_comment = "none" ir.censor_rules << censor_rule + + begin + receive_incoming_mail('incoming-request-two-same-name.email', ir.incoming_email) - receive_incoming_mail('incoming-request-two-same-name.email', ir.incoming_email) - - get :get_attachment, :incoming_message_id => ir.incoming_messages[1].id, :id => ir.id, :part => 2, :file_name => ['hello.txt'] - response.content_type.should == "text/plain" - response.should have_text(/xxxxxx hello/) + get :get_attachment, :incoming_message_id => ir.incoming_messages[1].id, :id => ir.id, :part => 2, :file_name => ['hello.txt'], :skip_cache => 1 + response.content_type.should == "text/plain" + response.should have_text(/xxxxxx hello/) + ensure + ir.censor_rules.clear + end end it "should censor with rules on the user (rather than the request)" do @@ -441,7 +446,7 @@ describe RequestController, "when changing prominence of a request" do fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :info_request_events, :track_things # all needed as integrating views before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data end it "should not show hidden requests" do @@ -507,11 +512,11 @@ describe RequestController, "when changing prominence of a request" do ir.save! receive_incoming_mail('incoming-request-two-same-name.email', ir.incoming_email) - get :get_attachment, :incoming_message_id => ir.incoming_messages[1].id, :id => ir.id, :part => 2 + get :get_attachment, :incoming_message_id => ir.incoming_messages[1].id, :id => ir.id, :part => 2, :skip_cache => 1 response.content_type.should == "text/html" response.should_not have_text(/Second hello/) response.should render_template('request/hidden') - get :get_attachment, :incoming_message_id => ir.incoming_messages[1].id, :id => ir.id, :part => 3 + get :get_attachment, :incoming_message_id => ir.incoming_messages[1].id, :id => ir.id, :part => 3, :skip_cache => 1 response.content_type.should == "text/html" response.should_not have_text(/First hello/) response.should render_template('request/hidden') @@ -755,7 +760,7 @@ describe RequestController, "when viewing an individual response for reply/follo fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things # all needed as integrating views before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data end it "should ask for login if you are logged in as wrong person" do @@ -805,7 +810,7 @@ describe RequestController, "when classifying an information request" do @dog_request = info_requests(:fancy_dog_request) @dog_request.stub!(:is_old_unclassified?).and_return(false) InfoRequest.stub!(:find).and_return(@dog_request) - load_raw_emails_data(raw_emails) + load_raw_emails_data end def post_status(status) @@ -1142,7 +1147,7 @@ describe RequestController, "when sending a followup message" do fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things # all needed as integrating views before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data end it "should require login" do @@ -1225,7 +1230,7 @@ describe RequestController, "sending overdue request alerts" do fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things # all needed as integrating views before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data end it "should send an overdue alert mail to creators of overdue requests" do @@ -1313,7 +1318,7 @@ describe RequestController, "sending unclassified new response reminder alerts" fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things # all needed as integrating views before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data end it "should send an alert" do @@ -1343,7 +1348,7 @@ describe RequestController, "clarification required alerts" do integrate_views fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things # all needed as integrating views before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data end it "should send an alert" do @@ -1397,7 +1402,7 @@ describe RequestController, "comment alerts" do integrate_views fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things # all needed as integrating views before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data end it "should send an alert (once and once only)" do @@ -1472,7 +1477,7 @@ describe RequestController, "when viewing comments" do integrate_views fixtures :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data end it "should link to the user who submitted it" do @@ -1584,7 +1589,7 @@ describe RequestController, "when showing JSON version for API" do fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data end it "should return data in JSON form" do @@ -1621,9 +1626,11 @@ describe RequestController, "when doing type ahead searches" do it "should return all requests matching any of the given keywords" do get :search_typeahead, :q => "money dog" response.should render_template('request/_search_ahead.rhtml') - assigns[:xapian_requests].results.size.should == 2 - assigns[:xapian_requests].results[0][:model].title.should == info_requests(:fancy_dog_request).title - assigns[:xapian_requests].results[1][:model].title.should == info_requests(:naughty_chicken_request).title + assigns[:xapian_requests].results.map{|x|x[:model].info_request}.should =~ [ + info_requests(:fancy_dog_request), + info_requests(:naughty_chicken_request), + info_requests(:another_boring_request), + ] end it "should not return matches for short words" do diff --git a/spec/controllers/request_game_controller_spec.rb b/spec/controllers/request_game_controller_spec.rb index cc0808ef3..1383554a1 100644 --- a/spec/controllers/request_game_controller_spec.rb +++ b/spec/controllers/request_game_controller_spec.rb @@ -4,7 +4,7 @@ describe RequestGameController, "when playing the game" do fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things # all needed as integrating views before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data end it "should show the game homepage" do diff --git a/spec/controllers/track_controller_spec.rb b/spec/controllers/track_controller_spec.rb index ad4d651cb..509aa6725 100644 --- a/spec/controllers/track_controller_spec.rb +++ b/spec/controllers/track_controller_spec.rb @@ -40,7 +40,7 @@ describe TrackController, "when sending alerts for a track" do include LinkToHelper # for main_url before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data rebuild_xapian_index end @@ -118,7 +118,7 @@ describe TrackController, "when viewing RSS feed for a track" do fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data rebuild_xapian_index end @@ -144,7 +144,7 @@ describe TrackController, "when viewing JSON version of a track feed" do fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data rebuild_xapian_index end diff --git a/spec/controllers/user_controller_spec.rb b/spec/controllers/user_controller_spec.rb index 81d3ff2e5..4e14aeaa3 100644 --- a/spec/controllers/user_controller_spec.rb +++ b/spec/controllers/user_controller_spec.rb @@ -10,7 +10,7 @@ describe UserController, "when showing a user" do integrate_views fixtures :users, :public_bodies, :public_body_translations, :public_body_versions, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data rebuild_xapian_index end @@ -46,10 +46,14 @@ describe UserController, "when showing a user" do it "should search the user's contributions" do get :show, :url_name => "bob_smith" - assigns[:xapian_requests].results.map{|x|x[:model].info_request}.should =~ InfoRequest.all(:conditions => "user_id = #{users(:bob_smith_user).id}") + assigns[:xapian_requests].results.map{|x|x[:model].info_request}.should =~ InfoRequest.all( + :conditions => "user_id = #{users(:bob_smith_user).id}") get :show, :url_name => "bob_smith", :user_query => "money" - assigns[:xapian_requests].results.map{|x|x[:model].info_request}.should == [info_requests(:naughty_chicken_request)] + assigns[:xapian_requests].results.map{|x|x[:model].info_request}.should =~ [ + info_requests(:naughty_chicken_request), + info_requests(:another_boring_request), + ] end it "should not show unconfirmed users" do diff --git a/spec/fixtures/files/useless_raw_email.email b/spec/fixtures/files/raw_emails/1.email index 2e4585af7..2e4585af7 100644 --- a/spec/fixtures/files/useless_raw_email.email +++ b/spec/fixtures/files/raw_emails/1.email diff --git a/spec/fixtures/files/raw_emails/2.email b/spec/fixtures/files/raw_emails/2.email new file mode 100644 index 000000000..eab0b0f8d --- /dev/null +++ b/spec/fixtures/files/raw_emails/2.email @@ -0,0 +1,20 @@ +From: "FOI Person" <foiperson@localhost> +To: "Bob Smith" <bob@localhost> +Date: Tue, 13 Nov 2007 11:39:55 +0000 +Bcc: +Subject: Re: Your email +Reply-To: +In-Reply-To: <471f1eae5d1cb_7347..fdbe67386164@cat.tmail> +Content-Type: text/plain; charset=utf-8 + +Dear “Bob”, + +In the financial year 2010–2011, this Department spent a +total of nine hundred of your earth pounds on the purchase +and repair of boring equipment. + +Yours most sincerely, + +Quentin Nobble-Boston, +Permanent Under-Secretary, +Department for Humpadinking diff --git a/spec/fixtures/files/raw_emails/3.email b/spec/fixtures/files/raw_emails/3.email new file mode 100644 index 000000000..a6e780fe5 --- /dev/null +++ b/spec/fixtures/files/raw_emails/3.email @@ -0,0 +1,19 @@ +From: "The Minister" <msw@localhost> +To: "Bob Smith" <bob@localhost> +Date: Tue, 13 Nov 2009 11:39:55 +0000 +Bcc: +Subject: Re: Your message +Reply-To: +In-Reply-To: <471f1eae5d1cb_7347..fdbe67386165@cat.tmail> +Content-Type: text/plain; charset=utf-8 + +Dear “Bob”, + +In the financial year 2010–2011, this Ministry spent precisely +no money at all on the purchase or repair of boring equipment. + +Yours most sincerely, + +Martin Kibble-von Scratsching, +Chief Assistant to the Assistant Chief, +Ministry of Silly Walks diff --git a/spec/fixtures/files/raw_emails/4.email b/spec/fixtures/files/raw_emails/4.email new file mode 100644 index 000000000..c778e5ba9 --- /dev/null +++ b/spec/fixtures/files/raw_emails/4.email @@ -0,0 +1,17 @@ +From: "The Minister" <msw@localhost> +To: robin@localhost +Date: Tue, 13 Nov 2008 11:39:55 +0000 +Bcc: +Subject: Re: v1agra +Reply-To: +In-Reply-To: <fdshjksdahhjkfsdahjkfsd@gfh.example.com> +Content-Type: text/plain; charset=utf-8 + +Thank you for your spam, which we have processed with pleasure. Please +accept herewith our order for six packs of the finest cheap v1agra. + +Yours most sincerely, + +Martin Kibble-von Scratsching, +Chief Assistant to the Assistant Chief, +Ministry of Silly Walks diff --git a/spec/fixtures/files/raw_emails/5.email b/spec/fixtures/files/raw_emails/5.email new file mode 100644 index 000000000..16b65610a --- /dev/null +++ b/spec/fixtures/files/raw_emails/5.email @@ -0,0 +1,17 @@ +From: "The Minister" <sensewalk@localhost> +To: robin@localhost +Date: Tue, 13 Nov 2008 12:39:55 +0000 +Bcc: +Subject: Re: v1agra +Reply-To: +In-Reply-To: <fdshjks+hihhjkfsdahjkfsd@gfh.example.com> +Content-Type: text/plain; charset=utf-8 + +Thank you for your spam, which we have processed with pleasure. Please +accept herewith our order for six packs of the finest cheap v1agra. + +Yours most sincerely, + +Martin Scratsching-von Kibble, +Assistant (Second class) to the Chief Assistant to the Assistant Chief, +Ministry of Sensible Walks diff --git a/spec/fixtures/incoming_messages.yml b/spec/fixtures/incoming_messages.yml index a32052413..fca5c716c 100644 --- a/spec/fixtures/incoming_messages.yml +++ b/spec/fixtures/incoming_messages.yml @@ -14,7 +14,20 @@ useful_incoming_message: another_useful_incoming_message: id: 3 - info_request_id: 105 - raw_email_id: 2 + info_request_id: 106 + raw_email_id: 3 created_at: 2007-11-13 18:09:20 updated_at: 2007-11-13 18:09:20 + +spam_1_incoming_message: + id: 4 + info_request_id: 107 + raw_email_id: 4 + created_at: 2008-11-13 18:09:20 + updated_at: 2008-11-13 18:09:20 +spam_2_incoming_message: + id: 5 + info_request_id: 108 + raw_email_id: 5 + created_at: 2008-11-13 19:09:20 + updated_at: 2008-11-13 19:09:20 diff --git a/spec/fixtures/info_request_events.yml b/spec/fixtures/info_request_events.yml index ec6abbf57..3266ec634 100644 --- a/spec/fixtures/info_request_events.yml +++ b/spec/fixtures/info_request_events.yml @@ -1,37 +1,37 @@ useless_outgoing_message_event: + id: 900 params_yaml: "--- \n\ :outgoing_message_id: 1\n" - id: 900 info_request_id: 101 event_type: sent created_at: 2007-10-12 01:56:58.586598 described_state: outgoing_message_id: 1 silly_outgoing_message_event: + id: 901 params_yaml: "--- \n\ :outgoing_message_id: 2\n" - id: 901 info_request_id: 103 event_type: sent created_at: 2007-10-14 10:41:12.686264 described_state: outgoing_message_id: 2 useless_incoming_message_event: + id: 902 params_yaml: "--- \n\ :incoming_message_id: 1\n" - id: 902 info_request_id: 101 event_type: response created_at: 2007-11-13 18:09:20.042061 described_state: incoming_message_id: 1 silly_comment_event: + id: 903 params_yaml: "--- \n\ :comment_id: 1\n" incoming_message_id: last_described_at: described_state: - id: "903" info_request_id: 101 comment_id: 1 calculated_state: @@ -39,9 +39,9 @@ silly_comment_event: outgoing_message_id: created_at: 2008-08-12 23:05:12.500942 badger_outgoing_message_event: + id: 904 params_yaml: "--- \n\ :outgoing_message_id: 3\n" - id: 904 info_request_id: 104 event_type: sent created_at: 2011-10-12 01:56:58.586598 @@ -60,14 +60,14 @@ boring_outgoing_message_event: created_at: 2006-01-12 01:56:58.586598 described_state: waiting_response calculated_state: waiting_response -useful_incoming_message_event: +useful_incoming_message_event: id: 906 params_yaml: "--- \n\ :incoming_message_id: 2\n" incoming_message_id: 2 info_request_id: 105 event_type: response - created_at: 2007-11-13 18:09:20.042061 + created_at: 2007-11-13 18:00:20 described_state: successful calculated_state: successful @@ -92,7 +92,7 @@ another_useful_incoming_message_event: described_state: successful calculated_state: successful -another_comment_event: +another_comment_event: id: 909 info_request_id: 105 comment_id: 2 @@ -106,3 +106,47 @@ another_comment_event: event_type: comment created_at: 2008-08-12 12:05:12.879634 + +# The spam requests were both successful +spam_1_outgoing_message_event: + id: 910 + params_yaml: "--- \n\ + :outgoing_message_id: 6\n" + outgoing_message_id: 6 + info_request_id: 107 + event_type: sent + created_at: 2001-01-02 01:23:45.6789100 + described_state: waiting_response + calculated_state: waiting_response +spam_1_incoming_message_event: + id: 911 + params_yaml: "--- \n\ + :incoming_message_id: 4\n" + incoming_message_id: 4 + info_request_id: 107 + event_type: response + created_at: 2001-01-03 01:23:45.6789100 + described_state: successful + calculated_state: successful + +spam_2_outgoing_message_event: + id: 912 + params_yaml: "--- \n\ + :outgoing_message_id: 7\n" + outgoing_message_id: 7 + info_request_id: 108 + event_type: sent + created_at: 2001-01-02 02:23:45.6789100 + described_state: waiting_response + calculated_state: waiting_response +spam_2_incoming_message_event: + id: 913 + params_yaml: "--- \n\ + :incoming_message_id: 5\n" + incoming_message_id: 5 + info_request_id: 108 + event_type: response + created_at: 2001-01-03 02:23:45.6789100 + described_state: successful + calculated_state: successful + diff --git a/spec/fixtures/info_requests.yml b/spec/fixtures/info_requests.yml index 89c53e9f3..33e9a16f2 100644 --- a/spec/fixtures/info_requests.yml +++ b/spec/fixtures/info_requests.yml @@ -45,11 +45,36 @@ boring_request: another_boring_request: id: 106 title: The cost of boring - url_title: the_cost_of_boring_2 + url_title: the_cost_of_boring_two # Not _2, because we want to avoid the search collapsing these two created_at: 2006-01-12 01:56:58.586598 updated_at: 2007-11-13 18:09:20.042061 public_body_id: 5 user_id: 1 described_state: successful awaiting_description: false - idhash: 173fd003 + idhash: 173fd004 + +# A pair of identical requests (with url_title differing only in the numeric suffix) +# used to test the request de-duplication features. +spam_1_request: + id: 107 + title: Cheap v1agra + url_title: spam_1 + created_at: 2010-01-01 01:23:45.6789100 + created_at: 2010-01-01 01:23:45.6789100 + public_body_id: 5 + user_id: 5 + described_state: successful + awaiting_description: false + idhash: 173fd005 +spam_2_request: + id: 108 + title: Cheap v1agra + url_title: spam_2 + created_at: 2010-01-01 02:23:45.6789100 + created_at: 2010-01-01 02:23:45.6789100 + public_body_id: 6 + user_id: 5 + described_state: successful + awaiting_description: false + idhash: 173fd005 diff --git a/spec/fixtures/outgoing_messages.yml b/spec/fixtures/outgoing_messages.yml index 3fb472630..d33ca4292 100644 --- a/spec/fixtures/outgoing_messages.yml +++ b/spec/fixtures/outgoing_messages.yml @@ -65,4 +65,24 @@ another_boring_outgoing_message: updated_at: 2006-01-12 01:56:58.586598 what_doing: normal_sort +spam_1_outgoing_message: + id: 6 + info_request_id: 107 + message_type: initial_request + status: sent + body: "Would you like some cheap v1agra?" + last_sent_at: 2007-01-12 01:57:58.586598 + created_at: 2007-01-12 01:56:58.586598 + updated_at: 2007-01-12 01:56:58.586598 + what_doing: normal_sort +spam_2_outgoing_message: + id: 7 + info_request_id: 108 + message_type: initial_request + status: sent + body: "Would you like some cheap v1agra?" + last_sent_at: 2007-01-12 02:57:58.586598 + created_at: 2007-01-12 02:56:58.586598 + updated_at: 2007-01-12 02:56:58.586598 + what_doing: normal_sort diff --git a/spec/fixtures/public_bodies.yml b/spec/fixtures/public_bodies.yml index 737fe203f..a0893f1e5 100644 --- a/spec/fixtures/public_bodies.yml +++ b/spec/fixtures/public_bodies.yml @@ -16,7 +16,7 @@ humpadink_public_body: updated_at: 2007-10-25 10:51:01.161639 last_edit_comment: Not sure what this new department does. request_email: humpadink-requests@localhost - id: "3" + id: 3 version: "2" last_edit_editor: "francis" short_name: DfH @@ -49,3 +49,16 @@ silly_walks_public_body: url_name: msw created_at: 2007-10-25 10:51:01.161639 notes: You know the one. +sensible_walks_public_body: + id: 6 + version: 1 + name: "Ministry of Sensible Walks" + first_letter: M + request_email: sensible-walks-requests@localhost + short_name: SenseWalk + url_name: sensible_walks + notes: I bet you’ve never heard of it. + updated_at: 2008-10-25 10:51:01.161639 + last_edit_comment: Another stunning innovation from your friendly national government + last_edit_editor: robin + created_at: 2008-10-25 10:51:01.161639 diff --git a/spec/fixtures/public_body_translations.yml b/spec/fixtures/public_body_translations.yml index b81dd35a3..cbb55bb0c 100644 --- a/spec/fixtures/public_body_translations.yml +++ b/spec/fixtures/public_body_translations.yml @@ -63,3 +63,14 @@ silly_walks_en_public_body_translation: short_name: MSW url_name: msw notes: You know the one. + +sensible_walks_en_public_body_translation: + id: 7 + public_body_id: 6 + locale: en + name: "Ministry of Sensible Walks" + first_letter: M + request_email: sensible-walks-requests@localhost + short_name: SenseWalk + url_name: sensible_walks + notes: I bet you’ve never heard of it. diff --git a/spec/fixtures/raw_emails.yml b/spec/fixtures/raw_emails.yml index 32e039cab..ad2bc0a63 100644 --- a/spec/fixtures/raw_emails.yml +++ b/spec/fixtures/raw_emails.yml @@ -1,5 +1,20 @@ -useless_raw_email: +# The actual email messages are in fixtures/files/raw_emails +# +# Note that the words "money" and "bob" are used in some tests +# of the search functions, so if you use either of these words +# in the email text then some tests will have to be updated. + +useless_raw_email: id: 1 -useful_raw_email: +useful_raw_email: id: 2 + +another_useful_raw_email: + id: 3 + +spam_1_raw_email: + id: 4 + +spam_2_raw_email: + id: 5 diff --git a/spec/fixtures/users.yml b/spec/fixtures/users.yml index 16ffec034..8620fb3de 100644 --- a/spec/fixtures/users.yml +++ b/spec/fixtures/users.yml @@ -53,3 +53,16 @@ unconfirmed_user: admin_level: 'none' ban_text: '' about_me: '' +robin_user: + id: 5 + name: Robin Houston + url_name: robin_houston + email: robin@localhost + salt: "-6116981980.392287733335677" + hashed_password: 6b7cd45a5f35fd83febc0452a799530398bfb6e8 # jonespassword + updated_at: 2012-01-01 10:39:15.491593 + created_at: 2012-01-01 10:39:15.491593 + email_confirmed: true + admin_level: 'none' + ban_text: '' + about_me: 'I am the best' diff --git a/spec/integration/errors_spec.rb b/spec/integration/errors_spec.rb index d03323445..ea9caaf12 100644 --- a/spec/integration/errors_spec.rb +++ b/spec/integration/errors_spec.rb @@ -17,7 +17,7 @@ describe "When rendering errors" do ] before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data ActionController::Base.consider_all_requests_local = false end diff --git a/spec/integration/search_request_spec.rb b/spec/integration/search_request_spec.rb index d0a457923..61f8df313 100644 --- a/spec/integration/search_request_spec.rb +++ b/spec/integration/search_request_spec.rb @@ -17,8 +17,7 @@ describe "When searching" do ] before(:each) do - emails = raw_emails.clone - load_raw_emails_data(emails) + load_raw_emails_data rebuild_xapian_index end @@ -35,21 +34,34 @@ describe "When searching" do end it "should correctly filter searches for requests" do - request_via_redirect("post", "/search/bob/requests") + request_via_redirect("post", "/search/bob/requests") response.body.should_not include("One person found") - response.body.should include("FOI requests 1 to 2 of 2") + n = 4 # The number of requests that contain the word "bob" somewhere + # in the email text. At present this is: + # - fancy_dog_request + # - naughty_chicken_request + # - boring_request + # - another_boring_request + # + # In other words it is all requests made by Bob Smith + # except for badger_request, which he did not sign. + response.body.should include("FOI requests 1 to #{n} of #{n}") end it "should correctly filter searches for users" do - request_via_redirect("post", "/search/bob/users") + request_via_redirect("post", "/search/bob/users") response.body.should include("One person found") - response.body.should_not include("FOI requests 1 to 2 of 2") + response.body.should_not include("FOI requests 1 to") end it "should correctly filter searches for successful requests" do request_via_redirect("post", "/search/requests", :query => "bob", :latest_status => ['successful']) - response.body.should include("no results matching your query") + n = 2 # The number of *successful* requests that contain the word "bob" somewhere + # in the email text. At present this is: + # - boring_request + # - another_boring_request + response.body.should include("FOI requests 1 to #{n} of #{n}") end it "should correctly filter searches for comments" do diff --git a/spec/integration/view_request_spec.rb b/spec/integration/view_request_spec.rb index cf1e4ca6c..0787644ff 100644 --- a/spec/integration/view_request_spec.rb +++ b/spec/integration/view_request_spec.rb @@ -17,8 +17,7 @@ describe "When viewing requests" do ] before(:each) do - emails = raw_emails.clone - load_raw_emails_data(emails) + load_raw_emails_data end it "should not make endlessly recursive JSON <link>s" do diff --git a/spec/models/foi_attachment_spec.rb b/spec/models/foi_attachment_spec.rb index 05c4fc5fd..0d122aa1b 100644 --- a/spec/models/foi_attachment_spec.rb +++ b/spec/models/foi_attachment_spec.rb @@ -4,7 +4,7 @@ describe FoiAttachment, " when calculating due date" do fixtures :incoming_messages, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :users, :foi_attachments before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data end it "sets the body" do diff --git a/spec/models/incoming_message_spec.rb b/spec/models/incoming_message_spec.rb index 7808ef24c..c096e61e0 100644 --- a/spec/models/incoming_message_spec.rb +++ b/spec/models/incoming_message_spec.rb @@ -6,7 +6,7 @@ describe IncomingMessage, " when dealing with incoming mail" do before(:each) do @im = incoming_messages(:useless_incoming_message) - load_raw_emails_data(raw_emails) + load_raw_emails_data end after(:all) do @@ -236,7 +236,7 @@ describe IncomingMessage, " when censoring data" do @censor_rule_2.last_edit_comment = "none" @im.info_request.censor_rules << @censor_rule_2 - load_raw_emails_data(raw_emails) + load_raw_emails_data end it "should do nothing to a JPEG" do @@ -336,7 +336,7 @@ describe IncomingMessage, " when censoring whole users" do @censor_rule_1.last_edit_editor = "unknown" @censor_rule_1.last_edit_comment = "none" @im.info_request.user.censor_rules << @censor_rule_1 - load_raw_emails_data(raw_emails) + load_raw_emails_data end it "should apply censor rules to HTML files" do @@ -357,7 +357,7 @@ describe IncomingMessage, " when uudecoding bad messages" do fixtures :incoming_messages, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :users, :foi_attachments before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data end it "should be able to do it at all" do @@ -401,7 +401,7 @@ describe IncomingMessage, "when messages are attached to messages" do fixtures :incoming_messages, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :users, :foi_attachments before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data end it "should flatten all the attachments out" do @@ -426,7 +426,7 @@ describe IncomingMessage, "when Outlook messages are attached to messages" do fixtures :incoming_messages, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :users, :foi_attachments before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data end it "should flatten all the attachments out" do @@ -449,7 +449,7 @@ describe IncomingMessage, "when TNEF attachments are attached to messages" do fixtures :incoming_messages, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :users, :foi_attachments before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data end it "should flatten all the attachments out" do diff --git a/spec/models/info_request_event_spec.rb b/spec/models/info_request_event_spec.rb index a75f4b232..9a340c125 100644 --- a/spec/models/info_request_event_spec.rb +++ b/spec/models/info_request_event_spec.rb @@ -58,7 +58,7 @@ describe InfoRequestEvent do fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data parse_all_incoming_messages end diff --git a/spec/models/info_request_spec.rb b/spec/models/info_request_spec.rb index b1baa66a2..3c8860285 100644 --- a/spec/models/info_request_spec.rb +++ b/spec/models/info_request_spec.rb @@ -7,7 +7,7 @@ describe InfoRequest do before(:each) do @im = incoming_messages(:useless_incoming_message) - load_raw_emails_data(raw_emails) + load_raw_emails_data end it 'should compute a hash' do diff --git a/spec/models/outgoing_mailer_spec.rb b/spec/models/outgoing_mailer_spec.rb index c5fde93fc..9b8fb5f98 100644 --- a/spec/models/outgoing_mailer_spec.rb +++ b/spec/models/outgoing_mailer_spec.rb @@ -6,7 +6,7 @@ describe OutgoingMailer, " when working out follow up addresses" do # mocks. Put parts of the tests in spec/lib/tmail_extensions.rb fixtures :public_bodies, :public_body_translations, :public_body_versions, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data end it "should parse them right" do @@ -77,7 +77,7 @@ describe OutgoingMailer, "when working out follow up subjects" do fixtures :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data end it "should prefix the title with 'Freedom of Information request -' for initial requests" do diff --git a/spec/models/request_mailer_spec.rb b/spec/models/request_mailer_spec.rb index 2888213d7..0dd0b749a 100644 --- a/spec/models/request_mailer_spec.rb +++ b/spec/models/request_mailer_spec.rb @@ -3,7 +3,7 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') describe RequestMailer, " when receiving incoming mail" do fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data ActionMailer::Base.deliveries = [] end diff --git a/spec/models/xapian_spec.rb b/spec/models/xapian_spec.rb index 6f4f49373..ba06f3eac 100644 --- a/spec/models/xapian_spec.rb +++ b/spec/models/xapian_spec.rb @@ -4,7 +4,7 @@ describe User, " when indexing users with Xapian" do fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data rebuild_xapian_index end @@ -39,7 +39,7 @@ end describe PublicBody, " when indexing public bodies with Xapian" do fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data rebuild_xapian_index end @@ -73,7 +73,7 @@ describe PublicBody, " when indexing requests by body they are to" do fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data rebuild_xapian_index end @@ -129,7 +129,7 @@ end describe User, " when indexing requests by user they are from" do fixtures :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things, :public_bodies, :public_body_versions, :public_body_translations before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data rebuild_xapian_index end @@ -208,7 +208,7 @@ end describe User, " when indexing comments by user they are by" do fixtures :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data rebuild_xapian_index end @@ -244,7 +244,7 @@ end describe InfoRequest, " when indexing requests by their title" do fixtures :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data rebuild_xapian_index end @@ -274,7 +274,7 @@ end describe InfoRequest, " when indexing requests by tag" do fixtures :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data rebuild_xapian_index end @@ -296,7 +296,7 @@ end describe PublicBody, " when indexing authorities by tag" do fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data rebuild_xapian_index end @@ -321,7 +321,7 @@ end describe PublicBody, " when only indexing selected things on a rebuild" do fixtures :public_bodies, :public_body_translations, :public_body_versions, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do - load_raw_emails_data(raw_emails) + load_raw_emails_data rebuild_xapian_index end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 33a28449e..065d9d080 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -107,7 +107,7 @@ def validate_html(html) File.open(tempfilename, "w+") do |f| f.puts html end - if not system($html_validation_script, tempfilename) + if not system($html_validation_script, *($html_validation_script_options +[tempfilename])) raise "HTML validation error in " + tempfilename + " HTTP status: " + @response.response_code.to_s end File.unlink(tempfilename) @@ -131,6 +131,7 @@ utility_search_path = MySociety::Config.get("UTILITY_SEARCH_PATH", ["/usr/bin", $html_validation_script_found = false utility_search_path.each do |d| $html_validation_script = File.join(d, "validate") + $html_validation_script_options = ["--charset=utf-8"] if File.file? $html_validation_script and File.executable? $html_validation_script $html_validation_script_found = true break @@ -178,13 +179,12 @@ def safe_mock_model(model, args = {}) mock end -def load_raw_emails_data(raw_emails) - raw_email = raw_emails(:useless_raw_email) - begin - raw_email.destroy_file_representation! - rescue Errno::ENOENT +def load_raw_emails_data + raw_emails_yml = File.join(Spec::Runner.configuration.fixture_path, "raw_emails.yml") + for raw_email_id in YAML::load_file(raw_emails_yml).map{|k,v| v["id"]} do + raw_email = RawEmail.find(raw_email_id) + raw_email.data = load_file_fixture("raw_emails/%d.email" % [raw_email_id]) end - raw_email.data = load_file_fixture("useless_raw_email.email") end def parse_all_incoming_messages |