From c1ee22fe8df92b03573a03e13f6fd692c9074dd0 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Wed, 13 Nov 2013 15:34:07 +0000 Subject: Reduce the memory used to serve /body/all-authorities.csv On WDTK, /body/all-authorities was using lots of memory - this commit reduces that by (a) fetching the public bodies in batches, rather than keeping them all in memory at one time and (b) writing the CSV to a file and then returning it with X-Sendfile (or equivalent), rather than returning the whole file from memory with send_data. There's a FIXME to do with the layout of download directories; if that's changed, the example nginx config, etc. will need to be updated too. This commit also adds a basic test for reasonable CSV being returned and switches from FasterCSV to CSV in order to fix this NotImplementedError under Ruby 1.9: Please switch to Ruby 1.9's standard CSV library. It's FasterCSV plus support for Ruby 1.9's m17n encoding engine. (The CSV version seems to still work fine under 1.8.7.) --- spec/controllers/public_body_controller_spec.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'spec/controllers/public_body_controller_spec.rb') diff --git a/spec/controllers/public_body_controller_spec.rb b/spec/controllers/public_body_controller_spec.rb index 6800765f2..2663f2a75 100644 --- a/spec/controllers/public_body_controller_spec.rb +++ b/spec/controllers/public_body_controller_spec.rb @@ -273,6 +273,20 @@ describe PublicBodyController, "when showing JSON version for API" do end +describe PublicBodyController, "when asked to export public bodies as CSV" do + + it "should return a valid CSV file with the right number of rows" do + get :list_all_csv + all_data = CSV.parse response.body + all_data.length.should == 8 + # Check that the header has the right number of columns: + all_data[0].length.should == 11 + # And an actual line of data: + all_data[1].length.should == 11 + end + +end + describe PublicBodyController, "when showing public body statistics" do it "should render the right template with the right data" do -- cgit v1.2.3