From c1ee22fe8df92b03573a03e13f6fd692c9074dd0 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Wed, 13 Nov 2013 15:34:07 +0000 Subject: Reduce the memory used to serve /body/all-authorities.csv On WDTK, /body/all-authorities was using lots of memory - this commit reduces that by (a) fetching the public bodies in batches, rather than keeping them all in memory at one time and (b) writing the CSV to a file and then returning it with X-Sendfile (or equivalent), rather than returning the whole file from memory with send_data. There's a FIXME to do with the layout of download directories; if that's changed, the example nginx config, etc. will need to be updated too. This commit also adds a basic test for reasonable CSV being returned and switches from FasterCSV to CSV in order to fix this NotImplementedError under Ruby 1.9: Please switch to Ruby 1.9's standard CSV library. It's FasterCSV plus support for Ruby 1.9's m17n encoding engine. (The CSV version seems to still work fine under 1.8.7.) --- app/models/public_body.rb | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/public_body.rb b/app/models/public_body.rb index fbe2956e3..db6359f6b 100644 --- a/app/models/public_body.rb +++ b/app/models/public_body.rb @@ -514,10 +514,8 @@ class PublicBody < ActiveRecord::Base end # Returns all public bodies (except for the internal admin authority) as csv - def self.export_csv - public_bodies = PublicBody.visible.find(:all, :order => 'url_name', - :include => [:translations, :tags]) - FasterCSV.generate() do |csv| + def self.export_csv(output_filename) + CSV.open(output_filename, "w") do |csv| csv << [ 'Name', 'Short name', @@ -532,7 +530,7 @@ class PublicBody < ActiveRecord::Base 'Updated at', 'Version', ] - public_bodies.each do |public_body| + PublicBody.visible.find_each(:include => [:translations, :tags]) do |public_body| # Skip bodies we use only for site admin next if public_body.has_tag?('site_administration') csv << [ -- cgit v1.2.3