diff options
Diffstat (limited to 'app/models/public_body.rb')
| -rw-r--r-- | app/models/public_body.rb | 291 |
1 files changed, 213 insertions, 78 deletions
diff --git a/app/models/public_body.rb b/app/models/public_body.rb index 168b9f4c7..7b1ded820 100644 --- a/app/models/public_body.rb +++ b/app/models/public_body.rb @@ -1,27 +1,32 @@ +# -*- coding: utf-8 -*- # == Schema Information -# Schema version: 20120919140404 +# Schema version: 20131024114346 # # Table name: public_bodies # -# id :integer not null, primary key -# name :text not null -# short_name :text not null -# request_email :text not null -# version :integer not null -# last_edit_editor :string(255) not null -# last_edit_comment :text not null -# created_at :datetime not null -# updated_at :datetime not null -# url_name :text not null -# home_page :text default(""), not null -# notes :text default(""), not null -# first_letter :string(255) not null -# publication_scheme :text default(""), not null -# api_key :string(255) not null -# info_requests_count :integer default(0), not null +# id :integer not null, primary key +# name :text not null +# short_name :text not null +# request_email :text not null +# version :integer not null +# last_edit_editor :string(255) not null +# last_edit_comment :text not null +# created_at :datetime not null +# updated_at :datetime not null +# url_name :text not null +# home_page :text default(""), not null +# notes :text default(""), not null +# first_letter :string(255) not null +# publication_scheme :text default(""), not null +# api_key :string(255) not null +# info_requests_count :integer default(0), not null +# disclosure_log :text default(""), not null +# info_requests_successful_count :integer +# info_requests_not_held_count :integer +# info_requests_overdue_count :integer +# info_requests_visible_classified_count :integer # -# -*- coding: utf-8 -*- require 'csv' require 'securerandom' require 'set' @@ -40,12 +45,13 @@ class PublicBody < ActiveRecord::Base has_many :info_requests, :order => 'created_at desc' has_many :track_things, :order => 'created_at desc' has_many :censor_rules, :order => 'created_at desc' + attr_accessor :no_xapian_reindex has_tag_string before_save :set_api_key, :set_default_publication_scheme # Every public body except for the internal admin one is visible - named_scope :visible, lambda { + scope :visible, lambda { { :conditions => "public_bodies.id <> #{PublicBody.internal_admin_body.id}" } @@ -54,18 +60,29 @@ class PublicBody < ActiveRecord::Base translates :name, :short_name, :request_email, :url_name, :notes, :first_letter, :publication_scheme # Convenience methods for creating/editing translations via forms - def translation(locale) + def find_translation_by_locale(locale) self.translations.find_by_locale(locale) end # XXX - Don't like repeating this! def calculate_cached_fields(t) - t.first_letter = t.name.scan(/^./mu)[0].upcase unless t.name.nil? or t.name.empty? + PublicBody.set_first_letter(t) short_long_name = t.name short_long_name = t.short_name if t.short_name and !t.short_name.empty? t.url_name = MySociety::Format.simplify_url_part(short_long_name, 'body') end + # Set the first letter on a public body or translation + def PublicBody.set_first_letter(instance) + unless instance.name.nil? or instance.name.empty? + # we use a regex to ensure it works with utf-8/multi-byte + first_letter = Unicode.upcase instance.name.scan(/^./mu)[0] + if first_letter != instance.first_letter + instance.first_letter = first_letter + end + end + end + def translated_versions translations end @@ -79,7 +96,7 @@ class PublicBody < ActiveRecord::Base if translation_attrs.respond_to? :each_value # Hash => updating translation_attrs.each_value do |attrs| next if skip?(attrs) - t = translation(attrs[:locale]) || PublicBody::Translation.new + t = translation_for(attrs[:locale]) || PublicBody::Translation.new t.attributes = attrs calculate_cached_fields(t) t.save! @@ -106,35 +123,31 @@ class PublicBody < ActiveRecord::Base # like find_by_url_name but also search historic url_name if none found def self.find_by_url_name_with_historic(name) - locale = self.locale || I18n.locale - PublicBody.with_locale(locale) do - found = PublicBody.find(:all, - :conditions => ["public_body_translations.url_name=?", name], - :joins => :translations, - :readonly => false) - # If many bodies are found (usually because the url_name is the same across - # locales) return any of them - return found.first if found.size >= 1 - - # If none found, then search the history of short names - old = PublicBody::Version.find_all_by_url_name(name) - # Find unique public bodies in it - old = old.map { |x| x.public_body_id } - old = old.uniq - # Maybe return the first one, so we show something relevant, - # rather than throwing an error? - raise "Two bodies with the same historical URL name: #{name}" if old.size > 1 - return unless old.size == 1 - # does acts_as_versioned provide a method that returns the current version? - return PublicBody.find(old.first) - end + found = PublicBody.find(:all, + :conditions => ["public_body_translations.url_name=?", name], + :joins => :translations, + :readonly => false) + # If many bodies are found (usually because the url_name is the same across + # locales) return any of them + return found.first if found.size >= 1 + + # If none found, then search the history of short names + old = PublicBody::Version.find_all_by_url_name(name) + # Find unique public bodies in it + old = old.map { |x| x.public_body_id } + old = old.uniq + # Maybe return the first one, so we show something relevant, + # rather than throwing an error? + raise "Two bodies with the same historical URL name: #{name}" if old.size > 1 + return unless old.size == 1 + # does acts_as_versioned provide a method that returns the current version? + return PublicBody.find(old.first) end # Set the first letter, which is used for faster queries before_save(:set_first_letter) def set_first_letter - # we use a regex to ensure it works with utf-8/multi-byte - self.first_letter = self.name.scan(/./mu)[0].upcase + PublicBody.set_first_letter(self) end # If tagged "not_apply", then FOI/EIR no longer applies to authority at all @@ -180,9 +193,13 @@ class PublicBody < ActiveRecord::Base end acts_as_versioned - self.non_versioned_columns << 'created_at' << 'updated_at' << 'first_letter' << 'api_key' << 'info_requests_count' + self.non_versioned_columns << 'created_at' << 'updated_at' << 'first_letter' << 'api_key' + self.non_versioned_columns << 'info_requests_count' << 'info_requests_successful_count' + self.non_versioned_columns << 'info_requests_count' << 'info_requests_visible_classified_count' + self.non_versioned_columns << 'info_requests_not_held_count' << 'info_requests_overdue' + self.non_versioned_columns << 'info_requests_overdue_count' + class Version - attr_accessor :created_at def last_edit_comment_for_html_display text = self.last_edit_comment.strip @@ -234,6 +251,7 @@ class PublicBody < ActiveRecord::Base def reindex_requested_from if self.changes.include?('url_name') for info_request in self.info_requests + for info_request_event in info_request.info_request_events info_request_event.xapian_mark_needs_index end @@ -243,13 +261,13 @@ class PublicBody < ActiveRecord::Base # When name or short name is changed, also change the url name def short_name=(short_name) - globalize.write(self.class.locale || I18n.locale, :short_name, short_name) + globalize.write(Globalize.locale, :short_name, short_name) self[:short_name] = short_name self.update_url_name end def name=(name) - globalize.write(self.class.locale || I18n.locale, :name, name) + globalize.write(Globalize.locale, :name, name) self[:name] = name self.update_url_name end @@ -329,22 +347,26 @@ class PublicBody < ActiveRecord::Base # The "internal admin" is a special body for internal use. def PublicBody.internal_admin_body - PublicBody.with_locale(I18n.default_locale) do - pb = PublicBody.find_by_url_name("internal_admin_authority") - if pb.nil? - pb = PublicBody.new( - :name => 'Internal admin authority', - :short_name => "", - :request_email => Configuration::contact_email, - :home_page => "", - :notes => "", - :publication_scheme => "", - :last_edit_editor => "internal_admin", - :last_edit_comment => "Made by PublicBody.internal_admin_body" - ) - pb.save! + # Use find_by_sql to avoid the search being specific to a + # locale, since url_name is a translated field: + sql = "SELECT * FROM public_bodies WHERE url_name = 'internal_admin_authority'" + matching_pbs = PublicBody.find_by_sql sql + case + when matching_pbs.empty? then + I18n.with_locale(I18n.default_locale) do + PublicBody.create!(:name => 'Internal admin authority', + :short_name => "", + :request_email => AlaveteliConfiguration::contact_email, + :home_page => "", + :notes => "", + :publication_scheme => "", + :last_edit_editor => "internal_admin", + :last_edit_comment => "Made by PublicBody.internal_admin_body") end - return pb + when matching_pbs.length == 1 then + matching_pbs[0] + else + raise "Multiple public bodies (#{matching_pbs.length}) found with url_name 'internal_admin_authority'" end end @@ -352,10 +374,24 @@ class PublicBody < ActiveRecord::Base class ImportCSVDryRun < StandardError end - # Import from CSV. Just tests things and returns messages if dry_run is true. - # Returns an array of [array of errors, array of notes]. If there are errors, - # always rolls back (as with dry_run). + # Import from a string in CSV format. + # Just tests things and returns messages if dry_run is true. + # Returns an array of [array of errors, array of notes]. If there + # are errors, always rolls back (as with dry_run). def self.import_csv(csv, tag, tag_behaviour, dry_run, editor, available_locales = []) + tmp_csv = nil + Tempfile.open('alaveteli') do |f| + f.write csv + tmp_csv = f + end + PublicBody.import_csv_from_file(tmp_csv.path, tag, tag_behaviour, dry_run, editor, available_locales) + end + + # Import from a CSV file. + # Just tests things and returns messages if dry_run is true. + # Returns an array of [array of errors, array of notes]. If there + # are errors, always rolls back (as with dry_run). + def self.import_csv_from_file(csv_filename, tag, tag_behaviour, dry_run, editor, available_locales = []) errors = [] notes = [] available_locales = [I18n.default_locale] if available_locales.empty? @@ -367,7 +403,7 @@ class PublicBody < ActiveRecord::Base # of updating them bodies_by_name = {} set_of_existing = Set.new() - PublicBody.with_locale(I18n.default_locale) do + I18n.with_locale(I18n.default_locale) do bodies = (tag.nil? || tag.empty?) ? PublicBody.find(:all) : PublicBody.find_by_tag(tag) for existing_body in bodies # Hide InternalAdminBody from import notes @@ -381,7 +417,8 @@ class PublicBody < ActiveRecord::Base set_of_importing = Set.new() field_names = { 'name'=>1, 'request_email'=>2 } # Default values in case no field list is given line = 0 - CSV.parse(csv) do |row| + + CSV.foreach(csv_filename) do |row| line = line + 1 # Parse the first line as a field list if it starts with '#' @@ -394,6 +431,8 @@ class PublicBody < ActiveRecord::Base fields = {} field_names.each{|name, i| fields[name] = row[i]} + yield line, fields if block_given? + name = row[field_names['name']] email = row[field_names['request_email']] next if name.nil? @@ -410,7 +449,7 @@ class PublicBody < ActiveRecord::Base if public_body = bodies_by_name[name] # Existing public body available_locales.each do |locale| - PublicBody.with_locale(locale) do + I18n.with_locale(locale) do changed = ActiveSupport::OrderedHash.new field_list.each do |field_name| localized_field_name = (locale.to_s == I18n.default_locale.to_s) ? field_name : "#{field_name}.#{locale}" @@ -445,7 +484,7 @@ class PublicBody < ActiveRecord::Base else # New public body public_body = PublicBody.new(:name=>"", :short_name=>"", :request_email=>"") available_locales.each do |locale| - PublicBody.with_locale(locale) do + I18n.with_locale(locale) do changed = ActiveSupport::OrderedHash.new field_list.each do |field_name| localized_field_name = (locale.to_s == I18n.default_locale.to_s) ? field_name : "#{field_name}.#{locale}" @@ -494,10 +533,8 @@ class PublicBody < ActiveRecord::Base end # Returns all public bodies (except for the internal admin authority) as csv - def self.export_csv - public_bodies = PublicBody.visible.find(:all, :order => 'url_name', - :include => [:translations, :tags]) - FasterCSV.generate() do |csv| + def self.export_csv(output_filename) + CSV.open(output_filename, "w") do |csv| csv << [ 'Name', 'Short name', @@ -512,7 +549,7 @@ class PublicBody < ActiveRecord::Base 'Updated at', 'Version', ] - public_bodies.each do |public_body| + PublicBody.visible.find_each(:include => [:translations, :tags]) do |public_body| # Skip bodies we use only for site admin next if public_body.has_tag?('site_administration') csv << [ @@ -551,7 +588,7 @@ class PublicBody < ActiveRecord::Base # Returns nil if configuration variable not set def override_request_email - e = Configuration::override_all_public_body_request_emails + e = AlaveteliConfiguration::override_all_public_body_request_emails e if e != "" end @@ -635,6 +672,104 @@ class PublicBody < ActiveRecord::Base end end + def self.where_clause_for_stats(minimum_requests, total_column) + # When producing statistics for public bodies, we want to + # exclude any that are tagged with 'test' - we use a + # sub-select to find the IDs of those public bodies. + test_tagged_query = "SELECT model_id FROM has_tag_string_tags" \ + " WHERE model = 'PublicBody' AND name = 'test'" + "#{total_column} >= #{minimum_requests} AND id NOT IN (#{test_tagged_query})" + end + + # Return data for the 'n' public bodies with the highest (or + # lowest) number of requests, but only returning data for those + # with at least 'minimum_requests' requests. + def self.get_request_totals(n, highest, minimum_requests) + ordering = "info_requests_count" + ordering += " DESC" if highest + where_clause = where_clause_for_stats minimum_requests, 'info_requests_count' + public_bodies = PublicBody.order(ordering).where(where_clause).limit(n) + public_bodies.reverse! if highest + y_values = public_bodies.map { |pb| pb.info_requests_count } + return { + 'public_bodies' => public_bodies, + 'y_values' => y_values, + 'y_max' => y_values.max, + 'totals' => y_values} + end + + # Return data for the 'n' public bodies with the highest (or + # lowest) score according to the metric of the value in 'column' + # divided by the total number of requests, expressed as a + # percentage. This only returns data for those public bodies with + # at least 'minimum_requests' requests. + def self.get_request_percentages(column, n, highest, minimum_requests) + total_column = "info_requests_visible_classified_count" + ordering = "y_value" + ordering += " DESC" if highest + y_value_column = "(cast(#{column} as float) / #{total_column})" + where_clause = where_clause_for_stats minimum_requests, total_column + where_clause += " AND #{column} IS NOT NULL" + public_bodies = PublicBody.select("*, #{y_value_column} AS y_value").order(ordering).where(where_clause).limit(n) + public_bodies.reverse! if highest + y_values = public_bodies.map { |pb| pb.y_value.to_f } + + original_values = public_bodies.map { |pb| pb.send(column) } + # If these are all nil, then probably the values have never + # been set; some have to be set by a rake task. In that case, + # just return nil: + return nil unless original_values.any? { |ov| !ov.nil? } + + original_totals = public_bodies.map { |pb| pb.send(total_column) } + # Calculate confidence intervals, as offsets from the proportion: + cis_below = [] + cis_above = [] + original_totals.each_with_index.map { |total, i| + lower_ci, higher_ci = ci_bounds original_values[i], total, 0.05 + cis_below.push(y_values[i] - lower_ci) + cis_above.push(higher_ci - y_values[i]) + } + # Turn the y values and confidence interval offsets into + # percentages: + [y_values, cis_below, cis_above].each { |l| + l.map! { |v| 100 * v } + } + return { + 'public_bodies' => public_bodies, + 'y_values' => y_values, + 'cis_below' => cis_below, + 'cis_above' => cis_above, + 'y_max' => 100, + 'totals' => original_totals} + end + def self.popular_bodies(locale) + # get some example searches and public bodies to display + # either from config, or based on a (slow!) query if not set + body_short_names = AlaveteliConfiguration::frontpage_publicbody_examples.split(/\s*;\s*/) + locale_condition = 'public_body_translations.locale = ?' + underscore_locale = locale.gsub '-', '_' + conditions = [locale_condition, underscore_locale] + bodies = [] + I18n.with_locale(locale) do + if body_short_names.empty? + # This is too slow + bodies = visible.find(:all, + :order => "info_requests_count desc", + :limit => 32, + :conditions => conditions, + :joins => :translations + ) + else + conditions[0] += " and public_bodies.url_name in (?)" + conditions << body_short_names + bodies = find(:all, :conditions => conditions, :joins => :translations) + end + end + return bodies + end + + private + def request_email_if_requestable # Request_email can be blank, meaning we don't have details if self.is_requestable? |