diff options
Diffstat (limited to 'app/models')
| -rw-r--r-- | app/models/foi_attachment.rb | 2 | ||||
| -rw-r--r-- | app/models/incoming_message.rb | 8 | ||||
| -rw-r--r-- | app/models/info_request.rb | 84 | ||||
| -rw-r--r-- | app/models/outgoing_message.rb | 2 | ||||
| -rw-r--r-- | app/models/public_body.rb | 124 | ||||
| -rw-r--r-- | app/models/user.rb | 2 |
6 files changed, 175 insertions, 47 deletions
diff --git a/app/models/foi_attachment.rb b/app/models/foi_attachment.rb index 914420a2b..acbfc8a34 100644 --- a/app/models/foi_attachment.rb +++ b/app/models/foi_attachment.rb @@ -69,7 +69,7 @@ class FoiAttachment < ActiveRecord::Base tries = 0 delay = 1 begin - binary_data = File.open(self.filepath, "rb" ).read + binary_data = File.open(self.filepath, "rb" ){ |file| file.read } if self.content_type =~ /^text/ @cached_body = convert_string_to_utf8_or_binary(binary_data, 'UTF-8') else diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb index 8b2aa87e7..bcf0b6ec9 100644 --- a/app/models/incoming_message.rb +++ b/app/models/incoming_message.rb @@ -129,15 +129,15 @@ class IncomingMessage < ActiveRecord::Base if (!force.nil? || self.last_parsed.nil?) ActiveRecord::Base.transaction do self.extract_attachments! - self.sent_at = self.mail.date || self.created_at - self.subject = self.mail.subject - self.mail_from = MailHandler.get_from_name(self.mail) + write_attribute(:sent_at, self.mail.date || self.created_at) + write_attribute(:subject, self.mail.subject) + write_attribute(:mail_from, MailHandler.get_from_name(self.mail)) if self.from_email self.mail_from_domain = PublicBody.extract_domain_from_email(self.from_email) else self.mail_from_domain = "" end - self.valid_to_reply_to = self._calculate_valid_to_reply_to + write_attribute(:valid_to_reply_to, self._calculate_valid_to_reply_to) self.last_parsed = Time.now self.foi_attachments reload=true self.save! diff --git a/app/models/info_request.rb b/app/models/info_request.rb index eba620f53..0a073dc79 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -1182,14 +1182,23 @@ public after_save :update_counter_cache after_destroy :update_counter_cache + # This method updates the count columns of the PublicBody that + # store the number of "not held", "to some extent successful" and + # "both visible and classified" requests when saving or destroying + # an InfoRequest associated with the body: def update_counter_cache PublicBody.skip_callback(:save, :after, :purge_in_cache) - self.public_body.info_requests_not_held_count = InfoRequest.where( - :public_body_id => self.public_body.id, - :described_state => 'not_held').count - self.public_body.info_requests_successful_count = InfoRequest.where( - :public_body_id => self.public_body.id, - :described_state => ['successful', 'partially_successful']).count + basic_params = { + :public_body_id => self.public_body_id, + :awaiting_description => false, + :prominence => 'normal' + } + [['info_requests_not_held_count', {:described_state => 'not_held'}], + ['info_requests_successful_count', {:described_state => ['successful', 'partially_successful']}], + ['info_requests_visible_classified_count', {}]].each do |column, extra_params| + params = basic_params.clone.update extra_params + self.public_body.send "#{column}=", InfoRequest.where(params).count + end self.public_body.without_revision do public_body.no_xapian_reindex = true public_body.save @@ -1203,6 +1212,69 @@ public end end + + # Get requests that have similar important terms + def similar_requests(limit=10) + xapian_similar = nil + xapian_similar_more = false + begin + xapian_similar = ActsAsXapian::Similar.new([InfoRequestEvent], + info_request_events, + :limit => limit, + :collapse_by_prefix => 'request_collapse') + xapian_similar_more = (xapian_similar.matches_estimated > limit) + rescue + end + return [xapian_similar, xapian_similar_more] + end + + def InfoRequest.recent_requests + request_events = [] + request_events_all_successful = false + # Get some successful requests + begin + query = 'variety:response (status:successful OR status:partially_successful)' + sortby = "newest" + max_count = 5 + + xapian_object = ActsAsXapian::Search.new([InfoRequestEvent], + query, + :offset => 0, + :limit => 5, + :sort_by_prefix => 'created_at', + :sort_by_ascending => true, + :collapse_by_prefix => 'request_title_collapse' + ) + xapian_object.results + request_events = xapian_object.results.map { |r| r[:model] } + + # If there are not yet enough successful requests, fill out the list with + # other requests + if request_events.count < max_count + query = 'variety:sent' + xapian_object = ActsAsXapian::Search.new([InfoRequestEvent], + query, + :offset => 0, + :limit => max_count-request_events.count, + :sort_by_prefix => 'created_at', + :sort_by_ascending => true, + :collapse_by_prefix => 'request_title_collapse' + ) + xapian_object.results + more_events = xapian_object.results.map { |r| r[:model] } + request_events += more_events + # Overall we still want the list sorted with the newest first + request_events.sort!{|e1,e2| e2.created_at <=> e1.created_at} + else + request_events_all_successful = true + end + rescue + request_events = [] + end + + return [request_events, request_events_all_successful] + end + private def set_defaults diff --git a/app/models/outgoing_message.rb b/app/models/outgoing_message.rb index e2ee696c5..6efc1d2ba 100644 --- a/app/models/outgoing_message.rb +++ b/app/models/outgoing_message.rb @@ -12,6 +12,8 @@ # last_sent_at :datetime # incoming_message_followup_id :integer # what_doing :string(255) not null +# prominence :string(255) default("normal"), not null +# prominence_reason :text # # models/outgoing_message.rb: diff --git a/app/models/public_body.rb b/app/models/public_body.rb index 485a794b0..933825d2a 100644 --- a/app/models/public_body.rb +++ b/app/models/public_body.rb @@ -3,23 +3,27 @@ # # Table name: public_bodies # -# id :integer not null, primary key -# name :text not null -# short_name :text not null -# request_email :text not null -# version :integer not null -# last_edit_editor :string(255) not null -# last_edit_comment :text not null -# created_at :datetime not null -# updated_at :datetime not null -# url_name :text not null -# home_page :text default(""), not null -# notes :text default(""), not null -# first_letter :string(255) not null -# publication_scheme :text default(""), not null -# api_key :string(255) not null -# info_requests_count :integer default(0), not null -# disclosure_log :text default(""), not null +# id :integer not null, primary key +# name :text not null +# short_name :text not null +# request_email :text not null +# version :integer not null +# last_edit_editor :string(255) not null +# last_edit_comment :text not null +# created_at :datetime not null +# updated_at :datetime not null +# url_name :text not null +# home_page :text default(""), not null +# notes :text default(""), not null +# first_letter :string(255) not null +# publication_scheme :text default(""), not null +# api_key :string(255) not null +# info_requests_count :integer default(0), not null +# disclosure_log :text default(""), not null +# info_requests_successful_count :integer +# info_requests_not_held_count :integer +# info_requests_overdue_count :integer +# info_requests_visible_classified_count :integer # require 'csv' @@ -190,11 +194,11 @@ class PublicBody < ActiveRecord::Base acts_as_versioned self.non_versioned_columns << 'created_at' << 'updated_at' << 'first_letter' << 'api_key' self.non_versioned_columns << 'info_requests_count' << 'info_requests_successful_count' + self.non_versioned_columns << 'info_requests_count' << 'info_requests_visible_classified_count' self.non_versioned_columns << 'info_requests_not_held_count' << 'info_requests_overdue' self.non_versioned_columns << 'info_requests_overdue_count' class Version - attr_accessor :created_at def last_edit_comment_for_html_display text = self.last_edit_comment.strip @@ -256,13 +260,13 @@ class PublicBody < ActiveRecord::Base # When name or short name is changed, also change the url name def short_name=(short_name) - globalize.write(I18n.locale, :short_name, short_name) + globalize.write(Globalize.locale, :short_name, short_name) self[:short_name] = short_name self.update_url_name end def name=(name) - globalize.write(I18n.locale, :name, name) + globalize.write(Globalize.locale, :name, name) self[:name] = name self.update_url_name end @@ -365,10 +369,24 @@ class PublicBody < ActiveRecord::Base class ImportCSVDryRun < StandardError end - # Import from CSV. Just tests things and returns messages if dry_run is true. - # Returns an array of [array of errors, array of notes]. If there are errors, - # always rolls back (as with dry_run). + # Import from a string in CSV format. + # Just tests things and returns messages if dry_run is true. + # Returns an array of [array of errors, array of notes]. If there + # are errors, always rolls back (as with dry_run). def self.import_csv(csv, tag, tag_behaviour, dry_run, editor, available_locales = []) + tmp_csv = nil + Tempfile.open('alaveteli') do |f| + f.write csv + tmp_csv = f + end + PublicBody.import_csv_from_file(tmp_csv.path, tag, tag_behaviour, dry_run, editor, available_locales) + end + + # Import from a CSV file. + # Just tests things and returns messages if dry_run is true. + # Returns an array of [array of errors, array of notes]. If there + # are errors, always rolls back (as with dry_run). + def self.import_csv_from_file(csv_filename, tag, tag_behaviour, dry_run, editor, available_locales = []) errors = [] notes = [] available_locales = [I18n.default_locale] if available_locales.empty? @@ -394,7 +412,8 @@ class PublicBody < ActiveRecord::Base set_of_importing = Set.new() field_names = { 'name'=>1, 'request_email'=>2 } # Default values in case no field list is given line = 0 - CSV.parse(csv) do |row| + + CSV.foreach(csv_filename) do |row| line = line + 1 # Parse the first line as a field list if it starts with '#' @@ -407,6 +426,8 @@ class PublicBody < ActiveRecord::Base fields = {} field_names.each{|name, i| fields[name] = row[i]} + yield line, fields if block_given? + name = row[field_names['name']] email = row[field_names['request_email']] next if name.nil? @@ -507,10 +528,8 @@ class PublicBody < ActiveRecord::Base end # Returns all public bodies (except for the internal admin authority) as csv - def self.export_csv - public_bodies = PublicBody.visible.find(:all, :order => 'url_name', - :include => [:translations, :tags]) - FasterCSV.generate() do |csv| + def self.export_csv(output_filename) + CSV.open(output_filename, "w") do |csv| csv << [ 'Name', 'Short name', @@ -525,7 +544,7 @@ class PublicBody < ActiveRecord::Base 'Updated at', 'Version', ] - public_bodies.each do |public_body| + PublicBody.visible.find_each(:include => [:translations, :tags]) do |public_body| # Skip bodies we use only for site admin next if public_body.has_tag?('site_administration') csv << [ @@ -648,20 +667,30 @@ class PublicBody < ActiveRecord::Base end end + def self.where_clause_for_stats(minimum_requests, total_column) + # When producing statistics for public bodies, we want to + # exclude any that are tagged with 'test' - we use a + # sub-select to find the IDs of those public bodies. + test_tagged_query = "SELECT model_id FROM has_tag_string_tags" \ + " WHERE model = 'PublicBody' AND name = 'test'" + "#{total_column} >= #{minimum_requests} AND id NOT IN (#{test_tagged_query})" + end + # Return data for the 'n' public bodies with the highest (or # lowest) number of requests, but only returning data for those # with at least 'minimum_requests' requests. def self.get_request_totals(n, highest, minimum_requests) ordering = "info_requests_count" ordering += " DESC" if highest - where_clause = "info_requests_count >= #{minimum_requests}" + where_clause = where_clause_for_stats minimum_requests, 'info_requests_count' public_bodies = PublicBody.order(ordering).where(where_clause).limit(n) public_bodies.reverse! if highest y_values = public_bodies.map { |pb| pb.info_requests_count } return { 'public_bodies' => public_bodies, 'y_values' => y_values, - 'y_max' => y_values.max} + 'y_max' => y_values.max, + 'totals' => y_values} end # Return data for the 'n' public bodies with the highest (or @@ -670,11 +699,12 @@ class PublicBody < ActiveRecord::Base # percentage. This only returns data for those public bodies with # at least 'minimum_requests' requests. def self.get_request_percentages(column, n, highest, minimum_requests) - total_column = "info_requests_count" + total_column = "info_requests_visible_classified_count" ordering = "y_value" ordering += " DESC" if highest y_value_column = "(cast(#{column} as float) / #{total_column})" - where_clause = "#{total_column} >= #{minimum_requests} AND #{column} IS NOT NULL" + where_clause = where_clause_for_stats minimum_requests, total_column + where_clause += " AND #{column} IS NOT NULL" public_bodies = PublicBody.select("*, #{y_value_column} AS y_value").order(ordering).where(where_clause).limit(n) public_bodies.reverse! if highest y_values = public_bodies.map { |pb| pb.y_value.to_f } @@ -704,7 +734,33 @@ class PublicBody < ActiveRecord::Base 'y_values' => y_values, 'cis_below' => cis_below, 'cis_above' => cis_above, - 'y_max' => 100} + 'y_max' => 100, + 'totals' => original_totals} + end + def self.popular_bodies(locale) + # get some example searches and public bodies to display + # either from config, or based on a (slow!) query if not set + body_short_names = AlaveteliConfiguration::frontpage_publicbody_examples.split(/\s*;\s*/) + locale_condition = 'public_body_translations.locale = ?' + underscore_locale = locale.gsub '-', '_' + conditions = [locale_condition, underscore_locale] + bodies = [] + I18n.with_locale(locale) do + if body_short_names.empty? + # This is too slow + bodies = visible.find(:all, + :order => "info_requests_count desc", + :limit => 32, + :conditions => conditions, + :joins => :translations + ) + else + conditions[0] += " and public_bodies.url_name in (?)" + conditions << body_short_names + bodies = find(:all, :conditions => conditions, :joins => :translations) + end + end + return bodies end private diff --git a/app/models/user.rb b/app/models/user.rb index d7c1c854e..2c4f87944 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -20,8 +20,6 @@ # email_bounce_message :text default(""), not null # no_limit :boolean default(FALSE), not null # receive_email_alerts :boolean default(TRUE), not null -# address :string(255) -# dob :date # require 'digest/sha1' |