diff options
Diffstat (limited to 'app/models')
| -rw-r--r-- | app/models/censor_rule.rb | 51 | ||||
| -rw-r--r-- | app/models/exim_log.rb | 2 | ||||
| -rw-r--r-- | app/models/foi_attachment.rb | 23 | ||||
| -rw-r--r-- | app/models/incoming_message.rb | 16 | ||||
| -rw-r--r-- | app/models/info_request.rb | 71 | ||||
| -rw-r--r-- | app/models/public_body.rb | 13 | ||||
| -rw-r--r-- | app/models/request_mailer.rb | 7 | ||||
| -rw-r--r-- | app/models/user.rb | 3 |
8 files changed, 107 insertions, 79 deletions
diff --git a/app/models/censor_rule.rb b/app/models/censor_rule.rb index a477d2568..da3f49760 100644 --- a/app/models/censor_rule.rb +++ b/app/models/censor_rule.rb @@ -9,6 +9,7 @@ # public_body_id :integer # text :text not null # replacement :text not null +# regexp :boolean # last_edit_editor :string(255) not null # last_edit_comment :text not null # created_at :datetime not null @@ -28,33 +29,59 @@ class CensorRule < ActiveRecord::Base belongs_to :user belongs_to :public_body - def binary_replacement - self.text.gsub(/./, 'x') + # a flag to allow the require_user_request_or_public_body validation to be skipped + attr_accessor :allow_global + validate :require_user_request_or_public_body, :unless => proc{ |rule| rule.allow_global == true } + validate :require_valid_regexp, :if => proc{ |rule| rule.regexp? == true } + validates_presence_of :text + + named_scope :global, {:conditions => {:info_request_id => nil, + :user_id => nil, + :public_body_id => nil}} + + def require_user_request_or_public_body + if self.info_request.nil? && self.user.nil? && self.public_body.nil? + errors.add("Censor must apply to an info request a user or a body; ") + end + end + + def require_valid_regexp + begin + self.make_regexp() + rescue RegexpError => e + errors.add(:text, e.message) + end + end + + def make_regexp + return Regexp.new(self.text, Regexp::MULTILINE) end def apply_to_text!(text) if text.nil? return nil end - text.gsub!(self.text, self.replacement) + to_replace = regexp? ? self.make_regexp() : self.text + text.gsub!(to_replace, self.replacement) end + def apply_to_binary!(binary) if binary.nil? return nil end - binary.gsub!(self.text, self.binary_replacement) + to_replace = regexp? ? self.make_regexp() : self.text + binary.gsub!(to_replace){ |match| match.gsub(/./, 'x') } end - - def validate - if self.info_request.nil? && self.user.nil? && self.public_body.nil? - errors.add("Censor must apply to an info request a user or a body; ") + def for_admin_column + self.class.content_columns.each do |column| + yield(column.human_name, self.send(column.name), column.type.to_s, column.name) end end - def for_admin_column - self.class.content_columns.each do |column| - yield(column.human_name, self.send(column.name), column.type.to_s, column.name) + def is_global? + return true if (info_request_id.nil? && user_id.nil? && public_body_id.nil?) + return false end - end + end diff --git a/app/models/exim_log.rb b/app/models/exim_log.rb index 60faa7f0b..82000efa1 100644 --- a/app/models/exim_log.rb +++ b/app/models/exim_log.rb @@ -94,7 +94,7 @@ class EximLog < ActiveRecord::Base # Get all requests sent for from 2 to 10 days ago. The 2 day gap is # because we load exim log lines via cron at best an hour after they # are made) - irs = InfoRequest.find(:all, :conditions => [ "created_at < ? and created_at > ?", Time.now() - 2.day, Time.now() - 10.days ] ) + irs = InfoRequest.find(:all, :conditions => [ "created_at < ? and created_at > ? and user_id is not null", Time.now() - 2.day, Time.now() - 10.days ] ) # Go through each request and check it ok = true diff --git a/app/models/foi_attachment.rb b/app/models/foi_attachment.rb index 9bbf0988f..a40898aef 100644 --- a/app/models/foi_attachment.rb +++ b/app/models/foi_attachment.rb @@ -1,3 +1,5 @@ +# encoding: UTF-8 + # == Schema Information # Schema version: 114 # @@ -14,8 +16,6 @@ # hexdigest :string(32) # -# encoding: UTF-8 - # models/foi_attachment.rb: # An attachment to an email (IncomingMessage) # @@ -315,14 +315,21 @@ class FoiAttachment < ActiveRecord::Base tempfile.print self.body tempfile.flush + html = nil if self.content_type == 'application/pdf' - html = AlaveteliExternalCommand.run("pdftohtml", "-nodrm", "-zoom", "1.0", "-stdout", "-enc", "UTF-8", "-noframes", tempfile.path) + # We set a timeout here, because pdftohtml can spiral out of control + # on some PDF files and we don’t want to crash the whole server. + html = AlaveteliExternalCommand.run("pdftohtml", "-nodrm", "-zoom", "1.0", "-stdout", "-enc", "UTF-8", "-noframes", tempfile.path, :timeout => 30) elsif self.content_type == 'application/rtf' - html = AlaveteliExternalCommand.run("unrtf", "--html", tempfile.path) - elsif self.has_google_docs_viewer? - html = '' # force error and using Google docs viewer - else - raise "No HTML conversion available for type " + self.content_type + html = AlaveteliExternalCommand.run("unrtf", "--html", tempfile.path, :timeout => 120) + end + + if html.nil? + if self.has_google_docs_viewer? + html = '' # force error and using Google docs viewer + else + raise "No HTML conversion available for type " + self.content_type + end end tempfile.close diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb index 593590fb8..13fc316cd 100644 --- a/app/models/incoming_message.rb +++ b/app/models/incoming_message.rb @@ -375,25 +375,10 @@ class IncomingMessage < ActiveRecord::Base # http://www.whatdotheyknow.com/request/common_purpose_training_graduate#incoming-774 text.gsub!(/(Mobile|Mob)([\s\/]*(Fax|Tel))*\s*:?[\s\d]*\d/, "[mobile number]") - # Specific removals # XXX remove these and turn them into censor rules in database - # http://www.whatdotheyknow.com/request/total_number_of_objects_in_the_n_6 - text.gsub!(/\*\*\*+\nPolly Tucker.*/ms, "") - # http://www.whatdotheyknow.com/request/cctv_data_retention_and_use - text.gsub!(/Andy 079.*/, "Andy [mobile number]") - # http://www.whatdotheyknow.com/request/how_do_the_pct_deal_with_retirin_113 - text.gsub!(/(Complaints and Corporate Affairs Officer)\s+Westminster Primary Care Trust.+/ms, "\\1") - # Remove WhatDoTheyKnow signup links domain = MySociety::Config.get('DOMAIN') text.gsub!(/http:\/\/#{domain}\/c\/[^\s]+/, "[WDTK login link]") - # Remove Home Office survey links - # e.g. http://www.whatdotheyknow.com/request/serious_crime_act_2007_section_7#incoming-12650 - if self.info_request.public_body.url_name == 'home_office' - text.gsub!(/Your password:-\s+[^\s]+/, '[password]') - text.gsub!(/Password=[^\s]+/, '[password]') - end - # Remove things from censor rules self.info_request.apply_censor_rules_to_text!(text) end @@ -599,7 +584,6 @@ class IncomingMessage < ActiveRecord::Base # Remove existing quoted sections folded_quoted_text = self.remove_lotus_quoting(text, 'FOLDED_QUOTED_SECTION') folded_quoted_text = IncomingMessage.remove_quoted_sections(text, "FOLDED_QUOTED_SECTION") - self.cached_main_body_text_unfolded = text self.cached_main_body_text_folded = folded_quoted_text self.save! diff --git a/app/models/info_request.rb b/app/models/info_request.rb index d09acbcf6..6f472c290 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -1,11 +1,10 @@ # == Schema Information -# Schema version: 114 # # Table name: info_requests # # id :integer not null, primary key # title :text not null -# user_id :integer not null +# user_id :integer # public_body_id :integer not null # created_at :datetime not null # updated_at :datetime not null @@ -17,10 +16,11 @@ # allow_new_responses_from :string(255) default("anybody"), not null # handle_rejected_responses :string(255) default("bounce"), not null # idhash :string(255) not null +# external_user_name :string(255) +# external_url :string(255) # attention_requested :boolean default(FALSE) # - require 'digest/sha1' class InfoRequest < ActiveRecord::Base @@ -104,7 +104,7 @@ class InfoRequest < ActiveRecord::Base errors.add(:described_state, "is not a valid state") if !InfoRequest.enumerate_states.include? described_state end - + # The request must either be internal, in which case it has # a foreign key reference to a User object and no external_url or external_user_name, # or else be external in which case it has no user_id but does have an external_url, @@ -120,15 +120,15 @@ class InfoRequest < ActiveRecord::Base errors.add(:external_url, "must be null for an internal request") if !external_url.nil? end end - + def is_external? !external_url.nil? end - + def user_name is_external? ? external_user_name : user.name end - + def user_name_slug if is_external? if external_user_name.nil? @@ -136,7 +136,7 @@ class InfoRequest < ActiveRecord::Base else fake_slug = external_user_name.parameterize end - public_body.url_name + "_"+fake_slug + (public_body.url_name || "") + "_" + fake_slug else user.url_name end @@ -223,7 +223,7 @@ class InfoRequest < ActiveRecord::Base incoming_message.clear_in_database_caches! end end - + # For debugging def InfoRequest.profile_search(query) t = Time.now.usec @@ -246,7 +246,9 @@ public # For request with same title as others, add on arbitary numeric identifier unique_url_title = url_title suffix_num = 2 # as there's already one without numeric suffix - while not InfoRequest.find_by_url_title(unique_url_title, :conditions => self.id.nil? ? nil : ["id <> ?", self.id] ).nil? + while not InfoRequest.find_by_url_title(unique_url_title, + :conditions => self.id.nil? ? nil : ["id <> ?", self.id] + ).nil? unique_url_title = url_title + "_" + suffix_num.to_s suffix_num = suffix_num + 1 end @@ -456,7 +458,7 @@ public if !allow if self.handle_rejected_responses == 'bounce' - RequestMailer.deliver_stopped_responses(self, email, raw_email_data) + RequestMailer.deliver_stopped_responses(self, email, raw_email_data) if !is_external? elsif self.handle_rejected_responses == 'holding_pen' InfoRequest.holding_pen_request.receive(email, raw_email_data, false, reason) elsif self.handle_rejected_responses == 'blackhole' @@ -566,7 +568,10 @@ public self.calculate_event_states if self.requires_admin? - RequestMailer.deliver_requires_admin(self, set_by) + # Check there is someone to send the message "from" + if !set_by.nil? || !self.user.nil? + RequestMailer.deliver_requires_admin(self, set_by) + end end end @@ -708,10 +713,10 @@ public return self.public_body.is_followupable? end def recipient_name_and_email - return TMail::Address.address_from_name_and_email( - _("{{law_used}} requests at {{public_body}}", - :law_used => self.law_used_short, - :public_body => self.public_body.short_or_long_name), + return TMail::Address.address_from_name_and_email( + _("{{law_used}} requests at {{public_body}}", + :law_used => self.law_used_short, + :public_body => self.public_body.short_or_long_name), self.recipient_email).to_s end @@ -942,7 +947,7 @@ public last_response_created_at = last_event_time_clause('response') age = extra_params[:age_in_days] ? extra_params[:age_in_days].days : OLD_AGE_IN_DAYS params = {:select => "*, #{last_response_created_at} as last_response_time", - :conditions => ["awaiting_description = ? and #{last_response_created_at} < ? and url_title != 'holding_pen'", + :conditions => ["awaiting_description = ? and #{last_response_created_at} < ? and url_title != 'holding_pen' and user_id is not null", true, Time.now() - age], :order => "last_response_time"} params[:limit] = extra_params[:limit] if extra_params[:limit] @@ -960,6 +965,7 @@ public end def is_old_unclassified? + return false if is_external? return false if !awaiting_description return false if url_title == 'holding_pen' last_response_event = get_last_response_event @@ -995,27 +1001,28 @@ public return ret.reverse end + # Get the list of censor rules that apply to this request + def applicable_censor_rules + applicable_rules = [self.censor_rules, self.public_body.censor_rules, CensorRule.global.all] + if self.user && !self.user.censor_rules.empty? + applicable_rules << self.user.censor_rules + end + return applicable_rules.flatten + end + # Call groups of censor rules def apply_censor_rules_to_text!(text) - for censor_rule in self.censor_rules + self.applicable_censor_rules.each do |censor_rule| censor_rule.apply_to_text!(text) end - if self.user # requests during construction have no user - for censor_rule in self.user.censor_rules - censor_rule.apply_to_text!(text) - end - end + return text end def apply_censor_rules_to_binary!(binary) - for censor_rule in self.censor_rules + self.applicable_censor_rules.each do |censor_rule| censor_rule.apply_to_binary!(binary) end - if self.user # requests during construction have no user - for censor_rule in self.user.censor_rules - censor_rule.apply_to_binary!(binary) - end - end + return binary end def is_owning_user?(user) @@ -1035,6 +1042,12 @@ public return true end + # Is this request visible to everyone? + def all_can_view? + return true if ['normal', 'backpage'].include?(self.prominence) + return false + end + def indexed_by_search? if self.prominence == 'backpage' || self.prominence == 'hidden' || self.prominence == 'requester_only' return false diff --git a/app/models/public_body.rb b/app/models/public_body.rb index bc8f084bb..fb30da234 100644 --- a/app/models/public_body.rb +++ b/app/models/public_body.rb @@ -1,5 +1,5 @@ +# -*- coding: utf-8 -*- # == Schema Information -# Schema version: 114 # # Table name: public_bodies # @@ -19,7 +19,6 @@ # publication_scheme :text default(""), not null # api_key :string(255) not null # - # models/public_body.rb: # A public body, from which information can be requested. # @@ -43,6 +42,7 @@ class PublicBody < ActiveRecord::Base has_many :info_requests, :order => 'created_at desc' has_many :track_things, :order => 'created_at desc' + has_many :censor_rules, :order => 'created_at desc' has_tag_string @@ -93,8 +93,9 @@ class PublicBody < ActiveRecord::Base # Make sure publication_scheme gets the correct default value. # (This would work automatically, were publication_scheme not a translated attribute) self.publication_scheme = "" if self.publication_scheme.nil? - - # Set an API key if there isn’t one + end + + def before_save self.api_key = SecureRandom.base64(33) if self.api_key.nil? end @@ -103,7 +104,7 @@ class PublicBody < ActiveRecord::Base locale = self.locale || I18n.locale PublicBody.with_locale(locale) do found = PublicBody.find(:all, - :conditions => ["public_body_translations.url_name='#{name}'"], + :conditions => ["public_body_translations.url_name=?", name], :joins => :translations, :readonly => false) # If many bodies are found (usually because the url_name is the same across @@ -583,5 +584,3 @@ class PublicBody < ActiveRecord::Base end end - - diff --git a/app/models/request_mailer.rb b/app/models/request_mailer.rb index 03d26f237..ba9285fc6 100644 --- a/app/models/request_mailer.rb +++ b/app/models/request_mailer.rb @@ -28,17 +28,17 @@ class RequestMailer < ApplicationMailer :filename => attachment_name end end - + # Used when a response is uploaded using the API def external_response(info_request, body, sent_at, attachments) @from = blackhole_email @recipients = info_request.incoming_name_and_email @body = { :body => body } - + # ActionMailer only works properly when the time is in the local timezone: # see https://rails.lighthouseapp.com/projects/8994/tickets/3113-actionmailer-only-works-correctly-with-sent_on-times-that-are-in-the-local-time-zone @sent_on = sent_at.dup.localtime - + attachments.each do |attachment_hash| attachment attachment_hash end @@ -392,6 +392,7 @@ class RequestMailer < ApplicationMailer ) for info_request in info_requests + next if info_request.is_external? # Count number of new comments to alert on earliest_unalerted_comment_event = nil last_comment_event = nil diff --git a/app/models/user.rb b/app/models/user.rb index a21676f68..657ea2a4a 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -1,5 +1,4 @@ # == Schema Information -# Schema version: 114 # # Table name: users # @@ -21,9 +20,7 @@ # email_bounce_message :text default(""), not null # no_limit :boolean default(FALSE), not null # receive_email_alerts :boolean default(TRUE), not null -# user_similarity_id :integer # - # models/user.rb: # Model of people who use the site to file requests, make comments etc. # |