From b3f46a4232e244e19c9dc1345c37322ea4a091da Mon Sep 17 00:00:00 2001 From: Robin Houston Date: Wed, 4 Jul 2012 11:27:43 +0100 Subject: WIP new requests feed --- app/models/info_request.rb | 6 +++--- app/models/public_body.rb | 4 ---- app/models/user.rb | 3 --- 3 files changed, 3 insertions(+), 10 deletions(-) (limited to 'app/models') diff --git a/app/models/info_request.rb b/app/models/info_request.rb index d09acbcf6..a41d6d2db 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -1,11 +1,10 @@ # == Schema Information -# Schema version: 114 # # Table name: info_requests # # id :integer not null, primary key # title :text not null -# user_id :integer not null +# user_id :integer # public_body_id :integer not null # created_at :datetime not null # updated_at :datetime not null @@ -17,10 +16,11 @@ # allow_new_responses_from :string(255) default("anybody"), not null # handle_rejected_responses :string(255) default("bounce"), not null # idhash :string(255) not null +# external_user_name :string(255) +# external_url :string(255) # attention_requested :boolean default(FALSE) # - require 'digest/sha1' class InfoRequest < ActiveRecord::Base diff --git a/app/models/public_body.rb b/app/models/public_body.rb index bc8f084bb..9efeadf55 100644 --- a/app/models/public_body.rb +++ b/app/models/public_body.rb @@ -1,5 +1,4 @@ # == Schema Information -# Schema version: 114 # # Table name: public_bodies # @@ -19,7 +18,6 @@ # publication_scheme :text default(""), not null # api_key :string(255) not null # - # models/public_body.rb: # A public body, from which information can be requested. # @@ -583,5 +581,3 @@ class PublicBody < ActiveRecord::Base end end - - diff --git a/app/models/user.rb b/app/models/user.rb index a21676f68..657ea2a4a 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -1,5 +1,4 @@ # == Schema Information -# Schema version: 114 # # Table name: users # @@ -21,9 +20,7 @@ # email_bounce_message :text default(""), not null # no_limit :boolean default(FALSE), not null # receive_email_alerts :boolean default(TRUE), not null -# user_similarity_id :integer # - # models/user.rb: # Model of people who use the site to file requests, make comments etc. # -- cgit v1.2.3 From 088bc961328f4d876971994102cde52c1ad49246 Mon Sep 17 00:00:00 2001 From: Seb Bacon Date: Thu, 12 Jul 2012 13:04:27 +0100 Subject: Support regular expressions in CensorRules; also support 'global' CensorRules that aren't attached to a User or Request or Public Body (but don't expose this in the admin UI). Fixes #33 --- app/models/censor_rule.rb | 10 +++++++--- app/models/incoming_message.rb | 16 ---------------- app/models/info_request.rb | 9 +++------ 3 files changed, 10 insertions(+), 25 deletions(-) (limited to 'app/models') diff --git a/app/models/censor_rule.rb b/app/models/censor_rule.rb index a477d2568..cedbd767e 100644 --- a/app/models/censor_rule.rb +++ b/app/models/censor_rule.rb @@ -9,6 +9,7 @@ # public_body_id :integer # text :text not null # replacement :text not null +# regexp :boolean # last_edit_editor :string(255) not null # last_edit_comment :text not null # created_at :datetime not null @@ -28,6 +29,8 @@ class CensorRule < ActiveRecord::Base belongs_to :user belongs_to :public_body + named_scope :regexps, {:conditions => {:regexp => true}} + def binary_replacement self.text.gsub(/./, 'x') end @@ -36,8 +39,10 @@ class CensorRule < ActiveRecord::Base if text.nil? return nil end - text.gsub!(self.text, self.replacement) + to_replace = regexp? ? Regexp.new(self.text, Regexp::MULTILINE) : self.text + text.gsub!(to_replace, self.replacement) end + def apply_to_binary!(binary) if binary.nil? return nil @@ -45,9 +50,8 @@ class CensorRule < ActiveRecord::Base binary.gsub!(self.text, self.binary_replacement) end - def validate - if self.info_request.nil? && self.user.nil? && self.public_body.nil? + if !self.regexp? && self.info_request.nil? && self.user.nil? && self.public_body.nil? errors.add("Censor must apply to an info request a user or a body; ") end end diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb index 593590fb8..13fc316cd 100644 --- a/app/models/incoming_message.rb +++ b/app/models/incoming_message.rb @@ -375,25 +375,10 @@ class IncomingMessage < ActiveRecord::Base # http://www.whatdotheyknow.com/request/common_purpose_training_graduate#incoming-774 text.gsub!(/(Mobile|Mob)([\s\/]*(Fax|Tel))*\s*:?[\s\d]*\d/, "[mobile number]") - # Specific removals # XXX remove these and turn them into censor rules in database - # http://www.whatdotheyknow.com/request/total_number_of_objects_in_the_n_6 - text.gsub!(/\*\*\*+\nPolly Tucker.*/ms, "") - # http://www.whatdotheyknow.com/request/cctv_data_retention_and_use - text.gsub!(/Andy 079.*/, "Andy [mobile number]") - # http://www.whatdotheyknow.com/request/how_do_the_pct_deal_with_retirin_113 - text.gsub!(/(Complaints and Corporate Affairs Officer)\s+Westminster Primary Care Trust.+/ms, "\\1") - # Remove WhatDoTheyKnow signup links domain = MySociety::Config.get('DOMAIN') text.gsub!(/http:\/\/#{domain}\/c\/[^\s]+/, "[WDTK login link]") - # Remove Home Office survey links - # e.g. http://www.whatdotheyknow.com/request/serious_crime_act_2007_section_7#incoming-12650 - if self.info_request.public_body.url_name == 'home_office' - text.gsub!(/Your password:-\s+[^\s]+/, '[password]') - text.gsub!(/Password=[^\s]+/, '[password]') - end - # Remove things from censor rules self.info_request.apply_censor_rules_to_text!(text) end @@ -599,7 +584,6 @@ class IncomingMessage < ActiveRecord::Base # Remove existing quoted sections folded_quoted_text = self.remove_lotus_quoting(text, 'FOLDED_QUOTED_SECTION') folded_quoted_text = IncomingMessage.remove_quoted_sections(text, "FOLDED_QUOTED_SECTION") - self.cached_main_body_text_unfolded = text self.cached_main_body_text_folded = folded_quoted_text self.save! diff --git a/app/models/info_request.rb b/app/models/info_request.rb index a41d6d2db..d73dd6c70 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -997,14 +997,11 @@ public # Call groups of censor rules def apply_censor_rules_to_text!(text) - for censor_rule in self.censor_rules - censor_rule.apply_to_text!(text) - end - if self.user # requests during construction have no user - for censor_rule in self.user.censor_rules + [self.censor_rules, self.user.try(:censor_rules), + CensorRule.regexps.all].flatten.compact.each do |censor_rule| censor_rule.apply_to_text!(text) end - end + return text end def apply_censor_rules_to_binary!(binary) -- cgit v1.2.3 From a3bee933105768f12e13fd1334ef12c159183212 Mon Sep 17 00:00:00 2001 From: Seb Bacon Date: Thu, 12 Jul 2012 13:10:27 +0100 Subject: Fix bug that only occurs when loading sample data - sometimes an InfoRequest doesn't have a PublicBody at the time its slug is being calculated. --- app/models/info_request.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/info_request.rb b/app/models/info_request.rb index d73dd6c70..4c8181faa 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -136,7 +136,7 @@ class InfoRequest < ActiveRecord::Base else fake_slug = external_user_name.parameterize end - public_body.url_name + "_"+fake_slug + (public_body.url_name || "") + "_" + fake_slug else user.url_name end -- cgit v1.2.3 From 3ef1dd4e9f4c5c184910602c8133f81d0b0f61dc Mon Sep 17 00:00:00 2001 From: Seb Bacon Date: Thu, 19 Jul 2012 13:06:53 +0100 Subject: Set api_key in before_save rather than after_initialize (to avoid errors when uploading via CSV) --- app/models/public_body.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/public_body.rb b/app/models/public_body.rb index 9efeadf55..97802c7e8 100644 --- a/app/models/public_body.rb +++ b/app/models/public_body.rb @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # == Schema Information # # Table name: public_bodies @@ -91,8 +92,9 @@ class PublicBody < ActiveRecord::Base # Make sure publication_scheme gets the correct default value. # (This would work automatically, were publication_scheme not a translated attribute) self.publication_scheme = "" if self.publication_scheme.nil? - - # Set an API key if there isn’t one + end + + def before_save self.api_key = SecureRandom.base64(33) if self.api_key.nil? end -- cgit v1.2.3 From 5ad7ebf25cc684d68325681db4471e522ac31569 Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Tue, 14 Aug 2012 11:20:51 +0100 Subject: Add concept of global censor rules as orthogonal to regex censor rules. Apply global rules to every request, not regex rules. --- app/models/censor_rule.rb | 28 ++++++++++++++++++++-------- app/models/info_request.rb | 27 +++++++++++++++------------ 2 files changed, 35 insertions(+), 20 deletions(-) (limited to 'app/models') diff --git a/app/models/censor_rule.rb b/app/models/censor_rule.rb index cedbd767e..4548a3a71 100644 --- a/app/models/censor_rule.rb +++ b/app/models/censor_rule.rb @@ -29,7 +29,19 @@ class CensorRule < ActiveRecord::Base belongs_to :user belongs_to :public_body - named_scope :regexps, {:conditions => {:regexp => true}} + # a flag to allow the require_user_request_or_public_body validation to be skipped + attr_accessor :allow_global + validate :require_user_request_or_public_body, :unless => proc{ |rule| rule.allow_global == true } + + named_scope :global, {:conditions => {:info_request_id => nil, + :user_id => nil, + :public_body_id => nil}} + + def require_user_request_or_public_body + if !self.regexp? && self.info_request.nil? && self.user.nil? && self.public_body.nil? + errors.add("Censor must apply to an info request a user or a body; ") + end + end def binary_replacement self.text.gsub(/./, 'x') @@ -50,15 +62,15 @@ class CensorRule < ActiveRecord::Base binary.gsub!(self.text, self.binary_replacement) end - def validate - if !self.regexp? && self.info_request.nil? && self.user.nil? && self.public_body.nil? - errors.add("Censor must apply to an info request a user or a body; ") + def for_admin_column + self.class.content_columns.each do |column| + yield(column.human_name, self.send(column.name), column.type.to_s, column.name) end end - def for_admin_column - self.class.content_columns.each do |column| - yield(column.human_name, self.send(column.name), column.type.to_s, column.name) + def is_global? + return true if (info_request_id.nil? && user_id.nil? && public_body_id.nil?) + return false end - end + end diff --git a/app/models/info_request.rb b/app/models/info_request.rb index 4c8181faa..9290ab77d 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -104,7 +104,7 @@ class InfoRequest < ActiveRecord::Base errors.add(:described_state, "is not a valid state") if !InfoRequest.enumerate_states.include? described_state end - + # The request must either be internal, in which case it has # a foreign key reference to a User object and no external_url or external_user_name, # or else be external in which case it has no user_id but does have an external_url, @@ -120,15 +120,15 @@ class InfoRequest < ActiveRecord::Base errors.add(:external_url, "must be null for an internal request") if !external_url.nil? end end - + def is_external? !external_url.nil? end - + def user_name is_external? ? external_user_name : user.name end - + def user_name_slug if is_external? if external_user_name.nil? @@ -708,10 +708,10 @@ public return self.public_body.is_followupable? end def recipient_name_and_email - return TMail::Address.address_from_name_and_email( - _("{{law_used}} requests at {{public_body}}", - :law_used => self.law_used_short, - :public_body => self.public_body.short_or_long_name), + return TMail::Address.address_from_name_and_email( + _("{{law_used}} requests at {{public_body}}", + :law_used => self.law_used_short, + :public_body => self.public_body.short_or_long_name), self.recipient_email).to_s end @@ -997,10 +997,13 @@ public # Call groups of censor rules def apply_censor_rules_to_text!(text) - [self.censor_rules, self.user.try(:censor_rules), - CensorRule.regexps.all].flatten.compact.each do |censor_rule| - censor_rule.apply_to_text!(text) - end + applicable_rules = [self.censor_rules, CensorRule.global.all] + if self.user && !self.user.censor_rules.empty? + applicable_rules << self.user.censor_rules + end + applicable_rules.flatten.each do |censor_rule| + censor_rule.apply_to_text!(text) + end return text end -- cgit v1.2.3 From 03a875ae16520d7a5fc1c5e8f7e607c7fa6aded2 Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Tue, 14 Aug 2012 11:34:24 +0100 Subject: Add association between public bodies and censor rules, have info_request apply rules from its public body. --- app/models/info_request.rb | 2 +- app/models/public_body.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/info_request.rb b/app/models/info_request.rb index 9290ab77d..83456b54f 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -997,7 +997,7 @@ public # Call groups of censor rules def apply_censor_rules_to_text!(text) - applicable_rules = [self.censor_rules, CensorRule.global.all] + applicable_rules = [self.censor_rules, self.public_body.censor_rules, CensorRule.global.all] if self.user && !self.user.censor_rules.empty? applicable_rules << self.user.censor_rules end diff --git a/app/models/public_body.rb b/app/models/public_body.rb index 97802c7e8..60ecb2781 100644 --- a/app/models/public_body.rb +++ b/app/models/public_body.rb @@ -42,6 +42,7 @@ class PublicBody < ActiveRecord::Base has_many :info_requests, :order => 'created_at desc' has_many :track_things, :order => 'created_at desc' + has_many :censor_rules, :order => 'created_at desc' has_tag_string -- cgit v1.2.3 From c6d5020a9a48226feaccd856df0c8f4584d5fd08 Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Tue, 14 Aug 2012 14:14:35 +0100 Subject: Handle regexp rules when running censor rules on binary files. --- app/models/censor_rule.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'app/models') diff --git a/app/models/censor_rule.rb b/app/models/censor_rule.rb index 4548a3a71..4e96dc55a 100644 --- a/app/models/censor_rule.rb +++ b/app/models/censor_rule.rb @@ -43,10 +43,6 @@ class CensorRule < ActiveRecord::Base end end - def binary_replacement - self.text.gsub(/./, 'x') - end - def apply_to_text!(text) if text.nil? return nil @@ -59,7 +55,8 @@ class CensorRule < ActiveRecord::Base if binary.nil? return nil end - binary.gsub!(self.text, self.binary_replacement) + to_replace = regexp? ? Regexp.new(self.text, Regexp::MULTILINE) : self.text + binary.gsub!(to_replace){ |match| match.gsub(/./, 'x') } end def for_admin_column -- cgit v1.2.3 From 30019ed58f0cd608f6196d03b9679bbca2ea0881 Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Tue, 14 Aug 2012 14:41:54 +0100 Subject: Apply global and public body censor rules to binary files. --- app/models/info_request.rb | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'app/models') diff --git a/app/models/info_request.rb b/app/models/info_request.rb index 83456b54f..dfaa524b2 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -995,27 +995,28 @@ public return ret.reverse end - # Call groups of censor rules - def apply_censor_rules_to_text!(text) + # Get the list of censor rules that apply to this request + def applicable_censor_rules applicable_rules = [self.censor_rules, self.public_body.censor_rules, CensorRule.global.all] if self.user && !self.user.censor_rules.empty? applicable_rules << self.user.censor_rules end - applicable_rules.flatten.each do |censor_rule| + return applicable_rules.flatten + end + + # Call groups of censor rules + def apply_censor_rules_to_text!(text) + self.applicable_censor_rules.each do |censor_rule| censor_rule.apply_to_text!(text) end return text end def apply_censor_rules_to_binary!(binary) - for censor_rule in self.censor_rules + self.applicable_censor_rules.each do |censor_rule| censor_rule.apply_to_binary!(binary) end - if self.user # requests during construction have no user - for censor_rule in self.user.censor_rules - censor_rule.apply_to_binary!(binary) - end - end + return binary end def is_owning_user?(user) -- cgit v1.2.3 From 2aa3d7bfd76e774555207a81cd5e39513f965bf1 Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Tue, 14 Aug 2012 15:29:45 +0100 Subject: Make global validation work correctly for regex censor rules. --- app/models/censor_rule.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/censor_rule.rb b/app/models/censor_rule.rb index 4e96dc55a..02b85dc70 100644 --- a/app/models/censor_rule.rb +++ b/app/models/censor_rule.rb @@ -38,7 +38,7 @@ class CensorRule < ActiveRecord::Base :public_body_id => nil}} def require_user_request_or_public_body - if !self.regexp? && self.info_request.nil? && self.user.nil? && self.public_body.nil? + if self.info_request.nil? && self.user.nil? && self.public_body.nil? errors.add("Censor must apply to an info request a user or a body; ") end end -- cgit v1.2.3 From 21082cc55d3d61edce660ea7a73ec80380359e2f Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Tue, 14 Aug 2012 16:31:03 +0100 Subject: Add basic validation for regexp censor rules that a valid regexp can be created with the text of the rule. --- app/models/censor_rule.rb | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/censor_rule.rb b/app/models/censor_rule.rb index 02b85dc70..3a15bf532 100644 --- a/app/models/censor_rule.rb +++ b/app/models/censor_rule.rb @@ -32,6 +32,7 @@ class CensorRule < ActiveRecord::Base # a flag to allow the require_user_request_or_public_body validation to be skipped attr_accessor :allow_global validate :require_user_request_or_public_body, :unless => proc{ |rule| rule.allow_global == true } + validate :require_valid_regexp, :if => proc{ |rule| rule.regexp? == true } named_scope :global, {:conditions => {:info_request_id => nil, :user_id => nil, @@ -43,11 +44,23 @@ class CensorRule < ActiveRecord::Base end end + def require_valid_regexp + begin + self.make_regexp() + rescue RegexpError => e + errors.add(:text, e.message) + end + end + + def make_regexp + return Regexp.new(self.text, Regexp::MULTILINE) + end + def apply_to_text!(text) if text.nil? return nil end - to_replace = regexp? ? Regexp.new(self.text, Regexp::MULTILINE) : self.text + to_replace = regexp? ? self.make_regexp() : self.text text.gsub!(to_replace, self.replacement) end @@ -55,7 +68,7 @@ class CensorRule < ActiveRecord::Base if binary.nil? return nil end - to_replace = regexp? ? Regexp.new(self.text, Regexp::MULTILINE) : self.text + to_replace = regexp? ? self.make_regexp() : self.text binary.gsub!(to_replace){ |match| match.gsub(/./, 'x') } end -- cgit v1.2.3 From 495a29cd593ac0a270ebb3bf1c1ff85f03b52e31 Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Tue, 14 Aug 2012 16:41:52 +0100 Subject: Validate presence of text to replace in censor rules. --- app/models/censor_rule.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/models') diff --git a/app/models/censor_rule.rb b/app/models/censor_rule.rb index 3a15bf532..da3f49760 100644 --- a/app/models/censor_rule.rb +++ b/app/models/censor_rule.rb @@ -33,6 +33,7 @@ class CensorRule < ActiveRecord::Base attr_accessor :allow_global validate :require_user_request_or_public_body, :unless => proc{ |rule| rule.allow_global == true } validate :require_valid_regexp, :if => proc{ |rule| rule.regexp? == true } + validates_presence_of :text named_scope :global, {:conditions => {:info_request_id => nil, :user_id => nil, -- cgit v1.2.3