diff options
Diffstat (limited to 'app/models')
| -rw-r--r-- | app/models/change_email_validator.rb | 12 | ||||
| -rw-r--r-- | app/models/comment.rb | 6 | ||||
| -rw-r--r-- | app/models/holiday.rb | 2 | ||||
| -rw-r--r-- | app/models/holiday_import.rb | 93 | ||||
| -rw-r--r-- | app/models/incoming_message.rb | 164 | ||||
| -rw-r--r-- | app/models/info_request.rb | 35 | ||||
| -rw-r--r-- | app/models/info_request_event.rb | 9 | ||||
| -rw-r--r-- | app/models/public_body.rb | 5 | ||||
| -rw-r--r-- | app/models/public_body_category.rb | 14 | ||||
| -rw-r--r-- | app/models/public_body_category_link.rb | 11 | ||||
| -rw-r--r-- | app/models/public_body_heading.rb | 9 | ||||
| -rw-r--r-- | app/models/user.rb | 8 |
12 files changed, 183 insertions, 185 deletions
diff --git a/app/models/change_email_validator.rb b/app/models/change_email_validator.rb index 7ee6654bb..65f2fd81c 100644 --- a/app/models/change_email_validator.rb +++ b/app/models/change_email_validator.rb @@ -55,10 +55,20 @@ class ChangeEmailValidator def check_email_is_present_and_valid(email) if !send(email).blank? && !MySociety::Validate.is_valid_email(send(email)) - errors.add(email, _("#{ email.to_s.humanize } doesn't look like a valid address")) + msg_string = check_email_is_present_and_valid_msg_string(email) + errors.add(email, msg_string) end end + def check_email_is_present_and_valid_msg_string(email) + case email.to_sym + when :old_email then _("Old email doesn't look like a valid address") + when :new_email then _("New email doesn't look like a valid address") + else + raise "Unsupported email type #{ email }" + end + end + def email_belongs_to_user?(email) email.downcase == logged_in_user.email.downcase end diff --git a/app/models/comment.rb b/app/models/comment.rb index a286aa1f5..cc8d0e94b 100644 --- a/app/models/comment.rb +++ b/app/models/comment.rb @@ -91,9 +91,9 @@ class Comment < ActiveRecord::Base def check_body_uses_mixed_capitals unless MySociety::Validate.uses_mixed_capitals(body) - msg = 'Please write your annotation using a mixture of capital and ' \ - 'lower case letters. This makes it easier for others to read.' - errors.add(:body, _(msg)) + msg = _('Please write your annotation using a mixture of capital and ' \ + 'lower case letters. This makes it easier for others to read.') + errors.add(:body, msg) end end diff --git a/app/models/holiday.rb b/app/models/holiday.rb index 4c4941589..34044683a 100644 --- a/app/models/holiday.rb +++ b/app/models/holiday.rb @@ -22,6 +22,8 @@ class Holiday < ActiveRecord::Base + validates_presence_of :day + def self.holidays @@holidays ||= all.collect { |h| h.day }.to_set end diff --git a/app/models/holiday_import.rb b/app/models/holiday_import.rb new file mode 100644 index 000000000..c6019fac0 --- /dev/null +++ b/app/models/holiday_import.rb @@ -0,0 +1,93 @@ +class HolidayImport + + include ActiveModel::Validations + + attr_accessor :holidays, + :ical_feed_url, + :start_year, + :end_year, + :start_date, + :end_date, + :source, + :populated + + validate :all_holidays_valid + validates_inclusion_of :source, :in => %w( suggestions feed ) + validates_presence_of :ical_feed_url, + :if => proc { |holiday_import| holiday_import.source == 'feed' } + + def initialize(opts = {}) + @populated = false + @start_year = opts.fetch(:start_year, Time.now.year).to_i + @end_year = opts.fetch(:end_year, Time.now.year).to_i + @start_date = Date.civil(start_year, 1, 1) + @end_date = Date.civil(end_year, 12, 31) + @source = opts.fetch(:source, 'suggestions') + @ical_feed_url = opts.fetch(:ical_feed_url, nil) + @country_code = AlaveteliConfiguration::iso_country_code.downcase + self.holidays_attributes = opts.fetch(:holidays_attributes, []) + end + + def populate + source == 'suggestions' ? populate_from_suggestions : populate_from_ical_feed + @populated = true + end + + def suggestions_country_name + IsoCountryCodes.find(@country_code).name if @country_code + end + + def period + start_year == end_year ? "#{start_year}" : "#{start_year}-#{end_year}" + end + + def save + holidays.all?(&:save) + end + + def holidays_attributes=(incoming_data) + incoming_data.each{ |offset, incoming| self.holidays << Holiday.new(incoming) } + end + + def holidays + @holidays ||= [] + end + + private + + def all_holidays_valid + errors.add(:base, 'These holidays could not be imported') unless holidays.all?(&:valid?) + end + + def populate_from_ical_feed + begin + cal_file = open(ical_feed_url) + cals = Icalendar.parse(cal_file, strict=false) + cal = cals.first + cal.events.each{ |cal_event| populate_from_ical_event(cal_event) } + rescue Errno::ENOENT, Exception => e + if e.message == 'Invalid line in calendar string!' + errors.add(:ical_feed_url, "Sorry, there's a problem with the format of that feed.") + elsif e.message.starts_with 'No such file or directory' + errors.add(:ical_feed_url, "Sorry we couldn't find that feed.") + else + raise e + end + end + end + + def populate_from_ical_event(cal_event) + if cal_event.dtstart >= start_date and cal_event.dtstart <= end_date + holidays << Holiday.new(:description => cal_event.summary, + :day => cal_event.dtstart) + end + end + + def populate_from_suggestions + holiday_info = Holidays.between(start_date, end_date, @country_code.to_sym, :observed) + holiday_info.each do |holiday_info_hash| + holidays << Holiday.new(:description => holiday_info_hash[:name], + :day => holiday_info_hash[:date]) + end + end +end diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb index 135a6bdaf..658ee969a 100644 --- a/app/models/incoming_message.rb +++ b/app/models/incoming_message.rb @@ -52,17 +52,6 @@ class IncomingMessage < ActiveRecord::Base has_prominence - # See binary_mask_stuff function below. It just test for inclusion - # in this hash, not the value of the right hand side. - DoNotBinaryMask = { - 'image/tiff' => 1, - 'image/gif' => 1, - 'image/jpeg' => 1, - 'image/png' => 1, - 'image/bmp' => 1, - 'application/zip' => 1, - } - # Given that there are in theory many info request events, a convenience method for # getting the response event def response_event @@ -218,111 +207,10 @@ class IncomingMessage < ActiveRecord::Base end end - # Converts email addresses we know about into textual descriptions of them - def mask_special_emails!(text) - # TODO: can later display some of these special emails as actual emails, - # if they are public anyway. For now just be precautionary and only - # put in descriptions of them in square brackets. - if self.info_request.public_body.is_followupable? - text.gsub!(self.info_request.public_body.request_email, _("[{{public_body}} request email]", :public_body => self.info_request.public_body.short_or_long_name)) - end - text.gsub!(self.info_request.incoming_email, _('[FOI #{{request}} email]', :request => self.info_request.id.to_s) ) - text.gsub!(AlaveteliConfiguration::contact_email, _("[{{site_name}} contact email]", :site_name => AlaveteliConfiguration::site_name) ) - end - - # Replaces all email addresses in (possibly binary data) with equal length alternative ones. - # Also replaces censor items - def binary_mask_stuff!(text, content_type) - # See if content type is one that we mask - things like zip files and - # images may get broken if we try to. We err on the side of masking too - # much, as many unknown types will really be text. - if DoNotBinaryMask.include?(content_type) - return - end - - # Special cases for some content types - if content_type == 'application/pdf' - uncompressed_text = nil - uncompressed_text = AlaveteliExternalCommand.run("pdftk", "-", "output", "-", "uncompress", :stdin_string => text) - # if we managed to uncompress the PDF... - if !uncompressed_text.nil? && !uncompressed_text.empty? - # then censor stuff (making a copy so can compare again in a bit) - censored_uncompressed_text = uncompressed_text.dup - self._binary_mask_stuff_internal!(censored_uncompressed_text) - # if the censor rule removed something... - if censored_uncompressed_text != uncompressed_text - # then use the altered file (recompressed) - recompressed_text = nil - if AlaveteliConfiguration::use_ghostscript_compression == true - command = ["gs", "-sDEVICE=pdfwrite", "-dCompatibilityLevel=1.4", "-dPDFSETTINGS=/screen", "-dNOPAUSE", "-dQUIET", "-dBATCH", "-sOutputFile=-", "-"] - else - command = ["pdftk", "-", "output", "-", "compress"] - end - recompressed_text = AlaveteliExternalCommand.run(*(command + [{:stdin_string=>censored_uncompressed_text}])) - if recompressed_text.nil? || recompressed_text.empty? - # buggy versions of pdftk sometimes fail on - # compression, I don't see it's a disaster in - # these cases to save an uncompressed version? - recompressed_text = censored_uncompressed_text - logger.warn "Unable to compress PDF; problem with your pdftk version?" - end - if !recompressed_text.nil? && !recompressed_text.empty? - text.replace recompressed_text - end - end - end - return - end - - self._binary_mask_stuff_internal!(text) - end - - # Used by binary_mask_stuff - replace text in place - def _binary_mask_stuff_internal!(text) - # Keep original size, so can check haven't resized it - orig_size = text.mb_chars.size - - # Replace ASCII email addresses... - text.gsub!(MySociety::Validate.email_find_regexp) do |email| - email.gsub(/[^@.]/, 'x') - end - - # And replace UCS-2 ones (for Microsoft Office documents)... - # Find emails, by finding them in parts of text that have ASCII - # equivalents to the UCS-2 - ascii_chars = text.gsub(/\0/, "") - emails = ascii_chars.scan(MySociety::Validate.email_find_regexp) - - # Convert back to UCS-2, making a mask at the same time - if String.method_defined?(:encode) - emails.map! do |email| - # We want the ASCII representation of UCS-2 - [email[0].encode('UTF-16LE').force_encoding('US-ASCII'), - email[0].gsub(/[^@.]/, 'x').encode('UTF-16LE').force_encoding('US-ASCII')] - end - else - emails.map! {|email| [ - Iconv.conv('ucs-2le', 'ascii', email[0]), - Iconv.conv('ucs-2le', 'ascii', email[0].gsub(/[^@.]/, 'x')) - ] } - end - - # Now search and replace the UCS-2 email with the UCS-2 mask - for email, mask in emails - text.gsub!(email, mask) - end - - # Replace censor items - self.info_request.apply_censor_rules_to_binary!(text) - - raise "internal error in binary_mask_stuff" if text.mb_chars.size != orig_size - return text - end - - # Removes censored stuff from from HTML conversion of downloaded binaries - def html_mask_stuff!(html) - self.mask_special_emails!(html) - self.remove_privacy_sensitive_things!(html) + def apply_masks!(text, content_type) + mask_options = { :censor_rules => info_request.applicable_censor_rules, + :masks => info_request.masks } + AlaveteliTextMasker.apply_masks!(text, content_type, mask_options) end # Lotus notes quoting yeuch! @@ -346,26 +234,6 @@ class IncomingMessage < ActiveRecord::Base end - # Remove emails, mobile phones and other details FOI officers ask us to remove. - def remove_privacy_sensitive_things!(text) - # Remove any email addresses - we don't want bounce messages to leak out - # either the requestor's email address or the request's response email - # address out onto the internet - text.gsub!(MySociety::Validate.email_find_regexp, "[email address]") - - # Mobile phone numbers - # http://www.whatdotheyknow.com/request/failed_test_purchases_off_licenc#incoming-1013 - # http://www.whatdotheyknow.com/request/selective_licensing_statistics_i#incoming-550 - # http://www.whatdotheyknow.com/request/common_purpose_training_graduate#incoming-774 - text.gsub!(/(Mobile|Mob)([\s\/]*(Fax|Tel))*\s*:?[\s\d]*\d/, "[mobile number]") - - # Remove WhatDoTheyKnow signup links - text.gsub!(/http:\/\/#{AlaveteliConfiguration::domain}\/c\/[^\s]+/, "[WDTK login link]") - - # Remove things from censor rules - self.info_request.apply_censor_rules_to_text!(text) - end - # Remove quoted sections from emails (eventually the aim would be for this # to do as good a job as GMail does) TODO: bet it needs a proper parser @@ -465,9 +333,8 @@ class IncomingMessage < ActiveRecord::Base raise "main body text more than 1 MB, need to implement clipping like for attachment text, or there is some other MIME decoding problem or similar" end - # remove emails for privacy/anti-spam reasons - self.mask_special_emails!(text) - self.remove_privacy_sensitive_things!(text) + # apply masks for this message + apply_masks!(text, 'text/html') # Remove existing quoted sections folded_quoted_text = self.remove_lotus_quoting(text, 'FOLDED_QUOTED_SECTION') @@ -693,8 +560,10 @@ class IncomingMessage < ActiveRecord::Base self.reload # get the main body part from the set of attachments we just created, - # not from the self.foi_attachments association - some of the total set of - # self.foi_attachments may now be obsolete + # not from the self.foi_attachments association - some of the total set + # of self.foi_attachments may now be obsolete. Sometimes (e.g. when + # parsing mail from Apple Mail) we can end up with less attachments + # because the hexdigest of an attachment is identical. main_part = get_main_body_text_part(attachments) # we don't use get_main_body_text_internal, as we want to avoid charset # conversions, since /usr/bin/uudecode needs to deal with those. @@ -733,7 +602,14 @@ class IncomingMessage < ActiveRecord::Base text = MySociety::Format.simplify_angle_bracketed_urls(text) text = CGI.escapeHTML(text) text = MySociety::Format.make_clickable(text, :contract => 1) - text.gsub!(/\[(email address|mobile number)\]/, '[<a href="/help/officers#mobiles">\1</a>]') + + # add a helpful link to email addresses and mobile numbers removed + # by apply_masks! + email_pattern = Regexp.escape(_("email address")) + mobile_pattern = Regexp.escape(_("mobile number")) + text.gsub!(/\[(#{email_pattern}|#{mobile_pattern})\]/, + '[<a href="/help/officers#mobiles">\1</a>]') + if collapse_quoted_sections text = text.gsub(/(\s*FOLDED_QUOTED_SECTION\s*)+/m, "FOLDED_QUOTED_SECTION") text.strip! @@ -771,8 +647,8 @@ class IncomingMessage < ActiveRecord::Base # Returns text version of attachment text def get_attachment_text_full text = self._get_attachment_text_internal - self.mask_special_emails!(text) - self.remove_privacy_sensitive_things!(text) + apply_masks!(text, 'text/html') + # This can be useful for memory debugging #STDOUT.puts 'xxx '+ MySociety::DebugHelpers::allocated_string_size_around_gc diff --git a/app/models/info_request.rb b/app/models/info_request.rb index d0052603a..2b60e13d8 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -292,13 +292,18 @@ public end # Subject lines for emails about the request - def email_subject_request - _('{{law_used_full}} request - {{title}}',:law_used_full=>self.law_used_full,:title=>self.title.html_safe) + def email_subject_request(opts = {}) + html = opts.fetch(:html, true) + _('{{law_used_full}} request - {{title}}', + :law_used_full => self.law_used_full, + :title => (html ? title : title.html_safe)) end - def email_subject_followup(incoming_message = nil) + def email_subject_followup(opts = {}) + incoming_message = opts.fetch(:incoming_message, nil) + html = opts.fetch(:html, true) if incoming_message.nil? || !incoming_message.valid_to_reply_to? || !incoming_message.subject - 'Re: ' + self.email_subject_request + 'Re: ' + self.email_subject_request(:html => html) else if incoming_message.subject.match(/^Re:/i) incoming_message.subject @@ -1148,6 +1153,22 @@ public return binary end + # Masks we apply to text associated with this request convert email addresses + # we know about into textual descriptions of them + def masks + masks = [{ :to_replace => incoming_email, + :replacement => _('[FOI #{{request}} email]', + :request => id.to_s) }, + { :to_replace => AlaveteliConfiguration::contact_email, + :replacement => _("[{{site_name}} contact email]", + :site_name => AlaveteliConfiguration::site_name)} ] + if public_body.is_followupable? + masks << { :to_replace => public_body.request_email, + :replacement => _("[{{public_body}} request email]", + :public_body => public_body.short_or_long_name) } + end + end + def is_owning_user?(user) !user.nil? && (user.id == user_id || user.owns_every_request?) end @@ -1345,9 +1366,9 @@ public end def InfoRequest.find_in_state(state) - find(:all, :select => '*, ' + last_event_time_clause + ' as last_event_time', - :conditions => ["described_state = ?", state], - :order => "last_event_time") + select("*, #{ last_event_time_clause } as last_event_time"). + where(:described_state => state). + order('last_event_time') end private diff --git a/app/models/info_request_event.rb b/app/models/info_request_event.rb index 9dde3ba80..635ba8f58 100644 --- a/app/models/info_request_event.rb +++ b/app/models/info_request_event.rb @@ -161,11 +161,10 @@ class InfoRequestEvent < ActiveRecord::Base end def incoming_message_selective_columns(fields) - message = IncomingMessage.find(:all, - :select => fields + ", incoming_messages.info_request_id", - :joins => "INNER JOIN info_request_events ON incoming_messages.id = incoming_message_id ", - :conditions => "info_request_events.id = #{self.id}" - ) + message = IncomingMessage.select("#{ fields }, incoming_messages.info_request_id"). + joins('INNER JOIN info_request_events ON incoming_messages.id = incoming_message_id'). + where('info_request_events.id = ?', id) + message = message[0] if !message.nil? message.info_request = InfoRequest.find(message.info_request_id) diff --git a/app/models/public_body.rb b/app/models/public_body.rb index 1929272ea..a9cdfeab2 100644 --- a/app/models/public_body.rb +++ b/app/models/public_body.rb @@ -454,11 +454,12 @@ class PublicBody < ActiveRecord::Base # of updating them bodies_by_name = {} set_of_existing = Set.new() + internal_admin_body_id = PublicBody.internal_admin_body.id I18n.with_locale(I18n.default_locale) do - bodies = (tag.nil? || tag.empty?) ? PublicBody.find(:all) : PublicBody.find_by_tag(tag) + bodies = (tag.nil? || tag.empty?) ? PublicBody.find(:all, :include => :translations) : PublicBody.find_by_tag(tag) for existing_body in bodies # Hide InternalAdminBody from import notes - next if existing_body.id == PublicBody.internal_admin_body.id + next if existing_body.id == internal_admin_body_id bodies_by_name[existing_body.name] = existing_body set_of_existing.add(existing_body.name) diff --git a/app/models/public_body_category.rb b/app/models/public_body_category.rb index 8eaecd596..c313e5734 100644 --- a/app/models/public_body_category.rb +++ b/app/models/public_body_category.rb @@ -2,11 +2,8 @@ # # Table name: public_body_categories # -# id :integer not null, primary key -# title :text not null -# category_tag :text not null -# description :text not null -# display_order :integer +# id :integer not null, primary key +# category_tag :text not null # require 'forwardable' @@ -19,9 +16,10 @@ class PublicBodyCategory < ActiveRecord::Base has_many :public_body_headings, :through => :public_body_category_links translates :title, :description - validates_uniqueness_of :category_tag, :message => N_('Tag is already taken') - validates_presence_of :title, :message => N_("Title can't be blank") - validates_presence_of :category_tag, :message => N_("Tag can't be blank") + validates_uniqueness_of :category_tag, :message => 'Tag is already taken' + validates_presence_of :title, :message => "Title can't be blank" + validates_presence_of :category_tag, :message => "Tag can't be blank" + validates_presence_of :description, :message => "Description can't be blank" def self.get locale = I18n.locale.to_s || default_locale.to_s || "" diff --git a/app/models/public_body_category_link.rb b/app/models/public_body_category_link.rb index eb233b56f..8c3eb8060 100644 --- a/app/models/public_body_category_link.rb +++ b/app/models/public_body_category_link.rb @@ -1,10 +1,11 @@ # == Schema Information # -# Table name: public_body_category_link +# Table name: public_body_category_links # -# public_body_category_id :integer not null -# public_body_heading_id :integer not null -# category_display_order :integer +# public_body_category_id :integer not null +# public_body_heading_id :integer not null +# category_display_order :integer +# id :integer not null, primary key # class PublicBodyCategoryLink < ActiveRecord::Base @@ -15,7 +16,7 @@ class PublicBodyCategoryLink < ActiveRecord::Base validates_presence_of :public_body_category validates_presence_of :public_body_heading validates :category_display_order, :numericality => { :only_integer => true, - :message => N_('Display order must be a number') } + :message => 'Display order must be a number' } before_validation :on => :create do unless self.category_display_order diff --git a/app/models/public_body_heading.rb b/app/models/public_body_heading.rb index c38800561..f394c37c6 100644 --- a/app/models/public_body_heading.rb +++ b/app/models/public_body_heading.rb @@ -2,8 +2,7 @@ # # Table name: public_body_headings # -# id :integer not null, primary key -# name :text not null +# id :integer not null, primary key # display_order :integer # @@ -16,10 +15,10 @@ class PublicBodyHeading < ActiveRecord::Base translates :name - validates_uniqueness_of :name, :message => N_('Name is already taken') - validates_presence_of :name, :message => N_('Name can\'t be blank') + validates_uniqueness_of :name, :message => 'Name is already taken' + validates_presence_of :name, :message => 'Name can\'t be blank' validates :display_order, :numericality => { :only_integer => true, - :message => N_('Display order must be a number') } + :message => 'Display order must be a number' } before_validation :on => :create do unless self.display_order diff --git a/app/models/user.rb b/app/models/user.rb index 1c6dc0eb0..c953e52f2 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -264,11 +264,9 @@ class User < ActiveRecord::Base # Returns list of requests which the user hasn't described (and last # changed more than a day ago) def get_undescribed_requests - info_requests.find( - :all, - :conditions => [ 'awaiting_description = ? and ' + InfoRequest.last_event_time_clause + ' < ?', - true, Time.now() - 1.day - ] + info_requests.where( + "awaiting_description = ? and #{ InfoRequest.last_event_time_clause } < ?", + true, 1.day.ago ) end |