diff options
Diffstat (limited to 'app/models')
| -rw-r--r-- | app/models/holiday.rb | 2 | ||||
| -rw-r--r-- | app/models/holiday_import.rb | 93 | ||||
| -rw-r--r-- | app/models/incoming_message.rb | 158 | ||||
| -rw-r--r-- | app/models/info_request.rb | 46 | ||||
| -rw-r--r-- | app/models/info_request_event.rb | 9 | ||||
| -rw-r--r-- | app/models/public_body.rb | 30 | ||||
| -rw-r--r-- | app/models/public_body_category.rb | 7 | ||||
| -rw-r--r-- | app/models/public_body_category_link.rb | 9 | ||||
| -rw-r--r-- | app/models/public_body_heading.rb | 3 | ||||
| -rw-r--r-- | app/models/user.rb | 8 |
10 files changed, 167 insertions, 198 deletions
diff --git a/app/models/holiday.rb b/app/models/holiday.rb index 4c4941589..34044683a 100644 --- a/app/models/holiday.rb +++ b/app/models/holiday.rb @@ -22,6 +22,8 @@ class Holiday < ActiveRecord::Base + validates_presence_of :day + def self.holidays @@holidays ||= all.collect { |h| h.day }.to_set end diff --git a/app/models/holiday_import.rb b/app/models/holiday_import.rb new file mode 100644 index 000000000..c6019fac0 --- /dev/null +++ b/app/models/holiday_import.rb @@ -0,0 +1,93 @@ +class HolidayImport + + include ActiveModel::Validations + + attr_accessor :holidays, + :ical_feed_url, + :start_year, + :end_year, + :start_date, + :end_date, + :source, + :populated + + validate :all_holidays_valid + validates_inclusion_of :source, :in => %w( suggestions feed ) + validates_presence_of :ical_feed_url, + :if => proc { |holiday_import| holiday_import.source == 'feed' } + + def initialize(opts = {}) + @populated = false + @start_year = opts.fetch(:start_year, Time.now.year).to_i + @end_year = opts.fetch(:end_year, Time.now.year).to_i + @start_date = Date.civil(start_year, 1, 1) + @end_date = Date.civil(end_year, 12, 31) + @source = opts.fetch(:source, 'suggestions') + @ical_feed_url = opts.fetch(:ical_feed_url, nil) + @country_code = AlaveteliConfiguration::iso_country_code.downcase + self.holidays_attributes = opts.fetch(:holidays_attributes, []) + end + + def populate + source == 'suggestions' ? populate_from_suggestions : populate_from_ical_feed + @populated = true + end + + def suggestions_country_name + IsoCountryCodes.find(@country_code).name if @country_code + end + + def period + start_year == end_year ? "#{start_year}" : "#{start_year}-#{end_year}" + end + + def save + holidays.all?(&:save) + end + + def holidays_attributes=(incoming_data) + incoming_data.each{ |offset, incoming| self.holidays << Holiday.new(incoming) } + end + + def holidays + @holidays ||= [] + end + + private + + def all_holidays_valid + errors.add(:base, 'These holidays could not be imported') unless holidays.all?(&:valid?) + end + + def populate_from_ical_feed + begin + cal_file = open(ical_feed_url) + cals = Icalendar.parse(cal_file, strict=false) + cal = cals.first + cal.events.each{ |cal_event| populate_from_ical_event(cal_event) } + rescue Errno::ENOENT, Exception => e + if e.message == 'Invalid line in calendar string!' + errors.add(:ical_feed_url, "Sorry, there's a problem with the format of that feed.") + elsif e.message.starts_with 'No such file or directory' + errors.add(:ical_feed_url, "Sorry we couldn't find that feed.") + else + raise e + end + end + end + + def populate_from_ical_event(cal_event) + if cal_event.dtstart >= start_date and cal_event.dtstart <= end_date + holidays << Holiday.new(:description => cal_event.summary, + :day => cal_event.dtstart) + end + end + + def populate_from_suggestions + holiday_info = Holidays.between(start_date, end_date, @country_code.to_sym, :observed) + holiday_info.each do |holiday_info_hash| + holidays << Holiday.new(:description => holiday_info_hash[:name], + :day => holiday_info_hash[:date]) + end + end +end diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb index db6722976..658ee969a 100644 --- a/app/models/incoming_message.rb +++ b/app/models/incoming_message.rb @@ -52,17 +52,6 @@ class IncomingMessage < ActiveRecord::Base has_prominence - # See binary_mask_stuff function below. It just test for inclusion - # in this hash, not the value of the right hand side. - DoNotBinaryMask = { - 'image/tiff' => 1, - 'image/gif' => 1, - 'image/jpeg' => 1, - 'image/png' => 1, - 'image/bmp' => 1, - 'application/zip' => 1, - } - # Given that there are in theory many info request events, a convenience method for # getting the response event def response_event @@ -218,111 +207,10 @@ class IncomingMessage < ActiveRecord::Base end end - # Converts email addresses we know about into textual descriptions of them - def mask_special_emails!(text) - # TODO: can later display some of these special emails as actual emails, - # if they are public anyway. For now just be precautionary and only - # put in descriptions of them in square brackets. - if self.info_request.public_body.is_followupable? - text.gsub!(self.info_request.public_body.request_email, _("[{{public_body}} request email]", :public_body => self.info_request.public_body.short_or_long_name)) - end - text.gsub!(self.info_request.incoming_email, _('[FOI #{{request}} email]', :request => self.info_request.id.to_s) ) - text.gsub!(AlaveteliConfiguration::contact_email, _("[{{site_name}} contact email]", :site_name => AlaveteliConfiguration::site_name) ) - end - - # Replaces all email addresses in (possibly binary data) with equal length alternative ones. - # Also replaces censor items - def binary_mask_stuff!(text, content_type) - # See if content type is one that we mask - things like zip files and - # images may get broken if we try to. We err on the side of masking too - # much, as many unknown types will really be text. - if DoNotBinaryMask.include?(content_type) - return - end - - # Special cases for some content types - if content_type == 'application/pdf' - uncompressed_text = nil - uncompressed_text = AlaveteliExternalCommand.run("pdftk", "-", "output", "-", "uncompress", :stdin_string => text) - # if we managed to uncompress the PDF... - if !uncompressed_text.nil? && !uncompressed_text.empty? - # then censor stuff (making a copy so can compare again in a bit) - censored_uncompressed_text = uncompressed_text.dup - self._binary_mask_stuff_internal!(censored_uncompressed_text) - # if the censor rule removed something... - if censored_uncompressed_text != uncompressed_text - # then use the altered file (recompressed) - recompressed_text = nil - if AlaveteliConfiguration::use_ghostscript_compression == true - command = ["gs", "-sDEVICE=pdfwrite", "-dCompatibilityLevel=1.4", "-dPDFSETTINGS=/screen", "-dNOPAUSE", "-dQUIET", "-dBATCH", "-sOutputFile=-", "-"] - else - command = ["pdftk", "-", "output", "-", "compress"] - end - recompressed_text = AlaveteliExternalCommand.run(*(command + [{:stdin_string=>censored_uncompressed_text}])) - if recompressed_text.nil? || recompressed_text.empty? - # buggy versions of pdftk sometimes fail on - # compression, I don't see it's a disaster in - # these cases to save an uncompressed version? - recompressed_text = censored_uncompressed_text - logger.warn "Unable to compress PDF; problem with your pdftk version?" - end - if !recompressed_text.nil? && !recompressed_text.empty? - text.replace recompressed_text - end - end - end - return - end - - self._binary_mask_stuff_internal!(text) - end - - # Used by binary_mask_stuff - replace text in place - def _binary_mask_stuff_internal!(text) - # Keep original size, so can check haven't resized it - orig_size = text.mb_chars.size - - # Replace ASCII email addresses... - text.gsub!(MySociety::Validate.email_find_regexp) do |email| - email.gsub(/[^@.]/, 'x') - end - - # And replace UCS-2 ones (for Microsoft Office documents)... - # Find emails, by finding them in parts of text that have ASCII - # equivalents to the UCS-2 - ascii_chars = text.gsub(/\0/, "") - emails = ascii_chars.scan(MySociety::Validate.email_find_regexp) - - # Convert back to UCS-2, making a mask at the same time - if String.method_defined?(:encode) - emails.map! do |email| - # We want the ASCII representation of UCS-2 - [email[0].encode('UTF-16LE').force_encoding('US-ASCII'), - email[0].gsub(/[^@.]/, 'x').encode('UTF-16LE').force_encoding('US-ASCII')] - end - else - emails.map! {|email| [ - Iconv.conv('ucs-2le', 'ascii', email[0]), - Iconv.conv('ucs-2le', 'ascii', email[0].gsub(/[^@.]/, 'x')) - ] } - end - - # Now search and replace the UCS-2 email with the UCS-2 mask - for email, mask in emails - text.gsub!(email, mask) - end - - # Replace censor items - self.info_request.apply_censor_rules_to_binary!(text) - - raise "internal error in binary_mask_stuff" if text.mb_chars.size != orig_size - return text - end - - # Removes censored stuff from from HTML conversion of downloaded binaries - def html_mask_stuff!(html) - self.mask_special_emails!(html) - self.remove_privacy_sensitive_things!(html) + def apply_masks!(text, content_type) + mask_options = { :censor_rules => info_request.applicable_censor_rules, + :masks => info_request.masks } + AlaveteliTextMasker.apply_masks!(text, content_type, mask_options) end # Lotus notes quoting yeuch! @@ -346,26 +234,6 @@ class IncomingMessage < ActiveRecord::Base end - # Remove emails, mobile phones and other details FOI officers ask us to remove. - def remove_privacy_sensitive_things!(text) - # Remove any email addresses - we don't want bounce messages to leak out - # either the requestor's email address or the request's response email - # address out onto the internet - text.gsub!(MySociety::Validate.email_find_regexp, "[email address]") - - # Mobile phone numbers - # http://www.whatdotheyknow.com/request/failed_test_purchases_off_licenc#incoming-1013 - # http://www.whatdotheyknow.com/request/selective_licensing_statistics_i#incoming-550 - # http://www.whatdotheyknow.com/request/common_purpose_training_graduate#incoming-774 - text.gsub!(/(Mobile|Mob)([\s\/]*(Fax|Tel))*\s*:?[\s\d]*\d/, "[mobile number]") - - # Remove WhatDoTheyKnow signup links - text.gsub!(/http:\/\/#{AlaveteliConfiguration::domain}\/c\/[^\s]+/, "[WDTK login link]") - - # Remove things from censor rules - self.info_request.apply_censor_rules_to_text!(text) - end - # Remove quoted sections from emails (eventually the aim would be for this # to do as good a job as GMail does) TODO: bet it needs a proper parser @@ -465,9 +333,8 @@ class IncomingMessage < ActiveRecord::Base raise "main body text more than 1 MB, need to implement clipping like for attachment text, or there is some other MIME decoding problem or similar" end - # remove emails for privacy/anti-spam reasons - self.mask_special_emails!(text) - self.remove_privacy_sensitive_things!(text) + # apply masks for this message + apply_masks!(text, 'text/html') # Remove existing quoted sections folded_quoted_text = self.remove_lotus_quoting(text, 'FOLDED_QUOTED_SECTION') @@ -735,7 +602,14 @@ class IncomingMessage < ActiveRecord::Base text = MySociety::Format.simplify_angle_bracketed_urls(text) text = CGI.escapeHTML(text) text = MySociety::Format.make_clickable(text, :contract => 1) - text.gsub!(/\[(email address|mobile number)\]/, '[<a href="/help/officers#mobiles">\1</a>]') + + # add a helpful link to email addresses and mobile numbers removed + # by apply_masks! + email_pattern = Regexp.escape(_("email address")) + mobile_pattern = Regexp.escape(_("mobile number")) + text.gsub!(/\[(#{email_pattern}|#{mobile_pattern})\]/, + '[<a href="/help/officers#mobiles">\1</a>]') + if collapse_quoted_sections text = text.gsub(/(\s*FOLDED_QUOTED_SECTION\s*)+/m, "FOLDED_QUOTED_SECTION") text.strip! @@ -773,8 +647,8 @@ class IncomingMessage < ActiveRecord::Base # Returns text version of attachment text def get_attachment_text_full text = self._get_attachment_text_internal - self.mask_special_emails!(text) - self.remove_privacy_sensitive_things!(text) + apply_masks!(text, 'text/html') + # This can be useful for memory debugging #STDOUT.puts 'xxx '+ MySociety::DebugHelpers::allocated_string_size_around_gc diff --git a/app/models/info_request.rb b/app/models/info_request.rb index d0052603a..814057ef4 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -210,16 +210,6 @@ class InfoRequest < ActiveRecord::Base OLD_AGE_IN_DAYS = 21.days - def after_initialize - if self.described_state.nil? - self.described_state = 'waiting_response' - end - # FOI or EIR? - if !self.public_body.nil? && self.public_body.eir_only? - self.law_used = 'eir' - end - end - def visible_comments self.comments.find(:all, :conditions => 'visible') end @@ -292,13 +282,18 @@ public end # Subject lines for emails about the request - def email_subject_request - _('{{law_used_full}} request - {{title}}',:law_used_full=>self.law_used_full,:title=>self.title.html_safe) + def email_subject_request(opts = {}) + html = opts.fetch(:html, true) + _('{{law_used_full}} request - {{title}}', + :law_used_full => self.law_used_full, + :title => (html ? title : title.html_safe)) end - def email_subject_followup(incoming_message = nil) + def email_subject_followup(opts = {}) + incoming_message = opts.fetch(:incoming_message, nil) + html = opts.fetch(:html, true) if incoming_message.nil? || !incoming_message.valid_to_reply_to? || !incoming_message.subject - 'Re: ' + self.email_subject_request + 'Re: ' + self.email_subject_request(:html => html) else if incoming_message.subject.match(/^Re:/i) incoming_message.subject @@ -753,7 +748,6 @@ public # This is a long stop - even with UK public interest test extensions, 40 # days is a very long time. def date_very_overdue_after - last_sent = last_event_forming_initial_request if self.public_body.is_school? # schools have 60 working days maximum (even over a long holiday) Holiday.due_date_from(self.date_initial_request_last_sent_at, AlaveteliConfiguration::special_reply_very_late_after_days, AlaveteliConfiguration::working_or_calendar_days) @@ -1148,6 +1142,22 @@ public return binary end + # Masks we apply to text associated with this request convert email addresses + # we know about into textual descriptions of them + def masks + masks = [{ :to_replace => incoming_email, + :replacement => _('[FOI #{{request}} email]', + :request => id.to_s) }, + { :to_replace => AlaveteliConfiguration::contact_email, + :replacement => _("[{{site_name}} contact email]", + :site_name => AlaveteliConfiguration::site_name)} ] + if public_body.is_followupable? + masks << { :to_replace => public_body.request_email, + :replacement => _("[{{public_body}} request email]", + :public_body => public_body.short_or_long_name) } + end + end + def is_owning_user?(user) !user.nil? && (user.id == user_id || user.owns_every_request?) end @@ -1345,9 +1355,9 @@ public end def InfoRequest.find_in_state(state) - find(:all, :select => '*, ' + last_event_time_clause + ' as last_event_time', - :conditions => ["described_state = ?", state], - :order => "last_event_time") + select("*, #{ last_event_time_clause } as last_event_time"). + where(:described_state => state). + order('last_event_time') end private diff --git a/app/models/info_request_event.rb b/app/models/info_request_event.rb index 9dde3ba80..635ba8f58 100644 --- a/app/models/info_request_event.rb +++ b/app/models/info_request_event.rb @@ -161,11 +161,10 @@ class InfoRequestEvent < ActiveRecord::Base end def incoming_message_selective_columns(fields) - message = IncomingMessage.find(:all, - :select => fields + ", incoming_messages.info_request_id", - :joins => "INNER JOIN info_request_events ON incoming_messages.id = incoming_message_id ", - :conditions => "info_request_events.id = #{self.id}" - ) + message = IncomingMessage.select("#{ fields }, incoming_messages.info_request_id"). + joins('INNER JOIN info_request_events ON incoming_messages.id = incoming_message_id'). + where('info_request_events.id = ?', id) + message = message[0] if !message.nil? message.info_request = InfoRequest.find(message.info_request_id) diff --git a/app/models/public_body.rb b/app/models/public_body.rb index 829625cac..b8163b07d 100644 --- a/app/models/public_body.rb +++ b/app/models/public_body.rb @@ -349,33 +349,29 @@ class PublicBody < ActiveRecord::Base # Use tags to describe what type of thing this is def type_of_authority(html = false) - types = [] - first = true - for tag in self.tags + types = tags.each_with_index.map do |tag, index| if PublicBodyCategory.get().by_tag().include?(tag.name) desc = PublicBodyCategory.get().singular_by_tag()[tag.name] - if first - # terrible that Ruby/Rails doesn't have an equivalent of ucfirst - # (capitalize shockingly converts later characters to lowercase) - desc = desc[0,1].capitalize + desc[1,desc.size] - first = false + + if index.zero? + desc = desc.sub(/\S/) { |m| Unicode.upcase(m) } end + if html # TODO: this should call proper route helpers, but is in model sigh desc = '<a href="/body/list/' + tag.name + '">' + desc + '</a>' end - types.push(desc) + + desc end end - if types.size > 0 - ret = types[0, types.size - 1].join(", ") - if types.size > 1 - ret = ret + " and " - end - ret = ret + types[-1] - return ret.html_safe + + types.compact! + + if types.any? + types.to_sentence(:last_word_connector => ' and ').html_safe else - return _("A public authority") + _("A public authority") end end diff --git a/app/models/public_body_category.rb b/app/models/public_body_category.rb index bb83c4c82..c313e5734 100644 --- a/app/models/public_body_category.rb +++ b/app/models/public_body_category.rb @@ -2,11 +2,8 @@ # # Table name: public_body_categories # -# id :integer not null, primary key -# title :text not null -# category_tag :text not null -# description :text not null -# display_order :integer +# id :integer not null, primary key +# category_tag :text not null # require 'forwardable' diff --git a/app/models/public_body_category_link.rb b/app/models/public_body_category_link.rb index ba3ff1f95..8c3eb8060 100644 --- a/app/models/public_body_category_link.rb +++ b/app/models/public_body_category_link.rb @@ -1,10 +1,11 @@ # == Schema Information # -# Table name: public_body_category_link +# Table name: public_body_category_links # -# public_body_category_id :integer not null -# public_body_heading_id :integer not null -# category_display_order :integer +# public_body_category_id :integer not null +# public_body_heading_id :integer not null +# category_display_order :integer +# id :integer not null, primary key # class PublicBodyCategoryLink < ActiveRecord::Base diff --git a/app/models/public_body_heading.rb b/app/models/public_body_heading.rb index f1916d233..f394c37c6 100644 --- a/app/models/public_body_heading.rb +++ b/app/models/public_body_heading.rb @@ -2,8 +2,7 @@ # # Table name: public_body_headings # -# id :integer not null, primary key -# name :text not null +# id :integer not null, primary key # display_order :integer # diff --git a/app/models/user.rb b/app/models/user.rb index 1c6dc0eb0..c953e52f2 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -264,11 +264,9 @@ class User < ActiveRecord::Base # Returns list of requests which the user hasn't described (and last # changed more than a day ago) def get_undescribed_requests - info_requests.find( - :all, - :conditions => [ 'awaiting_description = ? and ' + InfoRequest.last_event_time_clause + ' < ?', - true, Time.now() - 1.day - ] + info_requests.where( + "awaiting_description = ? and #{ InfoRequest.last_event_time_clause } < ?", + true, 1.day.ago ) end |