aboutsummaryrefslogtreecommitdiffstats
path: root/app/models
diff options
context:
space:
mode:
Diffstat (limited to 'app/models')
-rw-r--r--app/models/holiday.rb2
-rw-r--r--app/models/holiday_import.rb93
-rw-r--r--app/models/incoming_message.rb158
-rw-r--r--app/models/info_request.rb45
-rw-r--r--app/models/info_request_event.rb9
-rw-r--r--app/models/public_body.rb30
-rw-r--r--app/models/public_body_category.rb7
-rw-r--r--app/models/public_body_category_link.rb9
-rw-r--r--app/models/public_body_heading.rb3
-rw-r--r--app/models/user.rb8
10 files changed, 167 insertions, 197 deletions
diff --git a/app/models/holiday.rb b/app/models/holiday.rb
index 4c4941589..34044683a 100644
--- a/app/models/holiday.rb
+++ b/app/models/holiday.rb
@@ -22,6 +22,8 @@
class Holiday < ActiveRecord::Base
+ validates_presence_of :day
+
def self.holidays
@@holidays ||= all.collect { |h| h.day }.to_set
end
diff --git a/app/models/holiday_import.rb b/app/models/holiday_import.rb
new file mode 100644
index 000000000..c6019fac0
--- /dev/null
+++ b/app/models/holiday_import.rb
@@ -0,0 +1,93 @@
+class HolidayImport
+
+ include ActiveModel::Validations
+
+ attr_accessor :holidays,
+ :ical_feed_url,
+ :start_year,
+ :end_year,
+ :start_date,
+ :end_date,
+ :source,
+ :populated
+
+ validate :all_holidays_valid
+ validates_inclusion_of :source, :in => %w( suggestions feed )
+ validates_presence_of :ical_feed_url,
+ :if => proc { |holiday_import| holiday_import.source == 'feed' }
+
+ def initialize(opts = {})
+ @populated = false
+ @start_year = opts.fetch(:start_year, Time.now.year).to_i
+ @end_year = opts.fetch(:end_year, Time.now.year).to_i
+ @start_date = Date.civil(start_year, 1, 1)
+ @end_date = Date.civil(end_year, 12, 31)
+ @source = opts.fetch(:source, 'suggestions')
+ @ical_feed_url = opts.fetch(:ical_feed_url, nil)
+ @country_code = AlaveteliConfiguration::iso_country_code.downcase
+ self.holidays_attributes = opts.fetch(:holidays_attributes, [])
+ end
+
+ def populate
+ source == 'suggestions' ? populate_from_suggestions : populate_from_ical_feed
+ @populated = true
+ end
+
+ def suggestions_country_name
+ IsoCountryCodes.find(@country_code).name if @country_code
+ end
+
+ def period
+ start_year == end_year ? "#{start_year}" : "#{start_year}-#{end_year}"
+ end
+
+ def save
+ holidays.all?(&:save)
+ end
+
+ def holidays_attributes=(incoming_data)
+ incoming_data.each{ |offset, incoming| self.holidays << Holiday.new(incoming) }
+ end
+
+ def holidays
+ @holidays ||= []
+ end
+
+ private
+
+ def all_holidays_valid
+ errors.add(:base, 'These holidays could not be imported') unless holidays.all?(&:valid?)
+ end
+
+ def populate_from_ical_feed
+ begin
+ cal_file = open(ical_feed_url)
+ cals = Icalendar.parse(cal_file, strict=false)
+ cal = cals.first
+ cal.events.each{ |cal_event| populate_from_ical_event(cal_event) }
+ rescue Errno::ENOENT, Exception => e
+ if e.message == 'Invalid line in calendar string!'
+ errors.add(:ical_feed_url, "Sorry, there's a problem with the format of that feed.")
+ elsif e.message.starts_with 'No such file or directory'
+ errors.add(:ical_feed_url, "Sorry we couldn't find that feed.")
+ else
+ raise e
+ end
+ end
+ end
+
+ def populate_from_ical_event(cal_event)
+ if cal_event.dtstart >= start_date and cal_event.dtstart <= end_date
+ holidays << Holiday.new(:description => cal_event.summary,
+ :day => cal_event.dtstart)
+ end
+ end
+
+ def populate_from_suggestions
+ holiday_info = Holidays.between(start_date, end_date, @country_code.to_sym, :observed)
+ holiday_info.each do |holiday_info_hash|
+ holidays << Holiday.new(:description => holiday_info_hash[:name],
+ :day => holiday_info_hash[:date])
+ end
+ end
+end
diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb
index db6722976..658ee969a 100644
--- a/app/models/incoming_message.rb
+++ b/app/models/incoming_message.rb
@@ -52,17 +52,6 @@ class IncomingMessage < ActiveRecord::Base
has_prominence
- # See binary_mask_stuff function below. It just test for inclusion
- # in this hash, not the value of the right hand side.
- DoNotBinaryMask = {
- 'image/tiff' => 1,
- 'image/gif' => 1,
- 'image/jpeg' => 1,
- 'image/png' => 1,
- 'image/bmp' => 1,
- 'application/zip' => 1,
- }
-
# Given that there are in theory many info request events, a convenience method for
# getting the response event
def response_event
@@ -218,111 +207,10 @@ class IncomingMessage < ActiveRecord::Base
end
end
- # Converts email addresses we know about into textual descriptions of them
- def mask_special_emails!(text)
- # TODO: can later display some of these special emails as actual emails,
- # if they are public anyway. For now just be precautionary and only
- # put in descriptions of them in square brackets.
- if self.info_request.public_body.is_followupable?
- text.gsub!(self.info_request.public_body.request_email, _("[{{public_body}} request email]", :public_body => self.info_request.public_body.short_or_long_name))
- end
- text.gsub!(self.info_request.incoming_email, _('[FOI #{{request}} email]', :request => self.info_request.id.to_s) )
- text.gsub!(AlaveteliConfiguration::contact_email, _("[{{site_name}} contact email]", :site_name => AlaveteliConfiguration::site_name) )
- end
-
- # Replaces all email addresses in (possibly binary data) with equal length alternative ones.
- # Also replaces censor items
- def binary_mask_stuff!(text, content_type)
- # See if content type is one that we mask - things like zip files and
- # images may get broken if we try to. We err on the side of masking too
- # much, as many unknown types will really be text.
- if DoNotBinaryMask.include?(content_type)
- return
- end
-
- # Special cases for some content types
- if content_type == 'application/pdf'
- uncompressed_text = nil
- uncompressed_text = AlaveteliExternalCommand.run("pdftk", "-", "output", "-", "uncompress", :stdin_string => text)
- # if we managed to uncompress the PDF...
- if !uncompressed_text.nil? && !uncompressed_text.empty?
- # then censor stuff (making a copy so can compare again in a bit)
- censored_uncompressed_text = uncompressed_text.dup
- self._binary_mask_stuff_internal!(censored_uncompressed_text)
- # if the censor rule removed something...
- if censored_uncompressed_text != uncompressed_text
- # then use the altered file (recompressed)
- recompressed_text = nil
- if AlaveteliConfiguration::use_ghostscript_compression == true
- command = ["gs", "-sDEVICE=pdfwrite", "-dCompatibilityLevel=1.4", "-dPDFSETTINGS=/screen", "-dNOPAUSE", "-dQUIET", "-dBATCH", "-sOutputFile=-", "-"]
- else
- command = ["pdftk", "-", "output", "-", "compress"]
- end
- recompressed_text = AlaveteliExternalCommand.run(*(command + [{:stdin_string=>censored_uncompressed_text}]))
- if recompressed_text.nil? || recompressed_text.empty?
- # buggy versions of pdftk sometimes fail on
- # compression, I don't see it's a disaster in
- # these cases to save an uncompressed version?
- recompressed_text = censored_uncompressed_text
- logger.warn "Unable to compress PDF; problem with your pdftk version?"
- end
- if !recompressed_text.nil? && !recompressed_text.empty?
- text.replace recompressed_text
- end
- end
- end
- return
- end
-
- self._binary_mask_stuff_internal!(text)
- end
-
- # Used by binary_mask_stuff - replace text in place
- def _binary_mask_stuff_internal!(text)
- # Keep original size, so can check haven't resized it
- orig_size = text.mb_chars.size
-
- # Replace ASCII email addresses...
- text.gsub!(MySociety::Validate.email_find_regexp) do |email|
- email.gsub(/[^@.]/, 'x')
- end
-
- # And replace UCS-2 ones (for Microsoft Office documents)...
- # Find emails, by finding them in parts of text that have ASCII
- # equivalents to the UCS-2
- ascii_chars = text.gsub(/\0/, "")
- emails = ascii_chars.scan(MySociety::Validate.email_find_regexp)
-
- # Convert back to UCS-2, making a mask at the same time
- if String.method_defined?(:encode)
- emails.map! do |email|
- # We want the ASCII representation of UCS-2
- [email[0].encode('UTF-16LE').force_encoding('US-ASCII'),
- email[0].gsub(/[^@.]/, 'x').encode('UTF-16LE').force_encoding('US-ASCII')]
- end
- else
- emails.map! {|email| [
- Iconv.conv('ucs-2le', 'ascii', email[0]),
- Iconv.conv('ucs-2le', 'ascii', email[0].gsub(/[^@.]/, 'x'))
- ] }
- end
-
- # Now search and replace the UCS-2 email with the UCS-2 mask
- for email, mask in emails
- text.gsub!(email, mask)
- end
-
- # Replace censor items
- self.info_request.apply_censor_rules_to_binary!(text)
-
- raise "internal error in binary_mask_stuff" if text.mb_chars.size != orig_size
- return text
- end
-
- # Removes censored stuff from from HTML conversion of downloaded binaries
- def html_mask_stuff!(html)
- self.mask_special_emails!(html)
- self.remove_privacy_sensitive_things!(html)
+ def apply_masks!(text, content_type)
+ mask_options = { :censor_rules => info_request.applicable_censor_rules,
+ :masks => info_request.masks }
+ AlaveteliTextMasker.apply_masks!(text, content_type, mask_options)
end
# Lotus notes quoting yeuch!
@@ -346,26 +234,6 @@ class IncomingMessage < ActiveRecord::Base
end
- # Remove emails, mobile phones and other details FOI officers ask us to remove.
- def remove_privacy_sensitive_things!(text)
- # Remove any email addresses - we don't want bounce messages to leak out
- # either the requestor's email address or the request's response email
- # address out onto the internet
- text.gsub!(MySociety::Validate.email_find_regexp, "[email address]")
-
- # Mobile phone numbers
- # http://www.whatdotheyknow.com/request/failed_test_purchases_off_licenc#incoming-1013
- # http://www.whatdotheyknow.com/request/selective_licensing_statistics_i#incoming-550
- # http://www.whatdotheyknow.com/request/common_purpose_training_graduate#incoming-774
- text.gsub!(/(Mobile|Mob)([\s\/]*(Fax|Tel))*\s*:?[\s\d]*\d/, "[mobile number]")
-
- # Remove WhatDoTheyKnow signup links
- text.gsub!(/http:\/\/#{AlaveteliConfiguration::domain}\/c\/[^\s]+/, "[WDTK login link]")
-
- # Remove things from censor rules
- self.info_request.apply_censor_rules_to_text!(text)
- end
-
# Remove quoted sections from emails (eventually the aim would be for this
# to do as good a job as GMail does) TODO: bet it needs a proper parser
@@ -465,9 +333,8 @@ class IncomingMessage < ActiveRecord::Base
raise "main body text more than 1 MB, need to implement clipping like for attachment text, or there is some other MIME decoding problem or similar"
end
- # remove emails for privacy/anti-spam reasons
- self.mask_special_emails!(text)
- self.remove_privacy_sensitive_things!(text)
+ # apply masks for this message
+ apply_masks!(text, 'text/html')
# Remove existing quoted sections
folded_quoted_text = self.remove_lotus_quoting(text, 'FOLDED_QUOTED_SECTION')
@@ -735,7 +602,14 @@ class IncomingMessage < ActiveRecord::Base
text = MySociety::Format.simplify_angle_bracketed_urls(text)
text = CGI.escapeHTML(text)
text = MySociety::Format.make_clickable(text, :contract => 1)
- text.gsub!(/\[(email address|mobile number)\]/, '[<a href="/help/officers#mobiles">\1</a>]')
+
+ # add a helpful link to email addresses and mobile numbers removed
+ # by apply_masks!
+ email_pattern = Regexp.escape(_("email address"))
+ mobile_pattern = Regexp.escape(_("mobile number"))
+ text.gsub!(/\[(#{email_pattern}|#{mobile_pattern})\]/,
+ '[<a href="/help/officers#mobiles">\1</a>]')
+
if collapse_quoted_sections
text = text.gsub(/(\s*FOLDED_QUOTED_SECTION\s*)+/m, "FOLDED_QUOTED_SECTION")
text.strip!
@@ -773,8 +647,8 @@ class IncomingMessage < ActiveRecord::Base
# Returns text version of attachment text
def get_attachment_text_full
text = self._get_attachment_text_internal
- self.mask_special_emails!(text)
- self.remove_privacy_sensitive_things!(text)
+ apply_masks!(text, 'text/html')
+
# This can be useful for memory debugging
#STDOUT.puts 'xxx '+ MySociety::DebugHelpers::allocated_string_size_around_gc
diff --git a/app/models/info_request.rb b/app/models/info_request.rb
index d0052603a..8d455e488 100644
--- a/app/models/info_request.rb
+++ b/app/models/info_request.rb
@@ -210,16 +210,6 @@ class InfoRequest < ActiveRecord::Base
OLD_AGE_IN_DAYS = 21.days
- def after_initialize
- if self.described_state.nil?
- self.described_state = 'waiting_response'
- end
- # FOI or EIR?
- if !self.public_body.nil? && self.public_body.eir_only?
- self.law_used = 'eir'
- end
- end
-
def visible_comments
self.comments.find(:all, :conditions => 'visible')
end
@@ -292,13 +282,18 @@ public
end
# Subject lines for emails about the request
- def email_subject_request
- _('{{law_used_full}} request - {{title}}',:law_used_full=>self.law_used_full,:title=>self.title.html_safe)
+ def email_subject_request(opts = {})
+ html = opts.fetch(:html, true)
+ _('{{law_used_full}} request - {{title}}',
+ :law_used_full => self.law_used_full,
+ :title => (html ? title : title.html_safe))
end
- def email_subject_followup(incoming_message = nil)
+ def email_subject_followup(opts = {})
+ incoming_message = opts.fetch(:incoming_message, nil)
+ html = opts.fetch(:html, true)
if incoming_message.nil? || !incoming_message.valid_to_reply_to? || !incoming_message.subject
- 'Re: ' + self.email_subject_request
+ 'Re: ' + self.email_subject_request(:html => html)
else
if incoming_message.subject.match(/^Re:/i)
incoming_message.subject
@@ -1148,6 +1143,22 @@ public
return binary
end
+ # Masks we apply to text associated with this request convert email addresses
+ # we know about into textual descriptions of them
+ def masks
+ masks = [{ :to_replace => incoming_email,
+ :replacement => _('[FOI #{{request}} email]',
+ :request => id.to_s) },
+ { :to_replace => AlaveteliConfiguration::contact_email,
+ :replacement => _("[{{site_name}} contact email]",
+ :site_name => AlaveteliConfiguration::site_name)} ]
+ if public_body.is_followupable?
+ masks << { :to_replace => public_body.request_email,
+ :replacement => _("[{{public_body}} request email]",
+ :public_body => public_body.short_or_long_name) }
+ end
+ end
+
def is_owning_user?(user)
!user.nil? && (user.id == user_id || user.owns_every_request?)
end
@@ -1345,9 +1356,9 @@ public
end
def InfoRequest.find_in_state(state)
- find(:all, :select => '*, ' + last_event_time_clause + ' as last_event_time',
- :conditions => ["described_state = ?", state],
- :order => "last_event_time")
+ select("*, #{ last_event_time_clause } as last_event_time").
+ where(:described_state => state).
+ order('last_event_time')
end
private
diff --git a/app/models/info_request_event.rb b/app/models/info_request_event.rb
index 9dde3ba80..635ba8f58 100644
--- a/app/models/info_request_event.rb
+++ b/app/models/info_request_event.rb
@@ -161,11 +161,10 @@ class InfoRequestEvent < ActiveRecord::Base
end
def incoming_message_selective_columns(fields)
- message = IncomingMessage.find(:all,
- :select => fields + ", incoming_messages.info_request_id",
- :joins => "INNER JOIN info_request_events ON incoming_messages.id = incoming_message_id ",
- :conditions => "info_request_events.id = #{self.id}"
- )
+ message = IncomingMessage.select("#{ fields }, incoming_messages.info_request_id").
+ joins('INNER JOIN info_request_events ON incoming_messages.id = incoming_message_id').
+ where('info_request_events.id = ?', id)
+
message = message[0]
if !message.nil?
message.info_request = InfoRequest.find(message.info_request_id)
diff --git a/app/models/public_body.rb b/app/models/public_body.rb
index 829625cac..b8163b07d 100644
--- a/app/models/public_body.rb
+++ b/app/models/public_body.rb
@@ -349,33 +349,29 @@ class PublicBody < ActiveRecord::Base
# Use tags to describe what type of thing this is
def type_of_authority(html = false)
- types = []
- first = true
- for tag in self.tags
+ types = tags.each_with_index.map do |tag, index|
if PublicBodyCategory.get().by_tag().include?(tag.name)
desc = PublicBodyCategory.get().singular_by_tag()[tag.name]
- if first
- # terrible that Ruby/Rails doesn't have an equivalent of ucfirst
- # (capitalize shockingly converts later characters to lowercase)
- desc = desc[0,1].capitalize + desc[1,desc.size]
- first = false
+
+ if index.zero?
+ desc = desc.sub(/\S/) { |m| Unicode.upcase(m) }
end
+
if html
# TODO: this should call proper route helpers, but is in model sigh
desc = '<a href="/body/list/' + tag.name + '">' + desc + '</a>'
end
- types.push(desc)
+
+ desc
end
end
- if types.size > 0
- ret = types[0, types.size - 1].join(", ")
- if types.size > 1
- ret = ret + " and "
- end
- ret = ret + types[-1]
- return ret.html_safe
+
+ types.compact!
+
+ if types.any?
+ types.to_sentence(:last_word_connector => ' and ').html_safe
else
- return _("A public authority")
+ _("A public authority")
end
end
diff --git a/app/models/public_body_category.rb b/app/models/public_body_category.rb
index bb83c4c82..c313e5734 100644
--- a/app/models/public_body_category.rb
+++ b/app/models/public_body_category.rb
@@ -2,11 +2,8 @@
#
# Table name: public_body_categories
#
-# id :integer not null, primary key
-# title :text not null
-# category_tag :text not null
-# description :text not null
-# display_order :integer
+# id :integer not null, primary key
+# category_tag :text not null
#
require 'forwardable'
diff --git a/app/models/public_body_category_link.rb b/app/models/public_body_category_link.rb
index ba3ff1f95..8c3eb8060 100644
--- a/app/models/public_body_category_link.rb
+++ b/app/models/public_body_category_link.rb
@@ -1,10 +1,11 @@
# == Schema Information
#
-# Table name: public_body_category_link
+# Table name: public_body_category_links
#
-# public_body_category_id :integer not null
-# public_body_heading_id :integer not null
-# category_display_order :integer
+# public_body_category_id :integer not null
+# public_body_heading_id :integer not null
+# category_display_order :integer
+# id :integer not null, primary key
#
class PublicBodyCategoryLink < ActiveRecord::Base
diff --git a/app/models/public_body_heading.rb b/app/models/public_body_heading.rb
index f1916d233..f394c37c6 100644
--- a/app/models/public_body_heading.rb
+++ b/app/models/public_body_heading.rb
@@ -2,8 +2,7 @@
#
# Table name: public_body_headings
#
-# id :integer not null, primary key
-# name :text not null
+# id :integer not null, primary key
# display_order :integer
#
diff --git a/app/models/user.rb b/app/models/user.rb
index 1c6dc0eb0..c953e52f2 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -264,11 +264,9 @@ class User < ActiveRecord::Base
# Returns list of requests which the user hasn't described (and last
# changed more than a day ago)
def get_undescribed_requests
- info_requests.find(
- :all,
- :conditions => [ 'awaiting_description = ? and ' + InfoRequest.last_event_time_clause + ' < ?',
- true, Time.now() - 1.day
- ]
+ info_requests.where(
+ "awaiting_description = ? and #{ InfoRequest.last_event_time_clause } < ?",
+ true, 1.day.ago
)
end
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-22 11:18:18 +0000