diff options
Diffstat (limited to 'app/models')
| -rw-r--r-- | app/models/exim_log.rb | 129 | ||||
| -rw-r--r-- | app/models/info_request.rb | 21 | ||||
| -rw-r--r-- | app/models/mail_server_log.rb | 201 | ||||
| -rw-r--r-- | app/models/mail_server_log_done.rb (renamed from app/models/exim_log_done.rb) | 11 | ||||
| -rw-r--r-- | app/models/public_body.rb | 49 | ||||
| -rw-r--r-- | app/models/request_mailer.rb | 37 | ||||
| -rw-r--r-- | app/models/user.rb | 11 |
7 files changed, 302 insertions, 157 deletions
diff --git a/app/models/exim_log.rb b/app/models/exim_log.rb deleted file mode 100644 index abe198493..000000000 --- a/app/models/exim_log.rb +++ /dev/null @@ -1,129 +0,0 @@ -# == Schema Information -# Schema version: 114 -# -# Table name: exim_logs -# -# id :integer not null, primary key -# exim_log_done_id :integer -# info_request_id :integer -# order :integer not null -# line :text not null -# created_at :datetime not null -# updated_at :datetime not null -# - -# models/exim_log.rb: -# We load log file lines for requests in here, for display in the admin interface. -# -# Copyright (c) 2009 UK Citizens Online Democracy. All rights reserved. -# Email: francis@mysociety.org; WWW: http://www.mysociety.org/ - -class EximLog < ActiveRecord::Base - belongs_to :info_request - belongs_to :exim_log_done - - # Load in exim log file from disk, or update if we already have it - # Assumes files are named with date, rather than cyclically. - # Doesn't do anything if file hasn't been modified since it was last loaded. - def EximLog.load_file(file_name) - file_name_db = file_name - is_gz = false - if file_name.include?(".gz") - is_gz = true - file_name_db = file_name.gsub(".gz", "") - end - - modified = File::stat(file_name).mtime - raise "EximLog.load_file: file not found " + file_name if modified.nil? - - ActiveRecord::Base.transaction do - # see if we already have it - done = EximLogDone.find_by_filename(file_name_db) - if !done.nil? - if modified.utc == done.last_stat.utc - # already have that, nothing to do - return - end - EximLog.delete_all "exim_log_done_id = " + done.id.to_s - end - if !done - done = EximLogDone.new - done.filename = file_name_db - end - done.last_stat = modified - - # scan the file - if is_gz - f = Zlib::GzipReader.open(file_name) - else - f = File.open(file_name, 'r') - end - order = 0 - for line in f - order = order + 1 - email_domain = Configuration::incoming_email_domain - emails = line.scan(/request-[^\s]+@#{email_domain}/).sort.uniq - for email in emails - info_request = InfoRequest.find_by_incoming_email(email) - if !info_request.nil? - exim_log = EximLog.new - exim_log.info_request = info_request - exim_log.exim_log_done = done - exim_log.line = line - exim_log.order = order - exim_log.save! - end - end - end - - # update done structure so we know when we last read this file - done.save! - end - end - - # Check that the last day of requests has been sent in Exim and we got the - # lines. Writes any errors to STDERR. This check is really mainly to - # check the envelope from is the request address, as Ruby is quite - # flaky with regard to that, and it is important for anti-spam reasons. - # XXX does this really check that, as the exim log just wouldn't pick - # up at all if the requests weren't sent that way as there would be - # no request- email in it? - def EximLog.check_recent_requests_have_been_sent - # Get all requests sent for from 2 to 10 days ago. The 2 day gap is - # because we load exim log lines via cron at best an hour after they - # are made) - irs = InfoRequest.find(:all, :conditions => [ "created_at < ? and created_at > ? and user_id is not null", Time.now() - 2.day, Time.now() - 10.days ] ) - - # Go through each request and check it - ok = true - for ir in irs - # Look for line showing request was sent - found = false - for exim_log in ir.exim_logs - test_outgoing = " <= " + ir.incoming_email + " " - if exim_log.line.include?(test_outgoing) - # Check the from value is the same (it always will be, but may as well - # be sure we are parsing the exim line right) - envelope_from = " from <" + ir.incoming_email + "> " - if !exim_log.line.include?(envelope_from) - $stderr.puts("unexpected parsing of exim line: [#{exim_log.line.chomp}]") - else - found = true - end - end - end - if !found - # It's very important the envelope from is set for avoiding spam filter reasons - this - # effectively acts as a check for that. - $stderr.puts("failed to find request sending Exim line for request id " + ir.id.to_s + " " + ir.url_title + " (check envelope from is being set to request address in Ruby, and load-exim-logs crontab is working)") # *** don't comment out this STDERR line, it is the point of the function! - ok = false - end - end - - return ok - end - -end - - - diff --git a/app/models/info_request.rb b/app/models/info_request.rb index 85168e6d4..47424e573 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -47,10 +47,12 @@ class InfoRequest < ActiveRecord::Base has_many :track_things, :order => 'created_at desc' has_many :comments, :order => 'created_at' has_many :censor_rules, :order => 'created_at desc' - has_many :exim_logs, :order => 'exim_log_done_id' + has_many :mail_server_logs, :order => 'mail_server_log_done_id' has_tag_string + named_scope :visible, :conditions => {:prominence => "normal"} + # user described state (also update in info_request_event, admin_request/edit.rhtml) validate :must_be_valid_state @@ -582,12 +584,11 @@ public # waiting_classification # waiting_response_overdue # waiting_response_very_overdue - def calculate_status - if @@custom_states_loaded - return self.theme_calculate_status - else - self.base_calculate_status + def calculate_status(cached_value_ok=false) + if cached_value_ok && @cached_calculated_status + return @cached_calculated_status end + @cached_calculated_status = @@custom_states_loaded ? self.theme_calculate_status : self.base_calculate_status end def base_calculate_status @@ -869,8 +870,8 @@ public end end - def display_status - InfoRequest.get_status_description(self.calculate_status) + def display_status(cached_value_ok=false) + InfoRequest.get_status_description(self.calculate_status(cached_value_ok)) end # Completely delete this request and all objects depending on it @@ -884,8 +885,8 @@ public info_request_event.track_things_sent_emails.each { |a| a.destroy } info_request_event.destroy end - self.exim_logs.each do |exim_log| - exim_log.destroy + self.mail_server_logs.each do |mail_server_log| + mail_server_log.destroy end self.outgoing_messages.each { |a| a.destroy } self.incoming_messages.each { |a| a.destroy } diff --git a/app/models/mail_server_log.rb b/app/models/mail_server_log.rb new file mode 100644 index 000000000..755584b90 --- /dev/null +++ b/app/models/mail_server_log.rb @@ -0,0 +1,201 @@ +# == Schema Information +# Schema version: 20121010214348 +# +# Table name: mail_server_logs +# +# id :integer not null, primary key +# mail_server_log_done_id :integer +# info_request_id :integer +# order :integer not null +# line :text not null +# created_at :datetime not null +# updated_at :datetime not null +# + +# We load log file lines for requests in here, for display in the admin interface. +# +# Copyright (c) 2009 UK Citizens Online Democracy. All rights reserved. +# Email: francis@mysociety.org; WWW: http://www.mysociety.org/ +# +# $Id: exim_log.rb,v 1.14 2009-09-17 21:10:05 francis Exp $ + +class MailServerLog < ActiveRecord::Base + belongs_to :info_request + belongs_to :mail_server_log_done + + # Load in exim or postfix log file from disk, or update if we already have it + # Assumes files are named with date, rather than cyclically. + # Doesn't do anything if file hasn't been modified since it was last loaded. + # Note: If you do use rotated log files (rather than files named by date), at some + # point old loaded log lines will get deleted in the database. + def MailServerLog.load_file(file_name) + is_gz = file_name.include?(".gz") + file_name_db = is_gz ? file_name.gsub(".gz", "") : file_name + + modified = File.stat(file_name).mtime + raise "MailServerLog.load_file: file not found " + file_name if modified.nil? + + ActiveRecord::Base.transaction do + # see if we already have it + done = MailServerLogDone.find_by_filename(file_name_db) + if done + if modified.utc == done.last_stat.utc + # already have that, nothing to do + return + else + MailServerLog.delete_all "mail_server_log_done_id = " + done.id.to_s + end + else + done = MailServerLogDone.new(:filename => file_name_db) + end + done.last_stat = modified + # update done structure so we know when we last read this file + done.save! + + f = is_gz ? Zlib::GzipReader.open(file_name) : File.open(file_name, 'r') + case(Configuration::mta_log_type.to_sym) + when :exim + load_exim_log_data(f, done) + when :postfix + load_postfix_log_data(f, done) + else + raise "Unexpected MTA type: #{type}" + end + end + end + + # Scan the file + def MailServerLog.load_exim_log_data(f, done) + order = 0 + f.each do |line| + order = order + 1 + emails = email_addresses_on_line(line) + for email in emails + info_request = InfoRequest.find_by_incoming_email(email) + if info_request + info_request.mail_server_logs.create!(:line => line, :order => order, :mail_server_log_done => done) + else + puts "Warning: Could not find request with email #{email}" + end + end + end + end + + def MailServerLog.load_postfix_log_data(f, done) + order = 0 + emails = scan_for_postfix_queue_ids(f) + # Go back to the beginning of the file + f.rewind + f.each do |line| + order = order + 1 + queue_id = extract_postfix_queue_id_from_syslog_line(line) + if emails.has_key?(queue_id) + emails[queue_id].each do |email| + info_request = InfoRequest.find_by_incoming_email(email) + if info_request + info_request.mail_server_logs.create!(:line => line, :order => order, :mail_server_log_done => done) + else + puts "Warning: Could not find request with email #{email}" + end + end + end + end + end + + def MailServerLog.scan_for_postfix_queue_ids(f) + result = {} + f.each do |line| + emails = email_addresses_on_line(line) + queue_id = extract_postfix_queue_id_from_syslog_line(line) + result[queue_id] = [] unless result.has_key?(queue_id) + result[queue_id] = (result[queue_id] + emails).uniq + end + result + end + + # Retuns nil if there is no queue id + def MailServerLog.extract_postfix_queue_id_from_syslog_line(line) + # Assume the log file was written using syslog and parse accordingly + m = SyslogProtocol.parse("<13>" + line).content.match(/^\S+: (\S+):/) + m[1] if m + end + + # We also check the email prefix so that we could, for instance, separately handle a staging and production + # instance running on the same server with different email prefixes. + def MailServerLog.email_addresses_on_line(line) + prefix = Regexp::quote(Configuration::incoming_email_prefix) + domain = Regexp::quote(Configuration::incoming_email_domain) + line.scan(/#{prefix}request-[^\s]+@#{domain}/).sort.uniq + end + + def MailServerLog.request_sent?(ir) + case(Configuration::mta_log_type.to_sym) + when :exim + request_exim_sent?(ir) + when :postfix + request_postfix_sent?(ir) + else + raise "Unexpected MTA type: #{type}" + end + end + + # Look at the log for a request and check that an email was delivered + def MailServerLog.request_exim_sent?(ir) + # Look for line showing request was sent + found = false + ir.mail_server_logs.each do |mail_server_log| + test_outgoing = " <= " + ir.incoming_email + " " + if mail_server_log.line.include?(test_outgoing) + # Check the from value is the same (it always will be, but may as well + # be sure we are parsing the exim line right) + envelope_from = " from <" + ir.incoming_email + "> " + if !mail_server_log.line.include?(envelope_from) + $stderr.puts("unexpected parsing of exim line: [#{mail_server_log.line.chomp}]") + else + found = true + end + end + end + found + end + + def MailServerLog.request_postfix_sent?(ir) + # dsn=2.0.0 is the magic word that says that postfix delivered the email + # See http://tools.ietf.org/html/rfc3464 + ir.mail_server_logs.any? { |l| l.line.include?("dsn=2.0.0") } + end + + # Check that the last day of requests has been sent in Exim or Postfix and we got the + # lines. Writes any errors to STDERR. This check is really mainly to + # check the envelope from is the request address, as Ruby is quite + # flaky with regard to that, and it is important for anti-spam reasons. + # XXX does this really check that, as the log just wouldn't pick + # up at all if the requests weren't sent that way as there would be + # no request- email in it? + # + # NB: There can be several emails involved in a request. This just checks that + # at least one of them has been succesfully sent. + # + def MailServerLog.check_recent_requests_have_been_sent + # Get all requests sent for from 2 to 10 days ago. The 2 day gap is + # because we load mail server log lines via cron at best an hour after they + # are made) + irs = InfoRequest.find(:all, :conditions => [ "created_at < ? and created_at > ? and user_id is not null", Time.now() - 2.day, Time.now() - 10.days ] ) + + # Go through each request and check it + ok = true + irs.each do |ir| + unless request_sent?(ir) + # It's very important the envelope from is set for avoiding spam filter reasons - this + # effectively acts as a check for that. + $stderr.puts("failed to find request sending in MTA logs for request id " + ir.id.to_s + " " + ir.url_title + " (check envelope from is being set to request address in Ruby, and load-mail-server-logs crontab is working)") # *** don't comment out this STDERR line, it is the point of the function! + ok = false + end + end + ok + end + +end + + + diff --git a/app/models/exim_log_done.rb b/app/models/mail_server_log_done.rb index 86574a4cd..3fb20f0b3 100644 --- a/app/models/exim_log_done.rb +++ b/app/models/mail_server_log_done.rb @@ -1,7 +1,7 @@ # == Schema Information -# Schema version: 114 +# Schema version: 20121010214348 # -# Table name: exim_log_dones +# Table name: mail_server_log_dones # # id :integer not null, primary key # filename :text not null @@ -10,14 +10,13 @@ # updated_at :datetime not null # -# models/exim_log_done.rb: -# Stores that a particular exim file has been loaded in, see exim_log.rb +# Stores that a particular mail server log file has been loaded in, see mail_server_log.rb # # Copyright (c) 2009 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ -class EximLogDone < ActiveRecord::Base - has_many :exim_logs +class MailServerLogDone < ActiveRecord::Base + has_many :mail_server_logs end diff --git a/app/models/public_body.rb b/app/models/public_body.rb index 34bcd332c..57fe27767 100644 --- a/app/models/public_body.rb +++ b/app/models/public_body.rb @@ -17,7 +17,7 @@ # notes :text default(""), not null # first_letter :string(255) not null # publication_scheme :text default(""), not null -# api_key :string(255) +# api_key :string(255) not null # info_requests_count :integer default(0), not null # @@ -42,6 +42,12 @@ class PublicBody < ActiveRecord::Base has_tag_string before_save :set_api_key, :set_default_publication_scheme + # Every public body except for the internal admin one is visible + named_scope :visible, lambda { + { + :conditions => "public_bodies.id <> #{PublicBody.internal_admin_body.id}" + } + } translates :name, :short_name, :request_email, :url_name, :notes, :first_letter, :publication_scheme @@ -407,7 +413,7 @@ class PublicBody < ActiveRecord::Base next end - field_list = ['name', 'short_name', 'request_email', 'notes', 'publication_scheme', 'home_page', 'tag_string'] + field_list = ['name', 'short_name', 'request_email', 'notes', 'publication_scheme', 'disclosure_log', 'home_page', 'tag_string'] if public_body = bodies_by_name[name] # Existing public body available_locales.each do |locale| @@ -494,6 +500,45 @@ class PublicBody < ActiveRecord::Base return [errors, notes] end + # Returns all public bodies (except for the internal admin authority) as csv + def self.export_csv + public_bodies = PublicBody.visible.find(:all, :order => 'url_name', + :include => [:translations, :tags]) + FasterCSV.generate() do |csv| + csv << [ + 'Name', + 'Short name', + # deliberately not including 'Request email' + 'URL name', + 'Tags', + 'Home page', + 'Publication scheme', + 'Disclosure log', + 'Notes', + 'Created at', + 'Updated at', + 'Version', + ] + public_bodies.each do |public_body| + csv << [ + public_body.name, + public_body.short_name, + # DO NOT include request_email (we don't want to make it + # easy to spam all authorities with requests) + public_body.url_name, + public_body.tag_string, + public_body.calculated_home_page, + public_body.publication_scheme, + public_body.disclosure_log, + public_body.notes, + public_body.created_at, + public_body.updated_at, + public_body.version, + ] + end + end + end + # Does this user have the power of FOI officer for this body? def is_foi_officer?(user) user_domain = user.email_domain diff --git a/app/models/request_mailer.rb b/app/models/request_mailer.rb index 413e93e25..90c4c6b53 100644 --- a/app/models/request_mailer.rb +++ b/app/models/request_mailer.rb @@ -256,24 +256,47 @@ class RequestMailer < ApplicationMailer def self.alert_overdue_requests() info_requests = InfoRequest.find(:all, :conditions => [ - "described_state = 'waiting_response' and awaiting_description = ? and user_id is not null", false + "described_state = 'waiting_response' + AND awaiting_description = ? + AND user_id is not null + AND (SELECT id + FROM user_info_request_sent_alerts + WHERE alert_type = 'very_overdue_1' + AND info_request_id = info_requests.id + AND user_id = info_requests.user_id + AND info_request_event_id = (SELECT max(id) + FROM info_request_events + WHERE event_type in ('sent', + 'followup_sent', + 'resent', + 'followup_resent') + AND info_request_id = info_requests.id) + ) IS NULL", false ], :include => [ :user ] ) for info_request in info_requests alert_event_id = info_request.last_event_forming_initial_request.id # Only overdue requests - if ['waiting_response_overdue', 'waiting_response_very_overdue'].include?(info_request.calculate_status) - if info_request.calculate_status == 'waiting_response_overdue' + calculated_status = info_request.calculate_status + if ['waiting_response_overdue', 'waiting_response_very_overdue'].include?(calculated_status) + if calculated_status == 'waiting_response_overdue' alert_type = 'overdue_1' - elsif info_request.calculate_status == 'waiting_response_very_overdue' + elsif calculated_status == 'waiting_response_very_overdue' alert_type = 'very_overdue_1' else raise "unknown request status" end # For now, just to the user who created the request - sent_already = UserInfoRequestSentAlert.find(:first, :conditions => [ "alert_type = ? and user_id = ? and info_request_id = ? and info_request_event_id = ?", alert_type, info_request.user_id, info_request.id, alert_event_id]) + sent_already = UserInfoRequestSentAlert.find(:first, :conditions => [ "alert_type = ? + AND user_id = ? + AND info_request_id = ? + AND info_request_event_id = ?", + alert_type, + info_request.user_id, + info_request.id, + alert_event_id]) if sent_already.nil? # Alert not yet sent for this user, so send it store_sent = UserInfoRequestSentAlert.new @@ -284,9 +307,9 @@ class RequestMailer < ApplicationMailer # Only send the alert if the user can act on it by making a followup # (otherwise they are banned, and there is no point sending it) if info_request.user.can_make_followup? - if info_request.calculate_status == 'waiting_response_overdue' + if calculated_status == 'waiting_response_overdue' RequestMailer.deliver_overdue_alert(info_request, info_request.user) - elsif info_request.calculate_status == 'waiting_response_very_overdue' + elsif calculated_status == 'waiting_response_very_overdue' RequestMailer.deliver_very_overdue_alert(info_request, info_request.user) else raise "unknown request status" diff --git a/app/models/user.rb b/app/models/user.rb index 59f6c971c..4a68d60d1 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -246,6 +246,11 @@ class User < ActiveRecord::Base # Does the user magically gain powers as if they owned every request? # e.g. Can classify it def owns_every_request? + self.super? + end + + # Does this user have extraordinary powers? + def super? self.admin_level == 'super' end @@ -255,18 +260,18 @@ class User < ActiveRecord::Base # Can the user see every request, even hidden ones? def User.view_hidden_requests?(user) - !user.nil? && user.admin_level == 'super' + !user.nil? && user.super? end # Should the user be kept logged into their own account # if they follow a /c/ redirect link belonging to another user? def User.stay_logged_in_on_redirect?(user) - !user.nil? && user.admin_level == 'super' + !user.nil? && user.super? end # Does the user get "(admin)" links on each page on the main site? def admin_page_links? - self.admin_level == 'super' + self.super? end # Is it public that they are banned? def public_banned? |