diff options
Diffstat (limited to 'app/models')
| -rw-r--r-- | app/models/exim_log.rb | 129 | ||||
| -rw-r--r-- | app/models/info_request.rb | 6 | ||||
| -rw-r--r-- | app/models/mail_server_log.rb | 201 | ||||
| -rw-r--r-- | app/models/mail_server_log_done.rb (renamed from app/models/exim_log_done.rb) | 11 | ||||
| -rw-r--r-- | app/models/public_body.rb | 2 |
5 files changed, 210 insertions, 139 deletions
diff --git a/app/models/exim_log.rb b/app/models/exim_log.rb deleted file mode 100644 index abe198493..000000000 --- a/app/models/exim_log.rb +++ /dev/null @@ -1,129 +0,0 @@ -# == Schema Information -# Schema version: 114 -# -# Table name: exim_logs -# -# id :integer not null, primary key -# exim_log_done_id :integer -# info_request_id :integer -# order :integer not null -# line :text not null -# created_at :datetime not null -# updated_at :datetime not null -# - -# models/exim_log.rb: -# We load log file lines for requests in here, for display in the admin interface. -# -# Copyright (c) 2009 UK Citizens Online Democracy. All rights reserved. -# Email: francis@mysociety.org; WWW: http://www.mysociety.org/ - -class EximLog < ActiveRecord::Base - belongs_to :info_request - belongs_to :exim_log_done - - # Load in exim log file from disk, or update if we already have it - # Assumes files are named with date, rather than cyclically. - # Doesn't do anything if file hasn't been modified since it was last loaded. - def EximLog.load_file(file_name) - file_name_db = file_name - is_gz = false - if file_name.include?(".gz") - is_gz = true - file_name_db = file_name.gsub(".gz", "") - end - - modified = File::stat(file_name).mtime - raise "EximLog.load_file: file not found " + file_name if modified.nil? - - ActiveRecord::Base.transaction do - # see if we already have it - done = EximLogDone.find_by_filename(file_name_db) - if !done.nil? - if modified.utc == done.last_stat.utc - # already have that, nothing to do - return - end - EximLog.delete_all "exim_log_done_id = " + done.id.to_s - end - if !done - done = EximLogDone.new - done.filename = file_name_db - end - done.last_stat = modified - - # scan the file - if is_gz - f = Zlib::GzipReader.open(file_name) - else - f = File.open(file_name, 'r') - end - order = 0 - for line in f - order = order + 1 - email_domain = Configuration::incoming_email_domain - emails = line.scan(/request-[^\s]+@#{email_domain}/).sort.uniq - for email in emails - info_request = InfoRequest.find_by_incoming_email(email) - if !info_request.nil? - exim_log = EximLog.new - exim_log.info_request = info_request - exim_log.exim_log_done = done - exim_log.line = line - exim_log.order = order - exim_log.save! - end - end - end - - # update done structure so we know when we last read this file - done.save! - end - end - - # Check that the last day of requests has been sent in Exim and we got the - # lines. Writes any errors to STDERR. This check is really mainly to - # check the envelope from is the request address, as Ruby is quite - # flaky with regard to that, and it is important for anti-spam reasons. - # XXX does this really check that, as the exim log just wouldn't pick - # up at all if the requests weren't sent that way as there would be - # no request- email in it? - def EximLog.check_recent_requests_have_been_sent - # Get all requests sent for from 2 to 10 days ago. The 2 day gap is - # because we load exim log lines via cron at best an hour after they - # are made) - irs = InfoRequest.find(:all, :conditions => [ "created_at < ? and created_at > ? and user_id is not null", Time.now() - 2.day, Time.now() - 10.days ] ) - - # Go through each request and check it - ok = true - for ir in irs - # Look for line showing request was sent - found = false - for exim_log in ir.exim_logs - test_outgoing = " <= " + ir.incoming_email + " " - if exim_log.line.include?(test_outgoing) - # Check the from value is the same (it always will be, but may as well - # be sure we are parsing the exim line right) - envelope_from = " from <" + ir.incoming_email + "> " - if !exim_log.line.include?(envelope_from) - $stderr.puts("unexpected parsing of exim line: [#{exim_log.line.chomp}]") - else - found = true - end - end - end - if !found - # It's very important the envelope from is set for avoiding spam filter reasons - this - # effectively acts as a check for that. - $stderr.puts("failed to find request sending Exim line for request id " + ir.id.to_s + " " + ir.url_title + " (check envelope from is being set to request address in Ruby, and load-exim-logs crontab is working)") # *** don't comment out this STDERR line, it is the point of the function! - ok = false - end - end - - return ok - end - -end - - - diff --git a/app/models/info_request.rb b/app/models/info_request.rb index 85168e6d4..d8c84fa9e 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -47,7 +47,7 @@ class InfoRequest < ActiveRecord::Base has_many :track_things, :order => 'created_at desc' has_many :comments, :order => 'created_at' has_many :censor_rules, :order => 'created_at desc' - has_many :exim_logs, :order => 'exim_log_done_id' + has_many :mail_server_logs, :order => 'mail_server_log_done_id' has_tag_string @@ -884,8 +884,8 @@ public info_request_event.track_things_sent_emails.each { |a| a.destroy } info_request_event.destroy end - self.exim_logs.each do |exim_log| - exim_log.destroy + self.mail_server_logs.each do |mail_server_log| + mail_server_log.destroy end self.outgoing_messages.each { |a| a.destroy } self.incoming_messages.each { |a| a.destroy } diff --git a/app/models/mail_server_log.rb b/app/models/mail_server_log.rb new file mode 100644 index 000000000..755584b90 --- /dev/null +++ b/app/models/mail_server_log.rb @@ -0,0 +1,201 @@ +# == Schema Information +# Schema version: 20121010214348 +# +# Table name: mail_server_logs +# +# id :integer not null, primary key +# mail_server_log_done_id :integer +# info_request_id :integer +# order :integer not null +# line :text not null +# created_at :datetime not null +# updated_at :datetime not null +# + +# We load log file lines for requests in here, for display in the admin interface. +# +# Copyright (c) 2009 UK Citizens Online Democracy. All rights reserved. +# Email: francis@mysociety.org; WWW: http://www.mysociety.org/ +# +# $Id: exim_log.rb,v 1.14 2009-09-17 21:10:05 francis Exp $ + +class MailServerLog < ActiveRecord::Base + belongs_to :info_request + belongs_to :mail_server_log_done + + # Load in exim or postfix log file from disk, or update if we already have it + # Assumes files are named with date, rather than cyclically. + # Doesn't do anything if file hasn't been modified since it was last loaded. + # Note: If you do use rotated log files (rather than files named by date), at some + # point old loaded log lines will get deleted in the database. + def MailServerLog.load_file(file_name) + is_gz = file_name.include?(".gz") + file_name_db = is_gz ? file_name.gsub(".gz", "") : file_name + + modified = File.stat(file_name).mtime + raise "MailServerLog.load_file: file not found " + file_name if modified.nil? + + ActiveRecord::Base.transaction do + # see if we already have it + done = MailServerLogDone.find_by_filename(file_name_db) + if done + if modified.utc == done.last_stat.utc + # already have that, nothing to do + return + else + MailServerLog.delete_all "mail_server_log_done_id = " + done.id.to_s + end + else + done = MailServerLogDone.new(:filename => file_name_db) + end + done.last_stat = modified + # update done structure so we know when we last read this file + done.save! + + f = is_gz ? Zlib::GzipReader.open(file_name) : File.open(file_name, 'r') + case(Configuration::mta_log_type.to_sym) + when :exim + load_exim_log_data(f, done) + when :postfix + load_postfix_log_data(f, done) + else + raise "Unexpected MTA type: #{type}" + end + end + end + + # Scan the file + def MailServerLog.load_exim_log_data(f, done) + order = 0 + f.each do |line| + order = order + 1 + emails = email_addresses_on_line(line) + for email in emails + info_request = InfoRequest.find_by_incoming_email(email) + if info_request + info_request.mail_server_logs.create!(:line => line, :order => order, :mail_server_log_done => done) + else + puts "Warning: Could not find request with email #{email}" + end + end + end + end + + def MailServerLog.load_postfix_log_data(f, done) + order = 0 + emails = scan_for_postfix_queue_ids(f) + # Go back to the beginning of the file + f.rewind + f.each do |line| + order = order + 1 + queue_id = extract_postfix_queue_id_from_syslog_line(line) + if emails.has_key?(queue_id) + emails[queue_id].each do |email| + info_request = InfoRequest.find_by_incoming_email(email) + if info_request + info_request.mail_server_logs.create!(:line => line, :order => order, :mail_server_log_done => done) + else + puts "Warning: Could not find request with email #{email}" + end + end + end + end + end + + def MailServerLog.scan_for_postfix_queue_ids(f) + result = {} + f.each do |line| + emails = email_addresses_on_line(line) + queue_id = extract_postfix_queue_id_from_syslog_line(line) + result[queue_id] = [] unless result.has_key?(queue_id) + result[queue_id] = (result[queue_id] + emails).uniq + end + result + end + + # Retuns nil if there is no queue id + def MailServerLog.extract_postfix_queue_id_from_syslog_line(line) + # Assume the log file was written using syslog and parse accordingly + m = SyslogProtocol.parse("<13>" + line).content.match(/^\S+: (\S+):/) + m[1] if m + end + + # We also check the email prefix so that we could, for instance, separately handle a staging and production + # instance running on the same server with different email prefixes. + def MailServerLog.email_addresses_on_line(line) + prefix = Regexp::quote(Configuration::incoming_email_prefix) + domain = Regexp::quote(Configuration::incoming_email_domain) + line.scan(/#{prefix}request-[^\s]+@#{domain}/).sort.uniq + end + + def MailServerLog.request_sent?(ir) + case(Configuration::mta_log_type.to_sym) + when :exim + request_exim_sent?(ir) + when :postfix + request_postfix_sent?(ir) + else + raise "Unexpected MTA type: #{type}" + end + end + + # Look at the log for a request and check that an email was delivered + def MailServerLog.request_exim_sent?(ir) + # Look for line showing request was sent + found = false + ir.mail_server_logs.each do |mail_server_log| + test_outgoing = " <= " + ir.incoming_email + " " + if mail_server_log.line.include?(test_outgoing) + # Check the from value is the same (it always will be, but may as well + # be sure we are parsing the exim line right) + envelope_from = " from <" + ir.incoming_email + "> " + if !mail_server_log.line.include?(envelope_from) + $stderr.puts("unexpected parsing of exim line: [#{mail_server_log.line.chomp}]") + else + found = true + end + end + end + found + end + + def MailServerLog.request_postfix_sent?(ir) + # dsn=2.0.0 is the magic word that says that postfix delivered the email + # See http://tools.ietf.org/html/rfc3464 + ir.mail_server_logs.any? { |l| l.line.include?("dsn=2.0.0") } + end + + # Check that the last day of requests has been sent in Exim or Postfix and we got the + # lines. Writes any errors to STDERR. This check is really mainly to + # check the envelope from is the request address, as Ruby is quite + # flaky with regard to that, and it is important for anti-spam reasons. + # XXX does this really check that, as the log just wouldn't pick + # up at all if the requests weren't sent that way as there would be + # no request- email in it? + # + # NB: There can be several emails involved in a request. This just checks that + # at least one of them has been succesfully sent. + # + def MailServerLog.check_recent_requests_have_been_sent + # Get all requests sent for from 2 to 10 days ago. The 2 day gap is + # because we load mail server log lines via cron at best an hour after they + # are made) + irs = InfoRequest.find(:all, :conditions => [ "created_at < ? and created_at > ? and user_id is not null", Time.now() - 2.day, Time.now() - 10.days ] ) + + # Go through each request and check it + ok = true + irs.each do |ir| + unless request_sent?(ir) + # It's very important the envelope from is set for avoiding spam filter reasons - this + # effectively acts as a check for that. + $stderr.puts("failed to find request sending in MTA logs for request id " + ir.id.to_s + " " + ir.url_title + " (check envelope from is being set to request address in Ruby, and load-mail-server-logs crontab is working)") # *** don't comment out this STDERR line, it is the point of the function! + ok = false + end + end + ok + end + +end + + + diff --git a/app/models/exim_log_done.rb b/app/models/mail_server_log_done.rb index 86574a4cd..3fb20f0b3 100644 --- a/app/models/exim_log_done.rb +++ b/app/models/mail_server_log_done.rb @@ -1,7 +1,7 @@ # == Schema Information -# Schema version: 114 +# Schema version: 20121010214348 # -# Table name: exim_log_dones +# Table name: mail_server_log_dones # # id :integer not null, primary key # filename :text not null @@ -10,14 +10,13 @@ # updated_at :datetime not null # -# models/exim_log_done.rb: -# Stores that a particular exim file has been loaded in, see exim_log.rb +# Stores that a particular mail server log file has been loaded in, see mail_server_log.rb # # Copyright (c) 2009 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ -class EximLogDone < ActiveRecord::Base - has_many :exim_logs +class MailServerLogDone < ActiveRecord::Base + has_many :mail_server_logs end diff --git a/app/models/public_body.rb b/app/models/public_body.rb index 34bcd332c..c70aff9b9 100644 --- a/app/models/public_body.rb +++ b/app/models/public_body.rb @@ -17,7 +17,7 @@ # notes :text default(""), not null # first_letter :string(255) not null # publication_scheme :text default(""), not null -# api_key :string(255) +# api_key :string(255) not null # info_requests_count :integer default(0), not null # |