From 5f4a2e316da8e9415b81b9fe1d8d5effe354803b Mon Sep 17 00:00:00 2001 From: Robin Houston Date: Wed, 7 Sep 2011 14:05:49 +0100 Subject: Script to handle mail replies Handle replies by marking bounces and forwarding everything else to the address specified in the config variable FORWARD_NONBOUNCE_RESPONSES_TO. --- script/handle-mail-replies | 68 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100755 script/handle-mail-replies (limited to 'script/handle-mail-replies') diff --git a/script/handle-mail-replies b/script/handle-mail-replies new file mode 100755 index 000000000..5762ddd31 --- /dev/null +++ b/script/handle-mail-replies @@ -0,0 +1,68 @@ +#!/usr/bin/ruby + +# Handle email responses sent to us. +# +# This script is invoked as a pipe command, i.e. with the raw email message on stdin. +# - If a message is identified as a permanent bounce, the user is marked as having a +# bounced address, and will not be sent any more messages. +# - If a message is identified as an out-of-office autoreply, it is discarded. +# - Any other messages are forwarded to config.get("FORWARD_NONBOUNCE_RESPONSES_TO") + + +# We want to avoid loading rails unless we need it, so we start by just loading the +# config file ourselves. +$alaveteli_dir = File.join(File.dirname(__FILE__), '..') +$:.push(File.join($alaveteli_dir, "commonlib", "rblib")) +load "config.rb" +MySociety::Config.set_file(File.join($alaveteli_dir, 'config', 'general'), true) +MySociety::Config.load_default + +def main + load_rails + raw_message = $stdin.read + pfa = permanently_failed_address(raw_message) + if pfa.nil? + not_a_bounce(raw_message) + else + record_bounce(pfa, raw_message) + end +end + +def permanently_failed_address(raw_message) + message = TMail::Mail.parse(raw_message) + + if message.header_string("Return-Path") == "<>" + # Some sort of auto-response + + # Check for Exim’s X-Failed-Recipients header + failed_recipients = message.header_string("X-Failed-Recipients") + if !failed_recipients.nil? + # The X-Failed-Recipients header contains the email address that failed + # Check for the words "This is a permanent error." in the body, to indicate + # a permanent failure + if message.body =~ /This is a permanent error./ + return failed_recipients + end + end + return nil + end +end + +def not_a_bounce(raw_message) + forward_non_bounces_to = MySociety::Config.get("FORWARD_NONBOUNCE_RESPONSES_TO", "user-support@localhost") + IO.popen("/usr/sbin/sendmail -i #{forward_non_bounces_to}", "w") do |f| + f.write(raw_message); + f.close; + end +end + +def load_rails + require File.join($alaveteli_dir, 'config', 'boot') + require RAILS_ROOT + '/config/environment' +end + +def record_bounce(email_address, bounce_message) + User.record_bounce_for_email(email_address, bounce_message) +end + +main -- cgit v1.2.3 From b9c785535a020c0bc1b871bca48cee71aa2143d9 Mon Sep 17 00:00:00 2001 From: Robin Houston Date: Wed, 7 Sep 2011 20:52:44 +0100 Subject: Detect multipart/report bounce messages Add support for multipart/report bounce messages to script/handle-mail-replies. Also add a spec test for this script. --- script/handle-mail-replies | 92 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 76 insertions(+), 16 deletions(-) (limited to 'script/handle-mail-replies') diff --git a/script/handle-mail-replies b/script/handle-mail-replies index 5762ddd31..93cdc8cfd 100755 --- a/script/handle-mail-replies +++ b/script/handle-mail-replies @@ -17,20 +17,37 @@ load "config.rb" MySociety::Config.set_file(File.join($alaveteli_dir, 'config', 'general'), true) MySociety::Config.load_default -def main - load_rails - raw_message = $stdin.read - pfa = permanently_failed_address(raw_message) - if pfa.nil? - not_a_bounce(raw_message) - else - record_bounce(pfa, raw_message) +def main(in_test_mode) + Dir.chdir($alaveteli_dir) do + load_rails + + raw_message = $stdin.read + message = TMail::Mail.parse(raw_message) + + pfas = permanently_failed_addresses(message) + if !pfas.empty? + if in_test_mode + puts pfas + else + pfas.each do |pfa| + record_bounce(pfa, raw_message) + end + end + return 1 + end + + if is_oof? message + # Discard out-of-office messages + return 2 + end + + # Otherwise forward the message on + forward_on(raw_message) unless in_test_mode + return 0 end end -def permanently_failed_address(raw_message) - message = TMail::Mail.parse(raw_message) - +def permanently_failed_addresses(message) if message.header_string("Return-Path") == "<>" # Some sort of auto-response @@ -41,14 +58,55 @@ def permanently_failed_address(raw_message) # Check for the words "This is a permanent error." in the body, to indicate # a permanent failure if message.body =~ /This is a permanent error./ - return failed_recipients + return failed_recipients.split(/,\s*/) end end - return nil + + # Next, look for multipart/report + if message.content_type == "multipart/report" + permanently_failed_recipients = [] + message.parts.each do |part| + if part.content_type == "message/delivery-status" + sections = part.body.split(/\r?\n\r?\n/) + # The first section is a generic header; subsequent sections + # represent a particular recipient. Since we + sections[1..-1].each do |section| + if section !~ /^Status: (\d)/ || $1 != '5' + # Either we couldn’t find the Status field, or it was a transient failure + break + end + if section =~ /^Final-Recipient: rfc822;(.+)/ + permanently_failed_recipients.push($1) + end + end + end + end + if !permanently_failed_recipients.empty? + return permanently_failed_recipients + end + end + end + + return [] +end + +def is_oof?(message) + # Check for out-of-office + + if message.header_string("Return-Path") == "<>" + subject = message.header_string("Subject") + if subject.start_with? "Out of Office: " + return true + end + if subject.start_with? "Automatic reply: " + return true + end end + + return false end -def not_a_bounce(raw_message) +def forward_on(raw_message) forward_non_bounces_to = MySociety::Config.get("FORWARD_NONBOUNCE_RESPONSES_TO", "user-support@localhost") IO.popen("/usr/sbin/sendmail -i #{forward_non_bounces_to}", "w") do |f| f.write(raw_message); @@ -57,7 +115,7 @@ def not_a_bounce(raw_message) end def load_rails - require File.join($alaveteli_dir, 'config', 'boot') + require File.join('config', 'boot') require RAILS_ROOT + '/config/environment' end @@ -65,4 +123,6 @@ def record_bounce(email_address, bounce_message) User.record_bounce_for_email(email_address, bounce_message) end -main +in_test_mode = (ARGV[0] == "--test") +status = main(in_test_mode) +exit(status) if in_test_mode -- cgit v1.2.3 From 71048a2bd639943801d626fa89e2dfcb2ebb7ee2 Mon Sep 17 00:00:00 2001 From: Robin Houston Date: Thu, 8 Sep 2011 01:56:43 +0100 Subject: Improved OOF detection Slightly improved detection of out-of-office messages, + tests. --- script/handle-mail-replies | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'script/handle-mail-replies') diff --git a/script/handle-mail-replies b/script/handle-mail-replies index 93cdc8cfd..fa45d135a 100755 --- a/script/handle-mail-replies +++ b/script/handle-mail-replies @@ -93,8 +93,8 @@ end def is_oof?(message) # Check for out-of-office + subject = message.header_string("Subject") if message.header_string("Return-Path") == "<>" - subject = message.header_string("Subject") if subject.start_with? "Out of Office: " return true end @@ -103,6 +103,9 @@ def is_oof?(message) end end + if subject.start_with? "Out of Office AutoReply:" + return true + end return false end -- cgit v1.2.3 From e6c088e6b72e5bfee26fda8d933e8d71ea672b30 Mon Sep 17 00:00:00 2001 From: Robin Houston Date: Thu, 8 Sep 2011 06:18:35 +0100 Subject: More out-of-office message detection --- script/handle-mail-replies | 3 +++ 1 file changed, 3 insertions(+) (limited to 'script/handle-mail-replies') diff --git a/script/handle-mail-replies b/script/handle-mail-replies index fa45d135a..b4846b440 100755 --- a/script/handle-mail-replies +++ b/script/handle-mail-replies @@ -106,6 +106,9 @@ def is_oof?(message) if subject.start_with? "Out of Office AutoReply:" return true end + if subject.end_with? "is out of the office" + return true + end return false end -- cgit v1.2.3 From bfef9fe4210a6057b0e19bafe6f5d8d9c756a53c Mon Sep 17 00:00:00 2001 From: Robin Houston Date: Thu, 8 Sep 2011 06:21:26 +0100 Subject: Malformed messages are just passed on verbatim --- script/handle-mail-replies | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'script/handle-mail-replies') diff --git a/script/handle-mail-replies b/script/handle-mail-replies index b4846b440..7304e0528 100755 --- a/script/handle-mail-replies +++ b/script/handle-mail-replies @@ -22,7 +22,13 @@ def main(in_test_mode) load_rails raw_message = $stdin.read - message = TMail::Mail.parse(raw_message) + begin + message = TMail::Mail.parse(raw_message) + rescue + # Error parsing message. Just pass it on, to be on the safe side. + forward_on(raw_message) unless in_test_mode + return 0 + end pfas = permanently_failed_addresses(message) if !pfas.empty? -- cgit v1.2.3 From 94ff0bd6139f1c5eae0809dcee5e77ea728a0e19 Mon Sep 17 00:00:00 2001 From: Robin Houston Date: Thu, 8 Sep 2011 06:28:44 +0100 Subject: More out-of-office message detection --- script/handle-mail-replies | 3 +++ 1 file changed, 3 insertions(+) (limited to 'script/handle-mail-replies') diff --git a/script/handle-mail-replies b/script/handle-mail-replies index 7304e0528..e0f18d166 100755 --- a/script/handle-mail-replies +++ b/script/handle-mail-replies @@ -112,6 +112,9 @@ def is_oof?(message) if subject.start_with? "Out of Office AutoReply:" return true end + if subject == "Out of Office" + return true + end if subject.end_with? "is out of the office" return true end -- cgit v1.2.3 From 79f566808b70bb5adfcb0c90ff0031fa935961fa Mon Sep 17 00:00:00 2001 From: Robin Houston Date: Thu, 8 Sep 2011 06:40:06 +0100 Subject: Defer the loading of Rails Do not load Rails till we need it, which makes this script run faster and use less memory. --- script/handle-mail-replies | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'script/handle-mail-replies') diff --git a/script/handle-mail-replies b/script/handle-mail-replies index e0f18d166..8cf9b21d7 100755 --- a/script/handle-mail-replies +++ b/script/handle-mail-replies @@ -17,10 +17,11 @@ load "config.rb" MySociety::Config.set_file(File.join($alaveteli_dir, 'config', 'general'), true) MySociety::Config.load_default +$:.push("vendor/rails/actionmailer/lib/action_mailer/vendor/tmail-1.2.7") +require 'tmail' + def main(in_test_mode) Dir.chdir($alaveteli_dir) do - load_rails - raw_message = $stdin.read begin message = TMail::Mail.parse(raw_message) @@ -135,6 +136,7 @@ def load_rails end def record_bounce(email_address, bounce_message) + load_rails User.record_bounce_for_email(email_address, bounce_message) end -- cgit v1.2.3 From a062176d2117d75f5bd9c07b4e2a4a63fbedf387 Mon Sep 17 00:00:00 2001 From: Robin Houston Date: Thu, 8 Sep 2011 08:26:59 +0100 Subject: Use absolute path to tmail --- script/handle-mail-replies | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'script/handle-mail-replies') diff --git a/script/handle-mail-replies b/script/handle-mail-replies index 8cf9b21d7..125d84634 100755 --- a/script/handle-mail-replies +++ b/script/handle-mail-replies @@ -17,7 +17,7 @@ load "config.rb" MySociety::Config.set_file(File.join($alaveteli_dir, 'config', 'general'), true) MySociety::Config.load_default -$:.push("vendor/rails/actionmailer/lib/action_mailer/vendor/tmail-1.2.7") +$:.push(File.join($alaveteli_dir, "vendor", "rails", "actionmailer", "lib", "action_mailer", "vendor", "tmail-1.2.7")) require 'tmail' def main(in_test_mode) -- cgit v1.2.3 From df5522a83e8f60832f68d874d3d92c1f429afaea Mon Sep 17 00:00:00 2001 From: Robin Houston Date: Thu, 8 Sep 2011 09:04:46 +0100 Subject: Make OOF-detection subject matches case-insensitive --- script/handle-mail-replies | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'script/handle-mail-replies') diff --git a/script/handle-mail-replies b/script/handle-mail-replies index 125d84634..90a4d319a 100755 --- a/script/handle-mail-replies +++ b/script/handle-mail-replies @@ -100,20 +100,20 @@ end def is_oof?(message) # Check for out-of-office - subject = message.header_string("Subject") + subject = message.header_string("Subject").downcase if message.header_string("Return-Path") == "<>" - if subject.start_with? "Out of Office: " + if subject.start_with? "out of office: " return true end - if subject.start_with? "Automatic reply: " + if subject.start_with? "automatic reply: " return true end end - if subject.start_with? "Out of Office AutoReply:" + if subject.start_with? "out of office autoreply:" return true end - if subject == "Out of Office" + if subject == "out of office" return true end if subject.end_with? "is out of the office" -- cgit v1.2.3 From 2442a5322ee560756f521983a0f7f0de4c4aa4e1 Mon Sep 17 00:00:00 2001 From: Robin Houston Date: Thu, 8 Sep 2011 22:12:15 +0100 Subject: Detect OOF messages that identify themselves by a X-POST-MessageClass header --- script/handle-mail-replies | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'script/handle-mail-replies') diff --git a/script/handle-mail-replies b/script/handle-mail-replies index 90a4d319a..9b1fb5b29 100755 --- a/script/handle-mail-replies +++ b/script/handle-mail-replies @@ -100,6 +100,10 @@ end def is_oof?(message) # Check for out-of-office + if message.header_string("X-POST-MessageClass") == "9; Autoresponder" + return true + end + subject = message.header_string("Subject").downcase if message.header_string("Return-Path") == "<>" if subject.start_with? "out of office: " -- cgit v1.2.3