Detect multipart/report bounce messages

Add support for multipart/report bounce messages to script/handle-mail-replies.
Also add a spec test for this script.

5 files changed, 289 insertions, 16 deletions

diff --git a/commonlib b/commonlib
-Subproject 0a70988be774245643e0fb704d48f24a729f55d
+Subproject 16e32f0575107068ae1f16c26e31c598e4fef41
diff --git a/script/handle-mail-replies b/script/handle-mail-replies
index 5762ddd31..93cdc8cfd 100755
--- a/script/handle-mail-replies
+++ b/script/handle-mail-replies
@@ -17,20 +17,37 @@ load "config.rb"
 MySociety::Config.set_file(File.join($alaveteli_dir, 'config', 'general'), true)
 MySociety::Config.load_default
 
-def main
-    load_rails
-    raw_message = $stdin.read
-    pfa = permanently_failed_address(raw_message)
-    if pfa.nil?
-        not_a_bounce(raw_message)
-    else
-        record_bounce(pfa, raw_message)
+def main(in_test_mode)
+    Dir.chdir($alaveteli_dir) do
+        load_rails
+        
+        raw_message = $stdin.read
+        message = TMail::Mail.parse(raw_message)
+        
+        pfas = permanently_failed_addresses(message)
+        if !pfas.empty?
+            if in_test_mode
+                puts pfas
+            else
+                pfas.each do |pfa|
+                    record_bounce(pfa, raw_message)
+                end
+            end
+            return 1
+        end
+        
+        if is_oof? message
+            # Discard out-of-office messages
+            return 2
+        end
+        
+        # Otherwise forward the message on
+        forward_on(raw_message) unless in_test_mode
+        return 0
     end
 end
 
-def permanently_failed_address(raw_message)
-    message = TMail::Mail.parse(raw_message)
-
+def permanently_failed_addresses(message)
     if message.header_string("Return-Path") == "<>"
         # Some sort of auto-response
     
@@ -41,14 +58,55 @@ def permanently_failed_address(raw_message)
             # Check for the words "This is a permanent error." in the body, to indicate
             # a permanent failure
             if message.body =~ /This is a permanent error./
-                return failed_recipients
+                return failed_recipients.split(/,\s*/)
             end
         end
-        return nil
+        
+        # Next, look for multipart/report
+        if message.content_type == "multipart/report"
+            permanently_failed_recipients = []
+            message.parts.each do |part|
+                if part.content_type == "message/delivery-status"
+                    sections = part.body.split(/\r?\n\r?\n/)
+                    # The first section is a generic header; subsequent sections
+                    # represent a particular recipient. Since we 
+                    sections[1..-1].each do |section|
+                        if section !~ /^Status: (\d)/ || $1 != '5'
+                            # Either we couldn’t find the Status field, or it was a transient failure
+                            break
+                        end
+                        if section =~ /^Final-Recipient: rfc822;(.+)/
+                            permanently_failed_recipients.push($1)
+                        end
+                    end
+                end
+            end
+            if !permanently_failed_recipients.empty?
+                return permanently_failed_recipients
+            end
+        end
+    end
+    
+    return []
+end
+
+def is_oof?(message)
+    # Check for out-of-office
+    
+    if message.header_string("Return-Path") == "<>"
+        subject = message.header_string("Subject")
+        if subject.start_with? "Out of Office: "
+            return true
+        end
+        if subject.start_with? "Automatic reply: "
+            return true
+        end
     end
+    
+    return false
 end
 
-def not_a_bounce(raw_message)
+def forward_on(raw_message)
     forward_non_bounces_to = MySociety::Config.get("FORWARD_NONBOUNCE_RESPONSES_TO", "user-support@localhost")
     IO.popen("/usr/sbin/sendmail -i #{forward_non_bounces_to}", "w") do |f|
         f.write(raw_message);
@@ -57,7 +115,7 @@ def not_a_bounce(raw_message)
 end
 
 def load_rails
-    require File.join($alaveteli_dir, 'config', 'boot')
+    require File.join('config', 'boot')
     require RAILS_ROOT + '/config/environment'
 end
 
@@ -65,4 +123,6 @@ def record_bounce(email_address, bounce_message)
     User.record_bounce_for_email(email_address, bounce_message)
 end
 
-main
+in_test_mode = (ARGV[0] == "--test")
+status = main(in_test_mode)
+exit(status) if in_test_mode
diff --git a/spec/fixtures/files/track-response-exim-bounce.email b/spec/fixtures/files/track-response-exim-bounce.email
new file mode 100644
index 000000000..8d40380b1
--- /dev/null
+++ b/spec/fixtures/files/track-response-exim-bounce.email
@@ -0,0 +1,67 @@
+Delivered-To: mysociety.robin@gmail.com
+Received: by 10.216.187.197 with SMTP id y47cs98510wem;
+        Tue, 6 Sep 2011 14:22:44 -0700 (PDT)
+Received: by 10.216.203.79 with SMTP id e57mr78207weo.42.1315344164092;
+        Tue, 06 Sep 2011 14:22:44 -0700 (PDT)
+Return-Path: <>
+Received: from wildfire.ukcod.org.uk (wildfire.ukcod.org.uk [89.238.145.74])
+        by mx.google.com with ESMTPS id n64si9483505weq.102.2011.09.06.14.22.42
+        (version=TLSv1/SSLv3 cipher=OTHER);
+        Tue, 06 Sep 2011 14:22:43 -0700 (PDT)
+Received-SPF: pass (google.com: best guess record for domain of wildfire.ukcod.org.uk designates 89.238.145.74 as permitted sender) client-ip=89.238.145.74;
+Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of wildfire.ukcod.org.uk designates 89.238.145.74 as permitted sender) smtp.mail=
+Received: from Debian-exim by wildfire.ukcod.org.uk with local (Exim 4.69)
+	id 1R136L-0003xr-1Q
+	for team@whatdotheyknow.com; Tue, 06 Sep 2011 22:22:37 +0100
+X-Failed-Recipients: user@example.com
+Auto-Submitted: auto-replied
+From: Mail Delivery System <Mailer-Daemon@wildfire.ukcod.org.uk>
+To: team@whatdotheyknow.com
+Subject: Mail delivery failed: returning message to sender
+Message-Id: <E1R136L-0003xr-1Q@wildfire.ukcod.org.uk>
+Date: Tue, 06 Sep 2011 22:22:37 +0100
+List-Id: Admin Team for What Do They Know <team@whatdotheyknow.com>
+
+This message was created automatically by mail delivery software.
+
+A message that you sent could not be delivered to one or more of its
+recipients. This is a permanent error. The following address(es) failed:
+
+  user@example.com
+    Unrouteable address
+
+------ This is a copy of the message, including all the headers. ------
+
+Return-path: <team@whatdotheyknow.com>
+Received: from foi by wildfire.ukcod.org.uk with local (Exim 4.69)
+	(envelope-from <team@whatdotheyknow.com>)
+	id 1R136J-0003xp-Td
+	for user@example.com; Tue, 06 Sep 2011 22:22:36 +0100
+Date: Tue, 6 Sep 2011 22:22:35 +0100
+From: WhatDoTheyKnow <team@whatdotheyknow.com>
+To: Nonexistent User <user@example.com>
+Subject: Your WhatDoTheyKnow email alert
+Mime-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Precedence: bulk
+Auto-Submitted: auto-generated
+Message-Id: <E1R136J-0003xp-Td@wildfire.ukcod.org.uk>
+
+FOI requests to 'Maritime and Coastguard Agency'
+================================================
+
+-- MCA & HM Coastguard Official Vehicles --
+Maritime and Coastguard Agency sent a response to Peter Smith (12 August 2011)
+     "Peter In response (reference F0000881) to your FOI questions the
+     MCA answer is:- 1. All MCA vehicles are purchased outright. 2. Yes
+     there is a volu..."
+http://www.whatdotheyknow.com/request/mca_hm_coastguard_official_vehic#incoming-201529
+
+
+Alter your subscription
+=======================
+
+Please click on the link below to cancel or alter these emails.
+http://www.whatdotheyknow.com/c/ie4pkpy70dl4b8flsig
+
+-- the WhatDoTheyKnow team
diff --git a/spec/fixtures/files/track-response-multipart-report.email b/spec/fixtures/files/track-response-multipart-report.email
new file mode 100644
index 000000000..4f8e6d86b
--- /dev/null
+++ b/spec/fixtures/files/track-response-multipart-report.email
@@ -0,0 +1,113 @@
+Delivered-To: mysociety.robin@gmail.com
+Received: by 10.216.187.197 with SMTP id y47cs96752wem;
+        Tue, 6 Sep 2011 13:37:26 -0700 (PDT)
+Received: by 10.216.212.37 with SMTP id x37mr3361871weo.35.1315341445852;
+        Tue, 06 Sep 2011 13:37:25 -0700 (PDT)
+Return-Path: <>
+Received: from wildfire.ukcod.org.uk (wildfire.ukcod.org.uk [89.238.145.74])
+        by mx.google.com with ESMTPS id h49si1800318wed.40.2011.09.06.13.37.25
+        (version=TLSv1/SSLv3 cipher=OTHER);
+        Tue, 06 Sep 2011 13:37:25 -0700 (PDT)
+Received-SPF: pass (google.com: best guess record for domain of wildfire.ukcod.org.uk designates 89.238.145.74 as permitted sender) client-ip=89.238.145.74;
+Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of wildfire.ukcod.org.uk designates 89.238.145.74 as permitted sender) smtp.mail=
+Received: from cluster-a.mailcontrol.com ([85.115.52.190]:43258)
+	by wildfire.ukcod.org.uk with esmtp (Exim 4.69)
+	id 1R12OV-0003KQ-9c
+	for team@whatdotheyknow.com; Tue, 06 Sep 2011 21:37:19 +0100
+Received: from mail.example.com ([62.6.240.178])
+	by rly22a.srv.mailcontrol.com (MailControl) with ESMTP id p86KbIZV025877
+	(version=TLSv1/SSLv3 cipher=AES128-SHA bits=128 verify=NO)
+	for <team@whatdotheyknow.com>; Tue, 6 Sep 2011 21:37:18 +0100
+Received: from MX05.example.com (10.100.14.57) by GCEX534.PHSGROUP.local
+ (10.100.21.78) with Microsoft SMTP Server id 14.1.270.1; Tue, 6 Sep 2011
+ 21:38:52 +0100
+From: <postmaster@xyz.local>
+To: <team@whatdotheyknow.com>
+Date: Tue, 6 Sep 2011 21:37:13 +0100
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+	boundary="9B095B5ADSN=_01CC68D9CD29F1E300015B60MX05.example.com"
+X-DSNContext: 7ce717b1 - 1148 - 00000002 - 00000000
+Message-ID: <DEU8FnRwh00000d1b@MX05.example.com>
+Subject: Delivery Status Notification (Failure)
+X-Scanned-By: MailControl A-12-01-02 (www.mailcontrol.com) on 10.65.0.132
+List-Id: Admin Team for What Do They Know <team@whatdotheyknow.com>
+
+--9B095B5ADSN=_01CC68D9CD29F1E300015B60MX05.example.com
+Content-Type: text/plain; charset="unicode-1-1-utf-7"
+
+This is an automatically generated Delivery Status Notification.
+
+Delivery to the following recipients failed.
+
+       FailedUser@example.com
+
+
+
+
+--9B095B5ADSN=_01CC68D9CD29F1E300015B60MX05.example.com
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns;MX05.example.com
+Received-From-MTA: dns;MX04.example.com
+Arrival-Date: Tue, 6 Sep 2011 21:37:13 +0100
+
+Final-Recipient: rfc822;FailedUser@example.com
+Action: failed
+Status: 5.2.2
+X-Display-Name: Failed User
+
+
+--9B095B5ADSN=_01CC68D9CD29F1E300015B60MX05.example.com
+Content-Type: message/rfc822
+
+Received: from MX04.example.com ([10.100.14.56]) by MX05.example.com with
+ Microsoft SMTPSVC(5.0.2195.6713);	 Tue, 6 Sep 2011 21:37:13 +0100
+Received: from DCEX553.example.com ([10.211.10.27]) by MX04.example.com with
+ Microsoft SMTPSVC(5.0.2195.6713);	 Tue, 6 Sep 2011 21:37:13 +0100
+Received: from cluster-a.mailcontrol.com (85.115.52.190) by mail.example.com
+ (10.211.10.27) with Microsoft SMTP Server id 14.1.270.1; Tue, 6 Sep 2011
+ 20:36:49 +0100
+Received: from wildfire.ukcod.org.uk (wildfire.ukcod.org.uk [89.238.145.74])
+	by rly01a.srv.mailcontrol.com (MailControl) with ESMTP id p86KbAZN016792
+	(version=TLSv1/SSLv3 cipher=AES256-SHA bits=256 verify=NO)	for
+ <faileduser@example.com>; Tue, 6 Sep 2011 21:37:11 +0100
+Received: from foi by wildfire.ukcod.org.uk with local (Exim 4.69)
+	(envelope-from <team@whatdotheyknow.com>)	id 1R12OL-0003K9-UE	for
+ faileduser@example.com; Tue, 06 Sep 2011 21:37:10 +0100
+Date: Tue, 6 Sep 2011 21:37:09 +0100
+From: WhatDoTheyKnow <team@whatdotheyknow.com>
+To: Failed <faileduser@example.com>
+Subject: Your WhatDoTheyKnow email alert
+MIME-Version: 1.0
+Content-Type: text/plain; charset="utf-8"
+Precedence: bulk
+Auto-Submitted: auto-generated
+Message-ID: <E1R12OL-0003K9-UE@wildfire.ukcod.org.uk>
+X-Mailcontrol-Inbound: 7DN0MnCsYCrl5jj2!rZ4BuxxWQ2q0l7t3Lrex4V7ScCE2eoC2RNzHw==
+X-Spam-Score: -0.857
+X-Scanned-By: MailControl A-12-01-02 (www.mailcontrol.com) on 10.65.0.111
+Return-Path: team@whatdotheyknow.com
+X-OriginalArrivalTime: 06 Sep 2011 20:37:13.0070 (UTC) FILETIME=[C20250E0:01CC6CD4]
+
+Requests or responses matching 'bottled water cooler'
+=====================================================
+
+-- HS2 meetings - Agendas and Minutes --
+Warwickshire County Council sent a response to Richard Jones (21 July 2011)
+     "Dear Mr Jones FREEDOM OF INFORMATION ACT 2000 - INFORMATION
+     REQUEST Your request for information has now been considered. The
+     information you have..."
+http://www.whatdotheyknow.com/request/hs2_meetings_agendas_and_minutes_2#incoming-195748
+
+
+Alter your subscription
+=======================
+
+Please click on the link below to cancel or alter these emails.
+http://www.whatdotheyknow.com/c/f76ffwifzlo5sk4egr3
+
+-- the WhatDoTheyKnow team
+
+
+--9B095B5ADSN=_01CC68D9CD29F1E300015B60MX05.example.com--
\ No newline at end of file
diff --git a/spec/script/handle-mail-replies_spec.rb b/spec/script/handle-mail-replies_spec.rb
new file mode 100644
index 000000000..856476c3f
--- /dev/null
+++ b/spec/script/handle-mail-replies_spec.rb
@@ -0,0 +1,33 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require "external_command"
+
+def mail_reply_test(email_filename)
+    Dir.chdir RAILS_ROOT do
+        xc = ExternalCommand.new("script/handle-mail-replies", "--test")
+        xc.run(load_file_fixture(email_filename))
+        
+        xc.err.should == ""
+        return xc
+    end
+end
+
+describe "When filtering" do
+    it "should detect an Exim bounce" do
+        r = mail_reply_test("track-response-exim-bounce.email")
+        r.status.should == 1
+        r.out.should == "user@example.com\n"
+    end
+    
+    it "should pass on a non-bounce message" do
+        r = mail_reply_test("incoming-request-bad-uuencoding.email")
+        r.status.should == 0
+        r.out.should == ""
+    end
+
+    it "should detect a multipart bounce" do
+        r = mail_reply_test("track-response-multipart-report.email")
+        r.status.should == 1
+        r.out.should == "FailedUser@example.com\n"
+    end
+end
+