Merge branch 'feature/ruby-19-compat-immediate' into develop

12 files changed, 206 insertions, 17 deletions

diff --git a/Gemfile b/Gemfile
index 650c27537..0b0814479 100644
--- a/Gemfile
+++ b/Gemfile
@@ -30,7 +30,7 @@ gem 'rake', '~> 0.9.2.2'
 gem 'rspec', '~> 1.3.2'
 gem 'rspec-rails', '~> 1.3.4'
 gem 'ruby-msg', '~> 1.5.0'
-gem 'test-unit', '~> 1.2.3' if RUBY_VERSION.to_f >= 1.9
+gem 'test-unit', '~> 1.2.3', :platforms => :ruby_19
 gem 'vpim'
 gem 'will_paginate', '~> 2.3.11'
 # when 1.2.9 is released by the maintainer, we can stop using this fork:
@@ -45,6 +45,7 @@ group :test do
 end
 
 group :develop do
-  gem 'ruby-debug'
+  gem 'ruby-debug', :platforms => :ruby_18
+  gem 'ruby-debug19', :platforms => :ruby_19
   gem 'annotate'
 end
diff --git a/Gemfile.lock b/Gemfile.lock
index f243ffaad..35edac1e9 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -25,6 +25,7 @@ GEM
       net-sftp (>= 2.0.0)
       net-ssh (>= 2.0.14)
       net-ssh-gateway (>= 1.1.0)
+    archive-tar-minitar (0.5.2)
     columnize (0.3.6)
     fakeweb (1.3.0)
     fast_gettext (0.6.8)
@@ -32,9 +33,13 @@ GEM
     gettext (2.2.1)
       locale
     highline (1.6.13)
+    hoe (3.0.8)
+      rake (~> 0.8)
     json (1.5.4)
     linecache (0.46)
       rbx-require-relative (> 0.0.4)
+    linecache19 (0.5.12)
+      ruby_core_source (>= 0.1.4)
     locale (2.0.5)
     mahoro (0.3)
     memcache-client (1.8.5)
@@ -72,10 +77,22 @@ GEM
       ruby-debug-base (~> 0.10.4.0)
     ruby-debug-base (0.10.4)
       linecache (>= 0.3)
+    ruby-debug-base19 (0.11.25)
+      columnize (>= 0.3.1)
+      linecache19 (>= 0.5.11)
+      ruby_core_source (>= 0.1.4)
+    ruby-debug19 (0.11.6)
+      columnize (>= 0.3.1)
+      linecache19 (>= 0.5.11)
+      ruby-debug-base19 (>= 0.11.19)
     ruby-msg (1.5.0)
       ruby-ole (>= 1.2.8)
       vpim (>= 0.360)
     ruby-ole (1.2.11.3)
+    ruby_core_source (0.1.5)
+      archive-tar-minitar (>= 0.5.2)
+    test-unit (1.2.3)
+      hoe (>= 1.5.1)
     vpim (0.695)
     will_paginate (2.3.16)
     xapian-full-alaveteli (1.2.9.5)
@@ -110,7 +127,9 @@ DEPENDENCIES
   rspec (~> 1.3.2)
   rspec-rails (~> 1.3.4)
   ruby-debug
+  ruby-debug19
   ruby-msg (~> 1.5.0)
+  test-unit (~> 1.2.3)
   vpim
   will_paginate (~> 2.3.11)
   xapian-full-alaveteli (~> 1.2.9.5)
diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb
index fcb4671c5..367909f57 100644
--- a/app/models/incoming_message.rb
+++ b/app/models/incoming_message.rb
@@ -75,8 +75,7 @@ class IncomingMessage < ActiveRecord::Base
     # Documentation at http://i.loveruby.net/en/projects/tmail/doc/
     def mail(force = nil)
         if (!force.nil? || @mail.nil?) && !self.raw_email.nil?
-            # Hack round bug in TMail's MIME decoding. Example request which provokes it:
-            # http://www.whatdotheyknow.com/request/reviews_of_unduly_lenient_senten#incoming-4830
+            # Hack round bug in TMail's MIME decoding.
             # Report of TMail bug:
             # http://rubyforge.org/tracker/index.php?func=detail&aid=21810&group_id=4512&atid=17370
             copy_of_raw_data = self.raw_email.data.gsub(/; boundary=\s+"/ims,'; boundary="')
@@ -345,8 +344,7 @@ class IncomingMessage < ActiveRecord::Base
         name = Regexp.escape(self.info_request.user_name)
 
         # To end of message sections
-        # http://www.whatdotheyknow.com/request/university_investment_in_the_arm
-        text.gsub!(/^#{name}[^\n]+\nSent by:[^\n]+\n.*/ims, "\n\n" + replacement)
+        text.gsub!(/^\s?#{name}[^\n]+\n([^\n]+\n)?\s?Sent by:[^\n]+\n.*/ims, "\n\n" + replacement)
 
         # Some other sort of forwarding quoting
         # http://www.whatdotheyknow.com/request/224/response/326
@@ -580,7 +578,8 @@ class IncomingMessage < ActiveRecord::Base
 
         # Remove existing quoted sections
         folded_quoted_text = self.remove_lotus_quoting(text, 'FOLDED_QUOTED_SECTION')
-        folded_quoted_text = IncomingMessage.remove_quoted_sections(text, "FOLDED_QUOTED_SECTION")
+        folded_quoted_text = IncomingMessage.remove_quoted_sections(folded_quoted_text, "FOLDED_QUOTED_SECTION")
+
         self.cached_main_body_text_unfolded = text
         self.cached_main_body_text_folded = folded_quoted_text
         self.save!
diff --git a/commonlib b/commonlib
-Subproject 131375c752c02aa20f1644240fe69720275da42
+Subproject 3bf8c34eed89a946f981363eb862f25ac843769
diff --git a/config/environment.rb b/config/environment.rb
index 3348ef92a..4ae3f26d9 100644
--- a/config/environment.rb
+++ b/config/environment.rb
@@ -1,5 +1,9 @@
 # Be sure to restart your web server when you modify this file.
 
+# the default encoding for IO is utf-8, and we use utf-8 internally
+if RUBY_VERSION.to_f >= 1.9
+    Encoding.default_external = Encoding.default_internal = Encoding::UTF_8
+end
 
 # Uncomment below to force Rails into production mode when
 # you don't control web/app server and can't set it the proper way
diff --git a/config/initializers/fast_gettext.rb b/config/initializers/fast_gettext.rb
index 721c49cd0..2ecf5cb5d 100644
--- a/config/initializers/fast_gettext.rb
+++ b/config/initializers/fast_gettext.rb
@@ -1,4 +1,3 @@
-Encoding.default_external = 'UTF-8' if RUBY_VERSION.to_f >= 1.9
 FastGettext.add_text_domain 'app', :path => File.join(Rails.root, 'locale'), :type => :po
 FastGettext.default_text_domain = 'app'
 
diff --git a/config/initializers/missing_source_file.rb b/config/initializers/missing_source_file.rb
new file mode 100644
index 000000000..a114fa972
--- /dev/null
+++ b/config/initializers/missing_source_file.rb
@@ -0,0 +1,2 @@
+# For Rails 2.3 on Ruby 1.9.3 @see https://github.com/rails/rails/pull/3745
+MissingSourceFile::REGEXPS << [/^cannot load such file -- (.+)$/i, 1]
diff --git a/spec/controllers/api_controller_spec.rb b/spec/controllers/api_controller_spec.rb
index ded9a040a..8d8a39950 100644
--- a/spec/controllers/api_controller_spec.rb
+++ b/spec/controllers/api_controller_spec.rb
@@ -1,3 +1,4 @@
+# coding: utf-8
 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
 def normalise_whitespace(s)
@@ -167,7 +168,7 @@ describe ApiController, "when using the API" do
         OutgoingMessage.count.should == n_outgoing_messages
     end
     
-    it "should not allow other people’s requests to be updated" do
+    it "should not allow other people's requests to be updated" do
         request_id = _create_request
         n_incoming_messages = IncomingMessage.count
         n_outgoing_messages = OutgoingMessage.count
diff --git a/spec/controllers/public_body_controller_spec.rb b/spec/controllers/public_body_controller_spec.rb
index 9eca43aee..d12818a1c 100644
--- a/spec/controllers/public_body_controller_spec.rb
+++ b/spec/controllers/public_body_controller_spec.rb
@@ -1,3 +1,4 @@
+# coding: utf-8
 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
 describe PublicBodyController, "when showing a body" do
diff --git a/spec/fixtures/files/lotus-notes-quoting.email b/spec/fixtures/files/lotus-notes-quoting.email
new file mode 100644
index 000000000..863a2dfef
--- /dev/null
+++ b/spec/fixtures/files/lotus-notes-quoting.email
@@ -0,0 +1,112 @@
+From foi@example.com Thu Mar 06 17:28:24 2008
+To: FOI Person <EMAIL_TO>
+Subject: Re: Freedom of Information request
+MIME-Version: 1.0
+X-Mailer: Lotus Notes Release 6.5.6 March 06, 2007
+From: foi <foi@example.com>
+Date: Thu, 6 Mar 2008 17:31:51 +0000
+Content-Type: multipart/alternative; boundary="=_alternative 005FFC2E80257404_="
+
+This is a multipart message in MIME format.
+--=_alternative 005FFC2E80257404_=
+Content-Type: text/plain; charset="ISO-8859-1"
+Content-Transfer-Encoding: quoted-printable
+
+Dear Requester,
+
+Some text.
+
+Yours sincerely
+
+
+FOI Team
+
+
+
+Bob Smith <request-xx-xxxxx@whatdotheyknow.com>=20
+Sent by: Bob Smith <request-xx-xxxxx@whatdotheyknow.com>
+06/03/08 10:00
+Please respond to
+Bob Smith <request-xx-xxxxx@whatdotheyknow.com>
+
+
+To
+FOI requests <foi@example.com>
+cc
+
+Subject
+Freedom of Information request
+
+
+
+
+
+
+     Original text
+
+
+
+
+--=_alternative 005FFC2E80257404_=
+Content-Type: text/html; charset="ISO-8859-1"
+Content-Transfer-Encoding: quoted-printable
+
+
+<div>
+<br><font size=3D2 face=3D"Arial">Dear Requester,</font>
+<br>
+<br><font size=3D2 face=3D"Arial">Some text. </font>
+<br>
+<br><font size=3D2 face=3D"Arial">Yours sincerely</font>
+<br>
+<br>
+<div>
+<br><font size=3D2 face=3D"Arial">FOI Team</font>
+<p>
+<br>
+<br>
+<table width=3D100%>
+<tr valign=3Dtop>
+<td width=3D40%><font size=3D1 face=3D"sans-serif"><b>Bob Smith &lt;re=
+quest-xx-xxxxx@whatdotheyknow.com&gt;</b>
+</font>
+<br><font size=3D1 face=3D"sans-serif">Sent by: Bob Smith &lt;request-=
+bounce-xx-xxxxx@whatdotheyknow.com&gt;</font>
+<p><font size=3D1 face=3D"sans-serif">06/03/08 10:00</font>
+<table border>
+<tr valign=3Dtop>
+<td bgcolor=3Dwhite>
+<div align=3Dcenter><font size=3D1 face=3D"sans-serif">Please respond to<br>
+Bob Smith &lt;request-xx-xxxxx@whatdotheyknow.com&gt;</font></div><=
+/table>
+<br>
+<td width=3D59%>
+<table width=3D100%>
+<tr valign=3Dtop>
+<td>
+<div align=3Dright><font size=3D1 face=3D"sans-serif">To</font></div>
+<td><font size=3D1 face=3D"sans-serif">FOI requests
+&lt;foi@example.com&gt;</font>
+<tr valign=3Dtop>
+<td>
+<div align=3Dright><font size=3D1 face=3D"sans-serif">cc</font></div>
+<td>
+<tr valign=3Dtop>
+<td>
+<div align=3Dright><font size=3D1 face=3D"sans-serif">Subject</font></div>
+<td><font size=3D1 face=3D"sans-serif">Freedom of Information request</font></table>
+<br>
+<table>
+<tr valign=3Dtop>
+<td>
+<td></table>
+<br></table>
+<br>
+<br>
+<br><tt><font size=3D2>&nbsp; &nbsp; &nbsp;Original text<br>
+<br>
+<br>
+</font></tt>
+<br></div></div></div>
+--=_alternative 005FFC2E80257404_=--
+
diff --git a/spec/fixtures/files/space-boundary.email b/spec/fixtures/files/space-boundary.email
new file mode 100644
index 000000000..37d65dbb5
--- /dev/null
+++ b/spec/fixtures/files/space-boundary.email
@@ -0,0 +1,37 @@
+From responder@example.com Wed Sep 03 10:44:18 2008
+Return-path: <responder@example.com>
+Envelope-to: foi@sandwich.ukcod.org.uk
+Delivery-date: Wed, 03 Sep 2008 10:44:18 +0100
+X-MimeOLE: Produced By Microsoft Exchange V6.5
+Content-class: urn:content-classes:message
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary=
+    "----_=_NextPart_001_01C90DA9.9B158D0D"
+Subject: A subject
+Date: Wed, 3 Sep 2008 10:44:05 +0100
+Message-ID: <D403C5D38B9BD6468928738951034DDE85A457@aglnex02.attorneygeneral.gsi.gov.uk>
+X-MS-Has-Attach: yes
+X-MS-TNEF-Correlator:
+Thread-Topic: Freedom of Information request - An information request about
+    stuff
+Thread-Index: AckNqOAr459KZB17Q1OIy1jxNVd9xAAAAnVAAAAWJ+A=
+From: "A responder" <responder@example.com>
+To: FOI Person <EMAIL_TO>
+
+This is a multi-part message in MIME format.
+
+------_=_NextPart_001_01C90DA9.9B158D0D
+Content-Type: text/plain; charset="us-ascii"
+Content-Transfer-Encoding: quoted-printable
+
+=20Apologies.=20Herewith.
+
+------_=_NextPart_001_01C90DA9.9B158D0D
+Content-Type: application/octet-stream; name="Mr Gradwick.pdf"
+Content-Transfer-Encoding: base64
+Content-Description: Mr Gradwick.pdf
+Content-Disposition: attachment; filename="Mr Gradwick.pdf"
+
+xxx
+
+------_=_NextPart_001_01C90DA9.9B158D0D--
diff --git a/spec/models/incoming_message_spec.rb b/spec/models/incoming_message_spec.rb
index cf84b4595..69a5da0e9 100644
--- a/spec/models/incoming_message_spec.rb
+++ b/spec/models/incoming_message_spec.rb
@@ -12,6 +12,14 @@ describe IncomingMessage, " when dealing with incoming mail" do
         ActionMailer::Base.deliveries.clear
     end
 
+    it 'should correctly parse multipart mails with a linebreak in the boundary marker' do
+        ir = info_requests(:fancy_dog_request)
+        receive_incoming_mail('space-boundary.email', ir.incoming_email)
+        message = ir.incoming_messages[1]
+        message.mail.parts.size.should == 2
+        message.mail.multipart?.should == true
+    end
+
     it "should return the mail Date header date for sent at" do
         @im.parse_raw_email!(true)
         @im.reload
@@ -60,7 +68,6 @@ describe IncomingMessage, " when dealing with incoming mail" do
         is not good utf-8' do
         ir = info_requests(:fancy_dog_request)
         receive_incoming_mail('no-part-charset-bad-utf8.email', ir.incoming_email)
-        puts ir.incoming_messages.inspect
         message = ir.incoming_messages[1]
         message.parse_raw_email!
         message.get_main_body_text_internal.should include("The above text was badly encoded")
@@ -125,16 +132,23 @@ end
 
 describe IncomingMessage, " folding quoted parts of emails" do
 
-    it "cope with [ in user names properly" do
-        @user = mock_model(User)
-        @user.stub!(:name).and_return("Sir [ Bobble")
-        @info_request = mock_model(InfoRequest)
-        @info_request.stub!(:user).and_return(@user)
-        @info_request.stub!(:user_name).and_return(@user.name)
+    it 'should fold an example lotus notes quoted part converted from HTML correctly' do
+        ir = info_requests(:fancy_dog_request)
+        receive_incoming_mail('lotus-notes-quoting.email', ir.incoming_email)
+        message = ir.incoming_messages[1]
+        message.get_main_body_text_folded.should match(/FOLDED_QUOTED_SECTION/)
+    end
 
+    it 'should fold a plain text lotus notes quoted part correctly' do
+        text = "FOI Team\n\n\nInfo Requester <xxx@whatdotheyknow.com>=20\nSent by: Info Requester <request-bounce-xxxxx@whatdotheyknow.com>\n06/03/08 10:00\nPlease respond to\nInfo Requester <request-xxxx@whatdotheyknow.com>"
         @incoming_message = IncomingMessage.new()
-        @incoming_message.info_request = @info_request
+        @incoming_message.stub_chain(:info_request, :user_name).and_return("Info Requester")
+        @incoming_message.remove_lotus_quoting(text).should match(/FOLDED_QUOTED_SECTION/)
+    end
 
+    it "cope with [ in user names properly" do
+        @incoming_message = IncomingMessage.new()
+        @incoming_message.stub_chain(:info_request, :user_name).and_return("Sir [ Bobble")
         # this gives a warning if [ is in the name
         text = @incoming_message.remove_lotus_quoting("Sir [ Bobble \nSent by: \n")
         text.should == "\n\nFOLDED_QUOTED_SECTION"