diff options
| -rw-r--r-- | Gemfile | 5 | ||||
| -rw-r--r-- | Gemfile.lock | 19 | ||||
| -rw-r--r-- | app/models/incoming_message.rb | 9 | ||||
| m--------- | commonlib | 0 | ||||
| -rw-r--r-- | config/environment.rb | 4 | ||||
| -rw-r--r-- | config/initializers/fast_gettext.rb | 1 | ||||
| -rw-r--r-- | config/initializers/missing_source_file.rb | 2 | ||||
| -rw-r--r-- | spec/controllers/api_controller_spec.rb | 3 | ||||
| -rw-r--r-- | spec/controllers/public_body_controller_spec.rb | 1 | ||||
| -rw-r--r-- | spec/fixtures/files/lotus-notes-quoting.email | 112 | ||||
| -rw-r--r-- | spec/fixtures/files/space-boundary.email | 37 | ||||
| -rw-r--r-- | spec/models/incoming_message_spec.rb | 30 |
12 files changed, 206 insertions, 17 deletions
@@ -30,7 +30,7 @@ gem 'rake', '~> 0.9.2.2' gem 'rspec', '~> 1.3.2' gem 'rspec-rails', '~> 1.3.4' gem 'ruby-msg', '~> 1.5.0' -gem 'test-unit', '~> 1.2.3' if RUBY_VERSION.to_f >= 1.9 +gem 'test-unit', '~> 1.2.3', :platforms => :ruby_19 gem 'vpim' gem 'will_paginate', '~> 2.3.11' # when 1.2.9 is released by the maintainer, we can stop using this fork: @@ -45,6 +45,7 @@ group :test do end group :develop do - gem 'ruby-debug' + gem 'ruby-debug', :platforms => :ruby_18 + gem 'ruby-debug19', :platforms => :ruby_19 gem 'annotate' end diff --git a/Gemfile.lock b/Gemfile.lock index f243ffaad..35edac1e9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -25,6 +25,7 @@ GEM net-sftp (>= 2.0.0) net-ssh (>= 2.0.14) net-ssh-gateway (>= 1.1.0) + archive-tar-minitar (0.5.2) columnize (0.3.6) fakeweb (1.3.0) fast_gettext (0.6.8) @@ -32,9 +33,13 @@ GEM gettext (2.2.1) locale highline (1.6.13) + hoe (3.0.8) + rake (~> 0.8) json (1.5.4) linecache (0.46) rbx-require-relative (> 0.0.4) + linecache19 (0.5.12) + ruby_core_source (>= 0.1.4) locale (2.0.5) mahoro (0.3) memcache-client (1.8.5) @@ -72,10 +77,22 @@ GEM ruby-debug-base (~> 0.10.4.0) ruby-debug-base (0.10.4) linecache (>= 0.3) + ruby-debug-base19 (0.11.25) + columnize (>= 0.3.1) + linecache19 (>= 0.5.11) + ruby_core_source (>= 0.1.4) + ruby-debug19 (0.11.6) + columnize (>= 0.3.1) + linecache19 (>= 0.5.11) + ruby-debug-base19 (>= 0.11.19) ruby-msg (1.5.0) ruby-ole (>= 1.2.8) vpim (>= 0.360) ruby-ole (1.2.11.3) + ruby_core_source (0.1.5) + archive-tar-minitar (>= 0.5.2) + test-unit (1.2.3) + hoe (>= 1.5.1) vpim (0.695) will_paginate (2.3.16) xapian-full-alaveteli (1.2.9.5) @@ -110,7 +127,9 @@ DEPENDENCIES rspec (~> 1.3.2) rspec-rails (~> 1.3.4) ruby-debug + ruby-debug19 ruby-msg (~> 1.5.0) + test-unit (~> 1.2.3) vpim will_paginate (~> 2.3.11) xapian-full-alaveteli (~> 1.2.9.5) diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb index fcb4671c5..367909f57 100644 --- a/app/models/incoming_message.rb +++ b/app/models/incoming_message.rb @@ -75,8 +75,7 @@ class IncomingMessage < ActiveRecord::Base # Documentation at http://i.loveruby.net/en/projects/tmail/doc/ def mail(force = nil) if (!force.nil? || @mail.nil?) && !self.raw_email.nil? - # Hack round bug in TMail's MIME decoding. Example request which provokes it: - # http://www.whatdotheyknow.com/request/reviews_of_unduly_lenient_senten#incoming-4830 + # Hack round bug in TMail's MIME decoding. # Report of TMail bug: # http://rubyforge.org/tracker/index.php?func=detail&aid=21810&group_id=4512&atid=17370 copy_of_raw_data = self.raw_email.data.gsub(/; boundary=\s+"/ims,'; boundary="') @@ -345,8 +344,7 @@ class IncomingMessage < ActiveRecord::Base name = Regexp.escape(self.info_request.user_name) # To end of message sections - # http://www.whatdotheyknow.com/request/university_investment_in_the_arm - text.gsub!(/^#{name}[^\n]+\nSent by:[^\n]+\n.*/ims, "\n\n" + replacement) + text.gsub!(/^\s?#{name}[^\n]+\n([^\n]+\n)?\s?Sent by:[^\n]+\n.*/ims, "\n\n" + replacement) # Some other sort of forwarding quoting # http://www.whatdotheyknow.com/request/224/response/326 @@ -580,7 +578,8 @@ class IncomingMessage < ActiveRecord::Base # Remove existing quoted sections folded_quoted_text = self.remove_lotus_quoting(text, 'FOLDED_QUOTED_SECTION') - folded_quoted_text = IncomingMessage.remove_quoted_sections(text, "FOLDED_QUOTED_SECTION") + folded_quoted_text = IncomingMessage.remove_quoted_sections(folded_quoted_text, "FOLDED_QUOTED_SECTION") + self.cached_main_body_text_unfolded = text self.cached_main_body_text_folded = folded_quoted_text self.save! diff --git a/commonlib b/commonlib -Subproject 131375c752c02aa20f1644240fe69720275da42 +Subproject 3bf8c34eed89a946f981363eb862f25ac843769 diff --git a/config/environment.rb b/config/environment.rb index 3348ef92a..4ae3f26d9 100644 --- a/config/environment.rb +++ b/config/environment.rb @@ -1,5 +1,9 @@ # Be sure to restart your web server when you modify this file. +# the default encoding for IO is utf-8, and we use utf-8 internally +if RUBY_VERSION.to_f >= 1.9 + Encoding.default_external = Encoding.default_internal = Encoding::UTF_8 +end # Uncomment below to force Rails into production mode when # you don't control web/app server and can't set it the proper way diff --git a/config/initializers/fast_gettext.rb b/config/initializers/fast_gettext.rb index 721c49cd0..2ecf5cb5d 100644 --- a/config/initializers/fast_gettext.rb +++ b/config/initializers/fast_gettext.rb @@ -1,4 +1,3 @@ -Encoding.default_external = 'UTF-8' if RUBY_VERSION.to_f >= 1.9 FastGettext.add_text_domain 'app', :path => File.join(Rails.root, 'locale'), :type => :po FastGettext.default_text_domain = 'app' diff --git a/config/initializers/missing_source_file.rb b/config/initializers/missing_source_file.rb new file mode 100644 index 000000000..a114fa972 --- /dev/null +++ b/config/initializers/missing_source_file.rb @@ -0,0 +1,2 @@ +# For Rails 2.3 on Ruby 1.9.3 @see https://github.com/rails/rails/pull/3745 +MissingSourceFile::REGEXPS << [/^cannot load such file -- (.+)$/i, 1] diff --git a/spec/controllers/api_controller_spec.rb b/spec/controllers/api_controller_spec.rb index ded9a040a..8d8a39950 100644 --- a/spec/controllers/api_controller_spec.rb +++ b/spec/controllers/api_controller_spec.rb @@ -1,3 +1,4 @@ +# coding: utf-8 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') def normalise_whitespace(s) @@ -167,7 +168,7 @@ describe ApiController, "when using the API" do OutgoingMessage.count.should == n_outgoing_messages end - it "should not allow other people’s requests to be updated" do + it "should not allow other people's requests to be updated" do request_id = _create_request n_incoming_messages = IncomingMessage.count n_outgoing_messages = OutgoingMessage.count diff --git a/spec/controllers/public_body_controller_spec.rb b/spec/controllers/public_body_controller_spec.rb index 9eca43aee..d12818a1c 100644 --- a/spec/controllers/public_body_controller_spec.rb +++ b/spec/controllers/public_body_controller_spec.rb @@ -1,3 +1,4 @@ +# coding: utf-8 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') describe PublicBodyController, "when showing a body" do diff --git a/spec/fixtures/files/lotus-notes-quoting.email b/spec/fixtures/files/lotus-notes-quoting.email new file mode 100644 index 000000000..863a2dfef --- /dev/null +++ b/spec/fixtures/files/lotus-notes-quoting.email @@ -0,0 +1,112 @@ +From foi@example.com Thu Mar 06 17:28:24 2008 +To: FOI Person <EMAIL_TO> +Subject: Re: Freedom of Information request +MIME-Version: 1.0 +X-Mailer: Lotus Notes Release 6.5.6 March 06, 2007 +From: foi <foi@example.com> +Date: Thu, 6 Mar 2008 17:31:51 +0000 +Content-Type: multipart/alternative; boundary="=_alternative 005FFC2E80257404_=" + +This is a multipart message in MIME format. +--=_alternative 005FFC2E80257404_= +Content-Type: text/plain; charset="ISO-8859-1" +Content-Transfer-Encoding: quoted-printable + +Dear Requester, + +Some text. + +Yours sincerely + + +FOI Team + + + +Bob Smith <request-xx-xxxxx@whatdotheyknow.com>=20 +Sent by: Bob Smith <request-xx-xxxxx@whatdotheyknow.com> +06/03/08 10:00 +Please respond to +Bob Smith <request-xx-xxxxx@whatdotheyknow.com> + + +To +FOI requests <foi@example.com> +cc + +Subject +Freedom of Information request + + + + + + + Original text + + + + +--=_alternative 005FFC2E80257404_= +Content-Type: text/html; charset="ISO-8859-1" +Content-Transfer-Encoding: quoted-printable + + +<div> +<br><font size=3D2 face=3D"Arial">Dear Requester,</font> +<br> +<br><font size=3D2 face=3D"Arial">Some text. </font> +<br> +<br><font size=3D2 face=3D"Arial">Yours sincerely</font> +<br> +<br> +<div> +<br><font size=3D2 face=3D"Arial">FOI Team</font> +<p> +<br> +<br> +<table width=3D100%> +<tr valign=3Dtop> +<td width=3D40%><font size=3D1 face=3D"sans-serif"><b>Bob Smith <re= +quest-xx-xxxxx@whatdotheyknow.com></b> +</font> +<br><font size=3D1 face=3D"sans-serif">Sent by: Bob Smith <request-= +bounce-xx-xxxxx@whatdotheyknow.com></font> +<p><font size=3D1 face=3D"sans-serif">06/03/08 10:00</font> +<table border> +<tr valign=3Dtop> +<td bgcolor=3Dwhite> +<div align=3Dcenter><font size=3D1 face=3D"sans-serif">Please respond to<br> +Bob Smith <request-xx-xxxxx@whatdotheyknow.com></font></div><= +/table> +<br> +<td width=3D59%> +<table width=3D100%> +<tr valign=3Dtop> +<td> +<div align=3Dright><font size=3D1 face=3D"sans-serif">To</font></div> +<td><font size=3D1 face=3D"sans-serif">FOI requests +<foi@example.com></font> +<tr valign=3Dtop> +<td> +<div align=3Dright><font size=3D1 face=3D"sans-serif">cc</font></div> +<td> +<tr valign=3Dtop> +<td> +<div align=3Dright><font size=3D1 face=3D"sans-serif">Subject</font></div> +<td><font size=3D1 face=3D"sans-serif">Freedom of Information request</font></table> +<br> +<table> +<tr valign=3Dtop> +<td> +<td></table> +<br></table> +<br> +<br> +<br><tt><font size=3D2> Original text<br> +<br> +<br> +</font></tt> +<br></div></div></div> +--=_alternative 005FFC2E80257404_=-- + diff --git a/spec/fixtures/files/space-boundary.email b/spec/fixtures/files/space-boundary.email new file mode 100644 index 000000000..37d65dbb5 --- /dev/null +++ b/spec/fixtures/files/space-boundary.email @@ -0,0 +1,37 @@ +From responder@example.com Wed Sep 03 10:44:18 2008 +Return-path: <responder@example.com> +Envelope-to: foi@sandwich.ukcod.org.uk +Delivery-date: Wed, 03 Sep 2008 10:44:18 +0100 +X-MimeOLE: Produced By Microsoft Exchange V6.5 +Content-class: urn:content-classes:message +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary= + "----_=_NextPart_001_01C90DA9.9B158D0D" +Subject: A subject +Date: Wed, 3 Sep 2008 10:44:05 +0100 +Message-ID: <D403C5D38B9BD6468928738951034DDE85A457@aglnex02.attorneygeneral.gsi.gov.uk> +X-MS-Has-Attach: yes +X-MS-TNEF-Correlator: +Thread-Topic: Freedom of Information request - An information request about + stuff +Thread-Index: AckNqOAr459KZB17Q1OIy1jxNVd9xAAAAnVAAAAWJ+A= +From: "A responder" <responder@example.com> +To: FOI Person <EMAIL_TO> + +This is a multi-part message in MIME format. + +------_=_NextPart_001_01C90DA9.9B158D0D +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: quoted-printable + +=20Apologies.=20Herewith. + +------_=_NextPart_001_01C90DA9.9B158D0D +Content-Type: application/octet-stream; name="Mr Gradwick.pdf" +Content-Transfer-Encoding: base64 +Content-Description: Mr Gradwick.pdf +Content-Disposition: attachment; filename="Mr Gradwick.pdf" + +xxx + +------_=_NextPart_001_01C90DA9.9B158D0D-- diff --git a/spec/models/incoming_message_spec.rb b/spec/models/incoming_message_spec.rb index cf84b4595..69a5da0e9 100644 --- a/spec/models/incoming_message_spec.rb +++ b/spec/models/incoming_message_spec.rb @@ -12,6 +12,14 @@ describe IncomingMessage, " when dealing with incoming mail" do ActionMailer::Base.deliveries.clear end + it 'should correctly parse multipart mails with a linebreak in the boundary marker' do + ir = info_requests(:fancy_dog_request) + receive_incoming_mail('space-boundary.email', ir.incoming_email) + message = ir.incoming_messages[1] + message.mail.parts.size.should == 2 + message.mail.multipart?.should == true + end + it "should return the mail Date header date for sent at" do @im.parse_raw_email!(true) @im.reload @@ -60,7 +68,6 @@ describe IncomingMessage, " when dealing with incoming mail" do is not good utf-8' do ir = info_requests(:fancy_dog_request) receive_incoming_mail('no-part-charset-bad-utf8.email', ir.incoming_email) - puts ir.incoming_messages.inspect message = ir.incoming_messages[1] message.parse_raw_email! message.get_main_body_text_internal.should include("The above text was badly encoded") @@ -125,16 +132,23 @@ end describe IncomingMessage, " folding quoted parts of emails" do - it "cope with [ in user names properly" do - @user = mock_model(User) - @user.stub!(:name).and_return("Sir [ Bobble") - @info_request = mock_model(InfoRequest) - @info_request.stub!(:user).and_return(@user) - @info_request.stub!(:user_name).and_return(@user.name) + it 'should fold an example lotus notes quoted part converted from HTML correctly' do + ir = info_requests(:fancy_dog_request) + receive_incoming_mail('lotus-notes-quoting.email', ir.incoming_email) + message = ir.incoming_messages[1] + message.get_main_body_text_folded.should match(/FOLDED_QUOTED_SECTION/) + end + it 'should fold a plain text lotus notes quoted part correctly' do + text = "FOI Team\n\n\nInfo Requester <xxx@whatdotheyknow.com>=20\nSent by: Info Requester <request-bounce-xxxxx@whatdotheyknow.com>\n06/03/08 10:00\nPlease respond to\nInfo Requester <request-xxxx@whatdotheyknow.com>" @incoming_message = IncomingMessage.new() - @incoming_message.info_request = @info_request + @incoming_message.stub_chain(:info_request, :user_name).and_return("Info Requester") + @incoming_message.remove_lotus_quoting(text).should match(/FOLDED_QUOTED_SECTION/) + end + it "cope with [ in user names properly" do + @incoming_message = IncomingMessage.new() + @incoming_message.stub_chain(:info_request, :user_name).and_return("Sir [ Bobble") # this gives a warning if [ is in the name text = @incoming_message.remove_lotus_quoting("Sir [ Bobble \nSent by: \n") text.should == "\n\nFOLDED_QUOTED_SECTION" |