diff options
| -rw-r--r-- | app/controllers/request_controller.rb | 4 | ||||
| -rw-r--r-- | app/models/incoming_message.rb | 41 | ||||
| -rw-r--r-- | app/models/outgoing_message.rb | 10 | ||||
| -rw-r--r-- | spec/models/incoming_message_spec.rb | 2 | ||||
| -rw-r--r-- | todo.txt | 13 |
5 files changed, 31 insertions, 39 deletions
diff --git a/app/controllers/request_controller.rb b/app/controllers/request_controller.rb index 89b9c05d8..6e3137dfd 100644 --- a/app/controllers/request_controller.rb +++ b/app/controllers/request_controller.rb @@ -4,7 +4,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: request_controller.rb,v 1.182 2009-09-15 17:45:50 francis Exp $ +# $Id: request_controller.rb,v 1.183 2009-09-15 18:26:23 francis Exp $ class RequestController < ApplicationController @@ -562,7 +562,7 @@ class RequestController < ApplicationController view_html_prefix = render_to_string :partial => "request/view_html_prefix" html.sub!("<prefix-here>", view_html_prefix) - html= @incoming_message.html_mask_stuff(html) + @incoming_message.html_mask_stuff!(html) response.content_type = 'text/html' render :text => html diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb index ee5c662b0..dbc0f6213 100644 --- a/app/models/incoming_message.rb +++ b/app/models/incoming_message.rb @@ -19,7 +19,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: incoming_message.rb,v 1.221 2009-09-15 17:45:51 francis Exp $ +# $Id: incoming_message.rb,v 1.222 2009-09-15 18:26:23 francis Exp $ # TODO # Move some of the (e.g. quoting) functions here into rblib, as they feel @@ -467,16 +467,15 @@ class IncomingMessage < ActiveRecord::Base end # Converts email addresses we know about into textual descriptions of them - def mask_special_emails(text) + def mask_special_emails!(text) # XXX can later display some of these special emails as actual emails, # if they are public anyway. For now just be precautionary and only # put in descriptions of them in square brackets. if self.info_request.public_body.is_followupable? - text = text.gsub(self.info_request.public_body.request_email, "[" + self.info_request.public_body.short_or_long_name + " request email]") + text.gsub!(self.info_request.public_body.request_email, "[" + self.info_request.public_body.short_or_long_name + " request email]") end - text = text.gsub(self.info_request.incoming_email, "[FOI #" + self.info_request.id.to_s + " email]") - text = text.gsub(MySociety::Config.get("CONTACT_EMAIL", 'contact@localhost'), "[WhatDoTheyKnow contact email]") - return text + text.gsub!(self.info_request.incoming_email, "[FOI #" + self.info_request.id.to_s + " email]") + text.gsub!(MySociety::Config.get("CONTACT_EMAIL", 'contact@localhost'), "[WhatDoTheyKnow contact email]") end # Replaces all email addresses in (possibly binary data) with equal length alternative ones. @@ -554,11 +553,9 @@ class IncomingMessage < ActiveRecord::Base end # Removes censored stuff from from HTML conversion of downloaded binaries - def html_mask_stuff(html) - html = self.mask_special_emails(html) - html = self.remove_privacy_sensitive_things(html) - - return html + def html_mask_stuff!(html) + self.mask_special_emails!(html) + self.remove_privacy_sensitive_things!(html) end # Lotus notes quoting yeuch! @@ -583,9 +580,7 @@ class IncomingMessage < ActiveRecord::Base end # Remove emails, mobile phones and other details FOI officers ask us to remove. - def remove_privacy_sensitive_things(text) - text = text.dup - + def remove_privacy_sensitive_things!(text) # Remove any email addresses - we don't want bounce messages to leak out # either the requestor's email address or the request's response email # address out onto the internet @@ -617,8 +612,6 @@ class IncomingMessage < ActiveRecord::Base # Remove things from censor rules self.info_request.apply_censor_rules_to_text!(text) - - return text end @@ -778,6 +771,7 @@ class IncomingMessage < ActiveRecord::Base end # Returns body text from main text part of email, converted to UTF-8, with uudecode removed + # XXX returns a .dup of the text, so calling functions can in place modify it def get_main_body_text # Cached as loading raw_email can be quite huge, and need this for just # search results @@ -789,6 +783,7 @@ class IncomingMessage < ActiveRecord::Base text = self.cached_main_body_text # Strip the uudecode parts from main text + # - this also effectively does a .dup as well, so text mods don't alter original text = text.split(/^begin.+^`\n^end\n/sm).join(" ") return text @@ -989,8 +984,8 @@ class IncomingMessage < ActiveRecord::Base def get_body_for_html_display(collapse_quoted_sections = true) # Find the body text and remove emails for privacy/anti-spam reasons text = get_main_body_text - text = self.mask_special_emails(text) - text = self.remove_privacy_sensitive_things(text) + self.mask_special_emails!(text) + self.remove_privacy_sensitive_things!(text) # Remove quoted sections, adding HTML. XXX The FOLDED_QUOTED_SECTION is # a nasty hack so we can escape other HTML before adding the unfold @@ -1031,8 +1026,8 @@ class IncomingMessage < ActiveRecord::Base def get_body_for_quoting # Find the body text and remove emails for privacy/anti-spam reasons text = get_main_body_text - text = self.mask_special_emails(text) - text = self.remove_privacy_sensitive_things(text) + self.mask_special_emails!(text) + self.remove_privacy_sensitive_things!(text) # Remove existing quoted sections text = self.remove_lotus_quoting(text, '') @@ -1048,11 +1043,11 @@ class IncomingMessage < ActiveRecord::Base end # Remove any privacy things - text = self.cached_attachment_text + text = self.cached_attachment_text.dup #STDOUT.puts 'before mask_special_emails ' + MySociety::DebugHelpers::allocated_string_size_around_gc - text = self.mask_special_emails(text) + self.mask_special_emails!(text) #STDOUT.puts 'after mask_special_emails ' + MySociety::DebugHelpers::allocated_string_size_around_gc - text = self.remove_privacy_sensitive_things(text) + self.remove_privacy_sensitive_things!(text) #STDOUT.puts 'after remove_privacy_sensitive_things ' + MySociety::DebugHelpers::allocated_string_size_around_gc return text end diff --git a/app/models/outgoing_message.rb b/app/models/outgoing_message.rb index 5dd125716..14b8c47f2 100644 --- a/app/models/outgoing_message.rb +++ b/app/models/outgoing_message.rb @@ -22,7 +22,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: outgoing_message.rb,v 1.89 2009-09-15 17:45:51 francis Exp $ +# $Id: outgoing_message.rb,v 1.90 2009-09-15 18:26:23 francis Exp $ class OutgoingMessage < ActiveRecord::Base strip_attributes! @@ -193,10 +193,8 @@ class OutgoingMessage < ActiveRecord::Base end # We hide emails from display in outgoing messages. - def remove_privacy_sensitive_things(text) - text = text.dup + def remove_privacy_sensitive_things!(text) text.gsub!(MySociety::Validate.email_find_regexp, "[email address]") - return text end # Returns text for indexing / text display @@ -207,7 +205,7 @@ class OutgoingMessage < ActiveRecord::Base text.sub!(/Dear .+,/, "") # Remove email addresses from display/index etc. - text = self.remove_privacy_sensitive_things(text) + self.remove_privacy_sensitive_things!(text) return text end @@ -215,7 +213,7 @@ class OutgoingMessage < ActiveRecord::Base # Return body for display as HTML def get_body_for_html_display text = self.body.strip - text = self.remove_privacy_sensitive_things(text) + self.remove_privacy_sensitive_things!(text) text = MySociety::Format.wrap_email_body(text) # reparagraph and wrap it so is good preview of emails text = CGI.escapeHTML(text) text = MySociety::Format.make_clickable(text, :contract => 1) diff --git a/spec/models/incoming_message_spec.rb b/spec/models/incoming_message_spec.rb index 9db7acc1b..58ee071ba 100644 --- a/spec/models/incoming_message_spec.rb +++ b/spec/models/incoming_message_spec.rb @@ -173,7 +173,7 @@ describe IncomingMessage, " when censoring data" do it "should apply censor rules to HTML files" do data = @test_data.dup - data = @im.html_mask_stuff(data) + @im.html_mask_stuff!(data) data.should == "There was a mouse called Jarlsberg, he wished that he was yellow." end @@ -1,10 +1,5 @@ -Make incoming mail script tell exim to hold messages and try redelivery when - it goes wrong -And email admins - -Things to make bots not crawl: - /request/13683/response?internal_review=1 - /request/febrile_neutropenia_154?unfold=1 +mask_special_emails +html_mask_stuff Next (things that will reduce admin time mainly) @@ -75,6 +70,10 @@ and resend messages to them Later ===== +Things to make bots not crawl somehow: + /request/13683/response?internal_review=1 + /request/febrile_neutropenia_154?unfold=1 + Maybe don't email you in alert subscriptions about annotations you made on other people's requests? |