Add a couple of email subject lines to the translations.

2 files changed, 872 insertions, 2 deletions

diff --git a/app/models/incoming_message_new.rb.new b/app/models/incoming_message_new.rb.new
new file mode 100644
index 000000000..62908f7fb
--- /dev/null
+++ b/app/models/incoming_message_new.rb.new
@@ -0,0 +1,870 @@
+# coding: utf-8
+
+# == Schema Information
+# Schema version: 114
+#
+# Table name: incoming_messages
+#
+#  id                             :integer         not null, primary key
+#  info_request_id                :integer         not null
+#  created_at                     :datetime        not null
+#  updated_at                     :datetime        not null
+#  raw_email_id                   :integer         not null
+#  cached_attachment_text_clipped :text
+#  cached_main_body_text_folded   :text
+#  cached_main_body_text_unfolded :text
+#  subject                        :text
+#  mail_from_domain               :text
+#  valid_to_reply_to              :boolean
+#  last_parsed                    :datetime
+#  mail_from                      :text
+#  sent_at                        :datetime
+
+# models/incoming_message.rb:
+# An (email) message from really anybody to be logged with a request. e.g. A
+# response from the public body.
+#
+# Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
+# Email: francis@mysociety.org; WWW: http://www.mysociety.org/
+#
+# $Id: incoming_message.rb,v 1.228 2009-10-21 11:24:14 francis Exp $
+
+# TODO
+# Move some of the (e.g. quoting) functions here into rblib, as they feel
+# general not specific to IncomingMessage.
+
+require 'alaveteli_file_types'
+require 'htmlentities'
+require 'rexml/document'
+require 'zip/zip'
+require 'mapi/msg'
+require 'mapi/convert'
+
+class IncomingMessageNew < ActiveRecord::Base
+    belongs_to :info_request
+    validates_presence_of :info_request
+
+    validates_presence_of :raw_email
+
+    has_many :outgoing_message_followups, :foreign_key => 'incoming_message_followup_id', :class_name => 'OutgoingMessage'
+    has_many :foi_attachments, :order => 'id'
+    has_many :info_request_events # never really has many, but could in theory
+
+    belongs_to :raw_email
+
+    # Note that this may be via either the tmail or mail gems, depending on if we
+    # are running ruby 1.8 or 1.9
+    include MailParsing
+
+    # See binary_mask_stuff function below. It just test for inclusion
+    # in this hash, not the value of the right hand side.
+    DoNotBinaryMask = {
+        'image/tiff' => 1,
+        'image/gif' => 1,
+        'image/jpeg' => 1,
+        'image/png' => 1,
+        'image/bmp' => 1,
+        'application/zip' => 1,
+    }
+
+    def mail(force = nil)
+        if (!force.nil? || @mail.nil?) && !self.raw_email.nil?
+            @mail = MailParsing.mail_from_raw_email(self.raw_email.data)
+        end
+        @mail
+    end
+
+    # Return false if for some reason this is a message that we shouldn't let them reply to
+    def _calculate_valid_to_reply_to
+
+        email = get_from_address()
+        return false if !email
+        return false if !MySociety::Validate.is_valid_email(email)
+
+        # reject postmaster - authorities seem to nearly always not respond to
+        # email to postmaster, and it tends to only happen after delivery failure.
+        # likewise Mailer-Daemon, Auto_Reply...
+        prefix = email
+        prefix =~ /^(.*)@/
+        prefix = $1
+        if !prefix.nil? && prefix.downcase.match(/^(postmaster|mailer-daemon|auto_reply|donotreply|no.reply)$/)
+            return false
+        end
+        if !self.mail['return-path'].nil? && self.mail['return-path'].addr == "<>"
+            return false
+        end
+        if !self.mail['auto-submitted'].nil?
+            return false
+        end
+        return true
+    end
+
+    def parse_raw_email!(force = nil)
+        # The following fields may be absent; we treat them as cached
+        # values in case we want to regenerate them (due to mail
+        # parsing bugs, etc).
+        if self.raw_email.nil?
+            raise "Incoming message id=#{id} has no raw_email"
+        end
+        if (!force.nil? || self.last_parsed.nil?)
+            ActiveRecord::Base.transaction do
+                self.extract_attachments!
+                self.sent_at = self.mail.date || self.created_at
+                self.subject = self.mail.subject
+                self.mail_from = get_from_name
+                from_address = get_from_address
+                if from_address
+                    self.mail_from_domain = PublicBody.extract_domain_from_email(get_from_address)
+                else
+                    self.mail_from_domain = ""
+                end
+                self.valid_to_reply_to = self._calculate_valid_to_reply_to
+                self.last_parsed = Time.now
+                self.save!
+            end
+        end
+    end
+
+    def valid_to_reply_to?
+        return self.valid_to_reply_to
+    end
+
+    # The cached fields mentioned in the previous comment
+    # XXX there must be a nicer way to do this without all that
+    # repetition.  I tried overriding method_missing but got some
+    # unpredictable results.
+    def valid_to_reply_to
+        parse_raw_email!
+        super
+    end
+    def sent_at
+        parse_raw_email!
+        super
+    end
+    def subject
+        parse_raw_email!
+        super
+    end
+
+    # Returns the name of the person the incoming message is from, or nil if
+    # there isn't one or if there is only an email address.
+    def mail_from
+        parse_raw_email!
+        super
+    end
+
+    def safe_mail_from
+        if !self.mail_from.nil?
+            mail_from = self.mail_from.dup
+            self.info_request.apply_censor_rules_to_text!(mail_from)
+            return mail_from
+        end
+    end
+    def mail_from_domain
+        parse_raw_email!
+        super
+    end
+
+    # And look up by URL part number to get an attachment
+    # XXX relies on extract_attachments calling ensure_parts_counted
+    def self.get_attachment_by_url_part_number(attachments, found_url_part_number)
+        attachments.each do |a|
+            if a.url_part_number == found_url_part_number
+                return a
+            end
+        end
+        return nil
+    end
+
+    # Converts email addresses we know about into textual descriptions of them
+    def mask_special_emails!(text)
+        # XXX can later display some of these special emails as actual emails,
+        # if they are public anyway.  For now just be precautionary and only
+        # put in descriptions of them in square brackets.
+        if self.info_request.public_body.is_followupable?
+            text.gsub!(self.info_request.public_body.request_email, _("[{{public_body}} request email]", :public_body => self.info_request.public_body.short_or_long_name))
+        end
+        text.gsub!(self.info_request.incoming_email, _('[FOI #{{request}} email]', :request => self.info_request.id.to_s) )
+        text.gsub!(MySociety::Config.get("CONTACT_EMAIL", 'contact@localhost'), _("[{{site_name}} contact email]", :site_name => MySociety::Config.get('SITE_NAME', 'Alaveteli')) )
+    end
+
+    # Replaces all email addresses in (possibly binary data) with equal length alternative ones.
+    # Also replaces censor items
+    def binary_mask_stuff!(text, content_type)
+        # See if content type is one that we mask - things like zip files and
+        # images may get broken if we try to. We err on the side of masking too
+        # much, as many unknown types will really be text.
+        if DoNotBinaryMask.include?(content_type)
+            return
+        end
+        # Special cases for some content types
+        if content_type == 'application/pdf'
+            uncompressed_text = nil
+            uncompressed_text = AlaveteliExternalCommand.run("pdftk", "-", "output", "-", "uncompress", :stdin_string => text)
+
+            # if we managed to uncompress the PDF...
+            if !uncompressed_text.nil? && !uncompressed_text.empty?
+                # then censor stuff (making a copy so can compare again in a bit)
+                censored_uncompressed_text = uncompressed_text.dup
+                self._binary_mask_stuff_internal!(censored_uncompressed_text)
+                # if the censor rule removed something...
+                if censored_uncompressed_text != uncompressed_text
+                    # then use the altered file (recompressed)
+                    recompressed_text = nil
+                    if MySociety::Config.get('USE_GHOSTSCRIPT_COMPRESSION') == true
+                        command = ["gs", "-sDEVICE=pdfwrite", "-dCompatibilityLevel=1.4", "-dPDFSETTINGS=/screen", "-dNOPAUSE", "-dQUIET", "-dBATCH", "-sOutputFile=-", "-"]
+                    else
+                        command = ["pdftk", "-", "output", "-", "compress"]
+                    end
+                    recompressed_text = AlaveteliExternalCommand.run(*(command + [{:stdin_string=>censored_uncompressed_text}]))
+                    if recompressed_text.nil? || recompressed_text.empty?
+                        # buggy versions of pdftk sometimes fail on
+                        # compression, I don't see it's a disaster in
+                        # these cases to save an uncompressed version?
+                        recompressed_text = censored_uncompressed_text
+                        logger.warn "Unable to compress PDF; problem with your pdftk version?"
+                    end
+                    if !recompressed_text.nil? && !recompressed_text.empty?
+                        text.replace recompressed_text
+                    end
+                end
+            end
+            return
+        end
+        self._binary_mask_stuff_internal!(text)
+    end
+
+    # Used by binary_mask_stuff - replace text in place
+    def _binary_mask_stuff_internal!(text)
+        # Keep original size, so can check haven't resized it
+        orig_size = text.size
+        # Replace ASCII email addresses...
+        text.gsub!(MySociety::Validate.email_find_regexp) do |email|
+            email.gsub(/[^@.]/, 'x')
+        end
+        # And replace UCS-2 ones (for Microsoft Office documents)...
+        # Find emails, by finding them in parts of text that have ASCII
+        # equivalents to the UCS-2
+        ascii_chars = text.gsub(/\0/, "")
+        emails = ascii_chars.scan(MySociety::Validate.email_find_regexp)
+        # Convert back to UCS-2, making a mask at the same time
+        emails.map! {|email| [
+                Iconv.conv('ucs-2le', 'ascii', email[0]),
+                Iconv.conv('ucs-2le', 'ascii', email[0].gsub(/[^@.]/, 'x'))
+        ] }
+        # Now search and replace the UCS-2 email with the UCS-2 mask
+        for email, mask in emails
+            text.gsub!(email, mask)
+        end
+        # Replace censor items
+        self.info_request.apply_censor_rules_to_binary!(text)
+
+        raise "internal error in binary_mask_stuff" if text.size != orig_size
+        return text
+    end
+
+    # Removes censored stuff from from HTML conversion of downloaded binaries
+    def html_mask_stuff!(html)
+        self.mask_special_emails!(html)
+        self.remove_privacy_sensitive_things!(html)
+    end
+
+    # Lotus notes quoting yeuch!
+    def remove_lotus_quoting(text, replacement = "FOLDED_QUOTED_SECTION")
+        text = text.dup
+        name = Regexp.escape(self.info_request.user_name)
+
+        # To end of message sections
+        # http://www.whatdotheyknow.com/request/university_investment_in_the_arm
+        text.gsub!(/^#{name}[^\n]+\nSent by:[^\n]+\n.*/im, "\n\n" + replacement)
+
+        # Some other sort of forwarding quoting
+        # http://www.whatdotheyknow.com/request/224/response/326
+        text.gsub!(/^#{name}[^\n]+\n[0-9\/:\s]+\s+To\s+FOI requests at.*/im, "\n\n" + replacement)
+
+        # http://www.whatdotheyknow.com/request/how_do_the_pct_deal_with_retirin_33#incoming-930
+        # http://www.whatdotheyknow.com/request/229/response/809
+        text.gsub!(/^From: [^\n]+\nSent: [^\n]+\nTo:\s+['"?]#{name}['"]?\nSubject:.*/im, "\n\n" + replacement)
+
+        return text
+
+    end
+
+    # Remove emails, mobile phones and other details FOI officers ask us to remove.
+    def remove_privacy_sensitive_things!(text)
+        # Remove any email addresses - we don't want bounce messages to leak out
+        # either the requestor's email address or the request's response email
+        # address out onto the internet
+        text.gsub!(MySociety::Validate.email_find_regexp, "[email address]")
+
+        # Mobile phone numbers
+        # http://www.whatdotheyknow.com/request/failed_test_purchases_off_licenc#incoming-1013
+        # http://www.whatdotheyknow.com/request/selective_licensing_statistics_i#incoming-550
+        # http://www.whatdotheyknow.com/request/common_purpose_training_graduate#incoming-774
+        text.gsub!(/(Mobile|Mob)([\s\/]*(Fax|Tel))*\s*:?[\s\d]*\d/, "[mobile number]")
+
+        # Remove WhatDoTheyKnow signup links
+        domain = MySociety::Config.get('DOMAIN')
+        text.gsub!(/http:\/\/#{domain}\/c\/[^\s]+/, "[WDTK login link]")
+
+        # Remove things from censor rules
+        self.info_request.apply_censor_rules_to_text!(text)
+    end
+
+
+    # Remove quoted sections from emails (eventually the aim would be for this
+    # to do as good a job as GMail does) XXX bet it needs a proper parser
+    # XXX and this FOLDED_QUOTED_SECTION stuff is a mess
+    def self.remove_quoted_sections(text, replacement = "FOLDED_QUOTED_SECTION")
+        text = text.dup
+        replacement = "\n" + replacement + "\n"
+
+        # First do this peculiar form of quoting, as the > single line quoting
+        # further below messes with it. Note the carriage return where it wraps -
+        # this can happen anywhere according to length of the name/email. e.g.
+        # >>> D K Elwell <[email address]> 17/03/2008
+        # 01:51:50 >>>
+        # http://www.whatdotheyknow.com/request/71/response/108
+        # http://www.whatdotheyknow.com/request/police_powers_to_inform_car_insu
+        # http://www.whatdotheyknow.com/request/secured_convictions_aided_by_cct
+        multiline_original_message = '(' + '''>>>.* \d\d/\d\d/\d\d\d\d\s+\d\d:\d\d(?::\d\d)?\s*>>>''' + ')'
+        text.gsub!(/^(#{multiline_original_message}\n.*)$/m, replacement)
+
+        # Single line sections
+        text.gsub!(/^(>.*\n)/, replacement)
+        text.gsub!(/^(On .+ (wrote|said):\n)/, replacement)
+
+        ['-', '_', '*', '#'].each do |scorechar|
+            score = /(?:[#{scorechar}]\s*){8,}/
+            text.sub!(/(Disclaimer\s+)?  # appears just before
+                        (
+                            \s*#{score}\n(?:(?!#{score}\n).)*? # top line
+                            (disclaimer:\n|confidential|received\sthis\semail\sin\serror|virus|intended\s+recipient|monitored\s+centrally|intended\s+(for\s+|only\s+for\s+use\s+by\s+)the\s+addressee|routinely\s+monitored|MessageLabs|unauthorised\s+use)
+                            .*?(?:#{score}|\z) # bottom line OR end of whole string (for ones with no terminator XXX risky)
+                        )
+                       /imx, replacement)
+        end
+
+        # Special paragraphs
+        # http://www.whatdotheyknow.com/request/identity_card_scheme_expenditure
+        text.gsub!(/^[^\n]+Government\s+Secure\s+Intranet\s+virus\s+scanning
+                    .*?
+                    virus\sfree\.
+                    /imx, replacement)
+        text.gsub!(/^Communications\s+via\s+the\s+GSi\s+
+                    .*?
+                    legal\spurposes\.
+                    /imx, replacement)
+        # http://www.whatdotheyknow.com/request/net_promoter_value_scores_for_bb
+        text.gsub!(/^http:\/\/www.bbc.co.uk
+                    .*?
+                    Further\s+communication\s+will\s+signify\s+your\s+consent\s+to\s+this\.
+                    /imx, replacement)
+
+
+        # To end of message sections
+        # http://www.whatdotheyknow.com/request/123/response/192
+        # http://www.whatdotheyknow.com/request/235/response/513
+        # http://www.whatdotheyknow.com/request/445/response/743
+        original_message =
+            '(' + '''----* This is a copy of the message, including all the headers. ----*''' +
+            '|' + '''----*\s*Original Message\s*----*''' +
+            '|' + '''----*\s*Forwarded message.+----*''' +
+            '|' + '''----*\s*Forwarded by.+----*''' +
+            ')'
+        # Could have a ^ at start here, but see messed up formatting here:
+        # http://www.whatdotheyknow.com/request/refuse_and_recycling_collection#incoming-842
+        text.gsub!(/(#{original_message}\n.*)$/mi, replacement)
+
+
+        # Some silly Microsoft XML gets into parts marked as plain text.
+        # e.g. http://www.whatdotheyknow.com/request/are_traffic_wardens_paid_commiss#incoming-401
+        # Don't replace with "replacement" as it's pretty messy
+        text.gsub!(/<\?xml:namespace[^>]*\/>/, " ")
+
+        return text
+    end
+
+
+    # Removes anything cached about the object in the database, and saves
+    def clear_in_database_caches!
+        self.cached_attachment_text_clipped = nil
+        self.cached_main_body_text_unfolded = nil
+        self.cached_main_body_text_folded = nil
+        self.save!
+    end
+
+    # Internal function to cache two sorts of main body text.
+    # Cached as loading raw_email can be quite huge, and need this for just
+    # search results
+    def _cache_main_body_text
+        text = self.get_main_body_text_internal
+        # Strip the uudecode parts from main text
+        # - this also effectively does a .dup as well, so text mods don't alter original
+        text = text.split(/^begin.+^`\n^end\n/m).join(" ")
+
+        if text.size > 1000000 # 1 MB ish
+            raise "main body text more than 1 MB, need to implement clipping like for attachment text, or there is some other MIME decoding problem or similar"
+        end
+
+        # remove emails for privacy/anti-spam reasons
+        self.mask_special_emails!(text)
+        self.remove_privacy_sensitive_things!(text)
+
+        # Remove existing quoted sections
+        folded_quoted_text = self.remove_lotus_quoting(text, 'FOLDED_QUOTED_SECTION')
+        folded_quoted_text = IncomingMessage.remove_quoted_sections(text, "FOLDED_QUOTED_SECTION")
+        self.cached_main_body_text_unfolded = text
+        self.cached_main_body_text_folded = folded_quoted_text
+        self.save!
+    end
+    # Returns body text from main text part of email, converted to UTF-8, with uudecode removed,
+    # emails and privacy sensitive things remove, censored, and folded to remove excess quoted text
+    # (marked with FOLDED_QUOTED_SECTION)
+    # XXX returns a .dup of the text, so calling functions can in place modify it
+    def get_main_body_text_folded
+        if self.cached_main_body_text_folded.nil?
+            self._cache_main_body_text
+        end
+        return self.cached_main_body_text_folded
+    end
+    def get_main_body_text_unfolded
+        if self.cached_main_body_text_unfolded.nil?
+            self._cache_main_body_text
+        end
+        return self.cached_main_body_text_unfolded
+    end
+    # Returns body text from main text part of email, converted to UTF-8
+    def get_main_body_text_internal
+        parse_raw_email!
+        main_part = get_main_body_text_part
+        return _convert_part_body_to_text(main_part)
+    end
+
+    def extract_attachments!
+        attachment_ids = _extract_attachments!
+        # now get rid of any attachments we no longer have
+        if !attachment_ids.empty?
+            FoiAttachment.destroy_all("id NOT IN (#{attachment_ids.join(',')}) AND incoming_message_id = #{self.id}")
+        end
+    end
+
+    # Given a main text part, converts it to text
+    def _convert_part_body_to_text(part)
+        if part.nil?
+            text = "[ Email has no body, please see attachments ]"
+            source_charset = "utf-8"
+        else
+            text = part.body # by default, TMail converts to UTF8 in this call
+            source_charset = part.charset
+            if part.content_type == 'text/html'
+                # e.g. http://www.whatdotheyknow.com/request/35/response/177
+                # XXX This is a bit of a hack as it is calling a
+                # convert to text routine.  Could instead call a
+                # sanitize HTML one.
+
+                # If the text isn't UTF8, it means TMail had a problem
+                # converting it (invalid characters, etc), and we
+                # should instead tell elinks to respect the source
+                # charset
+                use_charset = "utf-8"
+                begin
+                    text = Iconv.conv('utf-8', 'utf-8', text)
+                rescue Iconv::IllegalSequence
+                    use_charset = source_charset
+                end
+                text = self.class._get_attachment_text_internal_one_file(part.content_type, text, use_charset)
+            end
+        end
+
+        # If TMail can't convert text, it just returns it, so we sanitise it.
+        begin
+            # Test if it's good UTF-8
+            text = Iconv.conv('utf-8', 'utf-8', text)
+        rescue Iconv::IllegalSequence
+            # Text looks like unlabelled nonsense,
+            # strip out anything that isn't UTF-8
+            begin
+                text = Iconv.conv('utf-8//IGNORE', source_charset, text) +
+                    _("\n\n[ {{site_name}} note: The above text was badly encoded, and has had strange characters removed. ]",
+                      :site_name => MySociety::Config.get('SITE_NAME', 'Alaveteli'))
+            rescue Iconv::InvalidEncoding, Iconv::IllegalSequence
+                if source_charset != "utf-8"
+                    source_charset = "utf-8"
+                    retry
+                end
+            end
+        end
+
+        # Fix DOS style linefeeds to Unix style ones (or other later regexps won't work)
+        # Needed for e.g. http://www.whatdotheyknow.com/request/60/response/98
+        text = text.gsub(/\r\n/, "\n")
+
+        # Compress extra spaces down to save space, and to stop regular expressions
+        # breaking in strange extreme cases. e.g. for
+        # http://www.whatdotheyknow.com/request/spending_on_consultants
+        text = text.gsub(/ +/, " ")
+
+        return text
+    end
+    # Returns part which contains main body text, or nil if there isn't one
+    def get_main_body_text_part
+        leaves = self.foi_attachments
+
+        # Find first part which is text/plain or text/html
+        # (We have to include HTML, as increasingly there are mail clients that
+        # include no text alternative for the main part, and we don't want to
+        # instead use the first text attachment
+        # e.g. http://www.whatdotheyknow.com/request/list_of_public_authorties)
+        leaves.each do |p|
+            if p.content_type == 'text/plain' or p.content_type == 'text/html'
+                return p
+            end
+        end
+
+        # Otherwise first part which is any sort of text
+        leaves.each do |p|
+            if p.content_type.match(/^text/)
+                return p
+            end
+        end
+
+        # ... or if none, consider first part
+        p = leaves[0]
+        # if it is a known type then don't use it, return no body (nil)
+        if !p.nil? && AlaveteliFileTypes.mimetype_to_extension(p.content_type)
+            # this is guess of case where there are only attachments, no body text
+            # e.g. http://www.whatdotheyknow.com/request/cost_benefit_analysis_for_real_n
+            return nil
+        end
+        # otherwise return it assuming it is text (sometimes you get things
+        # like binary/octet-stream, or the like, which are really text - XXX if
+        # you find an example, put URL here - perhaps we should be always returning
+        # nil in this case)
+        return p
+    end
+    # Returns attachments that are uuencoded in main body part
+    def _uudecode_and_save_attachments(text)
+        # Find any uudecoded things buried in it, yeuchly
+        uus = text.scan(/^begin.+^`\n^end\n/m)
+        attachments = []
+        for uu in uus
+            # Decode the string
+            content = nil
+            tempfile = Tempfile.new('foiuu')
+            tempfile.print uu
+            tempfile.flush
+            content = AlaveteliExternalCommand.run("uudecode", "-o", "/dev/stdout", tempfile.path)
+            tempfile.close
+            # Make attachment type from it, working out filename and mime type
+            filename = uu.match(/^begin\s+[0-9]+\s+(.*)$/)[1]
+            calc_mime = AlaveteliFileTypes.filename_and_content_to_mimetype(filename, content)
+            if calc_mime
+                calc_mime = normalise_content_type(calc_mime)
+                content_type = calc_mime
+            else
+                content_type = 'application/octet-stream'
+            end
+            hexdigest = Digest::MD5.hexdigest(content)
+            attachment = self.foi_attachments.find_or_create_by_hexdigest(:hexdigest => hexdigest)
+            attachment.update_attributes(:filename => filename,
+                                         :content_type => content_type,
+                                         :body => content,
+                                         :display_size => "0K")
+            attachment.save!
+            attachments << attachment
+        end
+        return attachments
+    end
+
+    def get_attachments_for_display
+        parse_raw_email!
+        # return what user would consider attachments, i.e. not the main body
+        main_part = get_main_body_text_part
+        attachments = []
+        for attachment in self.foi_attachments
+            attachments << attachment if attachment != main_part
+        end
+        return attachments
+    end
+
+    # Returns body text as HTML with quotes flattened, and emails removed.
+    def get_body_for_html_display(collapse_quoted_sections = true)
+        # Find the body text and remove emails for privacy/anti-spam reasons
+        text = get_main_body_text_unfolded
+        folded_quoted_text = get_main_body_text_folded
+
+        # Remove quoted sections, adding HTML. XXX The FOLDED_QUOTED_SECTION is
+        # a nasty hack so we can escape other HTML before adding the unfold
+        # links, without escaping them. Rather than using some proper parser
+        # making a tree structure (I don't know of one that is to hand, that
+        # works well in this kind of situation, such as with regexps).
+        if collapse_quoted_sections
+            text = folded_quoted_text
+        end
+        text = MySociety::Format.simplify_angle_bracketed_urls(text)
+        text = CGI.escapeHTML(text)
+        text = MySociety::Format.make_clickable(text, :contract => 1)
+        text.gsub!(/\[(email address|mobile number)\]/, '[<a href="/help/officers#mobiles">\1</a>]')
+        if collapse_quoted_sections
+            text = text.gsub(/(\s*FOLDED_QUOTED_SECTION\s*)+/m, "FOLDED_QUOTED_SECTION")
+            text.strip!
+            # if there is nothing but quoted stuff, then show the subject
+            if text == "FOLDED_QUOTED_SECTION"
+                text = "[Subject only] " + CGI.escapeHTML(self.subject) + text
+            end
+            # and display link for quoted stuff
+            text = text.gsub(/FOLDED_QUOTED_SECTION/, "\n\n" + '<span class="unfold_link"><a href="?unfold=1#incoming-'+self.id.to_s+'">'+_("show quoted sections")+'</a></span>' + "\n\n")
+        else
+            if folded_quoted_text.include?('FOLDED_QUOTED_SECTION')
+                text = text + "\n\n" + '<span class="unfold_link"><a href="?#incoming-'+self.id.to_s+'">'+_("hide quoted sections")+'</a></span>'
+            end
+        end
+        text.strip!
+
+        text = text.gsub(/\n/, '<br>')
+        text = text.gsub(/(?:<br>\s*){2,}/, '<br><br>') # remove excess linebreaks that unnecessarily space it out
+        return text
+    end
+
+
+    # Returns text of email for using in quoted section when replying
+    def get_body_for_quoting
+        # Get the body text with emails and quoted sections removed
+        text = get_main_body_text_folded
+        text.gsub!("FOLDED_QUOTED_SECTION", " ")
+        text.strip!
+        raise "internal error" if text.nil?
+        return text
+    end
+
+    MAX_ATTACHMENT_TEXT_CLIPPED = 1000000 # 1Mb ish
+
+    # Returns text version of attachment text
+    def get_attachment_text_full
+        text = self._get_attachment_text_internal
+        self.mask_special_emails!(text)
+        self.remove_privacy_sensitive_things!(text)
+        # This can be useful for memory debugging
+        #STDOUT.puts 'xxx '+ MySociety::DebugHelpers::allocated_string_size_around_gc
+
+        # Save clipped version for snippets
+        if self.cached_attachment_text_clipped.nil?
+            self.cached_attachment_text_clipped = text[0..MAX_ATTACHMENT_TEXT_CLIPPED]
+            self.save!
+        end
+
+        return text
+    end
+    # Returns a version reduced to a sensible maximum size - this
+    # is for performance reasons when showing snippets in search results.
+    def get_attachment_text_clipped
+        if self.cached_attachment_text_clipped.nil?
+            # As side effect, get_attachment_text_full makes snippet text
+            attachment_text = self.get_attachment_text_full
+            raise "internal error" if self.cached_attachment_text_clipped.nil?
+        end
+
+        return self.cached_attachment_text_clipped
+    end
+    def IncomingMessage._get_attachment_text_internal_one_file(content_type, body, charset = 'utf-8')
+        # note re. charset: TMail always tries to convert email bodies
+        # to UTF8 by default, so normally it should already be that.
+        text = ''
+        # XXX - tell all these command line tools to return utf-8
+        if content_type == 'text/plain'
+            text += body + "\n\n"
+        else
+            tempfile = Tempfile.new('foiextract')
+            tempfile.print body
+            tempfile.flush
+            if content_type == 'application/vnd.ms-word'
+                AlaveteliExternalCommand.run("wvText", tempfile.path, tempfile.path + ".txt")
+                # Try catdoc if we get into trouble (e.g. for InfoRequestEvent 2701)
+                if not File.exists?(tempfile.path + ".txt")
+                    AlaveteliExternalCommand.run("catdoc", tempfile.path, :append_to => text)
+                else
+                    text += File.read(tempfile.path + ".txt") + "\n\n"
+                    File.unlink(tempfile.path + ".txt")
+                end
+            elsif content_type == 'application/rtf'
+                # catdoc on RTF prodcues less comments and extra bumf than --text option to unrtf
+                AlaveteliExternalCommand.run("catdoc", tempfile.path, :append_to => text)
+            elsif content_type == 'text/html'
+                # lynx wordwraps links in its output, which then don't
+                # get formatted properly by Alaveteli. We use elinks
+                # instead, which doesn't do that.
+                AlaveteliExternalCommand.run("elinks", "-eval", "set document.codepage.assume = \"#{charset}\"", "-eval", "set document.codepage.force_assumed = 1", "-dump-charset", "utf-8", "-force-html", "-dump",
+                    tempfile.path, :append_to => text, :env => {"LANG" => "C"})
+            elsif content_type == 'application/vnd.ms-excel'
+                # Bit crazy using /usr/bin/strings - but xls2csv, xlhtml and
+                # py_xls2txt only extract text from cells, not from floating
+                # notes. catdoc may be fooled by weird character sets, but will
+                # probably do for UK FOI requests.
+                AlaveteliExternalCommand.run("/usr/bin/strings", tempfile.path, :append_to => text)
+            elsif content_type == 'application/vnd.ms-powerpoint'
+                # ppthtml seems to catch more text, but only outputs HTML when
+                # we want text, so just use catppt for now
+                AlaveteliExternalCommand.run("catppt", tempfile.path, :append_to => text)
+            elsif content_type == 'application/pdf'
+                AlaveteliExternalCommand.run("pdftotext", tempfile.path, "-", :append_to => text)
+            elsif content_type == 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
+                # This is Microsoft's XML office document format.
+                # Just pull out the main XML file, and strip it of text.
+                xml = AlaveteliExternalCommand.run("/usr/bin/unzip", "-qq", "-c", tempfile.path, "word/document.xml")
+                if !xml.nil?
+                    doc = REXML::Document.new(xml)
+                    text += doc.each_element( './/text()' ){}.join(" ")
+                end
+            elsif content_type == 'application/zip'
+                # recurse into zip files
+                begin
+                    zip_file = Zip::ZipFile.open(tempfile.path)
+                    text += IncomingMessage._get_attachment_text_from_zip_file(zip_file)
+                    zip_file.close()
+                rescue
+                    $stderr.puts("Error processing zip file: #{$!.inspect}")
+                end
+            end
+            tempfile.close
+        end
+
+        if text.respond_to? :force_encoding
+            text.force_encoding('utf-8')
+        else
+            text
+        end
+    end
+    def IncomingMessage._get_attachment_text_from_zip_file(zip_file)
+        text = ""
+        for entry in zip_file
+            if entry.file?
+                filename = entry.to_s
+                begin
+                    body = entry.get_input_stream.read
+                rescue
+                    # move to next attachment silently if there were problems
+                    # XXX really should reduce this to specific exceptions?
+                    # e.g. password protected
+                    next
+                end
+                calc_mime = AlaveteliFileTypes.filename_to_mimetype(filename)
+                if calc_mime
+                    content_type = calc_mime
+                else
+                    content_type = 'application/octet-stream'
+                end
+
+                text += _get_attachment_text_internal_one_file(content_type, body)
+            end
+        end
+        return text
+    end
+    def _get_attachment_text_internal
+        # Extract text from each attachment
+        text = ''
+        attachments = self.get_attachments_for_display
+        for attachment in attachments
+            text += IncomingMessage._get_attachment_text_internal_one_file(attachment.content_type, attachment.body, attachment.charset)
+        end
+        # Remove any bad characters
+        text = Iconv.conv('utf-8//IGNORE', 'utf-8', text)
+        return text
+    end
+
+
+    # Returns text for indexing
+    def get_text_for_indexing_full
+        return get_body_for_quoting + "\n\n" + get_attachment_text_full
+    end
+    # Used for excerpts in search results, when loading full text would be too slow
+    def get_text_for_indexing_clipped
+        return get_body_for_quoting + "\n\n" + get_attachment_text_clipped
+    end
+
+    # Has message arrived "recently"?
+    def recently_arrived
+        (Time.now - self.created_at) <= 3.days
+    end
+
+    def fully_destroy
+        ActiveRecord::Base.transaction do
+            for o in self.outgoing_message_followups
+                o.incoming_message_followup = nil
+                o.save!
+            end
+            info_request_event = InfoRequestEvent.find_by_incoming_message_id(self.id)
+            info_request_event.track_things_sent_emails.each { |a| a.destroy }
+            info_request_event.user_info_request_sent_alerts.each { |a| a.destroy }
+            info_request_event.destroy
+            self.raw_email.destroy_file_representation!
+            self.destroy
+        end
+    end
+
+    # Search all info requests for
+    def IncomingMessage.find_all_unknown_mime_types
+        for incoming_message in IncomingMessage.find(:all)
+            for attachment in incoming_message.get_attachments_for_display
+                raise "internal error incoming_message " + incoming_message.id.to_s if attachment.content_type.nil?
+                if AlaveteliFileTypes.mimetype_to_extension(attachment.content_type).nil?
+                    $stderr.puts "Unknown type for /request/" + incoming_message.info_request.id.to_s + "#incoming-"+incoming_message.id.to_s
+                    $stderr.puts " " + attachment.filename.to_s + " " + attachment.content_type.to_s
+                end
+            end
+        end
+
+        return nil
+    end
+
+    # Returns space separated list of file extensions of attachments to this message. Defaults to
+    # the normal extension for known mime type, otherwise uses other extensions.
+    def get_present_file_extensions
+        ret = {}
+        for attachment in self.get_attachments_for_display
+            ext = AlaveteliFileTypes.mimetype_to_extension(attachment.content_type)
+            ext = File.extname(attachment.filename).gsub(/^[.]/, "") if ext.nil? && !attachment.filename.nil?
+            ret[ext] = 1 if !ext.nil?
+        end
+        return ret.keys.join(" ")
+    end
+    # Return space separated list of all file extensions known
+    def IncomingMessage.get_all_file_extensions
+        return AlaveteliFileTypes.all_extensions.join(" ")
+    end
+
+    def normalise_content_type(content_type)
+        # e.g. http://www.whatdotheyknow.com/request/93/response/250
+        if content_type == 'application/excel' or content_type == 'application/msexcel' or content_type == 'application/x-ms-excel'
+            content_type = 'application/vnd.ms-excel'
+        end
+        if content_type == 'application/mspowerpoint' or content_type == 'application/x-ms-powerpoint'
+            content_type = 'application/vnd.ms-powerpoint'
+        end
+        if content_type == 'application/msword' or content_type == 'application/x-ms-word'
+            content_type = 'application/vnd.ms-word'
+        end
+        if content_type == 'application/x-zip-compressed'
+            content_type = 'application/zip'
+        end
+
+        # e.g. http://www.whatdotheyknow.com/request/copy_of_current_swessex_scr_opt#incoming-9928
+        if content_type == 'application/acrobat'
+            content_type = 'application/pdf'
+        end
+
+        return content_type
+    end
+
+  def for_admin_column
+    self.class.content_columns.each do |column|
+      yield(column.human_name, self.send(column.name), column.type.to_s, column.name)
+    end
+  end
+
+  private :normalise_content_type
+
+end
+
+
+
diff --git a/app/models/request_mailer.rb b/app/models/request_mailer.rb
index ba9285fc6..6b38e3fa8 100644
--- a/app/models/request_mailer.rb
+++ b/app/models/request_mailer.rb
@@ -50,7 +50,7 @@ class RequestMailer < ApplicationMailer
         headers 'Return-Path' => blackhole_email, 'Reply-To' => @from, # we don't care about bounces, likely from spammers
                 'Auto-Submitted' => 'auto-replied' # http://tools.ietf.org/html/rfc3834
         @recipients = email.from_addrs[0].to_s
-        @subject = "Your response to an FOI request was not delivered"
+        @subject = _("Your response to an FOI request was not delivered")
         attachment :content_type => 'message/rfc822', :body => raw_email_data,
             :filename => "original.eml", :transfer_encoding => '7bit', :content_disposition => 'inline'
         @body = {
@@ -154,7 +154,7 @@ class RequestMailer < ApplicationMailer
                 'Auto-Submitted' => 'auto-generated', # http://tools.ietf.org/html/rfc3834
                 'X-Auto-Response-Suppress' => 'OOF'
         @recipients = info_request.user.name_and_email
-        @subject = "Someone has updated the status of your request"
+        @subject = _("Someone has updated the status of your request")
         url = main_url(request_url(info_request))
         @body = {:info_request => info_request, :url => url}
     end