diff options
Diffstat (limited to 'app/models')
| -rw-r--r-- | app/models/change_email_validator.rb | 9 | ||||
| -rw-r--r-- | app/models/exim_log.rb | 4 | ||||
| -rw-r--r-- | app/models/incoming_message.rb | 384 | ||||
| -rw-r--r-- | app/models/info_request.rb | 1 | ||||
| -rw-r--r-- | app/models/public_body.rb | 40 | ||||
| -rw-r--r-- | app/models/request_mailer.rb | 28 | ||||
| -rw-r--r-- | app/models/track_mailer.rb | 29 |
7 files changed, 196 insertions, 299 deletions
diff --git a/app/models/change_email_validator.rb b/app/models/change_email_validator.rb index 515747b22..f7ec6d17e 100644 --- a/app/models/change_email_validator.rb +++ b/app/models/change_email_validator.rb @@ -22,12 +22,17 @@ class ChangeEmailValidator < ActiveRecord::BaseWithoutTable column :old_email, :string column :new_email, :string column :password, :string + column :user_circumstance, :string attr_accessor :logged_in_user validates_presence_of :old_email, :message => N_("Please enter your old email address") validates_presence_of :new_email, :message => N_("Please enter your new email address") - validates_presence_of :password, :message => N_("Please enter your password") + validates_presence_of :password, :message => N_("Please enter your password"), :unless => :changing_email + + def changing_email() + self.user_circumstance == 'change_email' + end def validate if !self.old_email.blank? && !MySociety::Validate.is_valid_email(self.old_email) @@ -37,7 +42,7 @@ class ChangeEmailValidator < ActiveRecord::BaseWithoutTable if !errors[:old_email] if self.old_email.downcase != self.logged_in_user.email.downcase errors.add(:old_email, _("Old email address isn't the same as the address of the account you are logged in with")) - elsif !self.logged_in_user.has_this_password?(self.password) + elsif (!self.changing_email) && (!self.logged_in_user.has_this_password?(self.password)) if !errors[:password] errors.add(:password, _("Password is not correct")) end diff --git a/app/models/exim_log.rb b/app/models/exim_log.rb index 6bdcdf9c2..80535ab41 100644 --- a/app/models/exim_log.rb +++ b/app/models/exim_log.rb @@ -68,7 +68,6 @@ class EximLog < ActiveRecord::Base for email in emails info_request = InfoRequest.find_by_incoming_email(email) if !info_request.nil? - #STDERR.puts "adding log for " + info_request.url_title + " from " + file_name + " line " + line exim_log = EximLog.new exim_log.info_request = info_request exim_log.exim_log_done = done @@ -112,14 +111,13 @@ class EximLog < ActiveRecord::Base raise "unexpected parsing of exim line" end - #STDERR.puts "check_recent_requests_have_been_sent test: " + exim_log.line # debugging found = true end end if !found # It's very important the envelope from is set for avoiding spam filter reasons - this # effectively acts as a check for that. - STDERR.puts("failed to find request sending Exim line for request id " + ir.id.to_s + " " + ir.url_title + " (check envelope from is being set to request address in Ruby, and load-exim-logs crontab is working)") # *** don't comment out this STDERR line, it is the point of the function! + $stderr.puts("failed to find request sending Exim line for request id " + ir.id.to_s + " " + ir.url_title + " (check envelope from is being set to request address in Ruby, and load-exim-logs crontab is working)") # *** don't comment out this STDERR line, it is the point of the function! ok = false end end diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb index c808dc6a1..4d3c08df3 100644 --- a/app/models/incoming_message.rb +++ b/app/models/incoming_message.rb @@ -28,10 +28,11 @@ # Move some of the (e.g. quoting) functions here into rblib, as they feel # general not specific to IncomingMessage. +require 'alaveteli_file_types' +require 'external_command' require 'htmlentities' require 'rexml/document' require 'zip/zip' -require 'mahoro' require 'mapi/msg' require 'mapi/convert' @@ -44,128 +45,17 @@ module TMail end end -# To add an image, create a file with appropriate name corresponding to the -# mime type in public/images e.g. icon_image_tiff_large.png -$file_extension_to_mime_type = { - "txt" => 'text/plain', - "pdf" => 'application/pdf', - "rtf" => 'application/rtf', - "doc" => 'application/vnd.ms-word', - "docx" => 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - "xls" => 'application/vnd.ms-excel', - "xlsx" => 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - "ppt" => 'application/vnd.ms-powerpoint', - "pptx" => 'application/vnd.openxmlformats-officedocument.presentationml.presentation', - "oft" => 'application/vnd.ms-outlook', - "msg" => 'application/vnd.ms-outlook', - "tnef" => 'application/ms-tnef', - "tif" => 'image/tiff', - "gif" => 'image/gif', - "jpg" => 'image/jpeg', # XXX add jpeg - "png" => 'image/png', - "bmp" => 'image/bmp', - "html" => 'text/html', # XXX add htm - "vcf" => 'text/x-vcard', - "zip" => 'application/zip', - "delivery-status" => 'message/delivery-status' -} -# XXX doesn't have way of choosing default for inverse map - might want to add -# one when you need it -$file_extension_to_mime_type_rev = $file_extension_to_mime_type.invert - -# See binary_mask_stuff function below. It just test for inclusion -# in this hash, not the value of the right hand side. -$do_not_binary_mask = { - 'image/tiff' => 1, - 'image/gif' => 1, - 'image/jpeg' => 1, - 'image/png' => 1, - 'image/bmp' => 1, - 'application/zip' => 1, -} - -# Given file name and its content, return most likely type -def filename_and_content_to_mimetype(filename, content) - # Try filename - ret = filename_to_mimetype(filename) - if !ret.nil? - return ret - end - - # Otherwise look inside the file to work out the type. - # Mahoro is a Ruby binding for libmagic. - m = Mahoro.new(Mahoro::MIME) - mahoro_type = m.buffer(content) - mahoro_type.strip! - #STDERR.puts("mahoro", mahoro_type, "xxxok") - # XXX we shouldn't have to check empty? here, but Mahoro sometimes returns a blank line :( - # e.g. for InfoRequestEvent 17930 - if mahoro_type.nil? || mahoro_type.empty? - return nil - end - # text/plain types sometimes come with a charset - mahoro_type.match(/^(.*);/) - if $1 - mahoro_type = $1 - end - # see if looks like a content type, or has something in it that does - # and return that - # mahoro returns junk "\012- application/msword" as mime type. - mahoro_type.match(/([a-z0-9.-]+\/[a-z0-9.-]+)/) - if $1 - return $1 - end - # otherwise we got junk back from mahoro - return nil -end - -# XXX clearly this shouldn't be a global function, or the above global vars. -def filename_to_mimetype(filename) - if !filename - return nil - end - if filename.match(/\.([^.]+)$/i) - lext = $1.downcase - if $file_extension_to_mime_type.include?(lext) - return $file_extension_to_mime_type[lext] - end - end - return nil -end - -def mimetype_to_extension(mime) - if $file_extension_to_mime_type_rev.include?(mime) - return $file_extension_to_mime_type_rev[mime] - end - return nil -end - -def normalise_content_type(content_type) - # e.g. http://www.whatdotheyknow.com/request/93/response/250 - if content_type == 'application/excel' or content_type == 'application/msexcel' or content_type == 'application/x-ms-excel' - content_type = 'application/vnd.ms-excel' - end - if content_type == 'application/mspowerpoint' or content_type == 'application/x-ms-powerpoint' - content_type = 'application/vnd.ms-powerpoint' - end - if content_type == 'application/msword' or content_type == 'application/x-ms-word' - content_type = 'application/vnd.ms-word' - end - if content_type == 'application/x-zip-compressed' - content_type = 'application/zip' - end - - # e.g. http://www.whatdotheyknow.com/request/copy_of_current_swessex_scr_opt#incoming-9928 - if content_type == 'application/acrobat' - content_type = 'application/pdf' - end - - return content_type -end +# This is the type which is used to send data about attachments to the view +class FOIAttachment + attr_accessor :body + attr_accessor :content_type + attr_accessor :filename + attr_accessor :url_part_number + attr_accessor :within_rfc822_subject # we use the subject as the filename for email attachments -# List of DSN codes taken from RFC 3463 -# http://tools.ietf.org/html/rfc3463 -$dsn_to_message = { + # List of DSN codes taken from RFC 3463 + # http://tools.ietf.org/html/rfc3463 + DsnToMessage = { 'X.1.0' => 'Other address status', 'X.1.1' => 'Bad destination mailbox address', 'X.1.2' => 'Bad destination system address', @@ -213,15 +103,7 @@ $dsn_to_message = { 'X.7.5' => 'Cryptographic failure', 'X.7.6' => 'Cryptographic algorithm not supported', 'X.7.7' => 'Message integrity failure' -} - -# This is the type which is used to send data about attachments to the view -class FOIAttachment - attr_accessor :body - attr_accessor :content_type - attr_accessor :filename - attr_accessor :url_part_number - attr_accessor :within_rfc822_subject # we use the subject as the filename for email attachments + } # Returns HTML, of extra comment to put by attachment def extra_note @@ -235,8 +117,8 @@ class FOIAttachment dsn_part = 'X.' + $2 dsn_message = "" - if $dsn_to_message.include?(dsn_part) - dsn_message = " (" + $dsn_to_message[dsn_part] + ")" + if DsnToMessage.include?(dsn_part) + dsn_message = " (" + DsnToMessage[dsn_part] + ")" end return "<br><em>DSN: " + dsn + dsn_message + "</em>" @@ -279,7 +161,7 @@ class FOIAttachment end def _internal_display_filename - calc_ext = mimetype_to_extension(@content_type) + calc_ext = AlaveteliFileTypes.mimetype_to_extension(@content_type) if @filename # Put right extension on if missing @@ -312,65 +194,54 @@ class FOIAttachment end # Whether this type can be shown in the Google Docs Viewer. - # PDF, PowerPoint and TIFF are listed on https://docs.google.com/viewer - # .doc and .docx were added later http://gmailblog.blogspot.com/2010/06/view-doc-attachments-right-in-your.html - # .xls appears to work fine too + # The full list of supported types can be found at + # https://docs.google.com/support/bin/answer.py?hl=en&answer=1189935 def has_google_docs_viewer? - if self.content_type == 'application/vnd.ms-word' - return true - elsif self.content_type == 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' - return true - elsif self.content_type == 'application/pdf' - return true - elsif self.content_type == 'image/tiff' - return true - elsif self.content_type == 'application/vnd.ms-powerpoint' - return true - elsif self.content_type == 'application/vnd.ms-excel' - return true - end + return !! { + "application/pdf" => true, # .pdf + "image/tiff" => true, # .tiff + + "application/vnd.ms-word" => true, # .doc + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" => true, # .docx + + "application/vnd.ms-powerpoint" => true, # .ppt + "application/vnd.openxmlformats-officedocument.presentationml.presentation" => true, # .pptx + + "application/vnd.ms-excel" => true, # .xls + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" => true, # .xlsx + + } [self.content_type] end # Whether this type has a "View as HTML" def has_body_as_html? - if self.content_type == 'text/plain' - return true - elsif self.content_type == 'application/vnd.ms-word' - return true - elsif self.content_type == 'application/vnd.ms-excel' - return true - elsif self.content_type == 'application/pdf' - return true - elsif self.content_type == 'application/rtf' - return true - end - # We use the same "View as HTML" link to embed the Google Doc Viewer - # (when it can't do a conversion locally) - if self.has_google_docs_viewer? - return true - end - return false + return ( + !!{ + "text/plain" => true, + "application/rtf" => true, + }[self.content_type] or + self.has_google_docs_viewer? + ) end # Name of type of attachment type - only valid for things that has_body_as_html? def name_of_content_type - if self.content_type == 'text/plain' - return "Text file" - elsif self.content_type == 'application/vnd.ms-word' - return "Word document" - elsif self.content_type == 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' - return "Word document - XML" - elsif self.content_type == 'application/vnd.ms-excel' - return "Excel spreadsheet" - elsif self.content_type == 'application/pdf' - return "PDF file" - elsif self.content_type == 'application/rtf' - return "RTF file" - elsif self.content_type == 'application/vnd.ms-powerpoint' - return "PowerPoint presentation" - elsif self.content_type == 'image/tiff' - return "TIFF image" - end + return { + "text/plain" => "Text file", + 'application/rtf' => "RTF file", + + 'application/pdf' => "PDF file", + 'image/tiff' => "TIFF image", + + 'application/vnd.ms-word' => "Word document", + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' => "Word document", + + 'application/vnd.ms-powerpoint' => "PowerPoint presentation", + 'application/vnd.openxmlformats-officedocument.presentationml.presentation' => "PowerPoint presentation", + + 'application/vnd.ms-excel' => "Excel spreadsheet", + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' => "Excel spreadsheet", + }[self.content_type] end # For "View as HTML" of attachment @@ -395,20 +266,6 @@ class FOIAttachment tempfile.print self.body tempfile.flush - # Use google docs for the view for these - hanging server - # if self.content_type == 'application/vnd.ms-word' - # # XXX do something with PNG files this spits out so they view too :) - # system("/usr/bin/wvHtml --charset=UTF-8 " + tempfile.path + " " + tempfile.path + ".html") - # html = File.read(tempfile.path + ".html") - # File.unlink(tempfile.path + ".html") -# elsif self.content_type == 'application/vnd.ms-excel' -# # Don't colorise, e.g. otherwise this one comes out with white -# # text which is nasty: -# # http://www.whatdotheyknow.com/request/30485/response/74705/attach/html/2/Empty%20premises%20Sefton.xls.html -# IO.popen("/usr/bin/xlhtml -nc -a " + tempfile.path + "", "r") do |child| -# html = child.read() -# wrapper_id = "wrapper_xlhtml" -# end if self.content_type == 'application/pdf' IO.popen("/usr/bin/pdftohtml -nodrm -zoom 1.0 -stdout -enc UTF-8 -noframes " + tempfile.path + "", "r") do |child| html = child.read() @@ -455,6 +312,7 @@ class FOIAttachment end + class IncomingMessage < ActiveRecord::Base belongs_to :info_request validates_presence_of :info_request @@ -467,6 +325,17 @@ class IncomingMessage < ActiveRecord::Base belongs_to :raw_email + # See binary_mask_stuff function below. It just test for inclusion + # in this hash, not the value of the right hand side. + DoNotBinaryMask = { + 'image/tiff' => 1, + 'image/gif' => 1, + 'image/jpeg' => 1, + 'image/png' => 1, + 'image/bmp' => 1, + 'application/zip' => 1, + } + # Return the structured TMail::Mail object # Documentation at http://i.loveruby.net/en/projects/tmail/doc/ def mail @@ -513,7 +382,7 @@ class IncomingMessage < ActiveRecord::Base # An email attached as text # e.g. http://www.whatdotheyknow.com/request/64/response/102 part.rfc822_attachment = TMail::Mail.parse(part.body) - elsif part.content_type == 'application/vnd.ms-outlook' || part_filename && filename_to_mimetype(part_filename) == 'application/vnd.ms-outlook' + elsif part.content_type == 'application/vnd.ms-outlook' || part_filename && AlaveteliFileTypes.filename_to_mimetype(part_filename) == 'application/vnd.ms-outlook' # An email attached as an Outlook file # e.g. http://www.whatdotheyknow.com/request/chinese_names_for_british_politi msg = Mapi::Msg.open(StringIO.new(part.body)) @@ -571,7 +440,7 @@ class IncomingMessage < ActiveRecord::Base # See if content type is one that we mask - things like zip files and # images may get broken if we try to. We err on the side of masking too # much, as many unknown types will really be text. - if $do_not_binary_mask.include?(content_type) + if DoNotBinaryMask.include?(content_type) return end @@ -800,7 +669,6 @@ class IncomingMessage < ActiveRecord::Base return _get_attachment_leaves_recursive(self.mail) end def _get_attachment_leaves_recursive(curr_mail, within_rfc822_attachment = nil) - # STDERR.puts "_get_attachment_leaves_recursive", curr_mail.content_type, curr_mail.sub_type, curr_mail.multipart?, "\n" leaves_found = [] if curr_mail.multipart? if curr_mail.parts.size == 0 @@ -846,7 +714,7 @@ class IncomingMessage < ActiveRecord::Base # PDFs often come with this mime type, fix it up for view code if curr_mail.content_type == 'application/octet-stream' part_file_name = self._get_censored_part_file_name(curr_mail) - calc_mime = filename_and_content_to_mimetype(part_file_name, curr_mail.body) + calc_mime = AlaveteliFileTypes.filename_and_content_to_mimetype(part_file_name, curr_mail.body) if calc_mime curr_mail.content_type = calc_mime end @@ -949,7 +817,7 @@ class IncomingMessage < ActiveRecord::Base # e.g. http://www.whatdotheyknow.com/request/35/response/177 # XXX This is a bit of a hack as it is calling a convert to text routine. # Could instead call a sanitize HTML one. - text = IncomingMessage._get_attachment_text_internal_one_file(part.content_type, text) + text = self.class._get_attachment_text_internal_one_file(part.content_type, text) end end @@ -1027,7 +895,7 @@ class IncomingMessage < ActiveRecord::Base # ... or if none, consider first part p = leaves[0] # if it is a known type then don't use it, return no body (nil) - if mimetype_to_extension(p.content_type) + if AlaveteliFileTypes.mimetype_to_extension(p.content_type) # this is guess of case where there are only attachments, no body text # e.g. http://www.whatdotheyknow.com/request/cost_benefit_analysis_for_real_n return nil @@ -1067,7 +935,7 @@ class IncomingMessage < ActiveRecord::Base attachment.body = content attachment.filename = uu.match(/^begin\s+[0-9]+\s+(.*)$/)[1] self.info_request.apply_censor_rules_to_text!(attachment.filename) - calc_mime = filename_and_content_to_mimetype(attachment.filename, attachment.body) + calc_mime = AlaveteliFileTypes.filename_and_content_to_mimetype(attachment.filename, attachment.body) if calc_mime calc_mime = normalise_content_type(calc_mime) attachment.content_type = calc_mime @@ -1238,54 +1106,42 @@ class IncomingMessage < ActiveRecord::Base tempfile.print body tempfile.flush if content_type == 'application/vnd.ms-word' - system("/usr/bin/wvText " + tempfile.path + " " + tempfile.path + ".txt") + external_command("/usr/bin/wvText", tempfile.path, tempfile.path + ".txt") # Try catdoc if we get into trouble (e.g. for InfoRequestEvent 2701) if not File.exists?(tempfile.path + ".txt") - IO.popen("/usr/bin/catdoc " + tempfile.path, "r") do |child| - text += child.read() + "\n\n" - end + external_command("/usr/bin/catdoc", tempfile.path, :append_to => text) else text += File.read(tempfile.path + ".txt") + "\n\n" File.unlink(tempfile.path + ".txt") end elsif content_type == 'application/rtf' # catdoc on RTF prodcues less comments and extra bumf than --text option to unrtf - IO.popen("/usr/bin/catdoc " + tempfile.path, "r") do |child| - text += child.read() + "\n\n" - end + external_command("/usr/bin/catdoc", tempfile.path, :append_to => text) elsif content_type == 'text/html' # lynx wordwraps links in its output, which then don't get formatted properly # by WhatDoTheyKnow. We use elinks instead, which doesn't do that. - IO.popen("/usr/bin/elinks -dump-charset utf-8 -force-html -dump " + tempfile.path, "r") do |child| - text += child.read() + "\n\n" - end + external_command("/usr/bin/elinks", "-dump-charset", "utf-8", "-force-html", "-dump", + tempfile.path, :append_to => text) elsif content_type == 'application/vnd.ms-excel' # Bit crazy using /usr/bin/strings - but xls2csv, xlhtml and # py_xls2txt only extract text from cells, not from floating # notes. catdoc may be fooled by weird character sets, but will # probably do for UK FOI requests. - IO.popen("/usr/bin/strings " + tempfile.path, "r") do |child| - text += child.read() + "\n\n" - end + external_command("/usr/bin/strings", tempfile.path, :append_to => text) elsif content_type == 'application/vnd.ms-powerpoint' # ppthtml seems to catch more text, but only outputs HTML when # we want text, so just use catppt for now - IO.popen("/usr/bin/catppt " + tempfile.path, "r") do |child| - text += child.read() + "\n\n" - end + external_command("/usr/bin/catppt", tempfile.path, :append_to => text) elsif content_type == 'application/pdf' - IO.popen("/usr/bin/pdftotext " + tempfile.path + " -", "r") do |child| - text += child.read() + "\n\n" - end + external_command("/usr/bin/pdftotext", tempfile.path, "-", :append_to => text) elsif content_type == 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' # This is Microsoft's XML office document format. # Just pull out the main XML file, and strip it of text. - xml = '' - IO.popen("/usr/bin/unzip -qq -c " + tempfile.path + " word/document.xml", "r") do |child| - xml += child.read() + "\n\n" + xml = external_command("/usr/bin/unzip", "-qq", "-c", tempfile.path, "word/document.xml") + if !xml.nil? + doc = REXML::Document.new(xml) + text += doc.each_element( './/text()' ){}.join(" ") end - doc = REXML::Document.new(xml) - text += doc.each_element( './/text()' ){}.join(" ") elsif content_type == 'application/zip' # recurse into zip files zip_file = Zip::ZipFile.open(tempfile.path) @@ -1300,15 +1156,14 @@ class IncomingMessage < ActiveRecord::Base # e.g. password protected next end - calc_mime = filename_to_mimetype(filename) + calc_mime = AlaveteliFileTypes.filename_to_mimetype(filename) if calc_mime content_type = calc_mime else content_type = 'application/octet-stream' end - #STDERR.puts("doing file " + filename + " content type " + content_type) - text += IncomingMessage._get_attachment_text_internal_one_file(content_type, body) + text += _get_attachment_text_internal_one_file(content_type, body) end end end @@ -1383,9 +1238,9 @@ class IncomingMessage < ActiveRecord::Base for incoming_message in IncomingMessage.find(:all) for attachment in incoming_message.get_attachments_for_display raise "internal error incoming_message " + incoming_message.id.to_s if attachment.content_type.nil? - if mimetype_to_extension(attachment.content_type).nil? - STDERR.puts "Unknown type for /request/" + incoming_message.info_request.id.to_s + "#incoming-"+incoming_message.id.to_s - STDERR.puts " " + attachment.filename.to_s + " " + attachment.content_type.to_s + if AlaveteliFileTypes.mimetype_to_extension(attachment.content_type).nil? + $stderr.puts "Unknown type for /request/" + incoming_message.info_request.id.to_s + "#incoming-"+incoming_message.id.to_s + $stderr.puts " " + attachment.filename.to_s + " " + attachment.content_type.to_s end end end @@ -1398,15 +1253,15 @@ class IncomingMessage < ActiveRecord::Base def get_present_file_extensions ret = {} for attachment in self.get_attachments_for_display - ext = mimetype_to_extension(attachment.content_type) + ext = AlaveteliFileTypes.mimetype_to_extension(attachment.content_type) ext = File.extname(attachment.filename).gsub(/^[.]/, "") if ext.nil? && !attachment.filename.nil? ret[ext] = 1 if !ext.nil? end return ret.keys.join(" ") end # Return space separated list of all file extensions known - def IncomingMessage.get_all_file_extentions - return $file_extension_to_mime_type.keys.join(" ") + def IncomingMessage.get_all_file_extensions + return AlaveteliFileTypes.all_extensions.join(" ") end # Return false if for some reason this is a message that we shouldn't let them reply to @@ -1432,6 +1287,59 @@ class IncomingMessage < ActiveRecord::Base return true end + + def normalise_content_type(content_type) + # e.g. http://www.whatdotheyknow.com/request/93/response/250 + if content_type == 'application/excel' or content_type == 'application/msexcel' or content_type == 'application/x-ms-excel' + content_type = 'application/vnd.ms-excel' + end + if content_type == 'application/mspowerpoint' or content_type == 'application/x-ms-powerpoint' + content_type = 'application/vnd.ms-powerpoint' + end + if content_type == 'application/msword' or content_type == 'application/x-ms-word' + content_type = 'application/vnd.ms-word' + end + if content_type == 'application/x-zip-compressed' + content_type = 'application/zip' + end + + # e.g. http://www.whatdotheyknow.com/request/copy_of_current_swessex_scr_opt#incoming-9928 + if content_type == 'application/acrobat' + content_type = 'application/pdf' + end + + return content_type + end + private :normalise_content_type + + def self.external_command(program_name, *args) + # Run an external program, and return its output. + # Standard error is suppressed unless the program + # fails (i.e. returns a non-zero exit status). + opts = {} + if !args.empty? && args[-1].is_a?(Hash) + opts = args.pop + end + + xc = ExternalCommand.new(program_name, *args) + if opts.has_key? :append_to + xc.out = opts[:append_to] + end + xc.run() + if xc.status != 0 + # Error + $stderr.puts("Error from #{program_name} #{args.join(' ')}:") + $stderr.print(xc.err) + return nil + else + if opts.has_key? :append_to + opts[:append_to] << "\n\n" + else + return xc.out + end + end + end + private_class_method :external_command end diff --git a/app/models/info_request.rb b/app/models/info_request.rb index 4e669e5cd..a36b140b8 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -535,7 +535,6 @@ public if curr_state.nil? if !event.described_state.nil? curr_state = event.described_state - #STDERR.puts "curr_state " + curr_state end end diff --git a/app/models/public_body.rb b/app/models/public_body.rb index 3d2b180b8..b68722b81 100644 --- a/app/models/public_body.rb +++ b/app/models/public_body.rb @@ -46,28 +46,34 @@ class PublicBody < ActiveRecord::Base translates :name, :short_name, :request_email, :url_name, :notes, :first_letter, :publication_scheme + # Make sure publication_scheme gets the correct default value. + # (This would work automatically, were publication_scheme not a translated attribute) + def after_initialize + self.publication_scheme = "" if self.publication_scheme.nil? + end + # like find_by_url_name but also search historic url_name if none found def self.find_by_url_name_with_historic(name) @localer = I18n.locale.to_s PublicBody.with_locale(@locale) do found = PublicBody.find(:all, - :conditions => ["public_body_translations.url_name='#{name}' AND public_body_translations.locale = '#{@localer}'"], - :joins => :translations) - return found.first if found.size == 1 - # Shouldn't we just make url_name unique? - raise "Two bodies with the same URL name: #{name}" if found.size > 1 - # If none found, then search the history of short names - old = PublicBody::Version.find_all_by_url_name(name) - # Find unique public bodies in it - old = old.map { |x| x.public_body_id } - old = old.uniq - # Maybe return the first one, so we show something relevant, - # rather than throwing an error? - raise "Two bodies with the same historical URL name: #{name}" if old.size > 1 - return unless old.size == 1 - # does acts_as_versioned provide a method that returns the current version? - return PublicBody.find(old.first) - end + :conditions => ["public_body_translations.url_name='#{name}' AND public_body_translations.locale = '#{@localer}'"], + :joins => :translations) + return found.first if found.size == 1 + # Shouldn't we just make url_name unique? + raise "Two bodies with the same URL name: #{name}" if found.size > 1 + # If none found, then search the history of short names + old = PublicBody::Version.find_all_by_url_name(name) + # Find unique public bodies in it + old = old.map { |x| x.public_body_id } + old = old.uniq + # Maybe return the first one, so we show something relevant, + # rather than throwing an error? + raise "Two bodies with the same historical URL name: #{name}" if old.size > 1 + return unless old.size == 1 + # does acts_as_versioned provide a method that returns the current version? + return PublicBody.find(old.first) + end end diff --git a/app/models/request_mailer.rb b/app/models/request_mailer.rb index 910f38cbe..e73b153b9 100644 --- a/app/models/request_mailer.rb +++ b/app/models/request_mailer.rb @@ -6,6 +6,8 @@ # # $Id: request_mailer.rb,v 1.89 2009-10-04 21:53:54 francis Exp $ +require 'alaveteli_file_types' + class RequestMailer < ApplicationMailer # Used when an FOI officer uploads a response from their web browser - this is @@ -18,13 +20,7 @@ class RequestMailer < ApplicationMailer :body => body } if !attachment_name.nil? && !attachment_content.nil? - IncomingMessage # load global filename_to_mimetype XXX should move filename_to_mimetype to proper namespace - calc_mime = filename_to_mimetype(attachment_name) - if calc_mime - content_type = calc_mime - else - content_type = 'application/octet-stream' - end + content_type = AlaveteliFileTypes.filename_to_mimetype(attachment_name) || 'application/octet-stream' attachment :content_type => content_type, :body => attachment_content, @@ -230,7 +226,6 @@ class RequestMailer < ApplicationMailer # Send email alerts for overdue requests def self.alert_overdue_requests() - #STDERR.puts "alert_overdue_requests" info_requests = InfoRequest.find(:all, :conditions => [ "described_state = 'waiting_response' and awaiting_description = ?", false ], :include => [ :user ] ) for info_request in info_requests alert_event_id = info_request.last_event_forming_initial_request.id @@ -248,7 +243,6 @@ class RequestMailer < ApplicationMailer sent_already = UserInfoRequestSentAlert.find(:first, :conditions => [ "alert_type = ? and user_id = ? and info_request_id = ? and info_request_event_id = ?", alert_type, info_request.user_id, info_request.id, alert_event_id]) if sent_already.nil? # Alert not yet sent for this user, so send it - #STDERR.puts "sending overdue alert to info_request " + info_request.id.to_s + " user " + info_request.user_id.to_s + " event " + alert_event_id store_sent = UserInfoRequestSentAlert.new store_sent.info_request = info_request store_sent.user = info_request.user @@ -266,7 +260,6 @@ class RequestMailer < ApplicationMailer end end store_sent.save! - #STDERR.puts "sent " + info_request.user.email end end end @@ -280,7 +273,6 @@ class RequestMailer < ApplicationMailer self.alert_new_response_reminders_internal(24, 'new_response_reminder_3') end def self.alert_new_response_reminders_internal(days_since, type_code) - #STDERR.puts "alert_new_response_reminders_internal days:" + days_since.to_s + " type: " + type_code info_requests = InfoRequest.find_old_unclassified(:order => 'info_requests.id', :include => [:user], :age_in_days => days_since) @@ -295,7 +287,6 @@ class RequestMailer < ApplicationMailer sent_already = UserInfoRequestSentAlert.find(:first, :conditions => [ "alert_type = ? and user_id = ? and info_request_id = ? and info_request_event_id = ?", type_code, info_request.user_id, info_request.id, alert_event_id]) if sent_already.nil? # Alert not yet sent for this user - #STDERR.puts "sending " + type_code + " alert to info_request " + info_request.url_title + " user " + info_request.user.url_name + " event " + alert_event_id.to_s store_sent = UserInfoRequestSentAlert.new store_sent.info_request = info_request store_sent.user = info_request.user @@ -304,7 +295,6 @@ class RequestMailer < ApplicationMailer # XXX uses same template for reminder 1 and reminder 2 right now. RequestMailer.deliver_new_response_reminder_alert(info_request, last_response_message) store_sent.save! - #STDERR.puts "sent " + info_request.user.email end end end @@ -312,7 +302,6 @@ class RequestMailer < ApplicationMailer # Send email alerts for requests which need clarification. Goes out 3 days # after last update of event. def self.alert_not_clarified_request() - #STDERR.puts "alert_not_clarified_request" info_requests = InfoRequest.find(:all, :conditions => [ "awaiting_description = ? and described_state = 'waiting_clarification' and info_requests.updated_at < ?", false, Time.now() - 3.days ], :include => [ :user ], :order => "info_requests.id" ) for info_request in info_requests alert_event_id = info_request.get_last_response_event_id @@ -324,7 +313,6 @@ class RequestMailer < ApplicationMailer sent_already = UserInfoRequestSentAlert.find(:first, :conditions => [ "alert_type = 'not_clarified_1' and user_id = ? and info_request_id = ? and info_request_event_id = ?", info_request.user_id, info_request.id, alert_event_id]) if sent_already.nil? # Alert not yet sent for this user - #STDERR.puts "sending clarification reminder alert to info_request " + info_request.id.to_s + " user " + info_request.user_id.to_s + " event " + alert_event_id.to_s store_sent = UserInfoRequestSentAlert.new store_sent.info_request = info_request store_sent.user = info_request.user @@ -336,14 +324,12 @@ class RequestMailer < ApplicationMailer RequestMailer.deliver_not_clarified_alert(info_request, last_response_message) end store_sent.save! - #STDERR.puts "sent " + info_request.user.email end end end # Send email alert to request submitter for new comments on the request. def self.alert_comment_on_request() - #STDERR.puts "alert_comment_on_request" # We only check comments made in the last month - this means if the # cron jobs broke for more than a month events would be lost, but no @@ -359,22 +345,18 @@ class RequestMailer < ApplicationMailer info_requests = InfoRequest.find(:all, :conditions => [ "(select id from info_request_events where event_type = 'comment' and info_request_events.info_request_id = info_requests.id and created_at > ? limit 1) is not null", Time.now() - 1.month ], :include => [ { :info_request_events => :user_info_request_sent_alerts } ], :order => "info_requests.id, info_request_events.created_at" ) for info_request in info_requests - #STDERR.puts "considering request " + info_request.id.to_s # Count number of new comments to alert on earliest_unalerted_comment_event = nil last_comment_event = nil count = 0 for e in info_request.info_request_events.reverse - #STDERR.puts "event " + e.id.to_s + " type " + e.event_type # alert on comments, which were not made by the user who originally made the request if e.event_type == 'comment' && e.comment.user_id != info_request.user_id last_comment_event = e if last_comment_event.nil? alerted_for = e.user_info_request_sent_alerts.find(:first, :conditions => [ "alert_type = 'comment_1' and user_id = ?", info_request.user_id]) - #STDERR.puts "is comment by other user, alerted_for " + alerted_for.to_s + " comment user " + e.comment.user_id.to_s + " request user " + info_request.user_id.to_s + " body: " + e.comment.body if alerted_for.nil? - # STDERR.puts "nil!" count = count + 1 earliest_unalerted_comment_event = e else @@ -382,7 +364,6 @@ class RequestMailer < ApplicationMailer end end end - #STDERR.puts "earliest_unalerted_comment_event " + earliest_unalerted_comment_event.to_s # Alert needs sending if there are new comments if count > 0 @@ -392,16 +373,13 @@ class RequestMailer < ApplicationMailer store_sent.alert_type = 'comment_1' store_sent.info_request_event_id = last_comment_event.id if count > 1 - #STDERR.puts "sending multiple comment on request alert to info_request " + info_request.id.to_s + " user " + info_request.user_id.to_s + " count " + count.to_s + " earliest " + earliest_unalerted_comment_event.id.to_s RequestMailer.deliver_comment_on_alert_plural(info_request, count, earliest_unalerted_comment_event.comment) elsif count == 1 - #STDERR.puts "sending comment on request alert to info_request " + info_request.id.to_s + " user " + info_request.user_id.to_s + " event " + last_comment_event.id.to_s RequestMailer.deliver_comment_on_alert(info_request, last_comment_event.comment) else raise "internal error" end store_sent.save! - #STDERR.puts "sent " + info_request.user.email end end end diff --git a/app/models/track_mailer.rb b/app/models/track_mailer.rb index 6c9d9949b..1b37709b9 100644 --- a/app/models/track_mailer.rb +++ b/app/models/track_mailer.rb @@ -32,14 +32,13 @@ class TrackMailer < ApplicationMailer def self.alert_tracks now = Time.now() users = User.find(:all, :conditions => [ "last_daily_track_email < ?", now - 1.day ]) + if users.empty? + return false + end for user in users - #STDERR.puts Time.now.to_s + " user " + user.url_name - email_about_things = [] track_things = TrackThing.find(:all, :conditions => [ "tracking_user_id = ? and track_medium = ?", user.id, 'email_daily' ]) for track_thing in track_things - #STDERR.puts Time.now.to_s + " track " + track_thing.track_query - # What have we alerted on already? # # We only use track_things_sent_emails records which are less than 14 days old. @@ -81,15 +80,6 @@ class TrackMailer < ApplicationMailer # If we have anything to send, then send everything for the user in one mail if email_about_things.size > 0 - # Debugging - # STDERR.puts "sending email alert for user " + user.url_name - # for track_thing, alert_results, xapian_object in email_about_things - # STDERR.puts " tracking " + track_thing.track_query - # for result in alert_results.reverse - # STDERR.puts " result " + result[:model].class.to_s + " id " + result[:model].id.to_s - # end - # end - # Send the email TrackMailer.deliver_event_digest(user, email_about_things) end @@ -111,6 +101,19 @@ class TrackMailer < ApplicationMailer user.no_xapian_reindex = true user.save! end + return true + end + + def self.alert_tracks_loop + # Run alert_tracks in an endless loop, sleeping when there is nothing to do + while true + sleep_seconds = 1 + while !alert_tracks + sleep sleep_seconds + sleep_seconds *= 2 + sleep_seconds = 300 if sleep_seconds > 300 + end + end end end |