diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/alaveteli_external_command.rb | 69 | ||||
| -rw-r--r-- | lib/alaveteli_text_masker.rb | 127 | ||||
| -rw-r--r-- | lib/category_and_heading_migrator.rb | 91 | ||||
| -rw-r--r-- | lib/configuration.rb | 1 | ||||
| -rw-r--r-- | lib/languages.rb | 6 | ||||
| -rw-r--r-- | lib/mail_handler/mail_handler.rb | 4 | ||||
| -rw-r--r-- | lib/public_body_categories.rb | 11 | ||||
| -rw-r--r-- | lib/quiet_opener.rb | 15 | ||||
| -rw-r--r-- | lib/tasks/gettext.rake | 70 |
9 files changed, 252 insertions, 142 deletions
diff --git a/lib/alaveteli_external_command.rb b/lib/alaveteli_external_command.rb index 086a461c8..ddf968f90 100644 --- a/lib/alaveteli_external_command.rb +++ b/lib/alaveteli_external_command.rb @@ -5,61 +5,72 @@ module AlaveteliExternalCommand # Final argument can be a hash of options. # Valid options are: # :append_to - string to append the output of the process to + # :append_errors_to - string to append the errors produced by the process to # :stdin_string - stdin string to pass to the process - # :binary_output - boolean flag for treating the output as binary or text (only significant - # ruby 1.9 and above) + # :binary_output - boolean flag for treating the output as binary or text encoded with + # the default external encoding (only significant in ruby 1.9 and above) + # :binary_input - boolean flag for treating the input as binary or as text encoded with + # the default external encoding (only significant in ruby 1.9 and above) # :memory_limit - maximum amount of memory (in bytes) available to the process + # :timeout - maximum amount of time (in s) to allow the process to run for + # :env - hash of environment variables to set for the process def run(program_name, *args) # Run an external program, and return its output. # Standard error is suppressed unless the program # fails (i.e. returns a non-zero exit status). + # If the program fails, returns nil and writes any error to stderr. + # TODO: calling code should be able to specify error stream - may want to log it or + # otherwise act upon it. opts = {} - if !args.empty? && args[-1].is_a?(Hash) - opts = args.pop - end - - if program_name =~ %r(^/) - program_path = program_name - else - found = false - AlaveteliConfiguration::utility_search_path.each do |d| - program_path = File.join(d, program_name) - if File.file? program_path and File.executable? program_path - found = true - break - end - end - raise "Could not find #{program_name} in any of #{AlaveteliConfiguration::utility_search_path.join(', ')}" if !found + if !args.empty? && args.last.is_a?(Hash) + opts = args.last end + program_path = find_program(program_name) xc = ExternalCommand.new(program_path, *args) - if opts.has_key? :append_to - xc.out = opts[:append_to] - end - if opts.has_key? :binary_output - xc.binary_mode = opts[:binary_output] - end - if opts.has_key? :memory_limit - xc.memory_limit = opts[:memory_limit] + begin + xc.run + rescue ExternalCommand::ChildUnterminated => e + $stderr.puts(e.message) + return nil end - xc.run(opts[:stdin_string] || "", opts[:env] || {}) if !xc.exited # Crash or timeout - $stderr.puts("#{program_name} #{args.join(' ')}:exited abnormally") + if xc.timed_out + $stderr.puts(%Q[External Command: "#{program_name} #{args.join(' ')}" timed out at #{opts[:timeout]}s]) + else + $stderr.puts(%Q[External Command: "#{program_name} #{args.join(' ')}" exited abnormally]) + end + $stderr.print(xc.err) return nil + elsif xc.status != 0 # Error - $stderr.puts("Error from #{program_name} #{args.join(' ')}:") + $stderr.puts(%Q[External Command: Error from command "#{program_name} #{args.join(' ')}":]) $stderr.print(xc.err) return nil else if opts.has_key? :append_to opts[:append_to] << "\n\n" else + return xc.out end end end + + def find_program(program_name) + if program_name =~ %r(^/) + return program_name + else + search_path = AlaveteliConfiguration::utility_search_path + search_path.each do |d| + program_path = File.join(d, program_name) + return program_name if File.file? program_path and File.executable? program_path + end + raise "Could not find #{program_name} in any of #{search_path.join(', ')}" + end + end end end diff --git a/lib/alaveteli_text_masker.rb b/lib/alaveteli_text_masker.rb new file mode 100644 index 000000000..68ff0d318 --- /dev/null +++ b/lib/alaveteli_text_masker.rb @@ -0,0 +1,127 @@ +module AlaveteliTextMasker + extend self + DoNotBinaryMask = [ 'image/tiff', + 'image/gif', + 'image/jpeg', + 'image/png', + 'image/bmp', + 'application/zip' ] + + # Replaces all email addresses in (possibly binary) data + # Also applies custom masks and censor items + def apply_masks!(text, content_type, options = {}) + # See if content type is one that we mask - things like zip files and + # images may get broken if we try to. We err on the side of masking too + # much, as many unknown types will really be text. + + # Special cases for some content types + case content_type + when *DoNotBinaryMask + # do nothing + when 'text/html' + apply_text_masks!(text, options) + when 'application/pdf' + apply_pdf_masks!(text, options) + else + apply_binary_masks!(text, options) + end + end + + def apply_pdf_masks!(text, options = {}) + uncompressed_text = nil + uncompressed_text = AlaveteliExternalCommand.run("pdftk", "-", "output", "-", "uncompress", + :stdin_string => text) + # if we managed to uncompress the PDF... + if !uncompressed_text.blank? + # then censor stuff (making a copy so can compare again in a bit) + censored_uncompressed_text = uncompressed_text.dup + apply_binary_masks!(censored_uncompressed_text, options) + # if the censor rule removed something... + if censored_uncompressed_text != uncompressed_text + # then use the altered file (recompressed) + recompressed_text = nil + if AlaveteliConfiguration::use_ghostscript_compression == true + command = ["gs", "-sDEVICE=pdfwrite", "-dCompatibilityLevel=1.4", "-dPDFSETTINGS=/screen", "-dNOPAUSE", "-dQUIET", "-dBATCH", "-sOutputFile=-", "-"] + else + command = ["pdftk", "-", "output", "-", "compress"] + end + recompressed_text = AlaveteliExternalCommand.run(*(command + [{:stdin_string=>censored_uncompressed_text}])) + if recompressed_text.blank? + # buggy versions of pdftk sometimes fail on + # compression, I don't see it's a disaster in + # these cases to save an uncompressed version? + recompressed_text = censored_uncompressed_text + logger.warn "Unable to compress PDF; problem with your pdftk version?" + end + if !recompressed_text.blank? + text.replace recompressed_text + end + end + end + end + + private + + # Replace text in place + def apply_binary_masks!(text, options = {}) + # Keep original size, so can check haven't resized it + orig_size = text.mb_chars.size + + # Replace ASCII email addresses... + text.gsub!(MySociety::Validate.email_find_regexp) do |email| + email.gsub(/[^@.]/, 'x') + end + + # And replace UCS-2 ones (for Microsoft Office documents)... + # Find emails, by finding them in parts of text that have ASCII + # equivalents to the UCS-2 + ascii_chars = text.gsub(/\0/, "") + emails = ascii_chars.scan(MySociety::Validate.email_find_regexp) + + # Convert back to UCS-2, making a mask at the same time + if String.method_defined?(:encode) + emails.map! do |email| + # We want the ASCII representation of UCS-2 + [email[0].encode('UTF-16LE').force_encoding('US-ASCII'), + email[0].gsub(/[^@.]/, 'x').encode('UTF-16LE').force_encoding('US-ASCII')] + end + else + emails.map! {|email| [ + Iconv.conv('ucs-2le', 'ascii', email[0]), + Iconv.conv('ucs-2le', 'ascii', email[0].gsub(/[^@.]/, 'x')) + ] } + end + + # Now search and replace the UCS-2 email with the UCS-2 mask + for email, mask in emails + text.gsub!(email, mask) + end + + # Replace censor items + censor_rules = options[:censor_rules] || [] + censor_rules.each{ |censor_rule| censor_rule.apply_to_binary!(text) } + raise "internal error in apply_binary_masks!" if text.mb_chars.size != orig_size + return text + end + + # Remove any email addresses, login links and mobile phone numbers + def default_text_masks + [{ :to_replace => MySociety::Validate.email_find_regexp, + :replacement => "[#{_("email address")}]" }, + { :to_replace => /(Mobile|Mob)([\s\/]*(Fax|Tel))*\s*:?[\s\d]*\d/, + :replacement => "[#{_("mobile number")}]" }, + { :to_replace => /https?:\/\/#{AlaveteliConfiguration::domain}\/c\/[^\s]+/, + :replacement => "[#{_("{{site_name}} login link", + :site_name => AlaveteliConfiguration::site_name)}]" }] + end + + def apply_text_masks!(text, options = {}) + masks = options[:masks] || [] + masks += default_text_masks + censor_rules = options[:censor_rules] || [] + masks.each{ |mask| text.gsub!(mask[:to_replace], mask[:replacement]) } + censor_rules.each{ |censor_rule| censor_rule.apply_to_text!(text) } + text + end + +end diff --git a/lib/category_and_heading_migrator.rb b/lib/category_and_heading_migrator.rb deleted file mode 100644 index 402ea7204..000000000 --- a/lib/category_and_heading_migrator.rb +++ /dev/null @@ -1,91 +0,0 @@ -module CategoryAndHeadingMigrator - - # This module migrates data from public_body_categories_[locale].rb files - # into PublicBodyHeading and PublicBodyCategory models - - # Load all the data from public_body_categories_[locale].rb files. - def self.migrate_categories_and_headings - if PublicBodyCategory.count > 0 - puts "PublicBodyCategories exist already, not migrating." - else - @first_locale = true - I18n.available_locales.each do |locale| - begin - load "public_body_categories_#{locale}.rb" - rescue MissingSourceFile - end - @first_locale = false - end - end - end - - # Load the categories and headings for a locale - def self.add_categories_and_headings_from_list(locale, data_list) - # set the counter for headings loaded from this locale - @@locale_heading_display_order = 0 - current_heading = nil - data_list.each do |list_item| - if list_item.is_a?(Array) - # item is list of category data - add_category(list_item, current_heading, locale) - else - # item is heading name - current_heading = add_heading(list_item, locale, @first_locale) - end - end - end - - def self.add_category(category_data, heading, locale) - tag, title, description = category_data - category = PublicBodyCategory.find_by_category_tag(tag) - if category - add_category_in_locale(category, title, description, locale) - else - category = PublicBodyCategory.create(:category_tag => tag, - :title => title, - :description => description) - - # add the translation if this is not the default locale - # (occurs when a category is not defined in default locale) - unless category.translations.map { |t| t.locale }.include?(locale) - add_category_in_locale(category, title, description, locale) - end - end - heading.add_category(category) - end - - def self.add_category_in_locale(category, title, description, locale) - I18n.with_locale(locale) do - category.title = title - category.description = description - category.save - end - end - - def self.add_heading(name, locale, first_locale) - heading = nil - I18n.with_locale(locale) do - heading = PublicBodyHeading.find_by_name(name) - end - # For multi-locale installs, we assume that all public_body_[locale].rb files - # use the same headings in the same order, so we add translations to the heading - # that was in the same position in the list loaded from other public_body_[locale].rb - # files. - if heading.nil? && !@first_locale - heading = PublicBodyHeading.where(:display_order => @@locale_heading_display_order).first - end - - if heading - I18n.with_locale(locale) do - heading.name = name - heading.save - end - else - I18n.with_locale(locale) do - heading = PublicBodyHeading.create(:name => name) - end - end - @@locale_heading_display_order += 1 - heading - end -end diff --git a/lib/configuration.rb b/lib/configuration.rb index 2144f9954..90fd30d5f 100644 --- a/lib/configuration.rb +++ b/lib/configuration.rb @@ -19,6 +19,7 @@ module AlaveteliConfiguration :ADMIN_PASSWORD => '', :ADMIN_USERNAME => '', :ALLOW_BATCH_REQUESTS => false, + :AUTHORITY_MUST_RESPOND => true, :AVAILABLE_LOCALES => '', :BLACKHOLE_PREFIX => 'do-not-reply-to-this-address', :BLOG_FEED => '', diff --git a/lib/languages.rb b/lib/languages.rb index 42231ef56..a45071a67 100644 --- a/lib/languages.rb +++ b/lib/languages.rb @@ -36,7 +36,7 @@ class LanguageNames 'kw' => 'Kernewek', 'co' => 'corsu', 'cr' => 'ᓀᐦᐃᔭᐍᐏᐣ', - 'hr' => 'hrvatski', + 'hr' => 'Hrvatski', 'cs' => 'česky', 'da' => 'dansk', 'dv' => 'ދިވެހި', @@ -111,11 +111,11 @@ class LanguageNames 'mn' => 'монгол', 'na' => 'Ekakairũ Naoero', 'nv' => 'Diné bizaad', - 'nb' => 'Norsk bokmål', + 'nb' => 'Bokmål', 'nd' => 'isiNdebele', 'ne' => 'नेपाली', 'ng' => 'Owambo', - 'nn' => 'Norsk nynorsk', + 'nn' => 'Nynorsk', 'no' => 'Norsk', 'ii' => 'ꆈꌠ꒿ Nuosuhxop', 'nr' => 'isiNdebele', diff --git a/lib/mail_handler/mail_handler.rb b/lib/mail_handler/mail_handler.rb index 47015f207..33d939e22 100644 --- a/lib/mail_handler/mail_handler.rb +++ b/lib/mail_handler/mail_handler.rb @@ -78,7 +78,9 @@ module MailHandler tempfile.binmode tempfile.print body tempfile.flush - default_params = { :append_to => text, :binary_output => false } + default_params = { :append_to => text, + :binary_output => false, + :timeout => 1200 } if content_type == 'application/vnd.ms-word' AlaveteliExternalCommand.run("wvText", tempfile.path, tempfile.path + ".txt", { :memory_limit => 536870912, :timeout => 120 } ) diff --git a/lib/public_body_categories.rb b/lib/public_body_categories.rb deleted file mode 100644 index 3528e85b1..000000000 --- a/lib/public_body_categories.rb +++ /dev/null @@ -1,11 +0,0 @@ -# Allow the PublicBodyCategory model to be addressed using the same syntax -# as the old PublicBodyCategories class without needing to rename everything, -# make sure we're not going to break any themes -class PublicBodyCategories - - def self.method_missing(method, *args, &block) - warn 'Use of PublicBodyCategories is deprecated and will be removed in release 0.21. Please use PublicBodyCategory instead.' - PublicBodyCategory.send(method, *args, &block) - end - -end diff --git a/lib/quiet_opener.rb b/lib/quiet_opener.rb index 16ea27b8e..c6e259b93 100644 --- a/lib/quiet_opener.rb +++ b/lib/quiet_opener.rb @@ -7,8 +7,19 @@ end def quietly_try_to_open(url) begin result = open(url).read.strip - rescue OpenURI::HTTPError, SocketError, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ECONNRESET - Rails.logger.warn("Unable to open third-party URL #{url}") + rescue OpenURI::HTTPError, + SocketError, + Errno::ETIMEDOUT, + Errno::ECONNREFUSED, + Errno::EHOSTUNREACH, + Errno::ECONNRESET, + Timeout::Error => exception + e = Exception.new("Unable to open third-party URL #{url}: #{exception.message}") + e.set_backtrace(exception.backtrace) + if !AlaveteliConfiguration.exception_notifications_from.blank? && !AlaveteliConfiguration.exception_notifications_to.blank? + ExceptionNotifier::Notifier.exception_notification(request.env, e).deliver + end + Rails.logger.warn(e.message) result = "" end return result diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index 3f357213f..4df92b008 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -11,11 +11,7 @@ namespace :gettext do desc "Update pot/po files for a theme." task :find_theme => :environment do - theme = ENV['THEME'] - unless theme - puts "Usage: Specify an Alaveteli-theme with THEME=[theme directory name]" - exit(0) - end + theme = find_theme(ENV['THEME']) load_gettext msgmerge = Rails.application.config.gettext_i18n_rails.msgmerge msgmerge ||= %w[--sort-output --no-location --no-wrap] @@ -28,6 +24,70 @@ namespace :gettext do ) end + desc 'Rewrite theme .po files into a consistent msgmerge format' + task :clean_theme do + theme = find_theme(ENV['THEME']) + load_gettext + + Dir.glob("#{ theme_locale_path(theme) }/*/app.po") do |po_file| + GetText::msgmerge(po_file, po_file, 'alaveteli', + :msgmerge => [:sort_output, :no_location, :no_wrap]) + end + end + + desc 'Update locale files with slightly changed English msgids using a csv file of old to new strings' + task :update_msgids_from_csv do + mapping_file = find_mapping_file(ENV['MAPPING_FILE']) + mappings = {} + CSV.parse(clean_csv_mapping_file(mapping_file)) do |csv_line| + from,to = csv_line + mappings[from] = to + end + Dir.glob("locale/**/app.po").each do |po_file| + lang_mappings = mappings.clone + lines = [] + File.read(po_file).each_line do |line| + /^msgid "(.*)"/ =~ line + if $1 && mappings[$1] + lines << "msgid \"#{lang_mappings.delete($1)}\"" + else + lines << line + end + end + puts "Mappings unused in #{po_file}: #{lang_mappings.keys}" unless lang_mappings.empty? + File.open(po_file, "w") { |f| f.puts(lines) } + end + end + + # Use a quote for quote-escaping as CSV errors on the \" with "Missing or stray quote" + def clean_csv_mapping_file(file) + data = '' + File.foreach(file) do |line| + data += line.gsub('\"', '""') + end + data + end + + def find_theme(theme) + unless theme + puts "Usage: Specify an Alaveteli-theme with THEME=[theme directory name]" + exit(1) + end + theme + end + + def find_mapping_file(file) + unless file + puts "Usage: Specify a csv file mapping old to new strings with MAPPING_FILE=[file name]" + exit(1) + end + unless File.exists?(file) + puts "Error: MAPPING_FILE #{file} not found" + exit(1) + end + file + end + def theme_files_to_translate(theme) Dir.glob("{lib/themes/#{theme}/lib}/**/*.{rb,erb}") end |