diff options
Diffstat (limited to 'lib')
50 files changed, 476 insertions, 369 deletions
diff --git a/lib/ability.rb b/lib/ability.rb index f63845e84..607f55286 100644 --- a/lib/ability.rb +++ b/lib/ability.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- module Ability def self.can_update_request_state?(user, request) (user && request.is_old_unclassified?) || request.is_owning_user?(user) diff --git a/lib/activerecord_errors_extensions.rb b/lib/activerecord_errors_extensions.rb index 8135d34f7..19bede539 100644 --- a/lib/activerecord_errors_extensions.rb +++ b/lib/activerecord_errors_extensions.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- # Monkeypatch! activerecord/lib/active_record/validations.rb # Method to remove individual error messages from an ActiveRecord. module ActiveRecord diff --git a/lib/activesupport_cache_extensions.rb b/lib/activesupport_cache_extensions.rb index 2791d5996..4e9a6a69c 100644 --- a/lib/activesupport_cache_extensions.rb +++ b/lib/activesupport_cache_extensions.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- # lib/activesupport_cache_extensions.rb: # Extensions / fixes to ActiveSupport::Cache # diff --git a/lib/acts_as_xapian/acts_as_xapian.rb b/lib/acts_as_xapian/acts_as_xapian.rb index 9194e8cc4..e0c7c6ae7 100644 --- a/lib/acts_as_xapian/acts_as_xapian.rb +++ b/lib/acts_as_xapian/acts_as_xapian.rb @@ -1,4 +1,4 @@ -# encoding: utf-8 +# -*- encoding : utf-8 -*- # acts_as_xapian/lib/acts_as_xapian.rb: # Xapian full text search in Ruby on Rails. # @@ -39,7 +39,7 @@ module ActsAsXapian ###################################################################### # Module level variables # TODO: must be some kind of cattr_accessor that can do this better - def ActsAsXapian.bindings_available + def self.bindings_available $acts_as_xapian_bindings_available end class NoXapianRubyBindingsError < StandardError @@ -58,40 +58,40 @@ module ActsAsXapian $acts_as_xapian_class_var_init = true end - def ActsAsXapian.db + def self.db @@db end - def ActsAsXapian.db_path=(db_path) + def self.db_path=(db_path) @@db_path = db_path end - def ActsAsXapian.db_path + def self.db_path @@db_path end - def ActsAsXapian.writable_db + def self.writable_db @@writable_db end - def ActsAsXapian.stemmer + def self.stemmer @@stemmer end - def ActsAsXapian.term_generator + def self.term_generator @@term_generator end - def ActsAsXapian.enquire + def self.enquire @@enquire end - def ActsAsXapian.query_parser + def self.query_parser @@query_parser end - def ActsAsXapian.values_by_prefix + def self.values_by_prefix @@values_by_prefix end - def ActsAsXapian.config + def self.config @@config end ###################################################################### # Initialisation - def ActsAsXapian.init(classname = nil, options = nil) + def self.init(classname = nil, options = nil) if not classname.nil? # store class and options for use later, when we open the db in readable_init @@init_values.push([classname,options]) @@ -99,7 +99,7 @@ module ActsAsXapian end # Reads the config file (if any) and sets up the path to the database we'll be using - def ActsAsXapian.prepare_environment + def self.prepare_environment return unless @@db_path.nil? # barf if we can't figure out the environment @@ -130,17 +130,17 @@ module ActsAsXapian # Opens / reopens the db for reading # TODO: we perhaps don't need to rebuild database and enquire and queryparser - # but db.reopen wasn't enough by itself, so just do everything it's easier. - def ActsAsXapian.readable_init + def self.readable_init raise NoXapianRubyBindingsError.new("Xapian Ruby bindings not installed") unless ActsAsXapian.bindings_available raise "acts_as_xapian hasn't been called in any models" if @@init_values.empty? prepare_environment # We need to reopen the database each time, so Xapian gets changes to it. - # Calling reopen() does not always pick up changes for reasons that I can + # Calling reopen does not always pick up changes for reasons that I can # only speculate about at the moment. (It is easy to reproduce this by - # changing the code below to use reopen() rather than open() followed by - # close(), and running rake spec.) + # changing the code below to use reopen rather than open followed by + # close, and running rake spec.) if !@@db.nil? @@db.close end @@ -157,23 +157,20 @@ module ActsAsXapian end # Make a new query parser - def ActsAsXapian.init_query_parser + def self.init_query_parser # for queries @@query_parser = Xapian::QueryParser.new @@query_parser.stemmer = @@stemmer @@query_parser.stemming_strategy = Xapian::QueryParser::STEM_SOME @@query_parser.database = @@db @@query_parser.default_op = Xapian::Query::OP_AND - begin - @@query_parser.set_max_wildcard_expansion(1000) - rescue NoMethodError - # The set_max_wildcard_expansion method was introduced in Xapian 1.2.7, - # so may legitimately not be available. - # - # Large installations of Alaveteli should consider - # upgrading, because uncontrolled wildcard expansion - # can crash the whole server: see http://trac.xapian.org/ticket/350 - end + # The set_max_wildcard_expansion method was introduced in Xapian 1.2.7, + # so may legitimately not be available. + # + # Large installations of Alaveteli should consider + # upgrading, because uncontrolled wildcard expansion + # can crash the whole server: see http://trac.xapian.org/ticket/350 + @@query_parser.set_max_wildcard_expansion(1000) if @@query_parser.respond_to? :set_max_wildcard_expansion @@stopper = Xapian::SimpleStopper.new @@stopper.add("and") @@ -186,61 +183,68 @@ module ActsAsXapian @@values_by_prefix = {} @@value_ranges_store = [] - for init_value_pair in @@init_values - classname = init_value_pair[0] - options = init_value_pair[1] - + @@init_values.each do |classname, options| # go through the various field types, and tell query parser about them, # and error check them - i.e. check for consistency between models @@query_parser.add_boolean_prefix("model", "M") @@query_parser.add_boolean_prefix("modelid", "I") - if options[:terms] - for term in options[:terms] - raise "Use a single capital letter for term code" if not term[1].match(/^[A-Z]$/) - raise "M and I are reserved for use as the model/id term" if term[1] == "M" or term[1] == "I" - raise "model and modelid are reserved for use as the model/id prefixes" if term[2] == "model" or term[2] == "modelid" - raise "Z is reserved for stemming terms" if term[1] == "Z" - raise "Already have code '" + term[1] + "' in another model but with different prefix '" + @@terms_by_capital[term[1]] + "'" if @@terms_by_capital.include?(term[1]) && @@terms_by_capital[term[1]] != term[2] - @@terms_by_capital[term[1]] = term[2] - # TODO: use boolean here so doesn't stem our URL names in WhatDoTheyKnow - # If making acts_as_xapian generic, would really need to make the :terms have - # another option that lets people choose non-boolean for terms that need it - # (i.e. searching explicitly within a free text field) - @@query_parser.add_boolean_prefix(term[2], term[1]) - end + init_terms(options[:terms]) if options[:terms] + init_values(options[:values]) if options[:values] + end + end + + def self.init_values(values) + values.each do |method, index, prefix, value_type| + raise "Value index '#{index}' must be an Integer, is #{index.class}" unless index.is_a? Integer + if @@values_by_number.include?(index) && @@values_by_number[index] != prefix + raise "Already have value index '#{index}' in another model " \ + "but with different prefix '#{@@values_by_number[index]}'" end - if options[:values] - for value in options[:values] - raise "Value index '"+value[1].to_s+"' must be an integer, is " + value[1].class.to_s if value[1].class != 1.class - raise "Already have value index '" + value[1].to_s + "' in another model but with different prefix '" + @@values_by_number[value[1]].to_s + "'" if @@values_by_number.include?(value[1]) && @@values_by_number[value[1]] != value[2] - - # date types are special, mark them so the first model they're seen for - if !@@values_by_number.include?(value[1]) - if value[3] == :date - value_range = Xapian::DateValueRangeProcessor.new(value[1]) - elsif value[3] == :string - value_range = Xapian::StringValueRangeProcessor.new(value[1]) - elsif value[3] == :number - value_range = Xapian::NumberValueRangeProcessor.new(value[1]) - else - raise "Unknown value type '" + value[3].to_s + "'" - end - - @@query_parser.add_valuerangeprocessor(value_range) - - # stop it being garbage collected, as - # add_valuerangeprocessor ref is outside Ruby's GC - @@value_ranges_store.push(value_range) - end + # date types are special, mark them so the first model they're seen for + unless @@values_by_number.include?(index) + case value_type + when :date + value_range = Xapian::DateValueRangeProcessor.new(index) + when :string + value_range = Xapian::StringValueRangeProcessor.new(index) + when :number + value_range = Xapian::NumberValueRangeProcessor.new(index) + else + raise "Unknown value type '#{value_type}'" + end - @@values_by_number[value[1]] = value[2] - @@values_by_prefix[value[2]] = value[1] - end + @@query_parser.add_valuerangeprocessor(value_range) + + # stop it being garbage collected, as + # add_valuerangeprocessor ref is outside Ruby's GC + @@value_ranges_store.push(value_range) + end + + @@values_by_number[index] = prefix + @@values_by_prefix[prefix] = index + end + end + + def self.init_terms(terms) + terms.each do |method, term_code, prefix| + raise "Use a single capital letter for term code" if not term_code.match(/^[A-Z]$/) + raise "M and I are reserved for use as the model/id term" if term_code == "M" || term_code == "I" + raise "model and modelid are reserved for use as the model/id prefixes" if prefix == "model" || prefix == "modelid" + raise "Z is reserved for stemming terms" if term_code == "Z" + if @@terms_by_capital.include?(term_code) && @@terms_by_capital[term_code] != prefix + raise "Already have code '#{term_code}' in another model but with different prefix " \ + "'#{@@terms_by_capital[term_code]}'" end + @@terms_by_capital[term_code] = prefix + # TODO: use boolean here so doesn't stem our URL names in WhatDoTheyKnow + # If making acts_as_xapian generic, would really need to make the :terms have + # another option that lets people choose non-boolean for terms that need it + # (i.e. searching explicitly within a free text field) + @@query_parser.add_boolean_prefix(prefix, term_code) end end - def ActsAsXapian.writable_init(suffix = "") + def self.writable_init(suffix = "") raise NoXapianRubyBindingsError.new("Xapian Ruby bindings not installed") unless ActsAsXapian.bindings_available raise "acts_as_xapian hasn't been called in any models" if @@init_values.empty? @@ -256,7 +260,7 @@ module ActsAsXapian # for indexing @@writable_db = Xapian::WritableDatabase.new(full_path, Xapian::DB_CREATE_OR_OPEN) @@enquire = Xapian::Enquire.new(@@writable_db) - @@term_generator = Xapian::TermGenerator.new() + @@term_generator = Xapian::TermGenerator.new @@term_generator.set_flags(Xapian::TermGenerator::FLAG_SPELLING, 0) @@term_generator.database = @@writable_db @@term_generator.stemmer = @@stemmer @@ -332,7 +336,7 @@ module ActsAsXapian delay *= 2 delay = MSET_MAX_DELAY if delay > MSET_MAX_DELAY - ActsAsXapian.db.reopen() + ActsAsXapian.db.reopen retry else raise @@ -558,7 +562,7 @@ module ActsAsXapian matches = ActsAsXapian.enquire.mset(0, 100, 100) # TODO: so this whole method will only work with 100 docs # Get set of relevant terms for those documents - selection = Xapian::RSet.new() + selection = Xapian::RSet.new iter = matches._begin while not iter.equals(matches._end) selection.add_document(iter) @@ -610,30 +614,29 @@ module ActsAsXapian # flush your changes. Specifying flush will reduce performance, but make # sure that each index update is definitely saved to disk before # logging in the database that it has been. - def ActsAsXapian.update_index(flush = false, verbose = false) + def self.update_index(flush = false, verbose = false) # STDOUT.puts("start of ActsAsXapian.update_index") if verbose # Before calling writable_init we have to make sure every model class has been initialized. # i.e. has had its class code loaded, so acts_as_xapian has been called inside it, and # we have the info from acts_as_xapian. - model_classes = ActsAsXapianJob.find_by_sql("select model from acts_as_xapian_jobs group by model").map {|a| a.model.constantize} + model_classes = ActsAsXapianJob.pluck("DISTINCT model").map { |a| a.constantize } # If there are no models in the queue, then nothing to do - return if model_classes.size == 0 + return if model_classes.empty? ActsAsXapian.writable_init # Abort if full rebuild is going on new_path = ActsAsXapian.db_path + ".new" if File.exist?(new_path) - raise "aborting incremental index update while full index rebuild happens; found existing " + new_path + raise "aborting incremental index update while full index rebuild happens; found existing #{new_path}" end - ids_to_refresh = ActsAsXapianJob.find(:all).map() { |i| i.id } - for id in ids_to_refresh + ActsAsXapianJob.pluck(:id).each do |id| job = nil begin ActiveRecord::Base.transaction do begin - job = ActsAsXapianJob.find(id, :lock =>true) + job = ActsAsXapianJob.find(id, :lock => true) rescue ActiveRecord::RecordNotFound => e # This could happen if while we are working the model # was updated a second time by another process. In that case @@ -642,30 +645,7 @@ module ActsAsXapian #STDERR.puts("job with #{id} vanished under foot") if verbose next end - STDOUT.puts("ActsAsXapian.update_index #{job.action} #{job.model} #{job.model_id.to_s} #{Time.now.to_s}") if verbose - - begin - if job.action == 'update' - # TODO: Index functions may reference other models, so we could eager load here too? - model = job.model.constantize.find(job.model_id) # :include => cls.constantize.xapian_options[:include] - model.xapian_index - elsif job.action == 'destroy' - # Make dummy model with right id, just for destruction - model = job.model.constantize.new - model.id = job.model_id - model.xapian_destroy - else - raise "unknown ActsAsXapianJob action '" + job.action + "'" - end - rescue ActiveRecord::RecordNotFound => e - # this can happen if the record was hand deleted in the database - job.action = 'destroy' - retry - end - if flush - ActsAsXapian.writable_db.flush - end - job.destroy + run_job(job, flush, verbose) end rescue => detail # print any error, and carry on so other things are indexed @@ -678,7 +658,34 @@ module ActsAsXapian ActsAsXapian.writable_db.close end - def ActsAsXapian._is_xapian_db(path) + def self.run_job(job, flush, verbose) + STDOUT.puts("ActsAsXapian.update_index #{job.action} #{job.model} #{job.model_id.to_s} #{Time.now.to_s}") if verbose + + begin + if job.action == 'update' + # TODO: Index functions may reference other models, so we could eager load here too? + model = job.model.constantize.find(job.model_id) # :include => cls.constantize.xapian_options[:include] + model.xapian_index + elsif job.action == 'destroy' + # Make dummy model with right id, just for destruction + model = job.model.constantize.new + model.id = job.model_id + model.xapian_destroy + else + raise "unknown ActsAsXapianJob action '#{job.action}'" + end + rescue ActiveRecord::RecordNotFound => e + # this can happen if the record was hand deleted in the database + job.action = 'destroy' + retry + end + if flush + ActsAsXapian.writable_db.flush + end + job.destroy + end + + def self._is_xapian_db(path) is_db = File.exist?(File.join(path, "iamflint")) || File.exist?(File.join(path, "iamchert")) return is_db end @@ -690,7 +697,7 @@ module ActsAsXapian # happens (i.e. while the .new database is there) - any index update jobs # are left in the database, and will run after the rebuild has finished. - def ActsAsXapian.rebuild_index(model_classes, verbose = false, terms = true, values = true, texts = true, safe_rebuild = true) + def self.rebuild_index(model_classes, verbose = false, terms = true, values = true, texts = true, safe_rebuild = true) #raise "when rebuilding all, please call as first and only thing done in process / task" if not ActsAsXapian.writable_db.nil? prepare_environment @@ -751,7 +758,7 @@ module ActsAsXapian @@db_path = old_path end - def ActsAsXapian._rebuild_index_safely(model_classes, verbose, terms, values, texts) + def self._rebuild_index_safely(model_classes, verbose, terms, values, texts) batch_size = 1000 for model_class in model_classes model_class_count = model_class.count diff --git a/lib/acts_as_xapian/tasks/xapian.rake b/lib/acts_as_xapian/tasks/xapian.rake index c1986ce1e..52d94011a 100644 --- a/lib/acts_as_xapian/tasks/xapian.rake +++ b/lib/acts_as_xapian/tasks/xapian.rake @@ -9,7 +9,7 @@ namespace :xapian do # "verbose=true" to print model name as it is run. desc 'Updates Xapian search index with changes to models since last call' task :update_index => :environment do - ActsAsXapian.update_index(ENV['flush'] ? true : false, ENV['verbose'] ? true : false) + ActsAsXapian.update_index(ENV['flush'], ENV['verbose']) end # Parameters - specify 'models="PublicBody User"' to say which models diff --git a/lib/alaveteli_external_command.rb b/lib/alaveteli_external_command.rb index ddf968f90..36a709c16 100644 --- a/lib/alaveteli_external_command.rb +++ b/lib/alaveteli_external_command.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- require 'external_command' module AlaveteliExternalCommand diff --git a/lib/alaveteli_file_types.rb b/lib/alaveteli_file_types.rb index 617048c05..2b5aa3efb 100644 --- a/lib/alaveteli_file_types.rb +++ b/lib/alaveteli_file_types.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- class AlaveteliFileTypes # To add an image, create a file with appropriate name corresponding to the # mime type in public/images e.g. icon_image_tiff_large.png @@ -87,4 +88,4 @@ class AlaveteliFileTypes return nil end end -end
\ No newline at end of file +end diff --git a/lib/alaveteli_localization.rb b/lib/alaveteli_localization.rb index 2b6978c92..6e9dbeac3 100644 --- a/lib/alaveteli_localization.rb +++ b/lib/alaveteli_localization.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- class AlaveteliLocalization class << self def set_locales(available_locales, default_locale) diff --git a/lib/alaveteli_text_masker.rb b/lib/alaveteli_text_masker.rb index 68ff0d318..49dd15ae5 100644 --- a/lib/alaveteli_text_masker.rb +++ b/lib/alaveteli_text_masker.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- module AlaveteliTextMasker extend self DoNotBinaryMask = [ 'image/tiff', @@ -7,6 +8,21 @@ module AlaveteliTextMasker 'image/bmp', 'application/zip' ] + TextMask = [ 'text/css', + 'text/csv', + 'text/html', + 'text/plain', + 'text/rfc822-headers', + 'text/rtf', + 'text/tab-separated-values', + 'text/x-c', + 'text/x-diff', + 'text/x-fortran', + 'text/x-mail', + 'text/xml', + 'text/x-pascal', + 'text/x-vcard' ] + # Replaces all email addresses in (possibly binary) data # Also applies custom masks and censor items def apply_masks!(text, content_type, options = {}) @@ -18,7 +34,7 @@ module AlaveteliTextMasker case content_type when *DoNotBinaryMask # do nothing - when 'text/html' + when *TextMask apply_text_masks!(text, options) when 'application/pdf' apply_pdf_masks!(text, options) @@ -28,9 +44,7 @@ module AlaveteliTextMasker end def apply_pdf_masks!(text, options = {}) - uncompressed_text = nil - uncompressed_text = AlaveteliExternalCommand.run("pdftk", "-", "output", "-", "uncompress", - :stdin_string => text) + uncompressed_text = uncompress_pdf(text) # if we managed to uncompress the PDF... if !uncompressed_text.blank? # then censor stuff (making a copy so can compare again in a bit) @@ -39,19 +53,13 @@ module AlaveteliTextMasker # if the censor rule removed something... if censored_uncompressed_text != uncompressed_text # then use the altered file (recompressed) - recompressed_text = nil - if AlaveteliConfiguration::use_ghostscript_compression == true - command = ["gs", "-sDEVICE=pdfwrite", "-dCompatibilityLevel=1.4", "-dPDFSETTINGS=/screen", "-dNOPAUSE", "-dQUIET", "-dBATCH", "-sOutputFile=-", "-"] - else - command = ["pdftk", "-", "output", "-", "compress"] - end - recompressed_text = AlaveteliExternalCommand.run(*(command + [{:stdin_string=>censored_uncompressed_text}])) + recompressed_text = compress_pdf(censored_uncompressed_text) if recompressed_text.blank? # buggy versions of pdftk sometimes fail on # compression, I don't see it's a disaster in # these cases to save an uncompressed version? recompressed_text = censored_uncompressed_text - logger.warn "Unable to compress PDF; problem with your pdftk version?" + Rails.logger.warn "Unable to compress PDF; problem with your pdftk version?" end if !recompressed_text.blank? text.replace recompressed_text @@ -62,10 +70,31 @@ module AlaveteliTextMasker private + def uncompress_pdf(text) + AlaveteliExternalCommand.run("pdftk", "-", "output", "-", "uncompress", :stdin_string => text) + end + + def compress_pdf(text) + if AlaveteliConfiguration::use_ghostscript_compression + command = ["gs", + "-sDEVICE=pdfwrite", + "-dCompatibilityLevel=1.4", + "-dPDFSETTINGS=/screen", + "-dNOPAUSE", + "-dQUIET", + "-dBATCH", + "-sOutputFile=-", + "-"] + else + command = ["pdftk", "-", "output", "-", "compress"] + end + AlaveteliExternalCommand.run(*(command + [ :stdin_string => text ])) + end + # Replace text in place def apply_binary_masks!(text, options = {}) # Keep original size, so can check haven't resized it - orig_size = text.mb_chars.size + orig_size = text.size # Replace ASCII email addresses... text.gsub!(MySociety::Validate.email_find_regexp) do |email| @@ -100,7 +129,7 @@ module AlaveteliTextMasker # Replace censor items censor_rules = options[:censor_rules] || [] censor_rules.each{ |censor_rule| censor_rule.apply_to_binary!(text) } - raise "internal error in apply_binary_masks!" if text.mb_chars.size != orig_size + raise "internal error in apply_binary_masks!" if text.size != orig_size return text end diff --git a/lib/attachment_to_html/adapter.rb b/lib/attachment_to_html/adapter.rb new file mode 100644 index 000000000..ac8a16411 --- /dev/null +++ b/lib/attachment_to_html/adapter.rb @@ -0,0 +1,67 @@ +module AttachmentToHTML + class Adapter + attr_reader :attachment + + # Public: Initialize a converter + # + # attachment - the FoiAttachment to convert to HTML + # opts - a Hash of options (default: {}): + # No options currently accepted + def initialize(attachment, opts = {}) + @attachment = attachment + end + + # Public: The title to use in the <title> tag + # + # Returns a String + def title + @title ||= attachment.display_filename + end + + # Public: The contents of the extracted html <body> tag + # + # Returns a String + def body + @body ||= parse_body + end + + def parse_body + convert + end + + # Public: Was the document conversion successful? + # + # Returns true + def success? + true + end + + def has_content? + !body.gsub(/\s+/,"").gsub(/\<[^\>]*\>/, "").empty? + end + + def contains_images? + body.match(/<img[^>]*>/mi) + end + + def create_tempfile(text) + tempfile = if RUBY_VERSION.to_f >= 1.9 + Tempfile.new('foiextract', '.', :encoding => text.encoding) + else + Tempfile.new('foiextract', '.') + end + tempfile.print(text) + tempfile.flush + tempfile + end + + def cleanup_tempfile(tempfile) + tempfile.close + tempfile.delete + end + + def attachment_body + @attachment_body ||= attachment.default_body + end + end +end diff --git a/lib/attachment_to_html/adapters/could_not_convert.rb b/lib/attachment_to_html/adapters/could_not_convert.rb index 8e4bf39dc..745a54114 100644 --- a/lib/attachment_to_html/adapters/could_not_convert.rb +++ b/lib/attachment_to_html/adapters/could_not_convert.rb @@ -1,49 +1,15 @@ +# -*- encoding : utf-8 -*- module AttachmentToHTML module Adapters - class CouldNotConvert - - attr_reader :attachment - - # Public: Initialize a PDF converter - # - # attachment - the FoiAttachment to convert to HTML - # opts - a Hash of options (default: {}): - # No options currently accepted - def initialize(attachment, opts = {}) - @attachment = attachment - end - - # Public: The title to use in the <title> tag - # - # Returns a String - def title - @title ||= attachment.display_filename - end - - # Public: The contents of the extracted html <body> tag - # - # Returns a String - def body - @body ||= parse_body - end - - - # Public: Was the document conversion successful? - # As this is a fallback option and not doing anything dynamic - # we're assuming this is successful whatever the case - # - # Returns true - def success? - true - end - + # As this is a fallback option and not doing anything dynamic + # we're assuming this is successful whatever the case + class CouldNotConvert < Adapter private def parse_body "<p>Sorry, we were unable to convert this file to HTML. " \ "Please use the download link at the top right.</p>" end - end end -end
\ No newline at end of file +end diff --git a/lib/attachment_to_html/adapters/google_docs_viewer.rb b/lib/attachment_to_html/adapters/google_docs_viewer.rb index 991fbb757..0817d08fd 100644 --- a/lib/attachment_to_html/adapters/google_docs_viewer.rb +++ b/lib/attachment_to_html/adapters/google_docs_viewer.rb @@ -1,9 +1,14 @@ +# -*- encoding : utf-8 -*- module AttachmentToHTML module Adapters # Renders the attachment in a Google Docs Viewer - class GoogleDocsViewer - - attr_reader :attachment, :attachment_url + # + # We can't really tell whether the document conversion has been + # successful as such; We're assuming that given a correctly + # constructed iframe (which is tested) that Google will make this + # Just Work. + class GoogleDocsViewer < Adapter + attr_reader :attachment_url # Public: Initialize a GoogleDocsViewer converter # @@ -12,35 +17,10 @@ module AttachmentToHTML # :attachment_url - a String url to the attachment for # Google to render (default: nil) def initialize(attachment, opts = {}) - @attachment = attachment + super @attachment_url = opts.fetch(:attachment_url, nil) end - # Public: The title to use in the <title> tag - # - # Returns a String - def title - @title ||= attachment.display_filename - end - - # Public: The contents of the extracted html <body> tag - # - # Returns a String - def body - @body ||= parse_body - end - - # Public: Was the document conversion successful? - # We can't really tell whether the document conversion has been - # successful as such; We're assuming that given a correctly - # constructed iframe (which is tested) that Google will make this - # Just Work. - # - # Returns true - def success? - true - end - private def parse_body @@ -50,7 +30,6 @@ module AttachmentToHTML def protocol AlaveteliConfiguration.force_ssl ? 'https' : 'http' end - end end end diff --git a/lib/attachment_to_html/adapters/pdf.rb b/lib/attachment_to_html/adapters/pdf.rb index a010b0342..afc8fbcb0 100644 --- a/lib/attachment_to_html/adapters/pdf.rb +++ b/lib/attachment_to_html/adapters/pdf.rb @@ -1,10 +1,11 @@ +# -*- encoding : utf-8 -*- module AttachmentToHTML module Adapters # Convert application/pdf documents in to HTML - class PDF + class PDF < Adapter TOO_MANY_IMAGES = 51 - attr_reader :attachment, :tmpdir + attr_reader :tmpdir # Public: Initialize a PDF converter # @@ -13,24 +14,10 @@ module AttachmentToHTML # :tmpdir - String name of directory to store the # converted document def initialize(attachment, opts = {}) - @attachment = attachment + super @tmpdir = opts.fetch(:tmpdir, ::Rails.root.join('tmp')) end - # Public: The title to use in the <title> tag - # - # Returns a String - def title - @title ||= attachment.display_filename - end - - # Public: The contents of the extracted html <body> tag - # - # Returns a String - def body - @body ||= parse_body - end - # Public: Was the document conversion successful? # # Returns a Boolean @@ -47,14 +34,6 @@ module AttachmentToHTML match ? match[1] : '' end - def has_content? - !body.gsub(/\s+/,"").gsub(/\<[^\>]*\>/, "").empty? - end - - def contains_images? - body.match(/<img[^>]*>/mi) ? true : false - end - # Works around https://bugs.freedesktop.org/show_bug.cgi?id=77932 in pdftohtml def contains_too_many_images? number_of_images_in_body >= TOO_MANY_IMAGES @@ -81,28 +60,6 @@ module AttachmentToHTML html end end - - def create_tempfile(text) - tempfile = if RUBY_VERSION.to_f >= 1.9 - Tempfile.new('foiextract', '.', - :encoding => text.encoding) - else - Tempfile.new('foiextract', '.') - end - tempfile.print(text) - tempfile.flush - tempfile - end - - def cleanup_tempfile(tempfile) - tempfile.close - tempfile.delete - end - - def attachment_body - @attachment_body ||= attachment.body - end - end end end diff --git a/lib/attachment_to_html/adapters/rtf.rb b/lib/attachment_to_html/adapters/rtf.rb index 95f499689..4a08bf618 100644 --- a/lib/attachment_to_html/adapters/rtf.rb +++ b/lib/attachment_to_html/adapters/rtf.rb @@ -1,9 +1,10 @@ +# -*- encoding : utf-8 -*- module AttachmentToHTML module Adapters # Convert application/rtf documents in to HTML - class RTF + class RTF < Adapter - attr_reader :attachment, :tmpdir + attr_reader :tmpdir # Public: Initialize a RTF converter # @@ -12,24 +13,10 @@ module AttachmentToHTML # :tmpdir - String name of directory to store the # converted document def initialize(attachment, opts = {}) - @attachment = attachment + super @tmpdir = opts.fetch(:tmpdir, ::Rails.root.join('tmp')) end - # Public: The title to use in the <title> tag - # - # Returns a String - def title - @title ||= attachment.display_filename - end - - # Public: The contents of the extracted html <body> tag - # - # Returns a String - def body - @body ||= parse_body - end - # Public: Was the document conversion successful? # # Returns a Boolean @@ -44,14 +31,6 @@ module AttachmentToHTML match ? match[1] : '' end - def has_content? - !body.gsub(/\s+/,"").gsub(/\<[^\>]*\>/, "").empty? - end - - def contains_images? - body.match(/<img[^>]*>/mi) ? true : false - end - def convert # Get the attachment body outside of the chdir call as getting # the body may require opening files too @@ -82,28 +61,6 @@ module AttachmentToHTML end html end - - def create_tempfile(text) - tempfile = if RUBY_VERSION.to_f >= 1.9 - Tempfile.new('foiextract', '.', - :encoding => text.encoding) - else - Tempfile.new('foiextract', '.') - end - tempfile.print(text) - tempfile.flush - tempfile - end - - def cleanup_tempfile(tempfile) - tempfile.close - tempfile.delete - end - - def attachment_body - @attachment_body ||= attachment.body - end - end end end diff --git a/lib/attachment_to_html/adapters/text.rb b/lib/attachment_to_html/adapters/text.rb index e99183f0e..61e4e57a8 100644 --- a/lib/attachment_to_html/adapters/text.rb +++ b/lib/attachment_to_html/adapters/text.rb @@ -1,33 +1,8 @@ +# -*- encoding : utf-8 -*- module AttachmentToHTML module Adapters # Convert text/plain documents in to HTML - class Text - - attr_reader :attachment - - # Public: Initialize a Text converter - # - # attachment - the FoiAttachment to convert to HTML - # opts - a Hash of options (default: {}): - # No options currently accepted - def initialize(attachment, opts = {}) - @attachment = attachment - end - - # Public: The title to use in the <title> tag - # - # Returns a String - def title - @title ||= attachment.display_filename - end - - # Public: The contents of the extracted html <body> tag - # - # Returns a String - def body - @body ||= parse_body - end - + class Text < Adapter # Public: Was the document conversion successful? # # Returns a Boolean @@ -43,19 +18,6 @@ module AttachmentToHTML text = MySociety::Format.make_clickable(text) text = text.gsub(/\n/, '<br>') end - - def parse_body - convert - end - - def has_content? - !body.gsub(/\s+/,"").gsub(/\<[^\>]*\>/, "").empty? - end - - def contains_images? - body.match(/<img[^>]*>/mi) ? true : false - end - end end end diff --git a/lib/attachment_to_html/attachment_to_html.rb b/lib/attachment_to_html/attachment_to_html.rb index 2f7c08264..2e8d35ca9 100644 --- a/lib/attachment_to_html/attachment_to_html.rb +++ b/lib/attachment_to_html/attachment_to_html.rb @@ -1,5 +1,8 @@ +# -*- encoding : utf-8 -*- require 'view' +require 'attachment_to_html/adapter' + Dir[File.dirname(__FILE__) + '/adapters/*.rb'].each do |file| require file end diff --git a/lib/attachment_to_html/view.rb b/lib/attachment_to_html/view.rb index e6991d44e..0d5b205b7 100644 --- a/lib/attachment_to_html/view.rb +++ b/lib/attachment_to_html/view.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- module AttachmentToHTML class View < ERB diff --git a/lib/confidence_intervals.rb b/lib/confidence_intervals.rb index 9fe38045a..83f736c63 100644 --- a/lib/confidence_intervals.rb +++ b/lib/confidence_intervals.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- # Calculate the confidence interval for a samples from a binonial # distribution using Wilson's score interval. For more theoretical # details, please see: diff --git a/lib/configuration.rb b/lib/configuration.rb index 90fd30d5f..ab7d1a65c 100644 --- a/lib/configuration.rb +++ b/lib/configuration.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- require File.dirname(__FILE__) + '/../commonlib/rblib/config' # Load intial mySociety config @@ -32,8 +33,9 @@ module AlaveteliConfiguration :DISABLE_EMERGENCY_USER => false, :DOMAIN => 'localhost:3000', :DONATION_URL => '', - :EXCEPTION_NOTIFICATIONS_FROM => '', - :EXCEPTION_NOTIFICATIONS_TO => '', + :ENABLE_WIDGETS => false, + :EXCEPTION_NOTIFICATIONS_FROM => 'errors@localhost', + :EXCEPTION_NOTIFICATIONS_TO => 'user-support@localhost', :FORCE_REGISTRATION_ON_NEW_REQUEST => false, :FORCE_SSL => true, :FORWARD_NONBOUNCE_RESPONSES_TO => 'user-support@localhost', @@ -52,6 +54,7 @@ module AlaveteliConfiguration :MTA_LOG_TYPE => 'exim', :NEW_RESPONSE_REMINDER_AFTER_DAYS => [3, 10, 24], :OVERRIDE_ALL_PUBLIC_BODY_REQUEST_EMAILS => '', + :PRODUCTION_MAILER_DELIVERY_METHOD => 'sendmail', :PUBLIC_BODY_STATISTICS_PAGE => false, :PUBLIC_BODY_LIST_FALLBACK_TO_DEFAULT_LOCALE => false, :RAW_EMAILS_LOCATION => 'files/raw_emails', @@ -63,6 +66,13 @@ module AlaveteliConfiguration :RESPONSIVE_STYLING => true, :SITE_NAME => 'Alaveteli', :SKIP_ADMIN_AUTH => false, + :SMTP_MAILER_ADDRESS => 'localhost', + :SMTP_MAILER_PORT => 25, + :SMTP_MAILER_DOMAIN => '', + :SMTP_MAILER_USER_NAME => '', + :SMTP_MAILER_PASSWORD => '', + :SMTP_MAILER_AUTHENTICATION => 'plain', + :SMTP_MAILER_ENABLE_STARTTLS_AUTO => true, :SPECIAL_REPLY_VERY_LATE_AFTER_DAYS => 60, :THEME_BRANCH => false, :THEME_URL => "", @@ -78,10 +88,11 @@ module AlaveteliConfiguration :UTILITY_SEARCH_PATH => ["/usr/bin", "/usr/local/bin"], :VARNISH_HOST => '', :WORKING_OR_CALENDAR_DAYS => 'working', + :USE_BULLET_IN_DEVELOPMENT => false } - end + end - def AlaveteliConfiguration.method_missing(name) + def self.method_missing(name) key = name.to_s.upcase if DEFAULTS.has_key?(key.to_sym) MySociety::Config.get(key, DEFAULTS[key.to_sym]) diff --git a/lib/date_quarter.rb b/lib/date_quarter.rb index ac159b420..b8cb03593 100644 --- a/lib/date_quarter.rb +++ b/lib/date_quarter.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- module DateQuarter extend self diff --git a/lib/generators/acts_as_xapian/acts_as_xapian_generator.rb b/lib/generators/acts_as_xapian/acts_as_xapian_generator.rb index 434c02cb5..0e525d868 100644 --- a/lib/generators/acts_as_xapian/acts_as_xapian_generator.rb +++ b/lib/generators/acts_as_xapian/acts_as_xapian_generator.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- require 'rails/generators/active_record/migration' class ActsAsXapianGenerator < Rails::Generators::Base diff --git a/lib/generators/acts_as_xapian/templates/migration.rb b/lib/generators/acts_as_xapian/templates/migration.rb index 84a9dd766..b390cd933 100644 --- a/lib/generators/acts_as_xapian/templates/migration.rb +++ b/lib/generators/acts_as_xapian/templates/migration.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- class CreateActsAsXapian < ActiveRecord::Migration def self.up create_table :acts_as_xapian_jobs do |t| diff --git a/lib/has_tag_string/has_tag_string.rb b/lib/has_tag_string/has_tag_string.rb index c28720f04..42d6c8898 100644 --- a/lib/has_tag_string/has_tag_string.rb +++ b/lib/has_tag_string/has_tag_string.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- # lib/has_tag_string.rb: # Lets a model have tags, represented as space separate strings in a public # interface, but stored in the database as keys. Each tag can have a value @@ -31,7 +32,7 @@ module HasTagString # Parses a text version of one single tag, such as "a:b" and returns # the name and value, with nil for value if there isn't one. - def HasTagStringTag.split_tag_into_name_value(tag) + def self.split_tag_into_name_value(tag) sections = tag.split(/:/) name = sections[0] if sections[1] @@ -151,7 +152,7 @@ module HasTagString ###################################################################### # Main entry point, add has_tag_string to your model. module HasMethods - def has_tag_string() + def has_tag_string has_many :tags, :conditions => "model = '" + self.to_s + "'", :foreign_key => "model_id", :class_name => 'HasTagString::HasTagStringTag' include InstanceMethods diff --git a/lib/health_checks/checks/days_ago_check.rb b/lib/health_checks/checks/days_ago_check.rb index 793fff586..3c1cb784f 100644 --- a/lib/health_checks/checks/days_ago_check.rb +++ b/lib/health_checks/checks/days_ago_check.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- module HealthChecks module Checks class DaysAgoCheck @@ -19,7 +20,7 @@ module HealthChecks "#{ super }: #{ subject.call }" end - def check + def ok? subject.call >= days.days.ago end diff --git a/lib/health_checks/health_checkable.rb b/lib/health_checks/health_checkable.rb index 5d674ca32..1e324c1c7 100644 --- a/lib/health_checks/health_checkable.rb +++ b/lib/health_checks/health_checkable.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- module HealthChecks module HealthCheckable @@ -12,12 +13,8 @@ module HealthChecks self.class.to_s end - def check - raise NotImplementedError - end - def ok? - check ? true : false + raise NotImplementedError end def message diff --git a/lib/health_checks/health_checks.rb b/lib/health_checks/health_checks.rb index 6f0c9de8e..6c98365fc 100644 --- a/lib/health_checks/health_checks.rb +++ b/lib/health_checks/health_checks.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- require 'health_checkable' Dir[File.dirname(__FILE__) + '/checks/*.rb'].each do |file| @@ -31,7 +32,7 @@ module HealthChecks private def assert_valid_check(check) - check.respond_to?(:check) + check.respond_to?(:ok?) end end diff --git a/lib/i18n_fixes.rb b/lib/i18n_fixes.rb index 64c370477..6c7cb8877 100644 --- a/lib/i18n_fixes.rb +++ b/lib/i18n_fixes.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- # Some of the monkeypatches in this file should possibly be submitted # as patches, but most are here because they should go away when we # upgrade to Rails 3.x diff --git a/lib/languages.rb b/lib/languages.rb index a45071a67..4f51f9bee 100644 --- a/lib/languages.rb +++ b/lib/languages.rb @@ -1,4 +1,4 @@ -# coding: utf-8 +# -*- encoding : utf-8 -*- class LanguageNames def self.get_language_name(locale) language_names = { @@ -32,6 +32,7 @@ class LanguageNames 'ce' => 'нохчийн мотт', 'ny' => 'chiCheŵa', 'zh' => '中文 (Zhōngwén)', + 'zh-HK' => '中文(香港)', 'cv' => 'чӑваш чӗлхи', 'kw' => 'Kernewek', 'co' => 'corsu', @@ -188,6 +189,7 @@ class LanguageNames 'zu' => 'isiZulu' } locale = locale.sub("_", "-") # normalize + return language_names[locale] if language_names[locale] main_part = I18n::Locale::Tag::Simple.tag(locale).subtags[0] return language_names[main_part] end diff --git a/lib/mail_handler/backends/mail_backend.rb b/lib/mail_handler/backends/mail_backend.rb index 974873b30..19f502275 100644 --- a/lib/mail_handler/backends/mail_backend.rb +++ b/lib/mail_handler/backends/mail_backend.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- require 'mail' require 'mapi/msg' require 'mapi/convert' @@ -35,7 +36,7 @@ module MailHandler module Backends module MailBackend - def backend() + def backend 'Mail' end @@ -64,7 +65,12 @@ module MailHandler # Return a copy of the file name for the mail part def get_part_file_name(part) part_file_name = part.filename - part_file_name.nil? ? nil : part_file_name.dup + part_file_name = part_file_name.nil? ? nil : part_file_name.dup + if part_file_name + part_file_name = CGI.unescape(part_file_name) + part_file_name = convert_string_to_utf8(part_file_name, part.charset).string + end + part_file_name end # Get the body of a mail part diff --git a/lib/mail_handler/backends/mail_extensions.rb b/lib/mail_handler/backends/mail_extensions.rb index f778cbc14..b39e54d08 100644 --- a/lib/mail_handler/backends/mail_extensions.rb +++ b/lib/mail_handler/backends/mail_extensions.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- require 'mail/message' require 'mail/part' require 'mail/fields/common/parameter_hash' @@ -37,7 +38,7 @@ module Mail # Can be removed when we no longer support Ruby 1.8 class Ruby18 - def Ruby18.b_value_decode(str) + def self.b_value_decode(str) match = str.match(/\=\?(.+)?\?[Bb]\?(.+)?\?\=/m) if match encoding = match[1] @@ -55,7 +56,7 @@ module Mail str end - def Ruby18.q_value_decode(str) + def self.q_value_decode(str) match = str.match(/\=\?(.+)?\?[Qq]\?(.+)?\?\=/m) if match encoding = match[1] @@ -75,7 +76,7 @@ module Mail private - def Ruby18.fix_encoding(encoding) + def self.fix_encoding(encoding) case encoding.upcase when 'UTF8' 'UTF-8' @@ -86,7 +87,7 @@ module Mail end class Ruby19 - def Ruby19.b_value_decode(str) + def self.b_value_decode(str) match = str.match(/\=\?(.+)?\?[Bb]\?(.+)?\?\=/m) if match charset = match[1] diff --git a/lib/mail_handler/mail_handler.rb b/lib/mail_handler/mail_handler.rb index 33d939e22..313869d16 100644 --- a/lib/mail_handler/mail_handler.rb +++ b/lib/mail_handler/mail_handler.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- # Handles the parsing of email require 'tmpdir' @@ -133,7 +134,7 @@ module MailHandler begin zip_file = Zip::ZipFile.open(tempfile.path) text += get_attachment_text_from_zip_file(zip_file) - zip_file.close() + zip_file.close rescue $stderr.puts("Error processing zip file: #{$!.inspect}") end diff --git a/lib/memory_profiler.rb b/lib/memory_profiler.rb index 15e8457ee..fd056dc0d 100644 --- a/lib/memory_profiler.rb +++ b/lib/memory_profiler.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- # Taken from # http://scottstuff.net/blog/2006/08/17/memory-leak-profiling-with-rails diff --git a/lib/message_prominence.rb b/lib/message_prominence.rb index 8f54fcc95..8f8008784 100644 --- a/lib/message_prominence.rb +++ b/lib/message_prominence.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- module MessageProminence def has_prominence diff --git a/lib/no_constraint_disabling.rb b/lib/no_constraint_disabling.rb index 32a4a6bfe..539b491b3 100644 --- a/lib/no_constraint_disabling.rb +++ b/lib/no_constraint_disabling.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- # In order to work around the problem of the database use not having # the permission to disable referential integrity when loading fixtures, # we redefine disable_referential_integrity so that it doesn't try to diff --git a/lib/normalize_string.rb b/lib/normalize_string.rb index 3b6116970..69853fd6e 100644 --- a/lib/normalize_string.rb +++ b/lib/normalize_string.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- require 'iconv' unless String.method_defined?(:encode) require 'charlock_holmes' @@ -72,6 +73,29 @@ def convert_string_to_utf8_or_binary(s, suggested_character_encoding=nil) result end +class StringConversionResult < Struct.new(:string, :scrubbed) + alias_method :scrubbed?, :scrubbed +end + +def convert_string_to_utf8(s, suggested_character_encoding=nil) + begin + result = normalize_string_to_utf8 s, suggested_character_encoding + StringConversionResult.new(result, false) + rescue EncodingNormalizationError + result = scrub(s) + StringConversionResult.new(result, true) + end +end + +def scrub(string) + if String.method_defined?(:encode) + string = string.force_encoding("utf-8") + string.valid_encoding? ? string : string.encode("utf-16le", :invalid => :replace, :replace => "").encode("utf-8") + else + Iconv.conv('UTF-8//IGNORE', 'UTF-8', string) + end +end + def log_text_details(message, text) if String.method_defined?(:encode) STDERR.puts "#{message}, we have text: #{text}, of class #{text.class} and encoding #{text.encoding}" diff --git a/lib/public_body_csv.rb b/lib/public_body_csv.rb index afb5d9043..3dab7b805 100644 --- a/lib/public_body_csv.rb +++ b/lib/public_body_csv.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- require 'csv' # Public: Generate a CSV representation of PublicBody instances diff --git a/lib/quiet_opener.rb b/lib/quiet_opener.rb index c6e259b93..9745e22a4 100644 --- a/lib/quiet_opener.rb +++ b/lib/quiet_opener.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- require 'open-uri' require 'net-purge' if RUBY_VERSION.to_f < 2.0 diff --git a/lib/routing_filters.rb b/lib/routing_filters.rb index 5b5da6870..2aad6f437 100644 --- a/lib/routing_filters.rb +++ b/lib/routing_filters.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- module RoutingFilter class Conditionallyprependlocale < RoutingFilter::Locale # Override core Locale filter not to prepend locale path segment diff --git a/lib/ruby19.rb b/lib/ruby19.rb deleted file mode 100644 index 39f48d74e..000000000 --- a/lib/ruby19.rb +++ /dev/null @@ -1,8 +0,0 @@ -if RUBY_VERSION.to_f == 1.9 - class String - # @see syck/lib/syck/rubytypes.rb - def is_binary_data? - self.count("\x00-\x7F", "^ -~\t\r\n").fdiv(self.size) > 0.3 || self.index("\x00") unless self.empty? - end - end -end
\ No newline at end of file diff --git a/lib/strip_attributes/strip_attributes.rb b/lib/strip_attributes/strip_attributes.rb index 12350277d..2e4383194 100644 --- a/lib/strip_attributes/strip_attributes.rb +++ b/lib/strip_attributes/strip_attributes.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- module StripAttributes # Strips whitespace from model fields and leaves nil values as nil. # TODO: this differs from official StripAttributes, as it doesn't make blank cells null. diff --git a/lib/strip_attributes/test/strip_attributes_test.rb b/lib/strip_attributes/test/strip_attributes_test.rb index 8158dc664..d06ef0671 100644 --- a/lib/strip_attributes/test/strip_attributes_test.rb +++ b/lib/strip_attributes/test/strip_attributes_test.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- require "#{File.dirname(__FILE__)}/test_helper" module MockAttributes diff --git a/lib/strip_attributes/test/test_helper.rb b/lib/strip_attributes/test/test_helper.rb index 7d06c40db..6a4f6136a 100644 --- a/lib/strip_attributes/test/test_helper.rb +++ b/lib/strip_attributes/test/test_helper.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- require 'test/unit' require 'rubygems' require 'active_record' @@ -9,7 +10,7 @@ require "#{PLUGIN_ROOT}/init" class ActiveRecord::Base alias_method :save, :valid? - def self.columns() + def self.columns @columns ||= [] end diff --git a/lib/tasks/config_files.rake b/lib/tasks/config_files.rake index 1528d7324..d0dc8f594 100644 --- a/lib/tasks/config_files.rake +++ b/lib/tasks/config_files.rake @@ -11,7 +11,7 @@ namespace :config_files do var = $1.to_sym replacement = replacements[var] if replacement == nil - raise "Unhandled variable in .ugly file: $#{var}" + raise "Unhandled variable in example file: $#{var}" else replacements[var] end @@ -21,9 +21,25 @@ namespace :config_files do converted_lines end - desc 'Convert Debian .ugly init script in config to a form suitable for installing in /etc/init.d' + desc 'Convert wrapper example in config to a form suitable for running mail handling scripts with rbenv' + task :convert_wrapper => :environment do + example = 'rake config_files:convert_wrapper DEPLOY_USER=deploy SCRIPT_FILE=config/run-with-rbenv-path.example' + check_for_env_vars(['DEPLOY_USER', + 'SCRIPT_FILE'], example) + + replacements = { + :user => ENV['DEPLOY_USER'], + } + + # Generate the template for potential further processing + convert_ugly(ENV['SCRIPT_FILE'], replacements).each do |line| + puts line + end + end + + desc 'Convert Debian example init script in config to a form suitable for installing in /etc/init.d' task :convert_init_script => :environment do - example = 'rake config_files:convert_init_script DEPLOY_USER=deploy VHOST_DIR=/dir/above/alaveteli VCSPATH=alaveteli SITE=alaveteli SCRIPT_FILE=config/alert-tracks-debian.ugly' + example = 'rake config_files:convert_init_script DEPLOY_USER=deploy VHOST_DIR=/dir/above/alaveteli VCSPATH=alaveteli SITE=alaveteli SCRIPT_FILE=config/alert-tracks-debian.example' check_for_env_vars(['DEPLOY_USER', 'VHOST_DIR', 'SCRIPT_FILE'], example) @@ -37,7 +53,7 @@ namespace :config_files do } # Use the filename for the $daemon_name ugly variable - daemon_name = File.basename(ENV['SCRIPT_FILE'], '-debian.ugly') + daemon_name = File.basename(ENV['SCRIPT_FILE'], '-debian.example') replacements.update(:daemon_name => "#{ replacements[:site] }-#{ daemon_name }") # Generate the template for potential further processing @@ -57,7 +73,7 @@ namespace :config_files do end end - desc 'Convert Debian .ugly crontab file in config to a form suitable for installing in /etc/cron.d' + desc 'Convert Debian example crontab file in config to a form suitable for installing in /etc/cron.d' task :convert_crontab => :environment do example = 'rake config_files:convert_crontab DEPLOY_USER=deploy VHOST_DIR=/dir/above/alaveteli VCSPATH=alaveteli SITE=alaveteli CRONTAB=config/crontab-example MAILTO=cron-alaveteli@example.org' check_for_env_vars(['DEPLOY_USER', diff --git a/lib/tasks/temp.rake b/lib/tasks/temp.rake index 67fa10174..d5f7e8b22 100644 --- a/lib/tasks/temp.rake +++ b/lib/tasks/temp.rake @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- namespace :temp do @@ -37,4 +38,99 @@ namespace :temp do end + desc 'Look for and fix invalid UTF-8 text in various models. Should be run under ruby 1.9 or above' + task :fix_invalid_utf8 => :environment do + + dryrun = ENV['DRYRUN'] != '0' + if dryrun + $stderr.puts "This is a dryrun - nothing will be changed" + end + + + PublicBody.find_each do |public_body| + unless public_body.name.valid_encoding? + name = convert_string_to_utf8(public_body.name) + puts "Bad encoding in PublicBody name, id: #{public_body.id}, " \ + "old name: #{public_body.name.force_encoding('UTF-8')}, new name #{name}" + unless dryrun + public_body.name_will_change! + public_body.name = name + public_body.last_edit_editor = 'system' + public_body.last_edit_comment = 'Invalid utf-8 encoding fixed by temp:fix_invalid_utf8' + public_body.save! + end + end + + # Editing old versions of public bodies - we don't want to affect the timestamp + PublicBody::Version.record_timestamps = false + public_body.versions.each do |public_body_version| + unless public_body_version.name.valid_encoding? + name = convert_string_to_utf8(public_body_version.name).string + puts "Bad encoding in PublicBody::Version name, " \ + "id: #{public_body_version.id}, old name: #{public_body_version.name.force_encoding('UTF-8')}, " \ + "new name: #{name}" + unless dryrun + public_body_version.name_will_change! + public_body_version.name = name + public_body_version.save! + end + end + end + PublicBody::Version.record_timestamps = true + + end + + IncomingMessage.find_each do |incoming_message| + if (incoming_message.cached_attachment_text_clipped && + !incoming_message.cached_attachment_text_clipped.valid_encoding?) || + (incoming_message.cached_main_body_text_folded && + !incoming_message.cached_main_body_text_folded.valid_encoding?) || + (incoming_message.cached_main_body_text_unfolded && + !incoming_message.cached_main_body_text_unfolded.valid_encoding?) + puts "Bad encoding in IncomingMessage cached fields, :id #{incoming_message.id} " + unless dryrun + incoming_message.clear_in_database_caches! + end + end + end + + FoiAttachment.find_each do |foi_attachment| + unescaped_filename = CGI.unescape(foi_attachment.filename) + unless unescaped_filename.valid_encoding? + filename = convert_string_to_utf8(unescaped_filename).string + puts "Bad encoding in FoiAttachment filename, id: #{foi_attachment.id} " \ + "old filename #{unescaped_filename.force_encoding('UTF-8')}, new filename #{filename}" + unless dryrun + foi_attachment.filename = filename + foi_attachment.save! + end + end + end + + OutgoingMessage.find_each do |outgoing_message| + unless outgoing_message.raw_body.valid_encoding? + + raw_body = convert_string_to_utf8(outgoing_message.raw_body).string + puts "Bad encoding in OutgoingMessage raw_body, id: #{outgoing_message.id} " \ + "old raw_body: #{outgoing_message.raw_body.force_encoding('UTF-8')}, new raw_body: #{raw_body}" + unless dryrun + outgoing_message.body = raw_body + outgoing_message.save! + end + end + end + + User.find_each do |user| + unless user.name.valid_encoding? + name = convert_string_to_utf8(user.name).string + puts "Bad encoding in User name, id: #{user.id}, " \ + "old name: #{user.name.force_encoding('UTF-8')}, new name: #{name}" + unless dryrun + user.name = name + user.save! + end + end + end + + end end diff --git a/lib/tasks/usage.rb b/lib/tasks/usage.rb index d6aac454d..350a6b07e 100644 --- a/lib/tasks/usage.rb +++ b/lib/tasks/usage.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- module Usage def usage_message message @@ -23,4 +24,4 @@ module Usage end end -end
\ No newline at end of file +end diff --git a/lib/theme.rb b/lib/theme.rb index 4f03b5d99..f2d4ba8b3 100644 --- a/lib/theme.rb +++ b/lib/theme.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- def theme_url_to_theme_name(theme_url) File.basename theme_url, '.git' end diff --git a/lib/use_spans_for_errors.rb b/lib/use_spans_for_errors.rb index 135453f78..ecc117964 100644 --- a/lib/use_spans_for_errors.rb +++ b/lib/use_spans_for_errors.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- # Monkeypatch! Use SPAN instead of DIV. # # Rails core refuse to fix this properly, by making it an official option. diff --git a/lib/whatdotheyknow/strip_empty_sessions.rb b/lib/whatdotheyknow/strip_empty_sessions.rb index 6d175ca98..1e5078172 100644 --- a/lib/whatdotheyknow/strip_empty_sessions.rb +++ b/lib/whatdotheyknow/strip_empty_sessions.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- module WhatDoTheyKnow class StripEmptySessions diff --git a/lib/world_foi_websites.rb b/lib/world_foi_websites.rb index fc2395986..a1e705c82 100644 --- a/lib/world_foi_websites.rb +++ b/lib/world_foi_websites.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- # -*- coding: utf-8 -*- class WorldFOIWebsites def self.world_foi_websites @@ -77,7 +78,11 @@ class WorldFOIWebsites {:name => "Слободен пристап", :country_name => "Република Македонија", :country_iso_code => "MK", - :url => "http://www.slobodenpristap.mk/"} + :url => "http://www.slobodenpristap.mk/"}, + {:name => "Imamo pravo znati", + :country_name => "Republika Hrvatska", + :country_iso_code => "HR", + :url => "http://imamopravoznati.org/"} ] return world_foi_websites end diff --git a/lib/xapian_queries.rb b/lib/xapian_queries.rb index b3599740a..dbb17f592 100644 --- a/lib/xapian_queries.rb +++ b/lib/xapian_queries.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- module XapianQueries # These methods take some filter criteria expressed in a hash and convert them @@ -76,10 +77,10 @@ module XapianQueries end def make_query_from_params(params) - query = params[:query] || "" if query.nil? + query = params.fetch(:query) { '' } query += get_date_range_from_params(params) query += get_request_variety_from_params(params) query += get_status_from_params(params) - return query + query end end |