diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/confidence_intervals.rb | 31 | ||||
| -rw-r--r-- | lib/configuration.rb | 3 | ||||
| -rw-r--r-- | lib/i18n_fixes.rb | 105 | ||||
| -rw-r--r-- | lib/mail_handler/mail_handler.rb | 2 | ||||
| -rw-r--r-- | lib/tasks/gettext.rake | 42 | ||||
| -rw-r--r-- | lib/tasks/stats.rake | 26 | ||||
| -rw-r--r-- | lib/tasks/submodules.rake | 28 | ||||
| -rw-r--r-- | lib/tasks/temp.rake | 55 |
8 files changed, 158 insertions, 134 deletions
diff --git a/lib/confidence_intervals.rb b/lib/confidence_intervals.rb new file mode 100644 index 000000000..9fe38045a --- /dev/null +++ b/lib/confidence_intervals.rb @@ -0,0 +1,31 @@ +# Calculate the confidence interval for a samples from a binonial +# distribution using Wilson's score interval. For more theoretical +# details, please see: +# +# http://en.wikipedia.org/wiki/Binomial_proportion_confidence_interval#Wilson%20score%20interval +# +# This is a variant of the function suggested here: +# +# http://www.evanmiller.org/how-not-to-sort-by-average-rating.html +# +# total: the total number of observations +# successes: the subset of those observations that were "successes" +# power: for a 95% confidence interval, this should be 0.05 +# +# The naive proportion is (successes / total). This returns an array +# with the proportions that represent the lower and higher confidence +# intervals around that. + +require 'statistics2' + +def ci_bounds(successes, total, power) + if total == 0 + raise RuntimeError, "Can't calculate the CI for 0 observations" + end + z = Statistics2.pnormaldist(1 - power/2) + phat = successes.to_f/total + offset = z*Math.sqrt((phat*(1 - phat) + z*z/(4*total))/total) + denominator = 1 + z*z/total + return [(phat + z*z/(2*total) - offset)/denominator, + (phat + z*z/(2*total) + offset)/denominator] +end diff --git a/lib/configuration.rb b/lib/configuration.rb index 03c4ac616..ab985c8bf 100644 --- a/lib/configuration.rb +++ b/lib/configuration.rb @@ -43,10 +43,13 @@ module AlaveteliConfiguration :INCOMING_EMAIL_PREFIX => '', :INCOMING_EMAIL_SECRET => 'dummysecret', :ISO_COUNTRY_CODE => 'GB', + :MINIMUM_REQUESTS_FOR_STATISTICS => 100, :MAX_REQUESTS_PER_USER_PER_DAY => '', :MTA_LOG_TYPE => 'exim', :NEW_RESPONSE_REMINDER_AFTER_DAYS => [3, 10, 24], :OVERRIDE_ALL_PUBLIC_BODY_REQUEST_EMAILS => '', + :PUBLIC_BODY_STATISTICS_PAGE => false, + :PUBLIC_BODY_LIST_FALLBACK_TO_DEFAULT_LOCALE => false, :RAW_EMAILS_LOCATION => 'files/raw_emails', :READ_ONLY => '', :RECAPTCHA_PRIVATE_KEY => 'x', diff --git a/lib/i18n_fixes.rb b/lib/i18n_fixes.rb index 9c1206215..9f0849e75 100644 --- a/lib/i18n_fixes.rb +++ b/lib/i18n_fixes.rb @@ -49,111 +49,6 @@ def gettext_interpolate(string, values) end -module I18n - # used by Globalize plugin. - # XXX much of this stuff should (might?) be in newer versions of Rails - @@fallbacks = nil - class << self - # Returns the current fallbacks implementation. Defaults to +I18n::Locale::Fallbacks+. - def fallbacks - @@fallbacks ||= I18n::Locale::Fallbacks.new - end - end - - module Locale - module Tag - class Simple - class << self - def tag(tag) - new(tag) - end - end - - attr_reader :tag - - def initialize(*tag) - @tag = tag.join('-').to_sym - end - - def subtags - @subtags = tag.to_s.split('-').map { |subtag| subtag.to_s } - end - - def to_sym - tag - end - - def to_s - tag.to_s - end - - def to_a - subtags - end - - def parent - @parent ||= begin - segs = to_a.compact - segs.length > 1 ? self.class.tag(*segs[0..(segs.length-2)].join('-')) : nil - end - end - - def self_and_parents - @self_and_parents ||= [self] + parents - end - - def parents - @parents ||= ([parent] + (parent ? parent.parents : [])).compact - end - - - end - end - class Fallbacks < Hash - def initialize(*mappings) - @map = {} - map(mappings.pop) if mappings.last.is_a?(Hash) - self.defaults = mappings.empty? ? [I18n.default_locale.to_sym] : mappings - end - - def defaults=(defaults) - @defaults = defaults.map { |default| compute(default, false) }.flatten - end - attr_reader :defaults - - def [](locale) - raise InvalidLocale.new(locale) if locale.nil? - locale = locale.to_sym - super || store(locale, compute(locale)) - end - - def map(mappings) - mappings.each do |from, to| - from, to = from.to_sym, Array(to) - to.each do |_to| - @map[from] ||= [] - @map[from] << _to.to_sym - end - end - end - - protected - - def compute(tags, include_defaults = true) - result = Array(tags).collect do |tag| - tags = I18n::Locale::Tag::Simple.tag(tag).self_and_parents.map! { |t| t.to_sym } - tags.each { |_tag| tags += compute(@map[_tag]) if @map[_tag] } - tags - end.flatten - result.push(*defaults) if include_defaults - result.uniq.compact - end - end - autoload :Fallbacks, 'i18n/locale/fallbacks' - end -end - - # this monkeypatch corrects inconsistency with gettext_i18n_rails # where the latter deals with strings but rails i18n deals with # symbols for locales diff --git a/lib/mail_handler/mail_handler.rb b/lib/mail_handler/mail_handler.rb index 9c955cccd..918f91180 100644 --- a/lib/mail_handler/mail_handler.rb +++ b/lib/mail_handler/mail_handler.rb @@ -81,7 +81,7 @@ module MailHandler default_params = { :append_to => text, :binary_output => false } if content_type == 'application/vnd.ms-word' AlaveteliExternalCommand.run("wvText", tempfile.path, tempfile.path + ".txt", - { :memory_limit => 536870912 } ) + { :memory_limit => 536870912, :timeout => 120 } ) # Try catdoc if we get into trouble (e.g. for InfoRequestEvent 2701) if not File.exists?(tempfile.path + ".txt") AlaveteliExternalCommand.run("catdoc", tempfile.path, default_params) diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index ace7205ae..366dfbe88 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -9,27 +9,31 @@ namespace :gettext do end end - desc "Update pot file only, without fuzzy guesses (these are done by Transifex)" - task :findpot => :environment do + desc "Update pot/po files for a theme." + task :find_theme => :environment do + theme = ENV['THEME'] + unless theme + puts "Usage: Specify an Alaveteli-theme with THEME=[theme directory name]" + exit(0) + end load_gettext - $LOAD_PATH << File.join(File.dirname(__FILE__),'..','..','lib') - require 'gettext_i18n_rails/haml_parser' - files = files_to_translate + msgmerge = Rails.application.config.gettext_i18n_rails.msgmerge + msgmerge ||= %w[--sort-output --no-location --no-wrap] + GetText.update_pofiles_org( + text_domain, + theme_files_to_translate(theme), + "version 0.0.1", + :po_root => theme_locale_path(theme), + :msgmerge => msgmerge + ) + end - #write found messages to tmp.pot - temp_pot = "tmp.pot" - GetText::rgettext(files, temp_pot) + def theme_files_to_translate(theme) + Dir.glob("{vendor/plugins/#{theme}/lib}/**/*.{rb,erb}") + end - #merge tmp.pot and existing pot - FileUtils.mkdir_p('locale') - GetText::msgmerge("locale/app.pot", temp_pot, "alaveteli", :po_root => 'locale', :msgmerge=>[ :no_wrap, :sort_output ]) - Dir.glob("locale/*/app.po") do |po_file| - GetText::msgmerge(po_file, temp_pot, "alaveteli", :po_root => 'locale', :msgmerge=>[ :no_wrap, :sort_output ]) - end - File.delete(temp_pot) - end + def theme_locale_path(theme) + File.join(Rails.root, "vendor", "plugins", theme, "locale-theme") + end - def files_to_translate - Dir.glob("{app,lib,config,locale}/**/*.{rb,erb,haml,rhtml}") - end end diff --git a/lib/tasks/stats.rake b/lib/tasks/stats.rake index 9d7d70540..4eda27289 100644 --- a/lib/tasks/stats.rake +++ b/lib/tasks/stats.rake @@ -91,4 +91,30 @@ namespace :stats do end end + desc 'Update statistics in the public_bodies table' + task :update_public_bodies_stats => :environment do + verbose = ENV['VERBOSE'] == '1' + PublicBody.all.each do |public_body| + puts "Counting overdue requests for #{public_body.name}" if verbose + + # Look for values of 'waiting_response_overdue' and + # 'waiting_response_very_overdue' which aren't directly in the + # described_state column, and instead need to be calculated: + overdue_count = 0 + very_overdue_count = 0 + InfoRequest.find_each(:conditions => {:public_body_id => public_body.id}) do |ir| + case ir.calculate_status + when 'waiting_response_very_overdue' + very_overdue_count += 1 + when 'waiting_response_overdue' + overdue_count += 1 + end + end + public_body.info_requests_overdue_count = overdue_count + very_overdue_count + public_body.no_xapian_reindex = true + public_body.without_revision do + public_body.save! + end + end + end end diff --git a/lib/tasks/submodules.rake b/lib/tasks/submodules.rake new file mode 100644 index 000000000..426192713 --- /dev/null +++ b/lib/tasks/submodules.rake @@ -0,0 +1,28 @@ + +namespace :submodules do + + desc "Check the status of the project's submodules" + task :check => :environment do + commit_info = `git submodule status commonlib` + case commit_info[0,1] + when '+' + $stderr.puts "Error: Currently checked out submodule commit for commonlib" + $stderr.puts "does not match the commit expected by this version of Alaveteli." + $stderr.puts "You can update it with 'git submodule update'." + exit(1) + when '-' + $stderr.puts "Error: Submodule commonlib needs to be initialized." + $stderr.puts "You can do this by running 'git submodule update --init'." + exit(1) + when 'U' + $stderr.puts "Error: Submodule commonlib has merge conflicts." + $stderr.puts "You'll need to resolve these to run Alaveteli." + exit(1) + when ' ' + exit(0) + else + raise "Unexpected status character in response to 'git submodule status commonlib': #{commit_info[0,1]}" + end + end + +end diff --git a/lib/tasks/temp.rake b/lib/tasks/temp.rake index fcabb23de..d371ad0dc 100644 --- a/lib/tasks/temp.rake +++ b/lib/tasks/temp.rake @@ -1,5 +1,36 @@ namespace :temp do + desc "Fix the history of requests where the described state doesn't match the latest status value + used by search, by adding an edit event that will correct the latest status" + task :fix_bad_request_states => :environment do + dryrun = ENV['DRYRUN'] != '0' + if dryrun + puts "This is a dryrun" + end + + InfoRequest.find_each() do |info_request| + next if info_request.url_title == 'holding_pen' + last_info_request_event = info_request.info_request_events[-1] + if last_info_request_event.latest_status != info_request.described_state + puts "#{info_request.id} #{info_request.url_title} #{last_info_request_event.latest_status} #{info_request.described_state}" + params = { :script => 'rake temp:fix_bad_request_states', + :user_id => nil, + :old_described_state => info_request.described_state, + :described_state => info_request.described_state + } + if ! dryrun + info_request.info_request_events.create!(:last_described_at => last_info_request_event.described_at + 1.second, + :event_type => 'status_update', + :described_state => info_request.described_state, + :calculated_state => info_request.described_state, + :params => params) + info_request.info_request_events.each{ |event| event.xapian_mark_needs_index } + end + end + + end + end + def disable_duplicate_account(user, count, dryrun) dupe_email = "duplicateemail#{count}@example.com" puts "Updating #{user.email} to #{dupe_email} for user #{user.id}" @@ -13,15 +44,21 @@ namespace :temp do total_messages = 0 messages_to_reparse = 0 IncomingMessage.find_each :include => :foi_attachments do |im| - reparse = im.foi_attachments.any? { |fa| ! File.exists? fa.filepath } - total_messages += 1 - messages_to_reparse += 1 if reparse - if total_messages % 1000 == 0 - puts "Considered #{total_messages} received emails." - end - unless dry_run - im.parse_raw_email! true if reparse - sleep 2 + begin + reparse = im.foi_attachments.any? { |fa| ! File.exists? fa.filepath } + total_messages += 1 + messages_to_reparse += 1 if reparse + if total_messages % 1000 == 0 + puts "Considered #{total_messages} received emails." + end + unless dry_run + im.parse_raw_email! true if reparse + sleep 2 + end + rescue StandardError => e + puts "There was a #{e.class} exception reparsing IncomingMessage with ID #{im.id}" + puts e.backtrace + puts e.message end end message = dry_run ? "Would reparse" : "Reparsed" |