diff options
28 files changed, 1131 insertions, 212 deletions
diff --git a/.tx/config b/.tx/config index cc0cd4bbd..bc7529856 100644 --- a/.tx/config +++ b/.tx/config @@ -4,5 +4,5 @@ source_lang = en source_file = locale/en/app.po [main] -host = https://www.transifex.net +host = https://www.transifex.com @@ -33,6 +33,7 @@ gem 'rmagick', :require => 'RMagick' gem 'ruby-msg', '~> 1.5.0' gem "statistics2", "~> 0.54" gem 'syslog_protocol' +gem 'thin' gem 'vpim' gem 'will_paginate' # when 1.2.9 is released by the maintainer, we can stop using this fork: diff --git a/Gemfile.lock b/Gemfile.lock index 9accf0283..339aa358d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -292,6 +292,7 @@ DEPENDENCIES spork-rails statistics2 (~> 0.54) syslog_protocol + thin unicode unidecoder vpim diff --git a/app/controllers/public_body_controller.rb b/app/controllers/public_body_controller.rb index 494e19353..9c3e46ded 100644 --- a/app/controllers/public_body_controller.rb +++ b/app/controllers/public_body_controller.rb @@ -196,6 +196,50 @@ class PublicBodyController < ApplicationController :disposition =>'attachment', :encoding => 'utf8') end + + # This is a helper method to take data returned by the PublicBody + # model's statistics-generating methods, and converting them to + # simpler data structure that can be rendered by a Javascript + # graph library. (This could be a class method except that we need + # access to the URL helper public_body_path.) + def simplify_stats_for_graphs(data, + column, + percentages, + graph_properties) + # Copy the data, only taking known-to-be-safe keys: + result = Hash.new { |h, k| h[k] = [] } + result.update Hash[data.select do |key, value| + ['y_values', + 'y_max', + 'totals', + 'cis_below', + 'cis_above'].include? key + end] + + # Extract data about the public bodies for the x-axis, + # tooltips, and so on: + data['public_bodies'].each_with_index do |pb, i| + result['x_values'] << i + result['x_ticks'] << [i, pb.name] + result['tooltips'] << "#{pb.name} (#{result['totals'][i]})" + result['public_bodies'] << { + 'name' => pb.name, + 'url' => public_body_path(pb) + } + end + + # Set graph metadata properties, like the title, axis labels, etc. + graph_id = "#{column}-" + graph_id += graph_properties[:highest] ? 'highest' : 'lowest' + result.update({ + 'id' => graph_id, + 'x_axis' => _('Public Bodies'), + 'y_axis' => graph_properties[:y_axis], + 'errorbars' => percentages, + 'title' => graph_properties[:title] + }) + end + def statistics unless AlaveteliConfiguration::public_body_statistics_page raise ActiveRecord::RecordNotFound.new("Page not enabled") @@ -252,28 +296,10 @@ class PublicBodyController < ApplicationController end if data - # We just need the URL and name of each public body: - data['public_bodies'].map! { |pb| - {'name' => pb.name, 'url' => public_body_path(pb)} - } - - data_to_draw = Hash.new { |h, k| h[k] = [] } - data_to_draw.update({ - 'id' => "#{column}-#{highest ? 'highest' : 'lowest'}", - 'x_axis' => _('Public Bodies'), - 'y_axis' => graph_properties[:y_axis], - 'errorbars' => percentages, - 'title' => graph_properties[:title] - }) - - data_to_draw.update(data) - data['public_bodies'].each_with_index { |pb, i| - data_to_draw['x_values'].push i - data_to_draw['x_ticks'].push [i, pb['name']] - data_to_draw['tooltips'].push pb['name'] - } - - @graph_list.push data_to_draw + @graph_list.push simplify_stats_for_graphs(data, + column, + percentages, + graph_properties) end end end diff --git a/app/models/info_request.rb b/app/models/info_request.rb index eba620f53..9463a236e 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -1182,14 +1182,23 @@ public after_save :update_counter_cache after_destroy :update_counter_cache + # This method updates the count columns of the PublicBody that + # store the number of "not held", "to some extent successful" and + # "both visible and classified" requests when saving or destroying + # an InfoRequest associated with the body: def update_counter_cache PublicBody.skip_callback(:save, :after, :purge_in_cache) - self.public_body.info_requests_not_held_count = InfoRequest.where( - :public_body_id => self.public_body.id, - :described_state => 'not_held').count - self.public_body.info_requests_successful_count = InfoRequest.where( - :public_body_id => self.public_body.id, - :described_state => ['successful', 'partially_successful']).count + basic_params = { + :public_body_id => self.public_body_id, + :awaiting_description => false, + :prominence => 'normal' + } + [['info_requests_not_held_count', {:described_state => 'not_held'}], + ['info_requests_successful_count', {:described_state => ['successful', 'partially_successful']}], + ['info_requests_visible_classified_count', {}]].each do |column, extra_params| + params = basic_params.clone.update extra_params + self.public_body.send "#{column}=", InfoRequest.where(params).count + end self.public_body.without_revision do public_body.no_xapian_reindex = true public_body.save diff --git a/app/models/public_body.rb b/app/models/public_body.rb index 9e77eb181..fbe2956e3 100644 --- a/app/models/public_body.rb +++ b/app/models/public_body.rb @@ -3,26 +3,27 @@ # # Table name: public_bodies # -# id :integer not null, primary key -# name :text not null -# short_name :text not null -# request_email :text not null -# version :integer not null -# last_edit_editor :string(255) not null -# last_edit_comment :text not null -# created_at :datetime not null -# updated_at :datetime not null -# url_name :text not null -# home_page :text default(""), not null -# notes :text default(""), not null -# first_letter :string(255) not null -# publication_scheme :text default(""), not null -# api_key :string(255) not null -# info_requests_count :integer default(0), not null -# disclosure_log :text default(""), not null -# info_requests_successful_count :integer -# info_requests_not_held_count :integer -# info_requests_overdue_count :integer +# id :integer not null, primary key +# name :text not null +# short_name :text not null +# request_email :text not null +# version :integer not null +# last_edit_editor :string(255) not null +# last_edit_comment :text not null +# created_at :datetime not null +# updated_at :datetime not null +# url_name :text not null +# home_page :text default(""), not null +# notes :text default(""), not null +# first_letter :string(255) not null +# publication_scheme :text default(""), not null +# api_key :string(255) not null +# info_requests_count :integer default(0), not null +# disclosure_log :text default(""), not null +# info_requests_successful_count :integer +# info_requests_not_held_count :integer +# info_requests_overdue_count :integer +# info_requests_visible_classified_count :integer # require 'csv' @@ -193,6 +194,7 @@ class PublicBody < ActiveRecord::Base acts_as_versioned self.non_versioned_columns << 'created_at' << 'updated_at' << 'first_letter' << 'api_key' self.non_versioned_columns << 'info_requests_count' << 'info_requests_successful_count' + self.non_versioned_columns << 'info_requests_count' << 'info_requests_visible_classified_count' self.non_versioned_columns << 'info_requests_not_held_count' << 'info_requests_overdue' self.non_versioned_columns << 'info_requests_overdue_count' @@ -653,20 +655,30 @@ class PublicBody < ActiveRecord::Base end end + def self.where_clause_for_stats(minimum_requests, total_column) + # When producing statistics for public bodies, we want to + # exclude any that are tagged with 'test' - we use a + # sub-select to find the IDs of those public bodies. + test_tagged_query = "SELECT model_id FROM has_tag_string_tags" \ + " WHERE model = 'PublicBody' AND name = 'test'" + "#{total_column} >= #{minimum_requests} AND id NOT IN (#{test_tagged_query})" + end + # Return data for the 'n' public bodies with the highest (or # lowest) number of requests, but only returning data for those # with at least 'minimum_requests' requests. def self.get_request_totals(n, highest, minimum_requests) ordering = "info_requests_count" ordering += " DESC" if highest - where_clause = "info_requests_count >= #{minimum_requests}" + where_clause = where_clause_for_stats minimum_requests, 'info_requests_count' public_bodies = PublicBody.order(ordering).where(where_clause).limit(n) public_bodies.reverse! if highest y_values = public_bodies.map { |pb| pb.info_requests_count } return { 'public_bodies' => public_bodies, 'y_values' => y_values, - 'y_max' => y_values.max} + 'y_max' => y_values.max, + 'totals' => y_values} end # Return data for the 'n' public bodies with the highest (or @@ -675,11 +687,12 @@ class PublicBody < ActiveRecord::Base # percentage. This only returns data for those public bodies with # at least 'minimum_requests' requests. def self.get_request_percentages(column, n, highest, minimum_requests) - total_column = "info_requests_count" + total_column = "info_requests_visible_classified_count" ordering = "y_value" ordering += " DESC" if highest y_value_column = "(cast(#{column} as float) / #{total_column})" - where_clause = "#{total_column} >= #{minimum_requests} AND #{column} IS NOT NULL" + where_clause = where_clause_for_stats minimum_requests, total_column + where_clause += " AND #{column} IS NOT NULL" public_bodies = PublicBody.select("*, #{y_value_column} AS y_value").order(ordering).where(where_clause).limit(n) public_bodies.reverse! if highest y_values = public_bodies.map { |pb| pb.y_value.to_f } @@ -709,7 +722,8 @@ class PublicBody < ActiveRecord::Base 'y_values' => y_values, 'cis_below' => cis_below, 'cis_above' => cis_above, - 'y_max' => 100} + 'y_max' => 100, + 'totals' => original_totals} end private diff --git a/config/Vagrantfile b/config/Vagrantfile new file mode 100644 index 000000000..4253215fc --- /dev/null +++ b/config/Vagrantfile @@ -0,0 +1,23 @@ +# This Vagrantfile should be used with the --no-color option, e.g. +# vagrant --no-color up +# Then you should be able to visit the site at: +# http://alaveteli.10.10.10.30.xip.io + +Vagrant::Config.run do |config| + config.vm.box = "precise64" + config.vm.box_url = "http://files.vagrantup.com/precise64.box" + config.vm.network :hostonly, "10.10.10.30" + # The bundle install fails unless you have quite a large amount of + # memory; insist on 1.5GiB: + config.vm.customize ["modifyvm", :id, "--memory", 1536] + # Fetch and run the install script: + config.vm.provision :shell, :inline => "wget -O install-site.sh https://raw.github.com/mysociety/commonlib/master/bin/install-site.sh" + config.vm.provision :shell, :inline => "chmod a+rx install-site.sh" + # This is only needed before the install-script branch is merged to + # master: + config.vm.provision :shell, :inline => "sed -i -e 's/BRANCH=master/BRANCH=install-script/' install-site.sh" + config.vm.provision :shell, :inline => "./install-site.sh " \ + "alaveteli " \ + "alaveteli " \ + "alaveteli.10.10.10.30.xip.io" +end diff --git a/config/crontab-example b/config/crontab-example index d2c3f6bd9..366624998 100644 --- a/config/crontab-example +++ b/config/crontab-example @@ -8,33 +8,33 @@ PATH=/usr/local/bin:/usr/bin:/bin MAILTO=cron-!!(*= $site *)!!@mysociety.org # Every 5 minutes -*/5 * * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/change-xapian-database.lock "/data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/update-xapian-index verbose=true" >> /data/vhost/!!(*= $vhost *)!!/logs/update-xapian-index.log || echo "stalled?" +*/5 * * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/change-xapian-database.lock "!!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/update-xapian-index verbose=true" >> !!(*= $vhost_dir *)!!/logs/update-xapian-index.log || echo "stalled?" # Every 10 minutes 5,15,25,35,45,55 * * * * !!(*= $user *)!! /etc/init.d/foi-alert-tracks check 5,15,25,35,45,55 * * * * !!(*= $user *)!! /etc/init.d/foi-purge-varnish check # Once an hour -09 * * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/alert-comment-on-request.lock /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/alert-comment-on-request || echo "stalled?" +09 * * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/alert-comment-on-request.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-comment-on-request || echo "stalled?" # Only root can read the log files -31 * * * * root run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/load-mail-server-logs.lock /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/load-mail-server-logs || echo "stalled?" +31 * * * * root run-with-lockfile -n !!(*= $vhost_dir *)!!/load-mail-server-logs.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/load-mail-server-logs || echo "stalled?" # Once a day, early morning -23 4 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/delete-old-things.lock /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/delete-old-things || echo "stalled?" -0 6 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/alert-overdue-requests.lock /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/alert-overdue-requests || echo "stalled?" -0 7 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/alert-new-response-reminders.lock /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/alert-new-response-reminders || echo "stalled?" -0 8 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/alert-not-clarified-request.lock /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/alert-not-clarified-request || echo "stalled?" -2 4 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/check-recent-requests-sent.lock /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/check-recent-requests-sent || echo "stalled?" -45 3 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/stop-new-responses-on-old-requests.lock /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/stop-new-responses-on-old-requests || echo "stalled?" -55 4 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/update-public-body-stats.lock /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/update-public-body-stats || echo "stalled?" +23 4 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/delete-old-things.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/delete-old-things || echo "stalled?" +0 6 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/alert-overdue-requests.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-overdue-requests || echo "stalled?" +0 7 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/alert-new-response-reminders.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-new-response-reminders || echo "stalled?" +0 8 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/alert-not-clarified-request.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-not-clarified-request || echo "stalled?" +2 4 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/check-recent-requests-sent.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/check-recent-requests-sent || echo "stalled?" +45 3 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/stop-new-responses-on-old-requests.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/stop-new-responses-on-old-requests || echo "stalled?" +55 4 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/update-public-body-stats.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/update-public-body-stats || echo "stalled?" # Only root can restart apache -31 1 * * * root run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/change-xapian-database.lock "/data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/compact-xapian-database production" || echo "stalled?" +31 1 * * * root run-with-lockfile -n !!(*= $vhost_dir *)!!/change-xapian-database.lock "!!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/compact-xapian-database production" || echo "stalled?" # Once a day on all servers -43 2 * * * !!(*= $user *)!! /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/request-creation-graph -48 2 * * * !!(*= $user *)!! /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/user-use-graph +43 2 * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/request-creation-graph +48 2 * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/user-use-graph # Once a year :) 0 0 1 11 * !!(*= $user *)!! /bin/echo "A year has passed, please update the bank holidays for the Freedom of Information site, thank you." diff --git a/config/environments/development.rb b/config/environments/development.rb index 54ab2977f..a912dd5de 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -16,9 +16,14 @@ Alaveteli::Application.configure do # Don't care if the mailer can't send config.action_mailer.raise_delivery_errors = false config.action_mailer.perform_deliveries = true - # Use mailcatcher in development - config.action_mailer.delivery_method = :smtp # so is queued, rather than giving immediate errors - config.action_mailer.smtp_settings = { :address => "localhost", :port => 1025 } + + if AlaveteliConfiguration::use_mailcatcher_in_development + # Use mailcatcher in development + config.action_mailer.delivery_method = :smtp # so is queued, rather than giving immediate errors + config.action_mailer.smtp_settings = { :address => "localhost", :port => 1025 } + else + config.action_mailer.delivery_method = :sendmail + end # Writes useful log files to debug memory leaks, of the sort where have # unintentionally kept references to objects, especially strings. diff --git a/config/general.yml-example b/config/general.yml-example index 8e749d9d6..60eb5ae1c 100644 --- a/config/general.yml-example +++ b/config/general.yml-example @@ -205,3 +205,7 @@ MINIMUM_REQUESTS_FOR_STATISTICS: 50 # available locale, you can allow a fallback to the default locale for # listing of public bodies. PUBLIC_BODY_LIST_FALLBACK_TO_DEFAULT_LOCALE: false + +# If true, while in development mode, try to send mail by SMTP to port +# 1025 (the port the mailcatcher listens on by default): +USE_MAILCATCHER_IN_DEVELOPMENT: true diff --git a/config/nginx.conf.example b/config/nginx.conf.example new file mode 100644 index 000000000..56e720abb --- /dev/null +++ b/config/nginx.conf.example @@ -0,0 +1,29 @@ +upstream alaveteli { + server 127.0.0.1:3300; +} + +server { + listen 80; + root /var/www/alaveteli/alaveteli/public; + + location / { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Proto http; + proxy_redirect off; + try_files $uri @ruby; + } + + location /download { + internal; + alias /var/www/alaveteli/alaveteli/cache/zips/development/download; + } + + location @ruby { + proxy_pass http://alaveteli; + proxy_set_header Host $http_host; + proxy_set_header X-Sendfile-Type X-Accel-Redirect; + proxy_set_header X-Accel-Mapping /var/www/alaveteli/alaveteli/cache/zips/development/download=/download; + } +} diff --git a/config/packages.debian-squeeze b/config/packages.debian-squeeze new file mode 100644 index 000000000..6cdf2f9d6 --- /dev/null +++ b/config/packages.debian-squeeze @@ -0,0 +1,39 @@ +ruby1.8 +ruby +libruby1.8 +rdoc1.8 +irb1.8 +wv +poppler-utils +pdftk +ghostscript +catdoc +links +elinks +unrtf +xlhtml +xapian-tools +gnuplot-nox +php5-cli +sharutils +unzip +mutt +tnef +gettext +python-yaml +wkhtmltopdf-static +libmagic-dev +libmagickwand-dev +libpq-dev +libxml2-dev +libxslt-dev +uuid-dev +ruby1.8-dev +rubygems/squeeze-backports +rake +build-essential +sqlite3 +libsqlite3-dev +libicu-dev +postgresql +postgresql-client diff --git a/config/packages.ubuntu-precise b/config/packages.ubuntu-precise new file mode 100644 index 000000000..177d504e2 --- /dev/null +++ b/config/packages.ubuntu-precise @@ -0,0 +1,35 @@ +ruby1.9.1 +wv +poppler-utils +pdftk +ghostscript +catdoc +links +elinks +unrtf +xlhtml +xapian-tools +gnuplot-nox +sharutils +unzip +mutt +tnef +gettext +python-yaml +wkhtmltopdf-static +libmagic-dev +libmagickwand-dev +libpq-dev +libxml2-dev +libxslt1-dev +uuid-dev +ruby1.9.1-dev +rubygems +rake +build-essential +ruby-bundler +sqlite3 +libsqlite3-dev +libicu-dev +postgresql +postgresql-client diff --git a/config/purge-varnish-debian.ugly b/config/purge-varnish-debian.ugly index 04458ea78..3f11344f2 100644 --- a/config/purge-varnish-debian.ugly +++ b/config/purge-varnish-debian.ugly @@ -22,6 +22,8 @@ DUSER=!!(*= $user *)!! # RAILS_ENV=your_rails_env # export RAILS_ENV +type varnishadm > /dev/null 2>&1 || exit + trap "" 1 export PIDFILE LOGFILE diff --git a/config/sysvinit.example b/config/sysvinit.example new file mode 100755 index 000000000..443e7c3fb --- /dev/null +++ b/config/sysvinit.example @@ -0,0 +1,53 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: application-thin-alaveteli +# Required-Start: $local_fs $network +# Required-Stop: $local_fs $network +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Starts the Thin web server for the "Alaveteli" site +# Description: The Thin web server for the "Alaveteli" site +### END INIT INFO + +# This example sysvinit script is based on the helpful example here: +# http://richard.wallman.org.uk/2010/02/howto-deploy-a-catalyst-application-using-fastcgi-and-nginx/ + +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin +SITE_HOME=/var/www/alaveteli +NAME=alaveteli +DESC="Alaveteli app server" +USER=fms + +echo $DAEMON +test -f $DAEMON || exit 0 + +set -e + +start_daemon() { + su -l -c "cd $SITE_HOME/alaveteli && bundle exec thin -d -p 3300 -e development start" $USER +} + +stop_daemon() { + pkill -f thin -u $USER || true +} + +case "$1" in + start) + start_daemon + ;; + stop) + stop_daemon + ;; + reload|restart|force-reload) + stop_daemon + sleep 5 + start_daemon + ;; + *) + N=/etc/init.d/$NAME + echo "Usage: $N {start|stop|reload|restart|force-reload}" >&2 + exit 1 + ;; +esac + +exit 0 diff --git a/db/migrate/20131101155844_add_stats_denominator.rb b/db/migrate/20131101155844_add_stats_denominator.rb new file mode 100644 index 000000000..7df4c8200 --- /dev/null +++ b/db/migrate/20131101155844_add_stats_denominator.rb @@ -0,0 +1,15 @@ +class AddStatsDenominator < ActiveRecord::Migration + def up + add_column :public_bodies, :info_requests_visible_classified_count, :integer + PublicBody.connection.execute("UPDATE public_bodies + SET info_requests_visible_classified_count = + (SELECT COUNT(*) FROM info_requests + WHERE awaiting_description = FALSE AND + prominence = 'normal' AND + public_body_id = public_bodies.id);") + end + + def down + remove_column :public_bodies, :info_requests_visible_classified_count + end +end diff --git a/doc/INSTALL.md b/doc/INSTALL.md index e466edd55..f39789936 100644 --- a/doc/INSTALL.md +++ b/doc/INSTALL.md @@ -1,3 +1,123 @@ +# Installation Script and AMI + +The easiest options for installating Alaveteli for evaluation +are to use our install script or to use the AMI (Amazon Machine +Image) to create an instance on Amazon EC2. These options are +described below. If you would prefer to install the site +manually, please go to the Manual Installation section below. + +## Installing from an AMI (Amazon Machine Image) + +To help people try out Alaveteli, we have created an AMI (Amazon +Machine Image) with a basic installation of Alaveteli, which you +can use to create a running server on an Amazon EC2 instance. +This creates an instance that runs in development mode, so we +wouldn't recommend you use it for a production system without +changing the configuration. + +If you haven't used Amazon Web Services before, then you can get +a Micro instance which will be +[free for a year](http://aws.amazon.com/free/). You will find +that a micro instance isn't powerful enough for anything other +very basic testing of Alaveteli, however. + +The AMI can be found in the EU West (Ireland) region, with the +ID ami-0f24c678 and name “Basic Alaveteli installation +2013-10-31”. You can launch an instance based on that AMI with +[this link](https://console.aws.amazon.com/ec2/home?region=eu-west-1#launchAmi=ami-0f24c678). + +When you create an EC2 instance based on that AMI, make sure +that you choose Security Groups that allows at least inbound +HTTP, HTTPS, SSH and, if you want to test incoming mail as well, +SMTP. + +When your EC2 instance is launched, you will be able to log in +as the `ubuntu` user. This user can `sudo` freely to run +commands as root. However, the code is actually owned by (and +runs as) the `alaveteli` user. After creating the instance, you +may want to edit a configuration file to customize the site's +configuration. That configuration file is +`/var/www/alaveteli/alaveteli/config/general.yml`, which can be +edited with: + + ubuntu@ip-10-58-191-98:~$ sudo su - alaveteli + alaveteli@ip-10-58-191-98:~$ cd alaveteli + alaveteli@ip-10-58-191-98:~/alaveteli$ nano config/general.yml + +Then you should restart the Thin webserver with: + + alaveteli@ip-10-58-191-98:~/alaveteli$ logout + ubuntu@ip-10-58-191-98:~$ sudo /etc/init.d/alaveteli restart + +If you find the hostname of your EC2 instance from the AWS +console, you should then be able to see the site at +`http://your-ec2-hostname.eu-west-1.compute.amazonaws.com` + +If you have any problems or questions, please ask on the +[Alaveteli Google Group](https://groups.google.com/forum/#!forum/alaveteli-dev) +or [report an issue](https://github.com/mysociety/alaveteli/issues?state=open). + +## Installing with the Installation Script + +If you have a clean installation of Debian squeeze or Ubuntu +precise, you can use an install script in our commonlib +repository to set up a working instance of Alaveteli. This is +not suitable for production (it runs in development mode, for +example) but should set up a functional installation of the +site. + +**Warning: only use this script on a newly installed server – it +will make significant changes to your server’s setup, including +modifying your nginx setup, creating a user account, creating a +database, installing new packages etc.** + +To download the script, run the following command: + + curl -O https://raw.github.com/mysociety/commonlib/master/bin/install-site.sh + +If you run this script with `sh install-site.sh`, you'll see its +usage message: + + Usage: ./install-site.sh [--default] <SITE-NAME> <UNIX-USER> [HOST] + HOST is only optional if you are running this on an EC2 instance. + --default means to install as the default site for this server, + rather than a virtualhost for HOST. + +In this case `<SITE-NAME>` should be `alaveteli`. `<UNIX-USER>` +is the name of the Unix user that you want to own and run the +code. (This user will be created by the script.) + +The `HOST` parameter is a hostname for the server that will be +usable externally – a virtualhost for this name will be created +by the script, unless you specified the `--default` option. This +parameter is optional if you are on an EC2 instance, in which +case the hostname of that instance will be used. + +For example, if you wish to use a new user called `alaveteli` +and the hostname `alaveteli.127.0.0.1.xip.io`, creating a +virtualhost just for that hostname, you could download and run +the script with: + + sudo sh install-site.sh alaveteli alaveteli alaveteli.127.0.0.1.xip.io + +([xip.io](http://xip.io/) is a helpful domain for development.) + +Or, if you want to set this up as the default site on an EC2 +instance, you could download the script, make it executable and +then invoke it with: + + sudo ./install-site.sh --default alaveteli alaveteli + +When the script has finished, you should have a working copy of +the website, accessible via the hostname you supplied to the +script. + +If you have any problems or questions, please ask on the +[Alaveteli Google Group](https://groups.google.com/forum/#!forum/alaveteli-dev) +or [report an issue](https://github.com/mysociety/alaveteli/issues?state=open). + +# Manual Installation + These instructions assume Debian Squeeze (64-bit) or Ubuntu 12.04 LTS (precise). [Install instructions for OS X](https://github.com/mysociety/alaveteli/wiki/OS-X-Quickstart) are under development. Debian Squeeze is the best supported @@ -9,7 +129,7 @@ As an aid to evaluation, there is an [Amazon AMI](https://github.com/mysociety/alaveteli/wiki/Alaveteli-ec2-ami) with all these steps configured. It is *not* production-ready. -# Get Alaveteli +## Get Alaveteli To start with, you may need to install git, e.g. with `sudo apt-get install git-core` @@ -25,7 +145,7 @@ master branch (which always contains the latest stable release): git checkout master -# Package pinning +## Package pinning You need to configure [apt-pinning](http://wiki.debian.org/AptPreferences#Pinning-1) preferences in order to prevent packages being pulled from the debian wheezy distribution in preference to the stable distribution once you have added the wheezy repository as described below. @@ -41,7 +161,7 @@ In order to configure apt-pinning and to keep most packages coming from the Debi sudo cp /tmp/preferences /etc/apt/ rm /tmp/preferences -# Install system dependencies +## Install system dependencies These are packages that the software depends on: third-party software used to parse documents, host the site, etc. There are also packages @@ -70,7 +190,7 @@ Some of the files also have a version number listed in config/packages - check that you have appropriate versions installed. Some also list "|" and offer a choice of packages. -# Install Ruby dependencies +## Install Ruby dependencies To install Alaveteli's Ruby dependencies, we need to install bundler. In Debian, this is provided as a package (installed as part @@ -79,7 +199,7 @@ gem: sudo gem1.8 install bundler -# Install mySociety libraries +## Install mySociety libraries You will also want to install mySociety's common ruby libraries and the Rails code. Run: @@ -88,7 +208,7 @@ code. Run: to fetch the contents of the submodules. -## Packages customised by mySociety +### Packages customised by mySociety Debian users should add the mySociety debian archive to their `/etc/apt/sources.list` as described above. Doing this and following @@ -115,7 +235,7 @@ use the Debian package compiled by mySociety (see link in [issue 305](https://github.com/mysociety/alaveteli/issues/305)) -# Configure Database +## Configure Database There has been a little work done in trying to make the code work with other databases (e.g. SQLite), but the currently supported database is @@ -155,7 +275,7 @@ data that may not be valid UTF (for example, data originating from various broken email clients that's not 8-bit clean), it's safer to be able to store *anything*, than reject data at runtime. -# Configure email +## Configure email You will need to set up an email server (MTA) to send and receive emails. Full configuration for an MTA is beyond the scope of this @@ -167,13 +287,13 @@ so that you can see the mails in a browser - see http://mailcatcher.me/ for more details. Start mailcatcher by running `bundle exec mailcatcher` in your application directory. -## Minimal +### Minimal If you just want to get the tests to pass, you will at a minimum need to allow sending emails via a `sendmail` command (a requirement met, for example, with `sudo apt-get install exim4`). -## Detailed +### Detailed When an authority receives an email, the email's `reply-to` field is a magic address which is parsed and consumed by the Rails app. @@ -206,7 +326,7 @@ A well-configured installation of this code will separately have had Exim make a backup copy of the email in a separate mailbox, just in case. -# Set up configs +## Set up configs Copy `config/general.yml-example` to `config/general.yml` and edit to your taste. @@ -225,7 +345,7 @@ performance management system. By default, monitoring is switched off by the `agent_enabled: false` setting. See https://github.com/newrelic/rpm for instructions on switching on local and remote performance analysis. -# Deployment +## Deployment In the 'alaveteli' directory, run: @@ -253,7 +373,7 @@ Next we need to create the index for the search engine (Xapian): If this fails, the site should still mostly run, but it's a core component so you should really try to get this working. -# Run the Tests +## Run the Tests Make sure everything looks OK: @@ -265,7 +385,7 @@ workaround). You might be able to move on to the next step, depending on how serious they are, but ideally you should try to find out what's gone wrong. -## glibc bug workaround +### glibc bug workaround There's a [bug in glibc](http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=637239) @@ -276,7 +396,7 @@ it's not as of version `2.11.3-2`. Until it's fixed (e.g. `libc6 2.13-26` does work), you can get the tests to pass by setting `export LD_PRELOAD=/lib/libuuid.so.1`. -# Run the Server +## Run the Server Run the following to get the server running: @@ -288,7 +408,7 @@ localhost interface by adding ` --binding=127.0.0.1` The server should have told you the URL to access in your browser to see the site in action. -# Administrator privileges +## Administrator privileges The administrative interface is at the URL `/admin`. @@ -311,7 +431,7 @@ in the front end. It is possible completely to override the administrator authentication by setting `SKIP_ADMIN_AUTH` to `true` in `general.yml`. -# Cron jobs and init scripts +## Cron jobs and init scripts `config/crontab-example` contains the cronjobs run on WhatDoTheyKnow. It's in a strange templating format they use in mySociety. mySociety @@ -336,8 +456,20 @@ like `!!(*= $this *)!!`. The variables are: * `user`: the user that the software runs as * `site`: a string to identify your alaveteli instance -There is a dumb python script at `script/make-crontab` which you can -edit and run to do some basic substitution for you. +There is a rake task that will help to rewrite this file into +one that is useful to you, which can be invoked with: + + bundle exec rake config_files:convert_crontab \ + DEPLOY_USER=deploy \ + VHOST_DIR=/dir/above/alaveteli \ + VCSPATH=alaveteli \ + SITE=alaveteli \ + CRONTAB=config/crontab-example > crontab + +You should change the `DEPLOY_USER`, `VHOST_DIR`, `VCSPATH` and +`SITE` environment variables to match your server and +installation. You should also edit the resulting `crontab` file +to customize the `MAILTO` variable. One of the cron jobs refers to a script at `/etc/init.d/foi-alert-tracks`. This is an init script, a copy of @@ -360,7 +492,7 @@ discussion of where to find this program, and how you might replace it. This [one line script](https://gist.github.com/3741194) can install this program system-wide. -# Set up production web server +## Set up production web server It is not recommended to run the website using the default Rails web server. There are various recommendations here: @@ -409,7 +541,7 @@ Some [production server best practice notes](https://github.com/mysociety/alaveteli/wiki/Production-Server-Best-Practices) are evolving on the wiki. -# Upgrading Alaveteli +## Upgrading Alaveteli The developer team policy is that the master branch in git should always contain the latest stable release. Therefore, in production, @@ -437,7 +569,7 @@ You should always run the script `scripts/rails-post-deploy` after each deployment. This runs any database migrations for you, plus various other things that can be automated for deployment. -# Troubleshooting +## Troubleshooting * **Incoming emails aren't appearing in my Alaveteli install** diff --git a/lib/configuration.rb b/lib/configuration.rb index ab985c8bf..fba70f27c 100644 --- a/lib/configuration.rb +++ b/lib/configuration.rb @@ -69,6 +69,7 @@ module AlaveteliConfiguration :TWITTER_WIDGET_ID => false, :USE_DEFAULT_BROWSER_LANGUAGE => true, :USE_GHOSTSCRIPT_COMPRESSION => false, + :USE_MAILCATCHER_IN_DEVELOPMENT => true, :UTILITY_SEARCH_PATH => ["/usr/bin", "/usr/local/bin"], :VARNISH_HOST => '', :WORKING_OR_CALENDAR_DAYS => 'working', diff --git a/lib/tasks/config_files.rake b/lib/tasks/config_files.rake index d3843f3a4..d0e4001f0 100644 --- a/lib/tasks/config_files.rake +++ b/lib/tasks/config_files.rake @@ -11,11 +11,7 @@ namespace :config_files do var = $1.to_sym replacement = replacements[var] if replacement == nil - if ! (skip[var] == true) - raise "Unhandled variable in .ugly file: $#{var}" - else - match - end + raise "Unhandled variable in .ugly file: $#{var}" else replacements[var] end @@ -52,5 +48,23 @@ namespace :config_files do end end + desc 'Convert Debian .ugly crontab file in config to a form suitable for installing in /etc/cron.d' + task :convert_crontab => :environment do + example = 'rake config_files:convert_crontab DEPLOY_USER=deploy VHOST_DIR=/dir/above/alaveteli VCSPATH=alaveteli SITE=alaveteli CRONTAB=config/crontab-example' + check_for_env_vars(['DEPLOY_USER', + 'VHOST_DIR', + 'VCSPATH', + 'SITE', + 'CRONTAB'], example) + replacements = { + :user => ENV['DEPLOY_USER'], + :vhost_dir => ENV['VHOST_DIR'], + :vcspath => ENV['VCSPATH'], + :site => ENV['SITE'] + } + convert_ugly(ENV['CRONTAB'], replacements).each do |line| + puts line + end + end -end
\ No newline at end of file +end diff --git a/lib/tasks/stats.rake b/lib/tasks/stats.rake index eb36204c6..38eb15996 100644 --- a/lib/tasks/stats.rake +++ b/lib/tasks/stats.rake @@ -1,8 +1,14 @@ namespace :stats do - desc 'Produce transaction stats' + desc 'Produce monthly transaction stats for a period starting START_YEAR' task :show => :environment do - month_starts = (Date.new(2009, 1)..Date.new(2011, 8)).select { |d| d.day == 1 } + example = 'rake stats:show START_YEAR=2009 [START_MONTH=3 END_YEAR=2012 END_MONTH=10]' + check_for_env_vars(['START_YEAR'], example) + start_year = (ENV['START_YEAR']).to_i + start_month = (ENV['START_MONTH'] || 1).to_i + end_year = (ENV['END_YEAR'] || Time.now.year).to_i + end_month = (ENV['END_MONTH'] || Time.now.month).to_i + month_starts = (Date.new(start_year, start_month)..Date.new(end_year, end_month)).select { |d| d.day == 1 } headers = ['Period', 'Requests sent', 'Annotations added', @@ -103,7 +109,11 @@ namespace :stats do overdue_count = 0 very_overdue_count = 0 InfoRequest.find_each(:batch_size => 200, - :conditions => {:public_body_id => public_body.id}) do |ir| + :conditions => { + :public_body_id => public_body.id, + :awaiting_description => false, + :prominence => 'normal' + }) do |ir| case ir.calculate_status when 'waiting_response_very_overdue' very_overdue_count += 1 diff --git a/script/install-as-user b/script/install-as-user new file mode 100755 index 000000000..4fc341fc6 --- /dev/null +++ b/script/install-as-user @@ -0,0 +1,159 @@ +#!/bin/bash + +set -e +set -x + +if [ $# -ne 3 ] +then + cat >&2 <<EOUSAGE +Usage: $0 <UNIX-USER> <HOST> <INSTALLATION-DIRECTORY> +EOUSAGE + exit 1 +fi + +UNIX_USER="$1" +HOST="$2" +DIRECTORY="$3" +DB_NAME="alaveteli" + +# Check that the arguments we've been passed are sensible: + +IP_ADDRESS_FOR_HOST="$(dig +short $HOST)" + +if [ x = x"$IP_ADDRESS_FOR_HOST" ] +then + echo "The hostname $HOST didn't resolve to an IP address" + exit 1 +fi + +if ! id "$UNIX_USER" 2> /dev/null > /dev/null +then + echo "The user '$UNIX_USER' didn't exist." + exit 1 +fi + +if [ "$(whoami)" != "$UNIX_USER" ] +then + echo "This script should be run by the user '$UNIX_USER'." + exit 1 +fi + +REPOSITORY="$DIRECTORY/alaveteli" +LINK_DESTINATION="$HOME/alaveteli" + +ln -sfn "$REPOSITORY" $LINK_DESTINATION +cd "$REPOSITORY" + +BASHRC="$HOME/.bashrc" + +BASHRC_GEM_COMMENT="Set up local gem directory for Alaveteli" +BASHRC_START="# START $BASHRC_GEM_COMMENT" +BASHRC_END="# END $BASHRC_GEM_COMMENT" + +# Remove the old lines we added: +sed -ibackup "/$BASHRC_START/,/$BASHRC_END/d" "$BASHRC" + +# Create a temporary file, so we can prepend the lines we need. They +# need to be prepended since the Ubuntu skeleton .bashrc begins with +# '[ -z "$PS1" ] && return', skipping the rest of the .bashrc for +# non-interactive use, but we need the gem settings when invoking +# commands in the shell non-interactively. +TMP_BASHRC="$(mktemp "$BASHRC.XXXXXXX")" + +cat >>$TMP_BASHRC <<EOBRC +$BASHRC_START +export GEM_HOME="$HOME/gems" +mkdir -p "\$GEM_HOME" +export GEM_PATH= +export PATH="\$GEM_HOME/bin:\$PATH" +$BASHRC_END +EOBRC + +cat "$BASHRC" >> "$TMP_BASHRC" +mv "$TMP_BASHRC" "$BASHRC" + +source "$BASHRC" + +# Speed up the installation of gems: +echo 'gem: --no-ri --no-rdoc' > "$HOME/.gemrc" + +# Write sensible values into the config file: + +function random_alphanumerics() { + < /dev/urandom tr -dc A-Za-z0-9 | head -c$1 +} + +RANDOM_EMAIL_SECRET=$(random_alphanumerics 32) +RANDOM_EMERGENCY_PASSWORD=$(random_alphanumerics 10) +RANDOM_COOKIE_SECRET=$(random_alphanumerics 100) + +if ! [ -f config/general.yml ] +then + sed -r \ + -e "s,^( *DOMAIN:).*,\\1 '$HOST'," \ + -e "s,^( *FORCE_SSL:).*,\\1 false," \ + -e "s,^( *TIME_ZONE:).*,\\1 'Europe/London'," \ + -e "s,^( *BLOG_FEED:).*,\\1 null," \ + -e "s,^( *TWITTER_USERNAME:).*,\\1 null," \ + -e "s,^( *INCLUDE_DEFAULT_LOCALE_IN_URLS:).*,\\1 false," \ + -e "s,^( *INCOMING_EMAIL_DOMAIN:).*,\\1 '$HOST'," \ + -e "s,^( *INCOMING_EMAIL_PREFIX:).*,\\1 'foi+'," \ + -e "s,^( *INCOMING_EMAIL_SECRET:).*,\\1 '$RANDOM_EMAIL_SECRET'," \ + -e "s,^( *ADMIN_USERNAME:).*,\\1 'emergency'," \ + -e "s,^( *ADMIN_PASSWORD:).*,\\1 '$RANDOM_EMERGENCY_PASSWORD'," \ + -e "s,^( *CONTACT_EMAIL:).*,\\1 'postmaster@$HOST'," \ + -e "s,^( *TRACK_SENDER_EMAIL:).*,\\1 'postmaster@$HOST'," \ + -e "s,^( *COOKIE_STORE_SESSION_SECRET:).*,\\1 '$RANDOM_COOKIE_SECRET'," \ + -e "s,^( *FORWARD_NONBOUNCE_RESPONSES_TO:).*,\\1 'user-support@$HOST'," \ + -e "s,^( *HTML_TO_PDF_COMMAND:).*,\\1 '/usr/bin/wkhtmltopdf-static'," \ + -e "s,^( *EXCEPTION_NOTIFICATIONS_FROM:).*,\\1 'do-not-reply-to-this-address@$HOST'," \ + -e "/EXCEPTION_NOTIFICATIONS_TO:/,/^$/c EXCEPTION_NOTIFICATIONS_TO:\n - team@$HOST\n" \ + -e "s,^( *VARNISH_HOST:).*,\\1 null," \ + -e "s,^( *MTA_LOG_PATH:).*,\\1 '/var/log/mail/mail.log-*'," \ + -e "s,^( *MTA_LOG_TYPE:).*,\\1 'postfix'," \ + -e "s,^( *DONATION_URL:).*,\\1 null," \ + -e "s,^( *THEME_BRANCH:).*,\\1 'develop'," \ + -e "s,^( *USE_MAILCATCHER_IN_DEVELOPMENT:).*,\\1 false," \ + config/general.yml-example > config/general.yml +fi + +# add database.yml +sed -r \ + -e "s,^( *database: *)foi_(.*),\\1${DB_NAME}_\\2," \ + -e "s,^( *username: *).*,\\1${UNIX_USER}," \ + -e "s,^( *password: *).*,\\1null," \ + -e "s,^( *host: *).*,\\1/var/run/postgresql/," \ + -e "s,# constraint_disabling: false, constraint_disabling: false," \ + config/database.yml-example > config/database.yml + +for SUFFIX in production test development +do + REAL_DB_NAME="${DB_NAME}_$SUFFIX" + echo Creating the database $REAL_DB_NAME + # Create each database if it doesn't exist: + if ! psql -l | egrep "^ *$REAL_DB_NAME *\|" > /dev/null + then + createdb -T template0 --owner "$UNIX_USER" "$REAL_DB_NAME" + fi +done + +# Bundler isn't packaged on Debian squeeze, so we have to install it +# as a gem: + +which bundle || gem install bundler + +echo Running rails-post-deploy +script/rails-post-deploy + +LOADED_INDICATOR="$HOME/.alaveteli-sample-data-loaded" + +if [ ! -f "$LOADED_INDICATOR" ] +then + echo Running load-sample-data + bundle exec script/load-sample-data + + echo Running rebuild-xapian-index + script/rebuild-xapian-index + + touch "$LOADED_INDICATOR" +fi diff --git a/script/make-crontab b/script/make-crontab deleted file mode 100755 index d214f1485..000000000 --- a/script/make-crontab +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python -import re - -mailto = "recipient-of-any-errors@localhost" -user = "user-to-run-as" -location = "/path/to/alaveteli" - -template = open("config/crontab-example").read() -template = re.sub(r"MAILTO=.*", "MAILTO=%s" % mailto, template) -template = template.replace("!!(*= $user *)!!", user) -template = re.sub(r"/data/vhost/.*/script", location + "/script", template) - -print template - - - diff --git a/script/rails-post-deploy b/script/rails-post-deploy index de950311c..c09868347 100755 --- a/script/rails-post-deploy +++ b/script/rails-post-deploy @@ -24,27 +24,32 @@ fi # read config file in for later (STAGING_SITE) if [ -e "config/general" ] || [ -e "config/general.yml" ] then - . commonlib/shlib/deployfns - read_conf config/general + . commonlib/shlib/deployfns + read_conf config/general else - OPTION_DOMAIN=127.0.0.1:3000 - OPTION_STAGING_SITE=1 + OPTION_DOMAIN=127.0.0.1:3000 + OPTION_STAGING_SITE=1 fi # create initial log files if [ -e $TOP_DIR/../logs ] then - # mySociety servers have logs dir in level above - rm -f log - ln -s $TOP_DIR/../logs log + # mySociety servers have logs dir in level above + if ! [ -h log ] && [ -d log ] + then + # If log is a directory rather than a symlink, move that + # directory out of the way: + mv log log.original + fi + ln -sfn $TOP_DIR/../logs log else - # otherwise just make the directory - if [ -h log ] - then - # remove any old-style symlink first - rm -f log - fi - mkdir -p log + # otherwise just make the directory + if [ -h log ] + then + # remove any old-style symlink first + rm -f log + fi + mkdir -p log fi cd log @@ -55,18 +60,18 @@ cd .. if [ "$OPTION_STAGING_SITE" = "0" ] then cat <<-END - - ***************************************************************** - WARNING: About to make config/rails_env.rb which, via special - code in config/boot.rb, forces the Rails environment to be - "production". If this is a development system, please edit your - config/general.yml file and set the STAGING_SITE option to 1, - and also delete the generated config/rails_env.rb file. - Alternatively, you can override config/rails_env.rb at any time - with an environment variable. - ***************************************************************** - - END + + ***************************************************************** + WARNING: About to make config/rails_env.rb which, via special + code in config/boot.rb, forces the Rails environment to be + "production". If this is a development system, please edit your + config/general.yml file and set the STAGING_SITE option to 1, + and also delete the generated config/rails_env.rb file. + Alternatively, you can override config/rails_env.rb at any time + with an environment variable. + ***************************************************************** + +END echo "ENV['RAILS_ENV'] ||= 'production'" > config/rails_env.rb fi @@ -81,7 +86,7 @@ then fi if [ "$TRAVIS" = "true" ] then - bundle_install_options="--without development develop --deployment" + bundle_install_options="--without development develop --deployment" fi bundle install $bundle_install_options diff --git a/script/site-specific-install.sh b/script/site-specific-install.sh new file mode 100755 index 000000000..8917fd577 --- /dev/null +++ b/script/site-specific-install.sh @@ -0,0 +1,163 @@ +#!/bin/sh + +# Set IDEAL_VERSION to the commitish we want to check out; typically +# this is the version tag. Since this may not exist before release, +# fall back to the master branch: +VERSIONS="origin/install-script 0.15 origin/master" + +PARENT_SCRIPT_URL=https://github.com/mysociety/commonlib/blob/master/bin/install-site.sh + +misuse() { + echo The variable $1 was not defined, and it should be. + echo This script should not be run directly - instead, please run: + echo $PARENT_SCRIPT_URL + exit 1 +} + +# Strictly speaking we don't need to check all of these, but it might +# catch some errors made when changing install-site.sh + +[ -z "$DIRECTORY" ] && misuse DIRECTORY +[ -z "$UNIX_USER" ] && misuse UNIX_USER +[ -z "$REPOSITORY" ] && misuse REPOSITORY +[ -z "$REPOSITORY_URL" ] && misuse REPOSITORY_URL +[ -z "$BRANCH" ] && misuse BRANCH +[ -z "$SITE" ] && misuse SITE +[ -z "$DEFAULT_SERVER" ] && misuse DEFAULT_SERVER +[ -z "$HOST" ] && misuse HOST +[ -z "$DISTRIBUTION" ] && misuse DISTRIBUTION +[ -z "$VERSIONS" ] && misuse VERSIONS +[ -z "$DEVELOPMENT_INSTALL" ] && misuse DEVELOPMENT_INSTALL +[ -z "$BIN_DIRECTORY" ] && misuse BIN_DIRECTORY + +update_mysociety_apt_sources + +if [ ! "$DEVELOPMENT_INSTALL" = true ]; then + install_nginx + add_website_to_nginx + # Check out the first available requested version: + su -l -c "cd '$REPOSITORY' && (for v in $VERSIONS; do git checkout $v && break; done)" \ + "$UNIX_USER" +fi + +install_postfix + +# Now there's quite a bit of Postfix configuration that we need to +# make sure is present: + +ensure_line_present \ + "^ *alaveteli *unix *" \ + "alaveteli unix - n n - 50 pipe flags=R user=$UNIX_USER argv=$REPOSITORY/script/mailin" \ + /etc/postfix/master.cf 644 + +ensure_line_present \ + "^ *transport_maps *=" \ + "transport_maps = regexp:/etc/postfix/transports" \ + /etc/postfix/main.cf 644 + +ensure_line_present \ + "^ *local_recipient_maps *=" \ + "local_recipient_maps = proxy:unix:passwd.byname regexp:/etc/postfix/recipients" \ + /etc/postfix/main.cf 644 + +ensure_line_present \ + "^ *mydestination *=" \ + "mydestination = $HOST, $(hostname --fqdn), localhost.localdomain, localhost" \ + /etc/postfix/main.cf 644 + +ensure_line_present \ + "^do-not-reply" \ + "do-not-reply-to-this-address: :blackhole:" \ + /etc/aliases 644 + +ensure_line_present \ + "^mail" \ + "mail.* -/var/log/mail/mail.log" \ + /etc/rsyslog.d/50-default.conf 644 + +cat > /etc/postfix/transports <<EOF +/^foi.*/ alaveteli +EOF + +cat > /etc/postfix/recipients <<EOF +/^foi.*/ this-is-ignored +/^postmaster@/ this-is-ignored +/^user-support@/ this-is-ignored +/^team@/ this-is-ignored +EOF + +if ! egrep '^ */var/log/mail/mail.log *{' /etc/logrotate.d/rsyslog +then + cat >> /etc/logrotate.d/rsyslog <<EOF +/var/log/mail/mail.log { + rotate 30 + daily + dateext + missingok + notifempty + compress + delaycompress + sharedscripts + postrotate + reload rsyslog >/dev/null 2>&1 || true + endscript +} +EOF +fi + +/etc/init.d/rsyslog restart + +newaliases +postmap /etc/postfix/transports +postmap /etc/postfix/recipients +postfix reload + +# (end of the Postfix configuration) + +install_website_packages + +# Make the PostgreSQL user a superuser to avoid the irritating error: +# PG::Error: ERROR: permission denied: "RI_ConstraintTrigger_16564" is a system trigger +# This is only needed for loading the sample data, so the superuser +# permissions are dropped below. +add_postgresql_user --superuser + +export DEVELOPMENT_INSTALL +su -l -c "$BIN_DIRECTORY/install-as-user '$UNIX_USER' '$HOST' '$DIRECTORY'" "$UNIX_USER" + +# Now that the install-as-user script has loaded the sample data, we +# no longer need the PostgreSQL user to be a superuser: +echo "ALTER USER \"$UNIX_USER\" WITH NOSUPERUSER;" | su -l -c 'psql' postgres + +if [ ! "$DEVELOPMENT_INSTALL" = true ]; then + install_sysvinit_script +fi + +# Set up root's crontab: + +cd "$REPOSITORY" + +echo -n "Creating /etc/cron.d/alaveteli... " +(su -l -c "cd '$REPOSITORY' && bundle exec rake config_files:convert_crontab DEPLOY_USER='$UNIX_USER' VHOST_DIR='$DIRECTORY' VCSPATH='$SITE' SITE='$SITE' CRONTAB=config/crontab-example" "$UNIX_USER") > /etc/cron.d/alaveteli +# There are some other parts to rewrite, so just do them with sed: +sed -r \ + -e "/foi-purge-varnish/d" \ + -e "s,^(MAILTO=).*,\1root@$HOST," \ + -e "s,run-with-lockfile,$REPOSITORY/commonlib/bin/run-with-lockfile.sh,g" \ + -i /etc/cron.d/alaveteli +echo $DONE_MSG + +echo -n "Creating /etc/init.d/foi-alert-tracks... " +(su -l -c "cd '$REPOSITORY' && bundle exec rake config_files:convert_init_script DEPLOY_USER='$UNIX_USER' VHOST_DIR='$DIRECTORY' SCRIPT_FILE=config/alert-tracks-debian.ugly" "$UNIX_USER") > /etc/init.d/foi-alert-tracks +chmod a+rx /etc/init.d/foi-alert-tracks +echo $DONE_MSG + +if [ $DEFAULT_SERVER = true ] && [ x != x$EC2_HOSTNAME ] +then + # If we're setting up as the default on an EC2 instance, make sure + # that the /etc/rc.local is set up to run the install script again + # to update the hostname: + overwrite_rc_local +fi + +done_msg "Installation complete"; echo diff --git a/spec/controllers/public_body_controller_spec.rb b/spec/controllers/public_body_controller_spec.rb index 025ebfdba..6800765f2 100644 --- a/spec/controllers/public_body_controller_spec.rb +++ b/spec/controllers/public_body_controller_spec.rb @@ -275,39 +275,126 @@ end describe PublicBodyController, "when showing public body statistics" do - it "should render the right template with the right data" do - config = MySociety::Config.load_default() - config['MINIMUM_REQUESTS_FOR_STATISTICS'] = 1 - config['PUBLIC_BODY_STATISTICS_PAGE'] = true - get :statistics - response.should render_template('public_body/statistics') - # There are 5 different graphs we're creating at the moment. - assigns[:graph_list].length.should == 5 - # The first is the only one with raw values, the rest are - # percentages with error bars: - assigns[:graph_list].each_with_index do |graph, index| - if index == 0 - graph['errorbars'].should be_false - graph['x_values'].length.should == 4 - graph['x_values'].should == [0, 1, 2, 3] - graph['y_values'].should == [1, 2, 2, 4] - else - graph['errorbars'].should be_true - # Just check the first one: - if index == 1 - graph['x_values'].should == [0, 1, 2, 3] - graph['y_values'].should == [0, 50, 100, 100] + it "should render the right template with the right data" do + config = MySociety::Config.load_default() + config['MINIMUM_REQUESTS_FOR_STATISTICS'] = 1 + config['PUBLIC_BODY_STATISTICS_PAGE'] = true + get :statistics + response.should render_template('public_body/statistics') + # There are 5 different graphs we're creating at the moment. + assigns[:graph_list].length.should == 5 + # The first is the only one with raw values, the rest are + # percentages with error bars: + assigns[:graph_list].each_with_index do |graph, index| + if index == 0 + graph['errorbars'].should be_false + graph['x_values'].length.should == 4 + graph['x_values'].should == [0, 1, 2, 3] + graph['y_values'].should == [1, 2, 2, 4] + else + graph['errorbars'].should be_true + # Just check the first one: + if index == 1 + graph['x_values'].should == [0, 1, 2, 3] + graph['y_values'].should == [0, 50, 100, 100] + end + # Check that at least every confidence interval value is + # a Float (rather than NilClass, say): + graph['cis_below'].each { |v| v.should be_instance_of(Float) } + graph['cis_above'].each { |v| v.should be_instance_of(Float) } + end + end + end + +end + +describe PublicBodyController, "when converting data for graphing" do + + before(:each) do + @raw_count_data = PublicBody.get_request_totals(n=3, + highest=true, + minimum_requests=1) + @percentages_data = PublicBody.get_request_percentages( + column='info_requests_successful_count', + n=3, + highest=false, + minimum_requests=1) + end + + it "should not include the real public body model instance" do + to_draw = controller.simplify_stats_for_graphs(@raw_count_data, + column='blah_blah', + percentages=false, + {} ) + to_draw['public_bodies'][0].class.should == Hash + to_draw['public_bodies'][0].has_key?('request_email').should be_false + end + + it "should generate the expected id" do + to_draw = controller.simplify_stats_for_graphs(@raw_count_data, + column='blah_blah', + percentages=false, + {:highest => true} ) + to_draw['id'].should == "blah_blah-highest" + to_draw = controller.simplify_stats_for_graphs(@raw_count_data, + column='blah_blah', + percentages=false, + {:highest => false} ) + to_draw['id'].should == "blah_blah-lowest" + end + + it "should have exactly the expected keys" do + to_draw = controller.simplify_stats_for_graphs(@raw_count_data, + column='blah_blah', + percentages=false, + {} ) + to_draw.keys.sort.should == ["errorbars", "id", "public_bodies", + "title", "tooltips", "totals", + "x_axis", "x_ticks", "x_values", + "y_axis", "y_max", "y_values"] + + to_draw = controller.simplify_stats_for_graphs(@percentages_data, + column='whatever', + percentages=true, + {}) + to_draw.keys.sort.should == ["cis_above", "cis_below", + "errorbars", "id", "public_bodies", + "title", "tooltips", "totals", + "x_axis", "x_ticks", "x_values", + "y_axis", "y_max", "y_values"] + end + + it "should have values of the expected class and length" do + [controller.simplify_stats_for_graphs(@raw_count_data, + column='blah_blah', + percentages=false, + {}), + controller.simplify_stats_for_graphs(@percentages_data, + column='whatever', + percentages=true, + {})].each do |to_draw| + per_pb_keys = ["cis_above", "cis_below", "public_bodies", + "tooltips", "totals", "x_ticks", "x_values", + "y_values"] + # These should be all be arrays with one element per public body: + per_pb_keys.each do |key| + if to_draw.has_key? key + to_draw[key].class.should == Array + to_draw[key].length.should eq(3), "for key #{key}" + end + end + # Just check that the rest aren't of class Array: + to_draw.keys.each do |key| + unless per_pb_keys.include? key + to_draw[key].class.should_not eq(Array), "for key #{key}" + end + end end - # Check that at least every confidence interval value is - # a Float (rather than NilClass, say): - graph['cis_below'].each { |v| v.should be_instance_of(Float) } - graph['cis_above'].each { |v| v.should be_instance_of(Float) } - end end - end end + describe PublicBodyController, "when doing type ahead searches" do render_views diff --git a/spec/fixtures/public_bodies.yml b/spec/fixtures/public_bodies.yml index e382b3cf6..d0eb572b3 100644 --- a/spec/fixtures/public_bodies.yml +++ b/spec/fixtures/public_bodies.yml @@ -2,26 +2,27 @@ # # Table name: public_bodies # -# id :integer not null, primary key -# name :text not null -# short_name :text not null -# request_email :text not null -# version :integer not null -# last_edit_editor :string(255) not null -# last_edit_comment :text not null -# created_at :datetime not null -# updated_at :datetime not null -# url_name :text not null -# home_page :text default(""), not null -# notes :text default(""), not null -# first_letter :string(255) not null -# publication_scheme :text default(""), not null -# api_key :string(255) not null -# info_requests_count :integer default(0), not null -# disclosure_log :text default(""), not null -# info_requests_successful_count :integer -# info_requests_not_held_count :integer -# info_requests_overdue_count :integer +# id :integer not null, primary key +# name :text not null +# short_name :text not null +# request_email :text not null +# version :integer not null +# last_edit_editor :string(255) not null +# last_edit_comment :text not null +# created_at :datetime not null +# updated_at :datetime not null +# url_name :text not null +# home_page :text default(""), not null +# notes :text default(""), not null +# first_letter :string(255) not null +# publication_scheme :text default(""), not null +# api_key :string(255) not null +# info_requests_count :integer default(0), not null +# disclosure_log :text default(""), not null +# info_requests_successful_count :integer +# info_requests_not_held_count :integer +# info_requests_overdue_count :integer +# info_requests_visible_classified_count :integer # geraldine_public_body: @@ -38,6 +39,7 @@ geraldine_public_body: created_at: 2007-10-24 10:51:01.161639 api_key: 1 info_requests_count: 4 + info_requests_visible_classified_count: 4 info_requests_successful_count: 0 info_requests_not_held_count: 0 info_requests_overdue_count: 3 @@ -56,6 +58,7 @@ humpadink_public_body: notes: An albatross told me!!! api_key: 2 info_requests_count: 2 + info_requests_visible_classified_count: 2 info_requests_successful_count: 1 info_requests_not_held_count: 0 info_requests_overdue_count: 1 @@ -74,6 +77,7 @@ forlorn_public_body: notes: A very lonely public body that no one has corresponded with api_key: 3 info_requests_count: 0 + info_requests_visible_classified_count: 0 info_requests_successful_count: 0 info_requests_not_held_count: 0 info_requests_overdue_count: 0 @@ -92,6 +96,7 @@ silly_walks_public_body: notes: You know the one. api_key: 4 info_requests_count: 2 + info_requests_visible_classified_count: 2 info_requests_successful_count: 2 info_requests_not_held_count: 0 info_requests_overdue_count: 0 @@ -110,6 +115,7 @@ sensible_walks_public_body: created_at: 2008-10-25 10:51:01.161639 api_key: 5 info_requests_count: 1 + info_requests_visible_classified_count: 1 info_requests_successful_count: 1 info_requests_not_held_count: 0 info_requests_overdue_count: 0 @@ -128,6 +134,7 @@ other_public_body: created_at: 2008-10-25 10:51:01.161639 api_key: 6 info_requests_count: 0 + info_requests_visible_classified_count: 0 info_requests_successful_count: 0 info_requests_not_held_count: 0 info_requests_overdue_count: 0 @@ -146,6 +153,7 @@ accented_public_body: created_at: 2008-10-25 10:51:01.161639 api_key: 7 info_requests_count: 0 + info_requests_visible_classified_count: 0 info_requests_successful_count: 0 info_requests_not_held_count: 0 info_requests_overdue_count: 0 diff --git a/spec/models/public_body_spec.rb b/spec/models/public_body_spec.rb index 78fc6e516..23842ccff 100644 --- a/spec/models/public_body_spec.rb +++ b/spec/models/public_body_spec.rb @@ -3,26 +3,27 @@ # # Table name: public_bodies # -# id :integer not null, primary key -# name :text not null -# short_name :text not null -# request_email :text not null -# version :integer not null -# last_edit_editor :string(255) not null -# last_edit_comment :text not null -# created_at :datetime not null -# updated_at :datetime not null -# url_name :text not null -# home_page :text default(""), not null -# notes :text default(""), not null -# first_letter :string(255) not null -# publication_scheme :text default(""), not null -# api_key :string(255) not null -# info_requests_count :integer default(0), not null -# disclosure_log :text default(""), not null -# info_requests_successful_count :integer -# info_requests_not_held_count :integer -# info_requests_overdue_count :integer +# id :integer not null, primary key +# name :text not null +# short_name :text not null +# request_email :text not null +# version :integer not null +# last_edit_editor :string(255) not null +# last_edit_comment :text not null +# created_at :datetime not null +# updated_at :datetime not null +# url_name :text not null +# home_page :text default(""), not null +# notes :text default(""), not null +# first_letter :string(255) not null +# publication_scheme :text default(""), not null +# api_key :string(255) not null +# info_requests_count :integer default(0), not null +# disclosure_log :text default(""), not null +# info_requests_successful_count :integer +# info_requests_not_held_count :integer +# info_requests_overdue_count :integer +# info_requests_visible_classified_count :integer # require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') @@ -529,3 +530,77 @@ describe PublicBody, " when override all public body request emails set" do @geraldine.request_email.should == "catch_all_test_email@foo.com" end end + +describe PublicBody, "when calculating statistics" do + + it "should not include unclassified or hidden requests in percentages" do + with_hidden_and_successful_requests do + totals_data = PublicBody.get_request_totals(n=3, + highest=true, + minimum_requests=1) + # For the total number of requests, we still include + # hidden or unclassified requests: + totals_data['public_bodies'][-1].name.should == "Geraldine Quango" + totals_data['totals'][-1].should == 4 + + # However, for percentages, don't include the hidden or + # unclassified requests. So, for the Geraldine Quango + # we've made sure that there are only two visible and + # classified requests, one of which is successful, so the + # percentage should be 50%: + + percentages_data = PublicBody.get_request_percentages(column='info_requests_successful_count', + n=3, + highest=false, + minimum_requests=1) + geraldine_index = percentages_data['public_bodies'].index do |pb| + pb.name == "Geraldine Quango" + end + + percentages_data['y_values'][geraldine_index].should == 50 + end + end + + it "should only return totals for those with at least a minimum number of requests" do + minimum_requests = 1 + with_enough_info_requests = PublicBody.where(["info_requests_count >= ?", + minimum_requests]).length + all_data = PublicBody.get_request_totals 4, true, minimum_requests + all_data['public_bodies'].length.should == with_enough_info_requests + end + + it "should only return percentages for those with at least a minimum number of requests" do + with_hidden_and_successful_requests do + # With minimum requests at 3, this should return nil + # (corresponding to zero public bodies) since the only + # public body with just more than 3 info requests (The + # Geraldine Quango) has a hidden and an unclassified + # request within this block: + minimum_requests = 3 + with_enough_info_requests = PublicBody.where(["info_requests_visible_classified_count >= ?", + minimum_requests]).length + all_data = PublicBody.get_request_percentages(column='info_requests_successful_count', + n=10, + true, + minimum_requests) + all_data.should be_nil + end + end + + it "should only return those with at least a minimum number of requests, but not tagged 'test'" do + hpb = PublicBody.find_by_name 'Department for Humpadinking' + + original_tag_string = hpb.tag_string + hpb.add_tag_if_not_already_present 'test' + + begin + minimum_requests = 1 + with_enough_info_requests = PublicBody.where(["info_requests_count >= ?", minimum_requests]) + all_data = PublicBody.get_request_totals 4, true, minimum_requests + all_data['public_bodies'].length.should == 3 + ensure + hpb.tag_string = original_tag_string + end + end + +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 6e65018f1..9d16f6387 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -158,6 +158,31 @@ Spork.prefork do ActiveRecord::Base.default_timezone = old_zone end + # To test the statistics calculations, it's helpful to have the + # request fixtures in different states, but changing the fixtures + # themselves disrupts many other tests. This function takes a + # block, and runs that block with the info requests for the + # Geraldine Quango altered so that one is hidden and there's a + # successful one. + def with_hidden_and_successful_requests + external = info_requests(:external_request) + chicken = info_requests(:naughty_chicken_request) + old_external_prominence = external.prominence + old_chicken_described_state = chicken.described_state + begin + external.prominence = 'hidden' + external.save! + chicken.described_state = 'successful' + chicken.save! + yield + ensure + external.prominence = old_external_prominence + external.save! + chicken.described_state = old_chicken_described_state + chicken.save! + end + end + def load_test_categories PublicBodyCategories.add(:en, [ "Local and regional", |