diff options
author | Mark Longair <mhl@pobox.com> | 2013-11-07 15:49:15 +0000 |
---|---|---|
committer | Mark Longair <mhl@pobox.com> | 2013-11-07 15:49:15 +0000 |
commit | 99c35e70de774a4ad8dfeb3e72badb4a7e5f94aa (patch) | |
tree | fd57b28710e28d9f6305d808b79f3bced87a7d8b | |
parent | 9541299e871bfd82d9c48745cce721a919dbde73 (diff) | |
parent | 0bd2e863f0f824aaa14cdabaeca865bea3c0b898 (diff) |
Merge branch 'install-script' into rails-3-develop
-rw-r--r-- | Gemfile | 1 | ||||
-rw-r--r-- | Gemfile.lock | 1 | ||||
-rw-r--r-- | config/Vagrantfile | 23 | ||||
-rw-r--r-- | config/crontab-example | 26 | ||||
-rw-r--r-- | config/environments/development.rb | 11 | ||||
-rw-r--r-- | config/general.yml-example | 4 | ||||
-rw-r--r-- | config/nginx.conf.example | 29 | ||||
-rw-r--r-- | config/packages.debian-squeeze | 39 | ||||
-rw-r--r-- | config/packages.ubuntu-precise | 35 | ||||
-rw-r--r-- | config/purge-varnish-debian.ugly | 2 | ||||
-rwxr-xr-x | config/sysvinit.example | 53 | ||||
-rw-r--r-- | doc/INSTALL.md | 176 | ||||
-rw-r--r-- | lib/configuration.rb | 1 | ||||
-rw-r--r-- | lib/tasks/config_files.rake | 26 | ||||
-rwxr-xr-x | script/install-as-user | 159 | ||||
-rwxr-xr-x | script/make-crontab | 16 | ||||
-rwxr-xr-x | script/rails-post-deploy | 59 | ||||
-rwxr-xr-x | script/site-specific-install.sh | 163 |
18 files changed, 737 insertions, 87 deletions
@@ -33,6 +33,7 @@ gem 'rmagick', :require => 'RMagick' gem 'ruby-msg', '~> 1.5.0' gem "statistics2", "~> 0.54" gem 'syslog_protocol' +gem 'thin' gem 'vpim' gem 'will_paginate' # when 1.2.9 is released by the maintainer, we can stop using this fork: diff --git a/Gemfile.lock b/Gemfile.lock index 9accf0283..339aa358d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -292,6 +292,7 @@ DEPENDENCIES spork-rails statistics2 (~> 0.54) syslog_protocol + thin unicode unidecoder vpim diff --git a/config/Vagrantfile b/config/Vagrantfile new file mode 100644 index 000000000..4253215fc --- /dev/null +++ b/config/Vagrantfile @@ -0,0 +1,23 @@ +# This Vagrantfile should be used with the --no-color option, e.g. +# vagrant --no-color up +# Then you should be able to visit the site at: +# http://alaveteli.10.10.10.30.xip.io + +Vagrant::Config.run do |config| + config.vm.box = "precise64" + config.vm.box_url = "http://files.vagrantup.com/precise64.box" + config.vm.network :hostonly, "10.10.10.30" + # The bundle install fails unless you have quite a large amount of + # memory; insist on 1.5GiB: + config.vm.customize ["modifyvm", :id, "--memory", 1536] + # Fetch and run the install script: + config.vm.provision :shell, :inline => "wget -O install-site.sh https://raw.github.com/mysociety/commonlib/master/bin/install-site.sh" + config.vm.provision :shell, :inline => "chmod a+rx install-site.sh" + # This is only needed before the install-script branch is merged to + # master: + config.vm.provision :shell, :inline => "sed -i -e 's/BRANCH=master/BRANCH=install-script/' install-site.sh" + config.vm.provision :shell, :inline => "./install-site.sh " \ + "alaveteli " \ + "alaveteli " \ + "alaveteli.10.10.10.30.xip.io" +end diff --git a/config/crontab-example b/config/crontab-example index d2c3f6bd9..366624998 100644 --- a/config/crontab-example +++ b/config/crontab-example @@ -8,33 +8,33 @@ PATH=/usr/local/bin:/usr/bin:/bin MAILTO=cron-!!(*= $site *)!!@mysociety.org # Every 5 minutes -*/5 * * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/change-xapian-database.lock "/data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/update-xapian-index verbose=true" >> /data/vhost/!!(*= $vhost *)!!/logs/update-xapian-index.log || echo "stalled?" +*/5 * * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/change-xapian-database.lock "!!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/update-xapian-index verbose=true" >> !!(*= $vhost_dir *)!!/logs/update-xapian-index.log || echo "stalled?" # Every 10 minutes 5,15,25,35,45,55 * * * * !!(*= $user *)!! /etc/init.d/foi-alert-tracks check 5,15,25,35,45,55 * * * * !!(*= $user *)!! /etc/init.d/foi-purge-varnish check # Once an hour -09 * * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/alert-comment-on-request.lock /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/alert-comment-on-request || echo "stalled?" +09 * * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/alert-comment-on-request.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-comment-on-request || echo "stalled?" # Only root can read the log files -31 * * * * root run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/load-mail-server-logs.lock /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/load-mail-server-logs || echo "stalled?" +31 * * * * root run-with-lockfile -n !!(*= $vhost_dir *)!!/load-mail-server-logs.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/load-mail-server-logs || echo "stalled?" # Once a day, early morning -23 4 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/delete-old-things.lock /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/delete-old-things || echo "stalled?" -0 6 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/alert-overdue-requests.lock /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/alert-overdue-requests || echo "stalled?" -0 7 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/alert-new-response-reminders.lock /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/alert-new-response-reminders || echo "stalled?" -0 8 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/alert-not-clarified-request.lock /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/alert-not-clarified-request || echo "stalled?" -2 4 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/check-recent-requests-sent.lock /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/check-recent-requests-sent || echo "stalled?" -45 3 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/stop-new-responses-on-old-requests.lock /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/stop-new-responses-on-old-requests || echo "stalled?" -55 4 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/update-public-body-stats.lock /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/update-public-body-stats || echo "stalled?" +23 4 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/delete-old-things.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/delete-old-things || echo "stalled?" +0 6 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/alert-overdue-requests.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-overdue-requests || echo "stalled?" +0 7 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/alert-new-response-reminders.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-new-response-reminders || echo "stalled?" +0 8 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/alert-not-clarified-request.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-not-clarified-request || echo "stalled?" +2 4 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/check-recent-requests-sent.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/check-recent-requests-sent || echo "stalled?" +45 3 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/stop-new-responses-on-old-requests.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/stop-new-responses-on-old-requests || echo "stalled?" +55 4 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/update-public-body-stats.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/update-public-body-stats || echo "stalled?" # Only root can restart apache -31 1 * * * root run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/change-xapian-database.lock "/data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/compact-xapian-database production" || echo "stalled?" +31 1 * * * root run-with-lockfile -n !!(*= $vhost_dir *)!!/change-xapian-database.lock "!!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/compact-xapian-database production" || echo "stalled?" # Once a day on all servers -43 2 * * * !!(*= $user *)!! /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/request-creation-graph -48 2 * * * !!(*= $user *)!! /data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!/script/user-use-graph +43 2 * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/request-creation-graph +48 2 * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/user-use-graph # Once a year :) 0 0 1 11 * !!(*= $user *)!! /bin/echo "A year has passed, please update the bank holidays for the Freedom of Information site, thank you." diff --git a/config/environments/development.rb b/config/environments/development.rb index 54ab2977f..a912dd5de 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -16,9 +16,14 @@ Alaveteli::Application.configure do # Don't care if the mailer can't send config.action_mailer.raise_delivery_errors = false config.action_mailer.perform_deliveries = true - # Use mailcatcher in development - config.action_mailer.delivery_method = :smtp # so is queued, rather than giving immediate errors - config.action_mailer.smtp_settings = { :address => "localhost", :port => 1025 } + + if AlaveteliConfiguration::use_mailcatcher_in_development + # Use mailcatcher in development + config.action_mailer.delivery_method = :smtp # so is queued, rather than giving immediate errors + config.action_mailer.smtp_settings = { :address => "localhost", :port => 1025 } + else + config.action_mailer.delivery_method = :sendmail + end # Writes useful log files to debug memory leaks, of the sort where have # unintentionally kept references to objects, especially strings. diff --git a/config/general.yml-example b/config/general.yml-example index 8e749d9d6..60eb5ae1c 100644 --- a/config/general.yml-example +++ b/config/general.yml-example @@ -205,3 +205,7 @@ MINIMUM_REQUESTS_FOR_STATISTICS: 50 # available locale, you can allow a fallback to the default locale for # listing of public bodies. PUBLIC_BODY_LIST_FALLBACK_TO_DEFAULT_LOCALE: false + +# If true, while in development mode, try to send mail by SMTP to port +# 1025 (the port the mailcatcher listens on by default): +USE_MAILCATCHER_IN_DEVELOPMENT: true diff --git a/config/nginx.conf.example b/config/nginx.conf.example new file mode 100644 index 000000000..56e720abb --- /dev/null +++ b/config/nginx.conf.example @@ -0,0 +1,29 @@ +upstream alaveteli { + server 127.0.0.1:3300; +} + +server { + listen 80; + root /var/www/alaveteli/alaveteli/public; + + location / { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Proto http; + proxy_redirect off; + try_files $uri @ruby; + } + + location /download { + internal; + alias /var/www/alaveteli/alaveteli/cache/zips/development/download; + } + + location @ruby { + proxy_pass http://alaveteli; + proxy_set_header Host $http_host; + proxy_set_header X-Sendfile-Type X-Accel-Redirect; + proxy_set_header X-Accel-Mapping /var/www/alaveteli/alaveteli/cache/zips/development/download=/download; + } +} diff --git a/config/packages.debian-squeeze b/config/packages.debian-squeeze new file mode 100644 index 000000000..6cdf2f9d6 --- /dev/null +++ b/config/packages.debian-squeeze @@ -0,0 +1,39 @@ +ruby1.8 +ruby +libruby1.8 +rdoc1.8 +irb1.8 +wv +poppler-utils +pdftk +ghostscript +catdoc +links +elinks +unrtf +xlhtml +xapian-tools +gnuplot-nox +php5-cli +sharutils +unzip +mutt +tnef +gettext +python-yaml +wkhtmltopdf-static +libmagic-dev +libmagickwand-dev +libpq-dev +libxml2-dev +libxslt-dev +uuid-dev +ruby1.8-dev +rubygems/squeeze-backports +rake +build-essential +sqlite3 +libsqlite3-dev +libicu-dev +postgresql +postgresql-client diff --git a/config/packages.ubuntu-precise b/config/packages.ubuntu-precise new file mode 100644 index 000000000..177d504e2 --- /dev/null +++ b/config/packages.ubuntu-precise @@ -0,0 +1,35 @@ +ruby1.9.1 +wv +poppler-utils +pdftk +ghostscript +catdoc +links +elinks +unrtf +xlhtml +xapian-tools +gnuplot-nox +sharutils +unzip +mutt +tnef +gettext +python-yaml +wkhtmltopdf-static +libmagic-dev +libmagickwand-dev +libpq-dev +libxml2-dev +libxslt1-dev +uuid-dev +ruby1.9.1-dev +rubygems +rake +build-essential +ruby-bundler +sqlite3 +libsqlite3-dev +libicu-dev +postgresql +postgresql-client diff --git a/config/purge-varnish-debian.ugly b/config/purge-varnish-debian.ugly index 04458ea78..3f11344f2 100644 --- a/config/purge-varnish-debian.ugly +++ b/config/purge-varnish-debian.ugly @@ -22,6 +22,8 @@ DUSER=!!(*= $user *)!! # RAILS_ENV=your_rails_env # export RAILS_ENV +type varnishadm > /dev/null 2>&1 || exit + trap "" 1 export PIDFILE LOGFILE diff --git a/config/sysvinit.example b/config/sysvinit.example new file mode 100755 index 000000000..443e7c3fb --- /dev/null +++ b/config/sysvinit.example @@ -0,0 +1,53 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: application-thin-alaveteli +# Required-Start: $local_fs $network +# Required-Stop: $local_fs $network +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Starts the Thin web server for the "Alaveteli" site +# Description: The Thin web server for the "Alaveteli" site +### END INIT INFO + +# This example sysvinit script is based on the helpful example here: +# http://richard.wallman.org.uk/2010/02/howto-deploy-a-catalyst-application-using-fastcgi-and-nginx/ + +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin +SITE_HOME=/var/www/alaveteli +NAME=alaveteli +DESC="Alaveteli app server" +USER=fms + +echo $DAEMON +test -f $DAEMON || exit 0 + +set -e + +start_daemon() { + su -l -c "cd $SITE_HOME/alaveteli && bundle exec thin -d -p 3300 -e development start" $USER +} + +stop_daemon() { + pkill -f thin -u $USER || true +} + +case "$1" in + start) + start_daemon + ;; + stop) + stop_daemon + ;; + reload|restart|force-reload) + stop_daemon + sleep 5 + start_daemon + ;; + *) + N=/etc/init.d/$NAME + echo "Usage: $N {start|stop|reload|restart|force-reload}" >&2 + exit 1 + ;; +esac + +exit 0 diff --git a/doc/INSTALL.md b/doc/INSTALL.md index e466edd55..f39789936 100644 --- a/doc/INSTALL.md +++ b/doc/INSTALL.md @@ -1,3 +1,123 @@ +# Installation Script and AMI + +The easiest options for installating Alaveteli for evaluation +are to use our install script or to use the AMI (Amazon Machine +Image) to create an instance on Amazon EC2. These options are +described below. If you would prefer to install the site +manually, please go to the Manual Installation section below. + +## Installing from an AMI (Amazon Machine Image) + +To help people try out Alaveteli, we have created an AMI (Amazon +Machine Image) with a basic installation of Alaveteli, which you +can use to create a running server on an Amazon EC2 instance. +This creates an instance that runs in development mode, so we +wouldn't recommend you use it for a production system without +changing the configuration. + +If you haven't used Amazon Web Services before, then you can get +a Micro instance which will be +[free for a year](http://aws.amazon.com/free/). You will find +that a micro instance isn't powerful enough for anything other +very basic testing of Alaveteli, however. + +The AMI can be found in the EU West (Ireland) region, with the +ID ami-0f24c678 and name “Basic Alaveteli installation +2013-10-31”. You can launch an instance based on that AMI with +[this link](https://console.aws.amazon.com/ec2/home?region=eu-west-1#launchAmi=ami-0f24c678). + +When you create an EC2 instance based on that AMI, make sure +that you choose Security Groups that allows at least inbound +HTTP, HTTPS, SSH and, if you want to test incoming mail as well, +SMTP. + +When your EC2 instance is launched, you will be able to log in +as the `ubuntu` user. This user can `sudo` freely to run +commands as root. However, the code is actually owned by (and +runs as) the `alaveteli` user. After creating the instance, you +may want to edit a configuration file to customize the site's +configuration. That configuration file is +`/var/www/alaveteli/alaveteli/config/general.yml`, which can be +edited with: + + ubuntu@ip-10-58-191-98:~$ sudo su - alaveteli + alaveteli@ip-10-58-191-98:~$ cd alaveteli + alaveteli@ip-10-58-191-98:~/alaveteli$ nano config/general.yml + +Then you should restart the Thin webserver with: + + alaveteli@ip-10-58-191-98:~/alaveteli$ logout + ubuntu@ip-10-58-191-98:~$ sudo /etc/init.d/alaveteli restart + +If you find the hostname of your EC2 instance from the AWS +console, you should then be able to see the site at +`http://your-ec2-hostname.eu-west-1.compute.amazonaws.com` + +If you have any problems or questions, please ask on the +[Alaveteli Google Group](https://groups.google.com/forum/#!forum/alaveteli-dev) +or [report an issue](https://github.com/mysociety/alaveteli/issues?state=open). + +## Installing with the Installation Script + +If you have a clean installation of Debian squeeze or Ubuntu +precise, you can use an install script in our commonlib +repository to set up a working instance of Alaveteli. This is +not suitable for production (it runs in development mode, for +example) but should set up a functional installation of the +site. + +**Warning: only use this script on a newly installed server – it +will make significant changes to your server’s setup, including +modifying your nginx setup, creating a user account, creating a +database, installing new packages etc.** + +To download the script, run the following command: + + curl -O https://raw.github.com/mysociety/commonlib/master/bin/install-site.sh + +If you run this script with `sh install-site.sh`, you'll see its +usage message: + + Usage: ./install-site.sh [--default] <SITE-NAME> <UNIX-USER> [HOST] + HOST is only optional if you are running this on an EC2 instance. + --default means to install as the default site for this server, + rather than a virtualhost for HOST. + +In this case `<SITE-NAME>` should be `alaveteli`. `<UNIX-USER>` +is the name of the Unix user that you want to own and run the +code. (This user will be created by the script.) + +The `HOST` parameter is a hostname for the server that will be +usable externally – a virtualhost for this name will be created +by the script, unless you specified the `--default` option. This +parameter is optional if you are on an EC2 instance, in which +case the hostname of that instance will be used. + +For example, if you wish to use a new user called `alaveteli` +and the hostname `alaveteli.127.0.0.1.xip.io`, creating a +virtualhost just for that hostname, you could download and run +the script with: + + sudo sh install-site.sh alaveteli alaveteli alaveteli.127.0.0.1.xip.io + +([xip.io](http://xip.io/) is a helpful domain for development.) + +Or, if you want to set this up as the default site on an EC2 +instance, you could download the script, make it executable and +then invoke it with: + + sudo ./install-site.sh --default alaveteli alaveteli + +When the script has finished, you should have a working copy of +the website, accessible via the hostname you supplied to the +script. + +If you have any problems or questions, please ask on the +[Alaveteli Google Group](https://groups.google.com/forum/#!forum/alaveteli-dev) +or [report an issue](https://github.com/mysociety/alaveteli/issues?state=open). + +# Manual Installation + These instructions assume Debian Squeeze (64-bit) or Ubuntu 12.04 LTS (precise). [Install instructions for OS X](https://github.com/mysociety/alaveteli/wiki/OS-X-Quickstart) are under development. Debian Squeeze is the best supported @@ -9,7 +129,7 @@ As an aid to evaluation, there is an [Amazon AMI](https://github.com/mysociety/alaveteli/wiki/Alaveteli-ec2-ami) with all these steps configured. It is *not* production-ready. -# Get Alaveteli +## Get Alaveteli To start with, you may need to install git, e.g. with `sudo apt-get install git-core` @@ -25,7 +145,7 @@ master branch (which always contains the latest stable release): git checkout master -# Package pinning +## Package pinning You need to configure [apt-pinning](http://wiki.debian.org/AptPreferences#Pinning-1) preferences in order to prevent packages being pulled from the debian wheezy distribution in preference to the stable distribution once you have added the wheezy repository as described below. @@ -41,7 +161,7 @@ In order to configure apt-pinning and to keep most packages coming from the Debi sudo cp /tmp/preferences /etc/apt/ rm /tmp/preferences -# Install system dependencies +## Install system dependencies These are packages that the software depends on: third-party software used to parse documents, host the site, etc. There are also packages @@ -70,7 +190,7 @@ Some of the files also have a version number listed in config/packages - check that you have appropriate versions installed. Some also list "|" and offer a choice of packages. -# Install Ruby dependencies +## Install Ruby dependencies To install Alaveteli's Ruby dependencies, we need to install bundler. In Debian, this is provided as a package (installed as part @@ -79,7 +199,7 @@ gem: sudo gem1.8 install bundler -# Install mySociety libraries +## Install mySociety libraries You will also want to install mySociety's common ruby libraries and the Rails code. Run: @@ -88,7 +208,7 @@ code. Run: to fetch the contents of the submodules. -## Packages customised by mySociety +### Packages customised by mySociety Debian users should add the mySociety debian archive to their `/etc/apt/sources.list` as described above. Doing this and following @@ -115,7 +235,7 @@ use the Debian package compiled by mySociety (see link in [issue 305](https://github.com/mysociety/alaveteli/issues/305)) -# Configure Database +## Configure Database There has been a little work done in trying to make the code work with other databases (e.g. SQLite), but the currently supported database is @@ -155,7 +275,7 @@ data that may not be valid UTF (for example, data originating from various broken email clients that's not 8-bit clean), it's safer to be able to store *anything*, than reject data at runtime. -# Configure email +## Configure email You will need to set up an email server (MTA) to send and receive emails. Full configuration for an MTA is beyond the scope of this @@ -167,13 +287,13 @@ so that you can see the mails in a browser - see http://mailcatcher.me/ for more details. Start mailcatcher by running `bundle exec mailcatcher` in your application directory. -## Minimal +### Minimal If you just want to get the tests to pass, you will at a minimum need to allow sending emails via a `sendmail` command (a requirement met, for example, with `sudo apt-get install exim4`). -## Detailed +### Detailed When an authority receives an email, the email's `reply-to` field is a magic address which is parsed and consumed by the Rails app. @@ -206,7 +326,7 @@ A well-configured installation of this code will separately have had Exim make a backup copy of the email in a separate mailbox, just in case. -# Set up configs +## Set up configs Copy `config/general.yml-example` to `config/general.yml` and edit to your taste. @@ -225,7 +345,7 @@ performance management system. By default, monitoring is switched off by the `agent_enabled: false` setting. See https://github.com/newrelic/rpm for instructions on switching on local and remote performance analysis. -# Deployment +## Deployment In the 'alaveteli' directory, run: @@ -253,7 +373,7 @@ Next we need to create the index for the search engine (Xapian): If this fails, the site should still mostly run, but it's a core component so you should really try to get this working. -# Run the Tests +## Run the Tests Make sure everything looks OK: @@ -265,7 +385,7 @@ workaround). You might be able to move on to the next step, depending on how serious they are, but ideally you should try to find out what's gone wrong. -## glibc bug workaround +### glibc bug workaround There's a [bug in glibc](http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=637239) @@ -276,7 +396,7 @@ it's not as of version `2.11.3-2`. Until it's fixed (e.g. `libc6 2.13-26` does work), you can get the tests to pass by setting `export LD_PRELOAD=/lib/libuuid.so.1`. -# Run the Server +## Run the Server Run the following to get the server running: @@ -288,7 +408,7 @@ localhost interface by adding ` --binding=127.0.0.1` The server should have told you the URL to access in your browser to see the site in action. -# Administrator privileges +## Administrator privileges The administrative interface is at the URL `/admin`. @@ -311,7 +431,7 @@ in the front end. It is possible completely to override the administrator authentication by setting `SKIP_ADMIN_AUTH` to `true` in `general.yml`. -# Cron jobs and init scripts +## Cron jobs and init scripts `config/crontab-example` contains the cronjobs run on WhatDoTheyKnow. It's in a strange templating format they use in mySociety. mySociety @@ -336,8 +456,20 @@ like `!!(*= $this *)!!`. The variables are: * `user`: the user that the software runs as * `site`: a string to identify your alaveteli instance -There is a dumb python script at `script/make-crontab` which you can -edit and run to do some basic substitution for you. +There is a rake task that will help to rewrite this file into +one that is useful to you, which can be invoked with: + + bundle exec rake config_files:convert_crontab \ + DEPLOY_USER=deploy \ + VHOST_DIR=/dir/above/alaveteli \ + VCSPATH=alaveteli \ + SITE=alaveteli \ + CRONTAB=config/crontab-example > crontab + +You should change the `DEPLOY_USER`, `VHOST_DIR`, `VCSPATH` and +`SITE` environment variables to match your server and +installation. You should also edit the resulting `crontab` file +to customize the `MAILTO` variable. One of the cron jobs refers to a script at `/etc/init.d/foi-alert-tracks`. This is an init script, a copy of @@ -360,7 +492,7 @@ discussion of where to find this program, and how you might replace it. This [one line script](https://gist.github.com/3741194) can install this program system-wide. -# Set up production web server +## Set up production web server It is not recommended to run the website using the default Rails web server. There are various recommendations here: @@ -409,7 +541,7 @@ Some [production server best practice notes](https://github.com/mysociety/alaveteli/wiki/Production-Server-Best-Practices) are evolving on the wiki. -# Upgrading Alaveteli +## Upgrading Alaveteli The developer team policy is that the master branch in git should always contain the latest stable release. Therefore, in production, @@ -437,7 +569,7 @@ You should always run the script `scripts/rails-post-deploy` after each deployment. This runs any database migrations for you, plus various other things that can be automated for deployment. -# Troubleshooting +## Troubleshooting * **Incoming emails aren't appearing in my Alaveteli install** diff --git a/lib/configuration.rb b/lib/configuration.rb index ab985c8bf..fba70f27c 100644 --- a/lib/configuration.rb +++ b/lib/configuration.rb @@ -69,6 +69,7 @@ module AlaveteliConfiguration :TWITTER_WIDGET_ID => false, :USE_DEFAULT_BROWSER_LANGUAGE => true, :USE_GHOSTSCRIPT_COMPRESSION => false, + :USE_MAILCATCHER_IN_DEVELOPMENT => true, :UTILITY_SEARCH_PATH => ["/usr/bin", "/usr/local/bin"], :VARNISH_HOST => '', :WORKING_OR_CALENDAR_DAYS => 'working', diff --git a/lib/tasks/config_files.rake b/lib/tasks/config_files.rake index d3843f3a4..d0e4001f0 100644 --- a/lib/tasks/config_files.rake +++ b/lib/tasks/config_files.rake @@ -11,11 +11,7 @@ namespace :config_files do var = $1.to_sym replacement = replacements[var] if replacement == nil - if ! (skip[var] == true) - raise "Unhandled variable in .ugly file: $#{var}" - else - match - end + raise "Unhandled variable in .ugly file: $#{var}" else replacements[var] end @@ -52,5 +48,23 @@ namespace :config_files do end end + desc 'Convert Debian .ugly crontab file in config to a form suitable for installing in /etc/cron.d' + task :convert_crontab => :environment do + example = 'rake config_files:convert_crontab DEPLOY_USER=deploy VHOST_DIR=/dir/above/alaveteli VCSPATH=alaveteli SITE=alaveteli CRONTAB=config/crontab-example' + check_for_env_vars(['DEPLOY_USER', + 'VHOST_DIR', + 'VCSPATH', + 'SITE', + 'CRONTAB'], example) + replacements = { + :user => ENV['DEPLOY_USER'], + :vhost_dir => ENV['VHOST_DIR'], + :vcspath => ENV['VCSPATH'], + :site => ENV['SITE'] + } + convert_ugly(ENV['CRONTAB'], replacements).each do |line| + puts line + end + end -end
\ No newline at end of file +end diff --git a/script/install-as-user b/script/install-as-user new file mode 100755 index 000000000..4fc341fc6 --- /dev/null +++ b/script/install-as-user @@ -0,0 +1,159 @@ +#!/bin/bash + +set -e +set -x + +if [ $# -ne 3 ] +then + cat >&2 <<EOUSAGE +Usage: $0 <UNIX-USER> <HOST> <INSTALLATION-DIRECTORY> +EOUSAGE + exit 1 +fi + +UNIX_USER="$1" +HOST="$2" +DIRECTORY="$3" +DB_NAME="alaveteli" + +# Check that the arguments we've been passed are sensible: + +IP_ADDRESS_FOR_HOST="$(dig +short $HOST)" + +if [ x = x"$IP_ADDRESS_FOR_HOST" ] +then + echo "The hostname $HOST didn't resolve to an IP address" + exit 1 +fi + +if ! id "$UNIX_USER" 2> /dev/null > /dev/null +then + echo "The user '$UNIX_USER' didn't exist." + exit 1 +fi + +if [ "$(whoami)" != "$UNIX_USER" ] +then + echo "This script should be run by the user '$UNIX_USER'." + exit 1 +fi + +REPOSITORY="$DIRECTORY/alaveteli" +LINK_DESTINATION="$HOME/alaveteli" + +ln -sfn "$REPOSITORY" $LINK_DESTINATION +cd "$REPOSITORY" + +BASHRC="$HOME/.bashrc" + +BASHRC_GEM_COMMENT="Set up local gem directory for Alaveteli" +BASHRC_START="# START $BASHRC_GEM_COMMENT" +BASHRC_END="# END $BASHRC_GEM_COMMENT" + +# Remove the old lines we added: +sed -ibackup "/$BASHRC_START/,/$BASHRC_END/d" "$BASHRC" + +# Create a temporary file, so we can prepend the lines we need. They +# need to be prepended since the Ubuntu skeleton .bashrc begins with +# '[ -z "$PS1" ] && return', skipping the rest of the .bashrc for +# non-interactive use, but we need the gem settings when invoking +# commands in the shell non-interactively. +TMP_BASHRC="$(mktemp "$BASHRC.XXXXXXX")" + +cat >>$TMP_BASHRC <<EOBRC +$BASHRC_START +export GEM_HOME="$HOME/gems" +mkdir -p "\$GEM_HOME" +export GEM_PATH= +export PATH="\$GEM_HOME/bin:\$PATH" +$BASHRC_END +EOBRC + +cat "$BASHRC" >> "$TMP_BASHRC" +mv "$TMP_BASHRC" "$BASHRC" + +source "$BASHRC" + +# Speed up the installation of gems: +echo 'gem: --no-ri --no-rdoc' > "$HOME/.gemrc" + +# Write sensible values into the config file: + +function random_alphanumerics() { + < /dev/urandom tr -dc A-Za-z0-9 | head -c$1 +} + +RANDOM_EMAIL_SECRET=$(random_alphanumerics 32) +RANDOM_EMERGENCY_PASSWORD=$(random_alphanumerics 10) +RANDOM_COOKIE_SECRET=$(random_alphanumerics 100) + +if ! [ -f config/general.yml ] +then + sed -r \ + -e "s,^( *DOMAIN:).*,\\1 '$HOST'," \ + -e "s,^( *FORCE_SSL:).*,\\1 false," \ + -e "s,^( *TIME_ZONE:).*,\\1 'Europe/London'," \ + -e "s,^( *BLOG_FEED:).*,\\1 null," \ + -e "s,^( *TWITTER_USERNAME:).*,\\1 null," \ + -e "s,^( *INCLUDE_DEFAULT_LOCALE_IN_URLS:).*,\\1 false," \ + -e "s,^( *INCOMING_EMAIL_DOMAIN:).*,\\1 '$HOST'," \ + -e "s,^( *INCOMING_EMAIL_PREFIX:).*,\\1 'foi+'," \ + -e "s,^( *INCOMING_EMAIL_SECRET:).*,\\1 '$RANDOM_EMAIL_SECRET'," \ + -e "s,^( *ADMIN_USERNAME:).*,\\1 'emergency'," \ + -e "s,^( *ADMIN_PASSWORD:).*,\\1 '$RANDOM_EMERGENCY_PASSWORD'," \ + -e "s,^( *CONTACT_EMAIL:).*,\\1 'postmaster@$HOST'," \ + -e "s,^( *TRACK_SENDER_EMAIL:).*,\\1 'postmaster@$HOST'," \ + -e "s,^( *COOKIE_STORE_SESSION_SECRET:).*,\\1 '$RANDOM_COOKIE_SECRET'," \ + -e "s,^( *FORWARD_NONBOUNCE_RESPONSES_TO:).*,\\1 'user-support@$HOST'," \ + -e "s,^( *HTML_TO_PDF_COMMAND:).*,\\1 '/usr/bin/wkhtmltopdf-static'," \ + -e "s,^( *EXCEPTION_NOTIFICATIONS_FROM:).*,\\1 'do-not-reply-to-this-address@$HOST'," \ + -e "/EXCEPTION_NOTIFICATIONS_TO:/,/^$/c EXCEPTION_NOTIFICATIONS_TO:\n - team@$HOST\n" \ + -e "s,^( *VARNISH_HOST:).*,\\1 null," \ + -e "s,^( *MTA_LOG_PATH:).*,\\1 '/var/log/mail/mail.log-*'," \ + -e "s,^( *MTA_LOG_TYPE:).*,\\1 'postfix'," \ + -e "s,^( *DONATION_URL:).*,\\1 null," \ + -e "s,^( *THEME_BRANCH:).*,\\1 'develop'," \ + -e "s,^( *USE_MAILCATCHER_IN_DEVELOPMENT:).*,\\1 false," \ + config/general.yml-example > config/general.yml +fi + +# add database.yml +sed -r \ + -e "s,^( *database: *)foi_(.*),\\1${DB_NAME}_\\2," \ + -e "s,^( *username: *).*,\\1${UNIX_USER}," \ + -e "s,^( *password: *).*,\\1null," \ + -e "s,^( *host: *).*,\\1/var/run/postgresql/," \ + -e "s,# constraint_disabling: false, constraint_disabling: false," \ + config/database.yml-example > config/database.yml + +for SUFFIX in production test development +do + REAL_DB_NAME="${DB_NAME}_$SUFFIX" + echo Creating the database $REAL_DB_NAME + # Create each database if it doesn't exist: + if ! psql -l | egrep "^ *$REAL_DB_NAME *\|" > /dev/null + then + createdb -T template0 --owner "$UNIX_USER" "$REAL_DB_NAME" + fi +done + +# Bundler isn't packaged on Debian squeeze, so we have to install it +# as a gem: + +which bundle || gem install bundler + +echo Running rails-post-deploy +script/rails-post-deploy + +LOADED_INDICATOR="$HOME/.alaveteli-sample-data-loaded" + +if [ ! -f "$LOADED_INDICATOR" ] +then + echo Running load-sample-data + bundle exec script/load-sample-data + + echo Running rebuild-xapian-index + script/rebuild-xapian-index + + touch "$LOADED_INDICATOR" +fi diff --git a/script/make-crontab b/script/make-crontab deleted file mode 100755 index d214f1485..000000000 --- a/script/make-crontab +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python -import re - -mailto = "recipient-of-any-errors@localhost" -user = "user-to-run-as" -location = "/path/to/alaveteli" - -template = open("config/crontab-example").read() -template = re.sub(r"MAILTO=.*", "MAILTO=%s" % mailto, template) -template = template.replace("!!(*= $user *)!!", user) -template = re.sub(r"/data/vhost/.*/script", location + "/script", template) - -print template - - - diff --git a/script/rails-post-deploy b/script/rails-post-deploy index de950311c..c09868347 100755 --- a/script/rails-post-deploy +++ b/script/rails-post-deploy @@ -24,27 +24,32 @@ fi # read config file in for later (STAGING_SITE) if [ -e "config/general" ] || [ -e "config/general.yml" ] then - . commonlib/shlib/deployfns - read_conf config/general + . commonlib/shlib/deployfns + read_conf config/general else - OPTION_DOMAIN=127.0.0.1:3000 - OPTION_STAGING_SITE=1 + OPTION_DOMAIN=127.0.0.1:3000 + OPTION_STAGING_SITE=1 fi # create initial log files if [ -e $TOP_DIR/../logs ] then - # mySociety servers have logs dir in level above - rm -f log - ln -s $TOP_DIR/../logs log + # mySociety servers have logs dir in level above + if ! [ -h log ] && [ -d log ] + then + # If log is a directory rather than a symlink, move that + # directory out of the way: + mv log log.original + fi + ln -sfn $TOP_DIR/../logs log else - # otherwise just make the directory - if [ -h log ] - then - # remove any old-style symlink first - rm -f log - fi - mkdir -p log + # otherwise just make the directory + if [ -h log ] + then + # remove any old-style symlink first + rm -f log + fi + mkdir -p log fi cd log @@ -55,18 +60,18 @@ cd .. if [ "$OPTION_STAGING_SITE" = "0" ] then cat <<-END - - ***************************************************************** - WARNING: About to make config/rails_env.rb which, via special - code in config/boot.rb, forces the Rails environment to be - "production". If this is a development system, please edit your - config/general.yml file and set the STAGING_SITE option to 1, - and also delete the generated config/rails_env.rb file. - Alternatively, you can override config/rails_env.rb at any time - with an environment variable. - ***************************************************************** - - END + + ***************************************************************** + WARNING: About to make config/rails_env.rb which, via special + code in config/boot.rb, forces the Rails environment to be + "production". If this is a development system, please edit your + config/general.yml file and set the STAGING_SITE option to 1, + and also delete the generated config/rails_env.rb file. + Alternatively, you can override config/rails_env.rb at any time + with an environment variable. + ***************************************************************** + +END echo "ENV['RAILS_ENV'] ||= 'production'" > config/rails_env.rb fi @@ -81,7 +86,7 @@ then fi if [ "$TRAVIS" = "true" ] then - bundle_install_options="--without development develop --deployment" + bundle_install_options="--without development develop --deployment" fi bundle install $bundle_install_options diff --git a/script/site-specific-install.sh b/script/site-specific-install.sh new file mode 100755 index 000000000..8917fd577 --- /dev/null +++ b/script/site-specific-install.sh @@ -0,0 +1,163 @@ +#!/bin/sh + +# Set IDEAL_VERSION to the commitish we want to check out; typically +# this is the version tag. Since this may not exist before release, +# fall back to the master branch: +VERSIONS="origin/install-script 0.15 origin/master" + +PARENT_SCRIPT_URL=https://github.com/mysociety/commonlib/blob/master/bin/install-site.sh + +misuse() { + echo The variable $1 was not defined, and it should be. + echo This script should not be run directly - instead, please run: + echo $PARENT_SCRIPT_URL + exit 1 +} + +# Strictly speaking we don't need to check all of these, but it might +# catch some errors made when changing install-site.sh + +[ -z "$DIRECTORY" ] && misuse DIRECTORY +[ -z "$UNIX_USER" ] && misuse UNIX_USER +[ -z "$REPOSITORY" ] && misuse REPOSITORY +[ -z "$REPOSITORY_URL" ] && misuse REPOSITORY_URL +[ -z "$BRANCH" ] && misuse BRANCH +[ -z "$SITE" ] && misuse SITE +[ -z "$DEFAULT_SERVER" ] && misuse DEFAULT_SERVER +[ -z "$HOST" ] && misuse HOST +[ -z "$DISTRIBUTION" ] && misuse DISTRIBUTION +[ -z "$VERSIONS" ] && misuse VERSIONS +[ -z "$DEVELOPMENT_INSTALL" ] && misuse DEVELOPMENT_INSTALL +[ -z "$BIN_DIRECTORY" ] && misuse BIN_DIRECTORY + +update_mysociety_apt_sources + +if [ ! "$DEVELOPMENT_INSTALL" = true ]; then + install_nginx + add_website_to_nginx + # Check out the first available requested version: + su -l -c "cd '$REPOSITORY' && (for v in $VERSIONS; do git checkout $v && break; done)" \ + "$UNIX_USER" +fi + +install_postfix + +# Now there's quite a bit of Postfix configuration that we need to +# make sure is present: + +ensure_line_present \ + "^ *alaveteli *unix *" \ + "alaveteli unix - n n - 50 pipe flags=R user=$UNIX_USER argv=$REPOSITORY/script/mailin" \ + /etc/postfix/master.cf 644 + +ensure_line_present \ + "^ *transport_maps *=" \ + "transport_maps = regexp:/etc/postfix/transports" \ + /etc/postfix/main.cf 644 + +ensure_line_present \ + "^ *local_recipient_maps *=" \ + "local_recipient_maps = proxy:unix:passwd.byname regexp:/etc/postfix/recipients" \ + /etc/postfix/main.cf 644 + +ensure_line_present \ + "^ *mydestination *=" \ + "mydestination = $HOST, $(hostname --fqdn), localhost.localdomain, localhost" \ + /etc/postfix/main.cf 644 + +ensure_line_present \ + "^do-not-reply" \ + "do-not-reply-to-this-address: :blackhole:" \ + /etc/aliases 644 + +ensure_line_present \ + "^mail" \ + "mail.* -/var/log/mail/mail.log" \ + /etc/rsyslog.d/50-default.conf 644 + +cat > /etc/postfix/transports <<EOF +/^foi.*/ alaveteli +EOF + +cat > /etc/postfix/recipients <<EOF +/^foi.*/ this-is-ignored +/^postmaster@/ this-is-ignored +/^user-support@/ this-is-ignored +/^team@/ this-is-ignored +EOF + +if ! egrep '^ */var/log/mail/mail.log *{' /etc/logrotate.d/rsyslog +then + cat >> /etc/logrotate.d/rsyslog <<EOF +/var/log/mail/mail.log { + rotate 30 + daily + dateext + missingok + notifempty + compress + delaycompress + sharedscripts + postrotate + reload rsyslog >/dev/null 2>&1 || true + endscript +} +EOF +fi + +/etc/init.d/rsyslog restart + +newaliases +postmap /etc/postfix/transports +postmap /etc/postfix/recipients +postfix reload + +# (end of the Postfix configuration) + +install_website_packages + +# Make the PostgreSQL user a superuser to avoid the irritating error: +# PG::Error: ERROR: permission denied: "RI_ConstraintTrigger_16564" is a system trigger +# This is only needed for loading the sample data, so the superuser +# permissions are dropped below. +add_postgresql_user --superuser + +export DEVELOPMENT_INSTALL +su -l -c "$BIN_DIRECTORY/install-as-user '$UNIX_USER' '$HOST' '$DIRECTORY'" "$UNIX_USER" + +# Now that the install-as-user script has loaded the sample data, we +# no longer need the PostgreSQL user to be a superuser: +echo "ALTER USER \"$UNIX_USER\" WITH NOSUPERUSER;" | su -l -c 'psql' postgres + +if [ ! "$DEVELOPMENT_INSTALL" = true ]; then + install_sysvinit_script +fi + +# Set up root's crontab: + +cd "$REPOSITORY" + +echo -n "Creating /etc/cron.d/alaveteli... " +(su -l -c "cd '$REPOSITORY' && bundle exec rake config_files:convert_crontab DEPLOY_USER='$UNIX_USER' VHOST_DIR='$DIRECTORY' VCSPATH='$SITE' SITE='$SITE' CRONTAB=config/crontab-example" "$UNIX_USER") > /etc/cron.d/alaveteli +# There are some other parts to rewrite, so just do them with sed: +sed -r \ + -e "/foi-purge-varnish/d" \ + -e "s,^(MAILTO=).*,\1root@$HOST," \ + -e "s,run-with-lockfile,$REPOSITORY/commonlib/bin/run-with-lockfile.sh,g" \ + -i /etc/cron.d/alaveteli +echo $DONE_MSG + +echo -n "Creating /etc/init.d/foi-alert-tracks... " +(su -l -c "cd '$REPOSITORY' && bundle exec rake config_files:convert_init_script DEPLOY_USER='$UNIX_USER' VHOST_DIR='$DIRECTORY' SCRIPT_FILE=config/alert-tracks-debian.ugly" "$UNIX_USER") > /etc/init.d/foi-alert-tracks +chmod a+rx /etc/init.d/foi-alert-tracks +echo $DONE_MSG + +if [ $DEFAULT_SERVER = true ] && [ x != x$EC2_HOSTNAME ] +then + # If we're setting up as the default on an EC2 instance, make sure + # that the /etc/rc.local is set up to run the install script again + # to update the hostname: + overwrite_rc_local +fi + +done_msg "Installation complete"; echo |