22 files changed, 554 insertions, 264 deletions
diff --git a/config/alert-tracks-debian.ugly b/config/alert-tracks-debian.ugly
index 29a350a0e..f1ca68b03 100644..100755
--- a/config/alert-tracks-debian.ugly
+++ b/config/alert-tracks-debian.ugly
@@ -13,10 +13,12 @@
 # !!(*= $daemon_name *)!! Start the Alaveteli email alert daemon
 
 NAME=!!(*= $daemon_name *)!!
-DAEMON=!!(*= $vhost_dir *)!!/alaveteli/script/runner
+DAEMON=!!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/runner
 DAEMON_ARGS="--daemon TrackMailer.alert_tracks_loop"
-PIDFILE=!!(*= $vhost_dir *)!!/alert-tracks.pid
-LOGFILE=!!(*= $vhost_dir *)!!/logs/alert-tracks.log
+PIDDIR=!!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/tmp/pids
+PIDFILE=$PIDDIR/!!(*= $daemon_name *)!!.pid
+LOGDIR=!!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/log
+LOGFILE=$LOGDIR/!!(*= $daemon_name *)!!.log
 DUSER=!!(*= $user *)!!
 # Set RAILS_ENV - not needed if using config/rails_env.rb
 # RAILS_ENV=your_rails_env
@@ -27,10 +29,14 @@ trap "" 1
 export PIDFILE LOGFILE
 
 quietly_start_daemon() {
+    mkdir -p {$LOGDIR,$PIDDIR}
+    chown $DUSER:$DUSER {$LOGDIR,$PIDDIR}
     /sbin/start-stop-daemon --quiet --start --pidfile "$PIDFILE" --chuid "$DUSER" --startas "$DAEMON" -- $DAEMON_ARGS
 }
 
 start_daemon() {
+    mkdir -p {$LOGDIR,$PIDDIR}
+    chown $DUSER:$DUSER {$LOGDIR,$PIDDIR}
     /sbin/start-stop-daemon --start --pidfile "$PIDFILE" --chuid "$DUSER" --startas "$DAEMON" -- $DAEMON_ARGS
 }
 
@@ -81,4 +87,3 @@ else
     echo " failed"
     exit 1
 fi
-
diff --git a/config/crontab-example b/config/crontab-example
index 8fe13151b..44e328e4e 100644
--- a/config/crontab-example
+++ b/config/crontab-example
@@ -5,32 +5,32 @@
 # Email: hello@mysociety.org. WWW: http://www.mysociety.org/
 
 PATH=/usr/local/bin:/usr/bin:/bin
-MAILTO=cron-!!(*= $site *)!!@mysociety.org
+MAILTO=!!(*= $mailto *)!!
 
 # Every 5 minutes
-*/5 * * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/change-xapian-database.lock "!!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/update-xapian-index verbose=true" >> !!(*= $vhost_dir *)!!/logs/update-xapian-index.log || echo "stalled?"
+*/5 * * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/change-xapian-database.lock "!!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/update-xapian-index verbose=true" >> !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/log/update-xapian-index.log || echo "stalled?"
 # Every 10 minutes
-5,15,25,35,45,55 * * * * !!(*= $user *)!! /etc/init.d/foi-alert-tracks check
-5,15,25,35,45,55 * * * * !!(*= $user *)!! /etc/init.d/foi-purge-varnish check
-0,10,20,30,40,50 * * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/send-batch-requests.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/send-batch-requests || echo "stalled?"
+5,15,25,35,45,55 * * * * !!(*= $user *)!! /etc/init.d/!!(*= $site *)!!-alert-tracks check
+5,15,25,35,45,55 * * * * !!(*= $user *)!! /etc/init.d/!!(*= $site *)!!-purge-varnish check
+0,10,20,30,40,50 * * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/send-batch-requests.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/send-batch-requests || echo "stalled?"
 
 # Once an hour
-09 * * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/alert-comment-on-request.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-comment-on-request || echo "stalled?"
+09 * * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/alert-comment-on-request.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-comment-on-request || echo "stalled?"
 
 # Only root can read the log files
-31 * * * * root run-with-lockfile -n !!(*= $vhost_dir *)!!/load-mail-server-logs.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/load-mail-server-logs || echo "stalled?"
+31 * * * * root !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/load-mail-server-logs.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/load-mail-server-logs || echo "stalled?"
 
 # Once a day, early morning
-23 4 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/delete-old-things.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/delete-old-things || echo "stalled?"
-0 6 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/alert-overdue-requests.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-overdue-requests || echo "stalled?"
-0 7 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/alert-new-response-reminders.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-new-response-reminders || echo "stalled?"
-0 8 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/alert-not-clarified-request.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-not-clarified-request || echo "stalled?"
-2 4 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/check-recent-requests-sent.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/check-recent-requests-sent || echo "stalled?"
-45 3 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/stop-new-responses-on-old-requests.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/stop-new-responses-on-old-requests || echo "stalled?"
-55 4 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/update-public-body-stats.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/update-public-body-stats || echo "stalled?"
+23 4 * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/delete-old-things.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/delete-old-things || echo "stalled?"
+0 6 * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/alert-overdue-requests.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-overdue-requests || echo "stalled?"
+0 7 * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/alert-new-response-reminders.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-new-response-reminders || echo "stalled?"
+0 8 * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/alert-not-clarified-request.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-not-clarified-request || echo "stalled?"
+2 4 * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/check-recent-requests-sent.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/check-recent-requests-sent || echo "stalled?"
+45 3 * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/stop-new-responses-on-old-requests.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/stop-new-responses-on-old-requests || echo "stalled?"
+55 4 * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/update-public-body-stats.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/update-public-body-stats || echo "stalled?"
 
 # Only root can restart apache
-31 1 * * * root run-with-lockfile -n !!(*= $vhost_dir *)!!/change-xapian-database.lock "!!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/compact-xapian-database production" || echo "stalled?"
+31 1 * * * root !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/change-xapian-database.lock "!!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/compact-xapian-database production" || echo "stalled?"
 
 
 # Once a day on all servers
@@ -38,7 +38,7 @@ MAILTO=cron-!!(*= $site *)!!@mysociety.org
 48 2 * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/user-use-graph
 
 # Once a year :)
-0 0 1 11 * !!(*= $user *)!! /bin/echo "A year has passed, please update the bank holidays for the Freedom of Information site, thank you."
+0 0 1 11 * !!(*= $user *)!! /bin/echo "A year has passed, please update the public holidays for the Freedom of Information site, thank you."
 
 
 
diff --git a/config/database.yml-example b/config/database.yml-example
index e48577f23..126e5a264 100644
--- a/config/database.yml-example
+++ b/config/database.yml-example
@@ -3,7 +3,8 @@
 
 development:
   adapter: postgresql
-  database: foi_development
+  template: template_utf8
+  database: alaveteli_development
   username: <username>
   password: <password>
   host: localhost
@@ -14,7 +15,8 @@ development:
 # Do not set this db to the same as development or production.
 test:
   adapter: postgresql
-  database: foi_test
+  template: template_utf8
+  database: alaveteli_test
   username: <username>
   password: <password>
   host: localhost
@@ -24,9 +26,9 @@ test:
 
 production:
   adapter: postgresql
-  database: foi_production
+  template: template_utf8
+  database: alaveteli_production
   username: <username>
   password: <password>
   host: localhost
   port: 5432
-
diff --git a/config/database.yml-test b/config/database.yml-test
index ff6e2013a..64eafd82d 100644
--- a/config/database.yml-test
+++ b/config/database.yml-test
@@ -1,4 +1,5 @@
 test:
   adapter: postgresql
+  template: template_utf8
   database: foi_test
   username: postgres
diff --git a/config/deploy.rb b/config/deploy.rb
index c73d16f53..c1954d058 100644
--- a/config/deploy.rb
+++ b/config/deploy.rb
@@ -33,14 +33,12 @@ namespace :xapian do
 end
 
 namespace :deploy do
-  desc "Restarting mod_rails with restart.txt"
-  task :restart, :roles => :app, :except => { :no_release => true } do
-    run "touch #{current_path}/tmp/restart.txt"
-  end
 
-  [:start, :stop].each do |t|
-    desc "#{t} task is a no-op with mod_rails"
-    task t, :roles => :app do ; end
+  [:start, :stop, :restart].each do |t|
+    desc "#{t.to_s.capitalize} Alaveteli service defined in /etc/init.d/alaveteli"
+    task t, :roles => :app, :except => { :no_release => true } do
+      run "/etc/init.d/alaveteli #{t}"
+    end
   end
 
   desc 'Link configuration after a code update'
@@ -56,6 +54,8 @@ namespace :deploy do
       "#{release_path}/public/foi-user-use.png" => "#{shared_path}/foi-user-use.png",
       "#{release_path}/files" => "#{shared_path}/files",
       "#{release_path}/cache" => "#{shared_path}/cache",
+      "#{release_path}/log" => "#{shared_path}/log",
+      "#{release_path}/tmp/pids" => "#{shared_path}/tmp/pids",
       "#{release_path}/lib/acts_as_xapian/xapiandbs" => "#{shared_path}/xapiandbs",
     }
 
@@ -66,6 +66,8 @@ namespace :deploy do
   after 'deploy:setup' do
     run "mkdir -p #{shared_path}/files"
     run "mkdir -p #{shared_path}/cache"
+    run "mkdir -p #{shared_path}/log"
+    run "mkdir -p #{shared_path}/tmp/pids"
     run "mkdir -p #{shared_path}/xapiandbs"
   end
 end
diff --git a/config/general.yml-example b/config/general.yml-example
index 6e223406e..0f32f6192 100644
--- a/config/general.yml-example
+++ b/config/general.yml-example
@@ -26,7 +26,7 @@ ISO_COUNTRY_CODE: GB
 TIME_ZONE: Australia/Sydney
 
 # These feeds are displayed accordingly on the Alaveteli "blog" page:
-BLOG_FEED: 'http://www.mysociety.org/category/projects/whatdotheyknow/feed/'
+BLOG_FEED: 'https://www.mysociety.org/category/projects/whatdotheyknow/feed/'
 TWITTER_USERNAME: 'whatdotheyknow'
 # Set the widget_id to get the Twitter sidebar on the blog page.
 # To get one https://twitter.com/settings/widgets
@@ -245,6 +245,8 @@ SHARED_DIRECTORIES:
  - files/
  - cache/
  - lib/acts_as_xapian/xapiandbs/
+ - log/
+ - tmp/pids
  - vendor/bundle
  - public/assets
 
@@ -255,4 +257,4 @@ SHARED_DIRECTORIES:
 ALLOW_BATCH_REQUESTS: false
 
 # Should we use the responsive stylesheets?
-RESPONSIVE_STYLING: false
+RESPONSIVE_STYLING: true
diff --git a/config/httpd-force-ssl.conf.example b/config/httpd-force-ssl.conf.example
new file mode 100644
index 000000000..0e89afe62
--- /dev/null
+++ b/config/httpd-force-ssl.conf.example
@@ -0,0 +1,5 @@
+# Rewrite all proxied HTTP requests to HTTPS.
+# Only include if FORCE_SSL: true is set in general.yml
+# Expected to be found at /etc/apache2/vhost.d/alaveteli
+RewriteCond %{HTTP:X-Forwarded-Proto} !https
+RewriteRule /(.*) https://www.example.com/$1 [L,R=permanent]
diff --git a/config/httpd-ssl.conf.example b/config/httpd-ssl.conf.example
new file mode 100644
index 000000000..8f099b8a6
--- /dev/null
+++ b/config/httpd-ssl.conf.example
@@ -0,0 +1,24 @@
+# VirtualHost for HTTPS requests
+<VirtualHost *:443>
+  ServerName www.example.com
+
+  ErrorLog /var/log/apache2/alaveteli_error.log
+  CustomLog /var/log/apache2/alaveteli_access.log combined
+
+  ProxyRequests       Off
+  ProxyPreserveHost   On
+  ProxyPass           /       http://localhost:80/
+  ProxyPassReverse    /       http://localhost:80/
+  RequestHeader       set X-Forwarded-Proto 'https'
+
+  SSLEngine on
+  SSLProtocol all -SSLv2
+  SSLCipherSuite ALL:!ADH:!EXPORT:!SSLv2:RC4+RSA:+HIGH:+MEDIUM
+
+  SSLCertificateFile /etc/ssl/certs/www.example.com.cert
+  SSLCertificateKeyFile /etc/ssl/private/www.example.com.key
+  # SSLCertificateChainFile /etc/ssl/sub.class2.server.ca.pem
+  # SSLCACertificateFile /etc/ssl/ca.pem
+
+  SetEnvIf User-Agent ".*MSIE.*" nokeepalive ssl-unclean-shutdown
+</VirtualHost>
diff --git a/config/httpd.conf-example b/config/httpd.conf-example
index dc2e4966e..2f6ca9c75 100644
--- a/config/httpd.conf-example
+++ b/config/httpd.conf-example
@@ -1,81 +1,145 @@
-# Apache configuration for FOI site.
-#
-# For development ignore this, you can just run ./scripts/server as for any
-# Ruby on Rails application.
-#
-# Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
-# Email: hello@mysociety.org; WWW: http://www.mysociety.org
-
-# This is needed for the PHP spell checker
-<Location /fcgi>
-    Options +ExecCGI
-    SetHandler fastcgi-script
-</Location>
-
-RewriteEngine On
-#RewriteLog /var/log/apache2/rewrite.log
-#RewriteLogLevel 9
-
-# XXX do we need this now we use Passenger?
-# Pass through the HTTP basic authentication to mongrel. See also
-# admin_http_auth_user in app/controllers/application.rb
-# Note: Apache 2 only. Doesn't work in Apache 1.3, you'll need to live without
-# it.
-RewriteCond %{LA-U:REMOTE_USER} (.+)
-RewriteRule . - [E=RU:%1]
-RequestHeader add X-Forwarded-User %{RU}e
-
-# Old /files URL to new subdomain (as can't use Alias with passenger,
-# so we do it on its own domain). This is for custom admin upload
-# files for archiving.
-RewriteRule /files/(.+) http://files.whatdotheyknow.com/$1
-
-# Serve attachments directly from the cache, if possible.
-#
-# The file names are URL-encoded on disk, and sharded by the first
-# three digits of the request id, which is why this is as complicated
-# as it is. The RewriteMap directive makes the URL-escaping function
-# available to use in the other directives.
-#
-# The condition means that the rule will fire only if the cached
-# file exists.
-RewriteMap escape int:escape
-RewriteCond %{DOCUMENT_ROOT}/views_cache/request/$2/$1/${escape:$3} -f
-RewriteRule ^/request/((\d{1,3})\d*)/(response/\d+/attach/(html/)?\d+/.+) /views_cache/request/$2/$1/${escape:$3} [L]
-
-
-<IfModule mod_passenger.c>
- # Set this to something like 100 if you have memory leak issues
- PassengerMaxRequests 500
- PassengerResolveSymlinksInDocumentRoot on
- # Recommend setting this to 3 or less on servers with 512MB RAM
- PassengerMaxPoolSize 6
- # The RackEnv variable applies to Rails 3 applications, while
- # the RailsEnv variable applies to applications for earlier
- # versions of Rails.  There doesn't seem to be any harm in
- # setting both, however.
- RailsEnv production
- RackEnv production
-</IfModule>
-
-# Gzip font resources
-<IfModule mod_deflate.c>
-    <IfModule mod_mime.c>
-        Addtype font/opentype .otf
-        Addtype font/opentype .woff
-        Addtype font/eot .eot
-        Addtype font/truetype .ttf
+# Redirect other domains to canonical domain
+<VirtualHost *:80>
+    ServerName example.com
+    ServerAlias example.org
+    ServerAlias www.example.org
+    RedirectPermanent / http://www.example.com/
+</VirtualHost>
+
+# Canonical domain vHost
+<VirtualHost *:80>
+    ServerName www.example.com
+    ServerAdmin webmaster@example.com
+    DocumentRoot /var/www/alaveteli/public
+
+    # TODO: Remove this and use PassengerUser when supported
+    # This is the user that runs the rails application processes
+    SuExecUserGroup alaveteli alaveteli
+    RewriteEngine On
+
+    ErrorLog /var/log/apache2/alaveteli_error.log
+    CustomLog /var/log/apache2/alaveteli_access.log combined
+
+    # App server configuration
+    <IfModule mod_passenger.c>
+        PassengerAppRoot /var/www/alaveteli
+        PassengerResolveSymlinksInDocumentRoot on
+
+        # See http://blog.phusion.nl/2013/03/12/tuning-phusion-passengers-concurrency-settings/
+        # and http://blog.scoutapp.com/articles/2009/12/08/production-rails-tuning-with-passenger-passengermaxprocesses
+        # for more information on tuning Passenger
+
+        # Set this to something like 100 if you have memory leak issues
+        PassengerMaxRequests 2000
+        # Passenger's default MaxPoolSize is 6. At the time of writing
+        # normal instances of Alaveteli seem to take 150-200MB per
+        # process, so we've set this conservatively at 3. Read the guides
+        # above to tune this for your system 
+        PassengerMaxPoolSize 3
+
+        # The RAILS_ENV that the app is running in. This can be any of
+        # the environments listed in APP_ROOT/config/environments.
+        RailsEnv production
+        RackEnv production
     </IfModule>
-    AddOutputFilterByType DEFLATE font/opentype font/truetype font/eot
-    AddOutputFilterByType DEFLATE image/svg+xml
-</IFModule>
-
-# Set the Sendfile header and switch sendfile on - Apache will
-# now handle send_file calls from Alaveteli
-<Location />
-    <IfModule mod_xsendfile.c>
-    RequestHeader Set X-Sendfile-Type X-Sendfile
-    XSendFile On
-    XSendFileAllowAbove On
+
+    # This is your Rails app's public directory
+    <Directory "/var/www/alaveteli/public">
+        Options +ExecCGI -MultiViews
+        AllowOverride All
+    </Directory>
+
+    # Maintenance Page
+        # Make a file down.html in the DocumentRoot to bring down the whole
+        # site and display itself.
+        RewriteEngine on
+        ErrorDocument 503 /down.html
+        Redirect 503 /down
+
+        # If down.html exists, and that's what's been asked for,
+        # just hand it over
+        RewriteCond %{DOCUMENT_ROOT}/down.html -s
+        RewriteRule /down.html /down.html [L]
+
+        RewriteCond %{DOCUMENT_ROOT}/down.html -s
+        RewriteRule /(.+).cgi /down [PT]
+        RewriteCond %{DOCUMENT_ROOT}/down.html -s
+        RewriteRule /(.+).php /down [PT]
+        # Mainly for Rails/Django type sites - anything without a . can go down
+        # TODO: could we just check that it is an HTML content-type?
+        RewriteCond %{DOCUMENT_ROOT}/down.html -s
+        RewriteRule /([^.]*)$ /down [PT]
+    # END Maintenance Page
+
+    # Set the Sendfile header and switch sendfile on - Apache will
+    # now handle send_file calls from Alaveteli
+    <Location />
+        <IfModule mod_xsendfile.c>
+            RequestHeader Set X-Sendfile-Type X-Sendfile
+            XSendFile On
+            XSendFilePath /var/www/alaveteli/public
+        </IfModule>
+    </Location>
+
+    # Commonlib is typically found in alaveteli/commonlib
+    Alias /jslib/ "/var/www/alaveteli/commonlib/jslib"
+    <Directory "/var/www/alaveteli/commonlib/jslib">
+        Options +ExecCGI
+        AddHandler fastcgi-script .cgi
+    </Directory>
+
+    # Serve attachments directly from the cache, if possible.
+    #
+    # The file names are URL-encoded on disk, and sharded by the first
+    # three digits of the request id, which is why this is as complicated
+    # as it is. The RewriteMap directive makes the URL-escaping function
+    # available to use in the other directives.
+    #
+    # The condition means that the rule will fire only if the cached
+    # file exists.
+    RewriteMap escape int:escape
+    RewriteCond %{DOCUMENT_ROOT}/views_cache/request/$2/$1/${escape:$3} -f
+    RewriteRule ^/request/((\d{1,3})\d*)/(response/\d+/attach/(html/)?\d+/.+) /views_cache/request/$2/$1/${escape:$3} [L]
+    RewriteCond %{DOCUMENT_ROOT}/views_cache/cy/request/$2/$1/${escape:$3} -f
+    RewriteRule ^/cy/request/((\d{1,3})\d*)/(response/\d+/attach/(html/)?\d+/.+) /views_cache/cy/request/$2/$1/${escape:$3} [L]
+
+    # Compress assets
+    <Location />
+        <IfModule mod_deflate.c>
+            AddOutputFilterByType DEFLATE text/css application/javascript text/plain
+        </IfModule>
+    </Location>
+
+    # Cache assets
+    ExpiresActive On
+    <LocationMatch "^/(assets).*\.(ico|gif|jpe?g|png|js|css|svg|ttf|otf|eot|woff)$">
+        ExpiresDefault "access plus 1 day"
+    </LocationMatch>
+
+    # Compress font resources
+    <IfModule mod_deflate.c>
+        <IfModule mod_mime.c>
+            Addtype font/opentype .otf
+            Addtype font/opentype .woff
+            Addtype font/eot .eot
+            Addtype font/truetype .ttf
+        </IfModule>
+        AddOutputFilterByType DEFLATE font/opentype font/truetype font/eot
+        AddOutputFilterByType DEFLATE image/svg+xml
     </IfModule>
-</Location>
+
+    # Include optional configuration
+    Include vhost.d/alaveteli/*.conf
+
+</VirtualHost>
+
+# Large / static files for WhatDoTheyKnow. Used for manual sysadmin uploads.
+# Is on its own files.example.com subdomain (instead of old /files URL)
+# as mod_alias doesn't work with Passenger.
+# <VirtualHost *:80>
+#     ServerName files.example.com
+#     DocumentRoot /var/www/alaveteli/files
+#     <Directory "/var/www/alaveteli/files">
+#         Options +Indexes
+#     </Directory>
+# </VirtualHost>
diff --git a/config/initializers/alaveteli.rb b/config/initializers/alaveteli.rb
index 6fb6b1420..9ea6428ba 100644
--- a/config/initializers/alaveteli.rb
+++ b/config/initializers/alaveteli.rb
@@ -10,7 +10,7 @@ load "debug_helpers.rb"
 load "util.rb"
 
 # Application version
-ALAVETELI_VERSION = '0.18'
+ALAVETELI_VERSION = '0.19'
 
 # Add new inflection rules using the following format
 # (all these examples are active by default):
@@ -53,6 +53,7 @@ require 'message_prominence'
 require 'theme'
 require 'xapian_queries'
 require 'date_quarter'
+require 'public_body_csv'
 
 AlaveteliLocalization.set_locales(AlaveteliConfiguration::available_locales,
                                   AlaveteliConfiguration::default_locale)
diff --git a/config/nginx-ssl.conf.example b/config/nginx-ssl.conf.example
new file mode 100644
index 000000000..af2e9e4ab
--- /dev/null
+++ b/config/nginx-ssl.conf.example
@@ -0,0 +1,44 @@
+upstream alaveteli {
+    server 127.0.0.1:3000;
+}
+
+# Redirect any http:// request to https://www.example.com
+server {
+  listen 80;
+  server_name www.example.com;
+  rewrite ^(.*) https://www.example.com$request_uri permanent;
+}
+
+server {
+    listen 443;
+    server_name www.example.com;
+    root /var/www/alaveteli/alaveteli/public;
+
+    server_tokens off;
+
+    try_files $uri/index.html $uri @alaveteli;
+
+    access_log /var/log/nginx/alaveteli_ssl_access.log;
+    error_log /var/log/nginx/alaveteli_ssl_error.log error;
+
+    location /download {
+        internal;
+        alias /var/www/alaveteli/alaveteli/cache/zips/production/download;
+    }
+
+    ssl on;
+    ssl_certificate /etc/ssl/certs/www.example.com.cert;
+    ssl_certificate_key /etc/ssl/private/www.example.com.key;
+    ssl_ciphers ECDHE-RSA-AES256-SHA384:AES256-SHA256:RC4:HIGH:!MD5:!aNULL:!EDH:!AESGCM;
+
+    location @alaveteli {
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Real-IP  $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto https;
+        proxy_set_header X-Sendfile-Type X-Accel-Redirect;
+        proxy_set_header X-Accel-Mapping /var/www/alaveteli/alaveteli/cache/zips/production/download=/download;
+        proxy_redirect off;
+        proxy_pass http://alaveteli;
+    }
+}
diff --git a/config/nginx.conf.example b/config/nginx.conf.example
index 56e720abb..7b8af2bd9 100644
--- a/config/nginx.conf.example
+++ b/config/nginx.conf.example
@@ -1,29 +1,43 @@
 upstream alaveteli {
-    server 127.0.0.1:3300;
+    server 127.0.0.1:3000;
 }
 
+# Example to redirect other domains to the canonical URL. Also redirects the
+# unqualified domain to the FQDN www.example.com, which is recommended.
+#
+# server {
+#   server_name example.com example.org www.example.org;
+#   rewrite ^(.*) http://www.example.com$1 permanent;
+# }
+
 server {
     listen 80;
+    # Set the server name to your domain name if you have multiple nginx servers
+    # running on your machine
+    # server_name www.example.com;
     root /var/www/alaveteli/alaveteli/public;
 
-    location / {
-        proxy_set_header X-Real-IP  $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header Host $http_host;
-        proxy_set_header X-Forwarded-Proto http;
-        proxy_redirect off;
-        try_files $uri @ruby;
-    }
+    server_tokens off;
+
+    access_log /var/log/nginx/alaveteli_access.log;
+    error_log /var/log/nginx/alaveteli_error.log error;
+
+    try_files $uri/index.html $uri @alaveteli;
+
 
     location /download {
         internal;
-        alias /var/www/alaveteli/alaveteli/cache/zips/development/download;
+        alias /var/www/alaveteli/alaveteli/cache/zips/production/download;
     }
 
-    location @ruby {
-        proxy_pass http://alaveteli;
+    location @alaveteli {
         proxy_set_header Host $http_host;
+        proxy_set_header X-Real-IP  $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto http;
         proxy_set_header X-Sendfile-Type X-Accel-Redirect;
-        proxy_set_header X-Accel-Mapping /var/www/alaveteli/alaveteli/cache/zips/development/download=/download;
+        proxy_set_header X-Accel-Mapping /var/www/alaveteli/alaveteli/cache/zips/production/download=/download;
+        proxy_redirect off;
+        proxy_pass http://alaveteli;
     }
 }
diff --git a/config/packages b/config/packages
index 9a07c5f20..a6eeb8079 100644
--- a/config/packages
+++ b/config/packages
@@ -1,41 +1,44 @@
 # please note that this package list currently forms part of the
 # mysociety deployment infrastructure -- please discuss on the
 # alavateli mailing list if you want to change it
-ruby1.8
-ruby
-rdoc | rdoc1.8
-irb | irb1.8
-wv
-poppler-utils
-pdftk (>> 1.41+dfsg-1) | pdftk (<< 1.41+dfsg-1) # that version has a non-functionining uncompress option
-ghostscript
+build-essential
+bundler
 catdoc
-links
 elinks
-unrtf
-xlhtml
-xapian-tools
-gnuplot-nox
-php5-cli
-sharutils
-unzip
-mutt
-tnef (>= 1.4.5)
 gettext
-python-yaml
-wkhtmltopdf-static
+ghostscript
+gnuplot-nox
+irb | irb1.8
+libapache2-mod-passenger
+libicu-dev
 libmagic-dev
 libmagickwand-dev
 libpq-dev
+libsqlite3-dev
 libxml2-dev
 libxslt-dev
-uuid-dev
+links
+lockfile-progs
+memcached
+mutt
+pdftk (>> 1.41+dfsg-1) | pdftk (<< 1.41+dfsg-1) # that version has a non-functionining uncompress option
+php5-cli
+poppler-utils
+python-yaml
+rake (>= 0.9.2.2)
+rdoc | rdoc1.8
+ruby
+ruby1.8
 ruby1.8-dev
 rubygems (>= 1.8.15)
-rake (>= 0.9.2.2)
-build-essential
-bundler
+sharutils
 sqlite3
-libsqlite3-dev
-libicu-dev
-memcached
+tnef (>= 1.4.5)
+ttf-bitstream-vera
+unrtf
+unzip
+uuid-dev
+wkhtmltopdf-static
+wv
+xapian-tools
+xlhtml
diff --git a/config/packages.debian-squeeze b/config/packages.debian-squeeze
index 6cdf2f9d6..c34abefde 100644
--- a/config/packages.debian-squeeze
+++ b/config/packages.debian-squeeze
@@ -1,39 +1,42 @@
-ruby1.8
-ruby
-libruby1.8
-rdoc1.8
-irb1.8
-wv
-poppler-utils
-pdftk
-ghostscript
+build-essential
+bundler
 catdoc
-links
 elinks
-unrtf
-xlhtml
-xapian-tools
-gnuplot-nox
-php5-cli
-sharutils
-unzip
-mutt
-tnef
 gettext
-python-yaml
-wkhtmltopdf-static
+ghostscript
+gnuplot-nox
+irb1.8
+libicu-dev
 libmagic-dev
 libmagickwand-dev
 libpq-dev
+libruby1.8
+libsqlite3-dev
 libxml2-dev
 libxslt-dev
-uuid-dev
+links
+lockfile-progs
+mutt
+pdftk
+php5-cli
+poppler-utils
+postgresql
+postgresql-client
+python-yaml
+rake
+rdoc1.8
+ruby
+ruby1.8
 ruby1.8-dev
 rubygems/squeeze-backports
-rake
-build-essential
+sharutils
 sqlite3
-libsqlite3-dev
-libicu-dev
-postgresql
-postgresql-client
+tnef
+ttf-bitstream-vera
+unrtf
+unzip
+uuid-dev
+wkhtmltopdf-static
+wv
+xapian-tools
+xlhtml
diff --git a/config/packages.debian-wheezy b/config/packages.debian-wheezy
new file mode 100644
index 000000000..4129aa930
--- /dev/null
+++ b/config/packages.debian-wheezy
@@ -0,0 +1,37 @@
+build-essential
+bundler
+catdoc
+elinks
+gettext
+ghostscript
+gnuplot-nox
+libicu-dev
+libmagic-dev
+libmagickwand-dev
+libpq-dev
+libsqlite3-dev
+libxml2-dev
+libxslt-dev
+links
+lockfile-progs
+mutt
+pdftk
+php5-cli
+poppler-utils
+postgresql
+postgresql-client
+python-yaml
+rake
+ruby-dev
+ruby1.9.3
+rubygems
+sharutils
+sqlite3
+tnef
+unrtf
+unzip
+uuid-dev
+wkhtmltopdf-static
+wv
+xapian-tools
+xlhtml
diff --git a/config/packages.ubuntu-precise b/config/packages.ubuntu-precise
index 177d504e2..d97579bc1 100644
--- a/config/packages.ubuntu-precise
+++ b/config/packages.ubuntu-precise
@@ -1,35 +1,37 @@
-ruby1.9.1
-wv
-poppler-utils
-pdftk
-ghostscript
+build-essential
 catdoc
-links
 elinks
-unrtf
-xlhtml
-xapian-tools
-gnuplot-nox
-sharutils
-unzip
-mutt
-tnef
 gettext
-python-yaml
-wkhtmltopdf-static
+ghostscript
+gnuplot-nox
+libicu-dev
 libmagic-dev
 libmagickwand-dev
 libpq-dev
+libsqlite3-dev
 libxml2-dev
 libxslt1-dev
-uuid-dev
-ruby1.9.1-dev
-rubygems
+links
+lockfile-progs
+mutt
+pdftk
+poppler-utils
+postgresql
+postgresql-client
+python-yaml
 rake
-build-essential
 ruby-bundler
+ruby1.9.1
+ruby1.9.1-dev
+rubygems
+sharutils
 sqlite3
-libsqlite3-dev
-libicu-dev
-postgresql
-postgresql-client
+tnef
+ttf-bitstream-vera
+unrtf
+unzip
+uuid-dev
+wkhtmltopdf-static
+wv
+xapian-tools
+xlhtml
diff --git a/config/purge-varnish-debian.ugly b/config/purge-varnish-debian.ugly
index 3f11344f2..dc3f74ff6 100644..100755
--- a/config/purge-varnish-debian.ugly
+++ b/config/purge-varnish-debian.ugly
@@ -13,10 +13,12 @@
 # !!(*= $daemon_name *)!! Start the Alaveteli email purge-varnish daemon
 
 NAME=!!(*= $daemon_name *)!!
-DAEMON=!!(*= $vhost_dir *)!!/alaveteli/script/runner
+DAEMON=!!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/runner
 DAEMON_ARGS="--daemon PurgeRequest.purge_all_loop"
-PIDFILE=!!(*= $vhost_dir *)!!/purge-varnish.pid
-LOGFILE=!!(*= $vhost_dir *)!!/logs/purge-varnish.log
+PIDDIR=!!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/tmp/pids
+PIDFILE=$PIDDIR/!!(*= $daemon_name *)!!.pid
+LOGDIR=!!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/log
+LOGFILE=$LOGDIR/!!(*= $daemon_name *)!!.log
 DUSER=!!(*= $user *)!!
 # Set RAILS_ENV - not needed if using config/rails_env.rb
 # RAILS_ENV=your_rails_env
@@ -29,10 +31,14 @@ trap "" 1
 export PIDFILE LOGFILE
 
 quietly_start_daemon() {
+    mkdir -p {$LOGDIR,$PIDDIR}
+    chown $DUSER:$DUSER {$LOGDIR,$PIDDIR}
     /sbin/start-stop-daemon --quiet --start --pidfile "$PIDFILE" --chuid "$DUSER" --startas "$DAEMON" -- $DAEMON_ARGS
 }
 
 start_daemon() {
+    mkdir -p {$LOGDIR,$PIDDIR}
+    chown $DUSER:$DUSER {$LOGDIR,$PIDDIR}
     /sbin/start-stop-daemon --start --pidfile "$PIDFILE" --chuid "$DUSER" --startas "$DAEMON" -- $DAEMON_ARGS
 }
 
diff --git a/config/routes.rb b/config/routes.rb
index d9d21f0bd..9f426fabf 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -16,7 +16,7 @@ Alaveteli::Application.routes.draw do
     match '/blog' => 'general#blog', :as => :blog
     match '/search' => 'general#search_redirect', :as => :search_redirect
     match '/search/all' => 'general#search_redirect', :as => :search_redirect
-    # XXX combined is the search query, and then if sorted a "/newest" at the end.
+    # `combined` is the search query, and then if sorted a "/newest" at the end.
     # Couldn't find a way to do this in routes which also picked up multiple other slashes
     # and dots and other characters that can appear in search query. So we sort it all
     # out in the controller.
@@ -130,7 +130,7 @@ Alaveteli::Application.routes.draw do
     match '/:feed/list/:view' => 'track#track_list', :as => :track_list, :view => nil, :feed => /(track|feed)/
     match '/:feed/body/:url_name' => 'track#track_public_body', :as => :track_public_body, :feed => /(track|feed)/
     match '/:feed/user/:url_name' => 'track#track_user', :as => :track_user, :feed => /(track|feed)/
-    # XXX :format doesn't work. See hacky code in the controller that makes up for this.
+    # TODO: :format doesn't work. See hacky code in the controller that makes up for this.
     match '/:feed/search/:query_array' => 'track#track_search_query',
           :as => :track_search,
           :feed => /(track|feed)/,
@@ -261,6 +261,7 @@ Alaveteli::Application.routes.draw do
 
     match '/api/v2/request/:id.json' => 'api#show_request', :as => :api_show_request, :via => :get
     match '/api/v2/request/:id.json' => 'api#add_correspondence', :as => :api_add_correspondence, :via => :post
+    match '/api/v2/request/:id/update.json' => 'api#update_state', :as => :api_update_state, :via => :post
 
     match '/api/v2/body/:id/request_events.:feed_type' => 'api#body_request_events', :as => :api_body_request_events, :feed_type => '^(json|atom)$'
     ####
diff --git a/config/sysvinit-passenger.ugly b/config/sysvinit-passenger.ugly
new file mode 100755
index 000000000..0940a4d63
--- /dev/null
+++ b/config/sysvinit-passenger.ugly
@@ -0,0 +1,59 @@
+#! /bin/sh
+### BEGIN INIT INFO
+# Provides:          application-passenger-!!(*= $site *)!!
+# Required-Start:    $local_fs $network
+# Required-Stop:     $local_fs $network
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Starts the Passenger app server for the "!!(*= $site *)!!" site
+# Description:       The Passenger app server for the "!!(*= $site *)!!" site
+### END INIT INFO
+
+# This example sysvinit script is based on the helpful example here:
+# http://richard.wallman.org.uk/2010/02/howto-deploy-a-catalyst-application-using-fastcgi-and-nginx/
+
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+NAME=!!(*= $site *)!!
+SITE_HOME=!!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!
+DESC="Alaveteli app server"
+USER=!!(*= $user *)!!
+
+set -e
+
+start_daemon() {
+  echo -n "Starting $DESC: "
+  rm -f "$SITE_HOME/public/down.html"
+  echo "$NAME."
+}
+
+stop_daemon() {
+  echo -n "Stopping $DESC: "
+  cp "$SITE_HOME/public/down.default.html" "$SITE_HOME/public/down.html"
+  echo "$NAME."
+}
+
+restart_daemon() {
+  echo -n "Restarting $DESC: "
+  rm -f "$SITE_HOME/public/down.html"
+  touch "$SITE_HOME/tmp/restart.txt"
+  echo "$NAME."
+}
+
+case "$1" in
+  start)
+    start_daemon
+    ;;
+  stop)
+    stop_daemon
+    ;;
+  reload|restart|force-reload)
+    restart_daemon
+    ;;
+  *)
+    N=/etc/init.d/$NAME
+    echo "Usage: $N {start|stop|reload|restart|force-reload}" >&2
+    exit 1
+    ;;
+esac
+
+exit 0
diff --git a/config/sysvinit-thin.ugly b/config/sysvinit-thin.ugly
new file mode 100755
index 000000000..cc604d994
--- /dev/null
+++ b/config/sysvinit-thin.ugly
@@ -0,0 +1,68 @@
+#! /bin/sh
+### BEGIN INIT INFO
+# Provides:          application-thin-!!(*= $site *)!!
+# Required-Start:    $local_fs $network
+# Required-Stop:     $local_fs $network
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Starts the Thin app server for the "!!(*= $site *)!!" site
+# Description:       The Thin app server for the "!!(*= $site *)!!" site
+### END INIT INFO
+
+# This example sysvinit script is based on the helpful example here:
+# http://richard.wallman.org.uk/2010/02/howto-deploy-a-catalyst-application-using-fastcgi-and-nginx/
+
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+NAME=!!(*= $site *)!!
+SITE_HOME=!!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!
+DESC="Alaveteli app server"
+USER=!!(*= $user *)!!
+
+set -e
+
+# Check that the Daemon can be run
+su -l -c "cd $SITE_HOME && bundle exec thin --version &> /dev/null || exit 0" $USER
+
+start_daemon() {
+  echo -n "Starting $DESC: "
+  cd "$SITE_HOME" && bundle exec thin \
+    --environment=production \
+    --user="$USER" \
+    --group="$USER" \
+    --address=127.0.0.1 \
+    --daemonize \
+    --quiet \
+    start || true
+  echo "$NAME."
+}
+
+stop_daemon() {
+  echo -n "Stopping $DESC: "
+  cd "$SITE_HOME" && bundle exec thin --quiet stop || true
+  echo "$NAME."
+}
+
+restart_daemon() {
+  echo -n "Restarting $DESC: "
+  cd "$SITE_HOME" && bundle exec thin --onebyone --quiet restart || true
+  echo "$NAME."
+}
+
+case "$1" in
+  start)
+    start_daemon
+    ;;
+  stop)
+    stop_daemon
+    ;;
+  reload|restart|force-reload)
+    restart_daemon
+    ;;
+  *)
+    N=/etc/init.d/$NAME
+    echo "Usage: $N {start|stop|reload|restart|force-reload}" >&2
+    exit 1
+    ;;
+esac
+
+exit 0
diff --git a/config/sysvinit.example b/config/sysvinit.example
deleted file mode 100755
index 443e7c3fb..000000000
--- a/config/sysvinit.example
+++ /dev/null
@@ -1,53 +0,0 @@
-#! /bin/sh
-### BEGIN INIT INFO
-# Provides:          application-thin-alaveteli
-# Required-Start:    $local_fs $network
-# Required-Stop:     $local_fs $network
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
-# Short-Description: Starts the Thin web server for the "Alaveteli" site
-# Description:       The Thin web server for the "Alaveteli" site
-### END INIT INFO
-
-# This example sysvinit script is based on the helpful example here:
-# http://richard.wallman.org.uk/2010/02/howto-deploy-a-catalyst-application-using-fastcgi-and-nginx/
-
-PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
-SITE_HOME=/var/www/alaveteli
-NAME=alaveteli
-DESC="Alaveteli app server"
-USER=fms
-
-echo $DAEMON
-test -f $DAEMON || exit 0
-
-set -e
-
-start_daemon() {
-  su -l -c "cd $SITE_HOME/alaveteli && bundle exec thin -d -p 3300 -e development start" $USER
-}
-
-stop_daemon() {
-  pkill -f thin -u $USER || true
-}
-
-case "$1" in
- start)
- start_daemon
- ;;
- stop)
- stop_daemon
- ;;
- reload|restart|force-reload)
- stop_daemon
- sleep 5
- start_daemon
- ;;
- *)
- N=/etc/init.d/$NAME
- echo "Usage: $N {start|stop|reload|restart|force-reload}" >&2
- exit 1
- ;;
-esac
-
-exit 0
diff --git a/config/varnish-alaveteli.vcl b/config/varnish-alaveteli.vcl
index 5dd0ac83c..d3726682c 100644
--- a/config/varnish-alaveteli.vcl
+++ b/config/varnish-alaveteli.vcl
@@ -92,7 +92,7 @@ sub vcl_recv {
       # ban lists, see
       # http://kristianlyng.wordpress.com/2010/07/28/smart-bans-with-varnish/
 
-      # XXX in Varnish 2.x, the following would be
+      # TODO: in Varnish 2.x, the following would be
       # purge("obj.http.x-url ~ " req.url);
       ban("obj.http.x-url ~ " + req.url);
       error 200 "Banned";