From 2dd8cf2a1e91703352228d8696e96c01806e71ca Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 16 Sep 2013 11:40:24 +0100 Subject: Add install scripts for Alaveteli site-specific-install.sh will be called by our generic site install script in commonlib/bin/install-site.sh These scripts assume that you have a new installation of Debian squeeze or Ubuntu precise and then will set up: - Alaveteli running in development mode with the Thin web server behing nginx - The cron jobs that are required for the site to work. - A basic Postfix configuration for sending and receiving mail. We also will use this script for generating new AMIs (Amazon Machin Images) for Alaveteli. The general.yml configuration file will be created if it doesn't exist, but if there is an existing copy it won't be overwritten, so it should be safe to customize that file and then re-run the install script. --- script/site-specific-install.sh | 161 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100755 script/site-specific-install.sh (limited to 'script/site-specific-install.sh') diff --git a/script/site-specific-install.sh b/script/site-specific-install.sh new file mode 100755 index 000000000..c7ec1e59e --- /dev/null +++ b/script/site-specific-install.sh @@ -0,0 +1,161 @@ +#!/bin/sh + +# Set IDEAL_VERSION to the commitish we want to check out; typically +# this is the version tag. Since this may not exist before release, +# fall back to the master branch: +IDEAL_VERSION=0.15 +FALLBACK_VERSION=origin/master + +PARENT_SCRIPT_URL=https://github.com/mysociety/commonlib/blob/master/bin/install-site.sh + +misuse() { + echo The variable $1 was not defined, and it should be. + echo This script should not be run directly - instead, please run: + echo $PARENT_SCRIPT_URL + exit 1 +} + +# Strictly speaking we don't need to check all of these, but it might +# catch some errors made when changing install-site.sh + +[ -z "$DIRECTORY" ] && misuse DIRECTORY +[ -z "$UNIX_USER" ] && misuse UNIX_USER +[ -z "$REPOSITORY" ] && misuse REPOSITORY +[ -z "$REPOSITORY_URL" ] && misuse REPOSITORY_URL +[ -z "$BRANCH" ] && misuse BRANCH +[ -z "$SITE" ] && misuse SITE +[ -z "$DEFAULT_SERVER" ] && misuse DEFAULT_SERVER +[ -z "$HOST" ] && misuse HOST +[ -z "$DISTRIBUTION" ] && misuse DISTRIBUTION +[ -z "$IDEAL_VERSION" ] && misuse VERSION +[ -z "$FALLBACK_VERSION" ] && misuse FALLBACK_VERSION +[ -z "$DEVELOPMENT_INSTALL" ] && misuse DEVELOPMENT_INSTALL +[ -z "$BIN_DIRECTORY" ] && misuse BIN_DIRECTORY + +update_mysociety_apt_sources + +if [ ! "$DEVELOPMENT_INSTALL" = true ]; then + install_nginx + add_website_to_nginx + # Check out the requested version: + su -l -c "cd '$REPOSITORY' && (git checkout '$IDEAL_VERSION' || + git checkout '$FALLBACK_VERSION')" \ + "$UNIX_USER" +fi + +install_postfix + +# Now there's quite a bit of Postfix configuration that we need to +# make sure is present: + +ensure_line_present \ + "^ *alaveteli *unix *" \ + "alaveteli unix - n n - 50 pipe flags=R user=$UNIX_USER argv=$REPOSITORY/script/mailin" \ + /etc/postfix/master.cf 644 + +ensure_line_present \ + "^ *transport_maps *= *regexp:/etc/postfix/regexp" \ + "transport_maps = regexp:/etc/postfix/regexp" \ + /etc/postfix/main.cf 644 + +ensure_line_present \ + "^ *local_recipient_maps *=" \ + "local_recipient_maps = " \ + /etc/postfix/main.cf 644 + +ensure_line_present \ + "^ *mydestination *=" \ + "mydestination = $HOST, $(hostname --fqdn), localhost.localdomain, localhost" \ + /etc/postfix/main.cf 644 + +ensure_line_present \ + "^.*alaveteli" \ + "/^foi.*/ alaveteli" \ + /etc/postfix/regexp 644 + +ensure_line_present \ + "^do-not-reply" \ + "do-not-reply-to-this-address: :blackhole:" \ + /etc/aliases 644 + +ensure_line_present \ + "^mail" \ + "mail.* -/var/log/mail/mail.log" \ + /etc/rsyslog.d/50-default.conf 644 + +if ! egrep '^ */var/log/mail/mail.log *{' /etc/logrotate.d/rsyslog +then + cat >> /etc/logrotate.d/rsyslog </dev/null 2>&1 || true + endscript +} +EOF +fi + +/etc/init.d/rsyslog restart + +newaliases +postmap /etc/postfix/regexp +postfix reload + +# (end of the Postfix configuration) + +install_website_packages + +# Make the PostgreSQL user a superuser to avoid the irritating error: +# PG::Error: ERROR: permission denied: "RI_ConstraintTrigger_16564" is a system trigger +add_postgresql_user --superuser + +export DEVELOPMENT_INSTALL +su -c "$BIN_DIRECTORY/install-as-user '$UNIX_USER' '$HOST' '$DIRECTORY'" "$UNIX_USER" + +if [ ! "$DEVELOPMENT_INSTALL" = true ]; then + install_sysvinit_script +fi + +# Set up root's crontab: + +cd "$REPOSITORY" + +sed -r \ + -e "s,^(MAILTO=).*,\1root@$HOST," \ + -e "s,\!\!\(\*= .user \*\)\!\!,$UNIX_USER,g" \ + -e "s,/data/vhost/\!\!\(\*= .vhost \*\)\!\!/\!\!\(\*= .vcspath \*\)\!\!,$REPOSITORY,g" \ + -e "s,/data/vhost/\!\!\(\*= .vhost \*\)\!\!,$DIRECTORY,g" \ + -e "s,run-with-lockfile,$REPOSITORY/commonlib/bin/run-with-lockfile.sh,g" \ + config/crontab-example > /etc/cron.d/alaveteli + +sed -r \ + -e "s,\!\!\(\*= .user \*\)\!\!,$UNIX_USER,g" \ + -e "s,\!\!\(\*= .daemon_name \*\)\!\!,foi-alert-tracks,g" \ + -e "s,\!\!\(\*= .vhost_dir \*\)\!\!,$DIRECTORY,g" \ + config/alert-tracks-debian.ugly > /etc/init.d/foi-alert-tracks + +sed -r \ + -e "s,\!\!\(\*= .user \*\)\!\!,$UNIX_USER,g" \ + -e "s,\!\!\(\*= .daemon_name \*\)\!\!,foi-alert-tracks,g" \ + -e "s,\!\!\(\*= .vhost_dir \*\)\!\!,$DIRECTORY,g" \ + config/purge-varnish-debian.ugly > /etc/init.d/foi-purge-varnish + +chmod a+rx /etc/init.d/foi-alert-tracks +chmod a+rx /etc/init.d/foi-purge-varnish + +if [ $DEFAULT_SERVER = true ] && [ x != x$EC2_HOSTNAME ] +then + # If we're setting up as the default on an EC2 instance, make sure + # that the /etc/rc.local is set up to run the install script again + # to update the hostname: + overwrite_rc_local +fi + +done_msg "Installation complete"; echo -- cgit v1.2.3 From d41c5a64379951953adc2542434a2d5ef83aef6d Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Wed, 30 Oct 2013 12:02:55 +0000 Subject: Drop PostgreSQL superuser permissions after loading sample data --- script/site-specific-install.sh | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'script/site-specific-install.sh') diff --git a/script/site-specific-install.sh b/script/site-specific-install.sh index c7ec1e59e..f23210f03 100755 --- a/script/site-specific-install.sh +++ b/script/site-specific-install.sh @@ -114,11 +114,17 @@ install_website_packages # Make the PostgreSQL user a superuser to avoid the irritating error: # PG::Error: ERROR: permission denied: "RI_ConstraintTrigger_16564" is a system trigger +# This is only needed for loading the sample data, so the superuser +# permissions are dropped below. add_postgresql_user --superuser export DEVELOPMENT_INSTALL su -c "$BIN_DIRECTORY/install-as-user '$UNIX_USER' '$HOST' '$DIRECTORY'" "$UNIX_USER" +# Now that the install-as-user script has loaded the sample data, we +# no longer need the PostgreSQL user to be a superuser: +echo "ALTER USER \"$UNIX_USER\" WITH NOSUPERUSER;" | su -l -c 'psql' postgres + if [ ! "$DEVELOPMENT_INSTALL" = true ]; then install_sysvinit_script fi -- cgit v1.2.3 From 0b809ac7df806698ddf936226a4baf245c284b2d Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Wed, 30 Oct 2013 12:10:56 +0000 Subject: We don't use foi-purge-varnish ourselves, so remove it We had various problems with the foi-purge-varnish script ourselves that led to us removing it, so it seems like a bad idea to be implicitly suggesting that others should use it. --- script/site-specific-install.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'script/site-specific-install.sh') diff --git a/script/site-specific-install.sh b/script/site-specific-install.sh index f23210f03..3c7e0a842 100755 --- a/script/site-specific-install.sh +++ b/script/site-specific-install.sh @@ -134,6 +134,7 @@ fi cd "$REPOSITORY" sed -r \ + -e "/foi-purge-varnish/d" \ -e "s,^(MAILTO=).*,\1root@$HOST," \ -e "s,\!\!\(\*= .user \*\)\!\!,$UNIX_USER,g" \ -e "s,/data/vhost/\!\!\(\*= .vhost \*\)\!\!/\!\!\(\*= .vcspath \*\)\!\!,$REPOSITORY,g" \ @@ -147,14 +148,7 @@ sed -r \ -e "s,\!\!\(\*= .vhost_dir \*\)\!\!,$DIRECTORY,g" \ config/alert-tracks-debian.ugly > /etc/init.d/foi-alert-tracks -sed -r \ - -e "s,\!\!\(\*= .user \*\)\!\!,$UNIX_USER,g" \ - -e "s,\!\!\(\*= .daemon_name \*\)\!\!,foi-alert-tracks,g" \ - -e "s,\!\!\(\*= .vhost_dir \*\)\!\!,$DIRECTORY,g" \ - config/purge-varnish-debian.ugly > /etc/init.d/foi-purge-varnish - chmod a+rx /etc/init.d/foi-alert-tracks -chmod a+rx /etc/init.d/foi-purge-varnish if [ $DEFAULT_SERVER = true ] && [ x != x$EC2_HOSTNAME ] then -- cgit v1.2.3 From c8289b5efae2b42bee2942a12038fff9c6582a76 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Thu, 31 Oct 2013 10:46:01 +0000 Subject: If the origin/install-script branch exists, use that in preference Once merged to master we can remove origin/install-script, but development is awkward without it. This commit also makes it easy to have an arbitrary number of fallbacks. --- script/site-specific-install.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'script/site-specific-install.sh') diff --git a/script/site-specific-install.sh b/script/site-specific-install.sh index 3c7e0a842..5231abc1d 100755 --- a/script/site-specific-install.sh +++ b/script/site-specific-install.sh @@ -3,8 +3,7 @@ # Set IDEAL_VERSION to the commitish we want to check out; typically # this is the version tag. Since this may not exist before release, # fall back to the master branch: -IDEAL_VERSION=0.15 -FALLBACK_VERSION=origin/master +VERSIONS="origin/install-script 0.15 origin/master" PARENT_SCRIPT_URL=https://github.com/mysociety/commonlib/blob/master/bin/install-site.sh @@ -27,8 +26,7 @@ misuse() { [ -z "$DEFAULT_SERVER" ] && misuse DEFAULT_SERVER [ -z "$HOST" ] && misuse HOST [ -z "$DISTRIBUTION" ] && misuse DISTRIBUTION -[ -z "$IDEAL_VERSION" ] && misuse VERSION -[ -z "$FALLBACK_VERSION" ] && misuse FALLBACK_VERSION +[ -z "$VERSIONS" ] && misuse VERSIONS [ -z "$DEVELOPMENT_INSTALL" ] && misuse DEVELOPMENT_INSTALL [ -z "$BIN_DIRECTORY" ] && misuse BIN_DIRECTORY @@ -37,9 +35,8 @@ update_mysociety_apt_sources if [ ! "$DEVELOPMENT_INSTALL" = true ]; then install_nginx add_website_to_nginx - # Check out the requested version: - su -l -c "cd '$REPOSITORY' && (git checkout '$IDEAL_VERSION' || - git checkout '$FALLBACK_VERSION')" \ + # Check out the first available requested version: + su -l -c "cd '$REPOSITORY' && (for v in $VERSIONS; do git checkout $v && break; done)" \ "$UNIX_USER" fi -- cgit v1.2.3 From 33fe4962c5da338f95dd30854dbdd3227d85f0d7 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Thu, 31 Oct 2013 12:57:50 +0000 Subject: Switch to using 'rake config_files:convert_init_script' Rather than reimplement the processing of the ugly file with sed, we can use the existing rake task for rewriting config/alert-tracks-debian.ugly. --- script/site-specific-install.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'script/site-specific-install.sh') diff --git a/script/site-specific-install.sh b/script/site-specific-install.sh index 5231abc1d..d7cce0dfb 100755 --- a/script/site-specific-install.sh +++ b/script/site-specific-install.sh @@ -139,13 +139,10 @@ sed -r \ -e "s,run-with-lockfile,$REPOSITORY/commonlib/bin/run-with-lockfile.sh,g" \ config/crontab-example > /etc/cron.d/alaveteli -sed -r \ - -e "s,\!\!\(\*= .user \*\)\!\!,$UNIX_USER,g" \ - -e "s,\!\!\(\*= .daemon_name \*\)\!\!,foi-alert-tracks,g" \ - -e "s,\!\!\(\*= .vhost_dir \*\)\!\!,$DIRECTORY,g" \ - config/alert-tracks-debian.ugly > /etc/init.d/foi-alert-tracks - +echo -n "Creating /etc/init.d/foi-alert-tracks... " +(su -c "cd '$REPOSITORY' && bundle exec rake config_files:convert_init_script DEPLOY_USER='$UNIX_USER' VHOST_DIR='$DIRECTORY' SCRIPT_FILE=config/alert-tracks-debian.ugly" "$UNIX_USER") > /etc/init.d/foi-alert-tracks chmod a+rx /etc/init.d/foi-alert-tracks +echo $DONE_MSG if [ $DEFAULT_SERVER = true ] && [ x != x$EC2_HOSTNAME ] then -- cgit v1.2.3 From aa9998404c976019de6355ec9a88c1884046a2ae Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Thu, 31 Oct 2013 13:06:59 +0000 Subject: Add a rake task to rewrite the crontab-example file As suggested by Louise Crow, this new rake task reuses the convert_ugly function to rewrite the crontab-example file into a usable crontab file. --- script/site-specific-install.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'script/site-specific-install.sh') diff --git a/script/site-specific-install.sh b/script/site-specific-install.sh index d7cce0dfb..da1b08b74 100755 --- a/script/site-specific-install.sh +++ b/script/site-specific-install.sh @@ -130,14 +130,15 @@ fi cd "$REPOSITORY" +echo -n "Creating /etc/cron.d/alaveteli... " +(su -c "cd '$REPOSITORY' && bundle exec rake config_files:convert_crontab DEPLOY_USER='$UNIX_USER' VHOST_DIR='$DIRECTORY' VCSPATH='$SITE' SITE='$SITE' CRONTAB=config/crontab-example" "$UNIX_USER") > /etc/cron.d/alaveteli +# There are some other parts to rewrite, so just do them with sed: sed -r \ -e "/foi-purge-varnish/d" \ -e "s,^(MAILTO=).*,\1root@$HOST," \ - -e "s,\!\!\(\*= .user \*\)\!\!,$UNIX_USER,g" \ - -e "s,/data/vhost/\!\!\(\*= .vhost \*\)\!\!/\!\!\(\*= .vcspath \*\)\!\!,$REPOSITORY,g" \ - -e "s,/data/vhost/\!\!\(\*= .vhost \*\)\!\!,$DIRECTORY,g" \ -e "s,run-with-lockfile,$REPOSITORY/commonlib/bin/run-with-lockfile.sh,g" \ - config/crontab-example > /etc/cron.d/alaveteli + -i /etc/cron.d/alaveteli +echo $DONE_MSG echo -n "Creating /etc/init.d/foi-alert-tracks... " (su -c "cd '$REPOSITORY' && bundle exec rake config_files:convert_init_script DEPLOY_USER='$UNIX_USER' VHOST_DIR='$DIRECTORY' SCRIPT_FILE=config/alert-tracks-debian.ugly" "$UNIX_USER") > /etc/init.d/foi-alert-tracks -- cgit v1.2.3 From 9aec34062f813dd5593815e0ca83aac5ff6766af Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Fri, 1 Nov 2013 12:09:40 +0000 Subject: Invoke commands as the unprivileged user in a login shell On some setups, the unprivileged user's .bashrc file won't be sourced unless you've started a login shell - we need that to be sourced to add the gem's bin directory to the PATH, or "bundle" won't be found, for example. --- script/site-specific-install.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'script/site-specific-install.sh') diff --git a/script/site-specific-install.sh b/script/site-specific-install.sh index da1b08b74..5fbab3322 100755 --- a/script/site-specific-install.sh +++ b/script/site-specific-install.sh @@ -116,7 +116,7 @@ install_website_packages add_postgresql_user --superuser export DEVELOPMENT_INSTALL -su -c "$BIN_DIRECTORY/install-as-user '$UNIX_USER' '$HOST' '$DIRECTORY'" "$UNIX_USER" +su -l -c "$BIN_DIRECTORY/install-as-user '$UNIX_USER' '$HOST' '$DIRECTORY'" "$UNIX_USER" # Now that the install-as-user script has loaded the sample data, we # no longer need the PostgreSQL user to be a superuser: @@ -131,7 +131,7 @@ fi cd "$REPOSITORY" echo -n "Creating /etc/cron.d/alaveteli... " -(su -c "cd '$REPOSITORY' && bundle exec rake config_files:convert_crontab DEPLOY_USER='$UNIX_USER' VHOST_DIR='$DIRECTORY' VCSPATH='$SITE' SITE='$SITE' CRONTAB=config/crontab-example" "$UNIX_USER") > /etc/cron.d/alaveteli +(su -l -c "cd '$REPOSITORY' && bundle exec rake config_files:convert_crontab DEPLOY_USER='$UNIX_USER' VHOST_DIR='$DIRECTORY' VCSPATH='$SITE' SITE='$SITE' CRONTAB=config/crontab-example" "$UNIX_USER") > /etc/cron.d/alaveteli # There are some other parts to rewrite, so just do them with sed: sed -r \ -e "/foi-purge-varnish/d" \ @@ -141,7 +141,7 @@ sed -r \ echo $DONE_MSG echo -n "Creating /etc/init.d/foi-alert-tracks... " -(su -c "cd '$REPOSITORY' && bundle exec rake config_files:convert_init_script DEPLOY_USER='$UNIX_USER' VHOST_DIR='$DIRECTORY' SCRIPT_FILE=config/alert-tracks-debian.ugly" "$UNIX_USER") > /etc/init.d/foi-alert-tracks +(su -l -c "cd '$REPOSITORY' && bundle exec rake config_files:convert_init_script DEPLOY_USER='$UNIX_USER' VHOST_DIR='$DIRECTORY' SCRIPT_FILE=config/alert-tracks-debian.ugly" "$UNIX_USER") > /etc/init.d/foi-alert-tracks chmod a+rx /etc/init.d/foi-alert-tracks echo $DONE_MSG -- cgit v1.2.3 From 5b1c747a0d86164cc15d13ef1dcbb90f8290ee8b Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Tue, 5 Nov 2013 14:31:05 +0000 Subject: Reduce "backscatter" bounce risk by setting local_recipient_maps Leaving the local_recipient_maps setting empty has a risk which is described in the Postfix manual: "[...] That is, an empty value. With this setting, the Postfix SMTP server will not reject mail with "User unknown in local recipient table". Don't do this on systems that receive mail directly from the Internet. With today's worms and viruses, Postfix will become a backscatter source: it accepts mail for non-existent recipients and then tries to return that mail as "undeliverable" to the often forged sender address." This commit changes the local_recipient_maps setting to only accept (and potentially bounce) emails where the local part is known (one that we've mentioned in general.yml) or to a Unix user that exists. Fixes #1166 --- script/site-specific-install.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'script/site-specific-install.sh') diff --git a/script/site-specific-install.sh b/script/site-specific-install.sh index 5fbab3322..e5a6f3f0b 100755 --- a/script/site-specific-install.sh +++ b/script/site-specific-install.sh @@ -57,7 +57,7 @@ ensure_line_present \ ensure_line_present \ "^ *local_recipient_maps *=" \ - "local_recipient_maps = " \ + "local_recipient_maps = proxy:unix:passwd.byname regexp:/etc/postfix/recipients" \ /etc/postfix/main.cf 644 ensure_line_present \ @@ -80,6 +80,13 @@ ensure_line_present \ "mail.* -/var/log/mail/mail.log" \ /etc/rsyslog.d/50-default.conf 644 +cat > /etc/postfix/recipients <> /etc/logrotate.d/rsyslog < Date: Tue, 5 Nov 2013 15:02:54 +0000 Subject: Rename /etc/postfix/regexp to /etc/postfix/transports This seems slightly less confusing, since both this file and /etc/postfix/recipients contain regexp patterns. This also switches to replacing any transport_maps file that was in use and overwrites the existing file rather than modifying it. --- script/site-specific-install.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'script/site-specific-install.sh') diff --git a/script/site-specific-install.sh b/script/site-specific-install.sh index e5a6f3f0b..8917fd577 100755 --- a/script/site-specific-install.sh +++ b/script/site-specific-install.sh @@ -51,8 +51,8 @@ ensure_line_present \ /etc/postfix/master.cf 644 ensure_line_present \ - "^ *transport_maps *= *regexp:/etc/postfix/regexp" \ - "transport_maps = regexp:/etc/postfix/regexp" \ + "^ *transport_maps *=" \ + "transport_maps = regexp:/etc/postfix/transports" \ /etc/postfix/main.cf 644 ensure_line_present \ @@ -65,11 +65,6 @@ ensure_line_present \ "mydestination = $HOST, $(hostname --fqdn), localhost.localdomain, localhost" \ /etc/postfix/main.cf 644 -ensure_line_present \ - "^.*alaveteli" \ - "/^foi.*/ alaveteli" \ - /etc/postfix/regexp 644 - ensure_line_present \ "^do-not-reply" \ "do-not-reply-to-this-address: :blackhole:" \ @@ -80,6 +75,10 @@ ensure_line_present \ "mail.* -/var/log/mail/mail.log" \ /etc/rsyslog.d/50-default.conf 644 +cat > /etc/postfix/transports < /etc/postfix/recipients <