diff options
Diffstat (limited to 'bin')
-rwxr-xr-x | bin/ec2-rewrite-conf | 20 | ||||
-rwxr-xr-x | bin/fetch-comments | 4 | ||||
-rwxr-xr-x | bin/gettext-extract | 12 | ||||
-rwxr-xr-x | bin/gettext-nget-patch | 18 | ||||
-rwxr-xr-x | bin/install-as-user | 111 | ||||
-rwxr-xr-x | bin/install_perl_modules | 8 | ||||
-rwxr-xr-x | bin/make_css | 18 | ||||
-rwxr-xr-x | bin/make_emptyhomes_welsh_po | 14 | ||||
-rwxr-xr-x | bin/make_po (renamed from bin/make_emptyhomes_po) | 18 | ||||
-rwxr-xr-x | bin/open311-populate-service-list | 13 | ||||
-rwxr-xr-x | bin/problem-creation-graph | 14 | ||||
-rwxr-xr-x | bin/problems-filed-graph | 6 | ||||
-rwxr-xr-x | bin/rotate-photos | 34 | ||||
-rw-r--r-- | bin/site-specific-install.sh | 53 |
14 files changed, 284 insertions, 59 deletions
diff --git a/bin/ec2-rewrite-conf b/bin/ec2-rewrite-conf new file mode 100755 index 000000000..0163ef511 --- /dev/null +++ b/bin/ec2-rewrite-conf @@ -0,0 +1,20 @@ +#!/bin/sh + +# This is a helper script for writing the current EC2 hostname into +# the FixMyStreet configuration file. Its intended usage is for lines +# like these to be added to /etc/rc.local: +# +# su -l -c /home/fms/fixmystreet/bin/ec2-rewrite-conf fms +# /etc/init.d/apache2 restart + +set -e + +BIN_DIR=$(dirname $(readlink -f $0)) +CONF_DIR=$BIN_DIR/../conf + +HOST=`curl -s http://169.254.169.254/latest/meta-data/public-hostname` + +sed -i -r \ + -e "s,^( *BASE_URL:).*,\\1 'http://$HOST'," \ + -e "s,^( *EMAIL_DOMAIN:).*,\\1 '$HOST'," \ + $CONF_DIR/general.yml diff --git a/bin/fetch-comments b/bin/fetch-comments index 4bbcc9d21..ef099fcc9 100755 --- a/bin/fetch-comments +++ b/bin/fetch-comments @@ -5,11 +5,13 @@ use warnings; require 5.8.0; use FixMyStreet::App; +use CronFns; +my ($verbose, $nomail) = CronFns::options(); use Open311; use Open311::GetServiceRequestUpdates; -my $updates = Open311::GetServiceRequestUpdates->new; +my $updates = Open311::GetServiceRequestUpdates->new( verbose => $verbose ); $updates->fetch; diff --git a/bin/gettext-extract b/bin/gettext-extract index 55623e86c..e77cf9cb0 100755 --- a/bin/gettext-extract +++ b/bin/gettext-extract @@ -31,23 +31,27 @@ rm -f $PO xgettext.pl --gnu-gettext --verbose --output $PO --plugin perl=* --plugin tt2 --directory perllib --directory templates/web --directory db --directory bin # Fix headers -TEMP=`tempfile` +# no such thing as tempfile on OS X +TEMP=`tempfile 2>/dev/null || mktemp /tmp/gettext-extract.XXXXXX` NOW=`date +"%Y-%m-%d %H:%M%z"` +# strictly POSIX sed on e.g. OS X doesn't let you used \n in replacements so we do this +nl=$'\n'; cat $PO | sed " s/SOME DESCRIPTIVE TITLE/FixMyStreet original .po file, autogenerated by gettext-extract/; s/YEAR THE PACKAGE'S COPYRIGHT HOLDER/2011 UK Citizens Online Democracy/; s/PACKAGE package/main FixMyStreet code/; s/FIRST AUTHOR <EMAIL@ADDRESS>, YEAR./Matthew Somerville <matthew@mysociety.org>, 2011-06-03./; - s/PACKAGE VERSION/1.0\\\n\"\n\"Report-Msgid-Bugs-To: matthew@mysociety.org/; + s/PACKAGE VERSION/1.0\\\n\"\\$nl\"Report-Msgid-Bugs-To: matthew@mysociety.org/; s/POT-Creation-Date: YEAR-MO-DA HO:MI+ZONE/POT-Creation-Date: $NOW/; s/LL@li.org/team@fixmystreet.com/; s/charset=CHARSET/charset=UTF-8/; - s/8bit/8bit\\\n\"\n\"Plural-Forms: nplurals=INTEGER; plural=EXPRESSION;/; + s/8bit/8bit\\\n\"\\$nl\"Plural-Forms: nplurals=2; plural=n != 1;/; " >> $TEMP mv $TEMP $PO echo "$( bin/gettext-nget-patch )" >> $PO -bin/make_emptyhomes_po +bin/make_po FixMyStreet-EmptyHomes +bin/make_po FixMyBarangay diff --git a/bin/gettext-nget-patch b/bin/gettext-nget-patch index 223bcc816..5ebd8bbcb 100755 --- a/bin/gettext-nget-patch +++ b/bin/gettext-nget-patch @@ -9,6 +9,7 @@ my %out; find( sub { next unless -f; + next if $File::Find::name =~ /ttc$/; open (FP, $_) or die $!; while (<FP>) { next unless /nget/; @@ -17,16 +18,17 @@ find( sub { do { $text .= <FP>; } until $text =~ /\)/; - $text =~ /nget\(\s*"(.*?)"\s*,\s*"(.*?)"\s*,\s*(.*?)\s*\)/s; - $out{$1} = { - file => $File::Find::name, - line => $line, - s => $1, - p => $2, - }; + if ($text =~ /nget\(\s*"(.*?)"\s*,\s*"(.*?)"\s*,\s*(.*?)\s*\)/s) { + $out{$1} = { + file => $File::Find::name, + line => $line, + s => $1, + p => $2, + }; + } } close FP; -}, 'templates'); +}, 'templates', 'perllib'); foreach (values %out) { print <<EOF; diff --git a/bin/install-as-user b/bin/install-as-user new file mode 100755 index 000000000..2656195f4 --- /dev/null +++ b/bin/install-as-user @@ -0,0 +1,111 @@ +#!/bin/sh + +set -e +set -x + +if [ $# -ne 3 ] +then + cat >&2 <<EOUSAGE +Usage: $0 <UNIX-USER> <HOST> <INSTALLATION-DIRECTORY> +EOUSAGE + exit 1 +fi + +UNIX_USER="$1" +HOST="$2" +DIRECTORY="$3" +DB_NAME="fixmystreet" + +# Check that the arguments we've been passed are sensible: + +IP_ADDRESS_FOR_HOST="$(dig +short $HOST)" + +if [ x = x"$IP_ADDRESS_FOR_HOST" ] +then + echo "The hostname $HOST didn't resolve to an IP address" + exit 1 +fi + +if ! id "$UNIX_USER" 2> /dev/null > /dev/null +then + echo "The user '$UNIX_USER' didn't exist." + exit 1 +fi + +if [ "$(whoami)" != "$UNIX_USER" ] +then + echo "This script should be run by the user '$UNIX_USER'." + exit 1 +fi + +REPOSITORY="$DIRECTORY/fixmystreet" +LINK_DESTINATION="$HOME/fixmystreet" + +ln -sfn "$REPOSITORY" $LINK_DESTINATION +cd "$REPOSITORY" + +# Add regularly scheduled tasks to cron: + +TEMPORARY_CRONTAB=$(mktemp) + +echo crontab file is $TEMPORARY_CRONTAB + +cp "$REPOSITORY"/conf/crontab.example "$TEMPORARY_CRONTAB" + +sed -i \ + -e 's,$FMS,'"$REPOSITORY,g" \ + -e 's,$LOCK_DIR,'"$DIRECTORY,g" \ + "$TEMPORARY_CRONTAB" + +crontab $TEMPORARY_CRONTAB + +# Install the compass gem locally - it's required for generating the +# CSS: + +export GEM_HOME="$DIRECTORY/gems" +mkdir -p "$GEM_HOME" +export GEM_PATH= +export PATH="$GEM_HOME/bin:$PATH" + +gem install --no-ri --no-rdoc compass + +# Use compass to generate the CSS, if it doesn't seem to already +# exist: + +if [ ! -f web/cobrands/default/base.css ] +then + bin/make_css +fi + +# Write sensible values into the config file: + +sed -r \ + -e "s,^( *FMS_DB_HOST:).*,\\1 ''," \ + -e "s,^( *FMS_DB_NAME:).*,\\1 '$DB_NAME'," \ + -e "s,^( *FMS_DB_USER:).*,\\1 '$UNIX_USER'," \ + -e "s,^( *BASE_URL:).*,\\1 'http://$HOST'," \ + -e "s,^( *EMAIL_DOMAIN:).*,\\1 '$HOST'," \ + -e "s,^( *CONTACT_EMAIL:).*,\\1 'help@$HOST'," \ + conf/general.yml-example > conf/general.yml + +# Create the database if it doesn't exist: +if ! psql -l | egrep "^ *$DB_NAME *\|" > /dev/null +then + createdb --owner "$UNIX_USER" "$DB_NAME" + echo 'CREATE LANGUAGE plpgsql;' | psql -U "$UNIX_USER" "$DB_NAME" || true + psql -U "$UNIX_USER" "$DB_NAME" < "$REPOSITORY"/db/schema.sql + psql -U "$UNIX_USER" "$DB_NAME" < "$REPOSITORY"/db/alert_types.sql + psql -U "$UNIX_USER" "$DB_NAME" < "$REPOSITORY"/db/generate_secret.sql +fi + +# Install the required Perl modules - this may take a very long time: + +cd "$FMS_REPOSITORY" +bin/install_perl_modules + +# Generate po and mo files (these invocations taken from Kagee's script): + +bin/cron-wrapper bin/make_po FixMyStreet-EmptyHomes +bin/cron-wrapper bin/make_emptyhomes_welsh_po + +commonlib/bin/gettext-makemo FixMyStreet diff --git a/bin/install_perl_modules b/bin/install_perl_modules index 2df4ffbf8..2311ae5f1 100755 --- a/bin/install_perl_modules +++ b/bin/install_perl_modules @@ -1,5 +1,7 @@ #!/bin/bash +set -e + DIR="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd | sed -e 's/\/bin$//' )" $DIR/bin/cpanm -l $DIR/local Carton @@ -9,11 +11,7 @@ export PERL5LIB=$DIR/local/lib/perl5 carton install --deployment -perl -MImage::Magick -e 'exit()' >/dev/null 2>&1 - -HAVE_IM=$? - -if [ $HAVE_IM -ne 0 ] +if ! perl -MImage::Magick -e 'exit()' >/dev/null 2>&1 then read -p "Image::Magick is not installed. Do you want to attempt to install it?" yn case $yn in diff --git a/bin/make_css b/bin/make_css index 4494c92e7..a86fd4f0d 100755 --- a/bin/make_css +++ b/bin/make_css @@ -14,15 +14,19 @@ DIRECTORY=$(cd `dirname $0`/../web && pwd) # FixMyStreet uses compass -compass compile --output-style compressed $DIRECTORY/cobrands/fixmystreet -compass compile --output-style compressed $DIRECTORY/cobrands/bromley -compass compile --output-style compressed $DIRECTORY/cobrands/barnet -compass compile --output-style compressed $DIRECTORY/cobrands/zurich -compass compile --output-style compressed $DIRECTORY/cobrands/stevenage -compass compile --output-style compressed $DIRECTORY/cobrands/default +NEWSTYLE=${1:-"fixmystreet bromley fixmybarangay barnet zurich default stevenage"} +NEWSTYLE_REGEX=${NEWSTYLE// /\\|} +for site in $NEWSTYLE; do + compass compile --output-style compressed $DIRECTORY/cobrands/$site +done + +# If given a command line argument, assume was a compass directory and exit +if [ -n "$1" ]; then + exit 0 +fi # The rest are plain sass -for scss in `find $DIRECTORY -name "*.scss" -exec dirname {} \; | uniq | grep -v "cobrands/\(fixmystreet\|bromley\|barnet\|stevenage\|default\|zurich\)"` +for scss in `find $DIRECTORY -name "*.scss" -exec dirname {} \; | uniq | grep -v "cobrands/\($NEWSTYLE_REGEX\)"` do sass --scss --update --style compressed $scss done diff --git a/bin/make_emptyhomes_welsh_po b/bin/make_emptyhomes_welsh_po index b3dd441a0..f4f6850d6 100755 --- a/bin/make_emptyhomes_welsh_po +++ b/bin/make_emptyhomes_welsh_po @@ -87,15 +87,21 @@ while(<MAINPO>) { $new_buffer =~ s/"\n"//g; if ($lookup{$new_buffer} && $lookup{$new_buffer}[1]) { - print OUTPO "#, fuzzy\n"; - } + print OUTPO "#, fuzzy\n"; + } print OUTPO $buffer; if ($lookup{$new_buffer}) { print OUTPO $lookup{$new_buffer}[0]; } else { - print __LINE__ . "\n"; - die "Failed to find match with buffer: $new_buffer"; + if (m/^msgstr\[0\] ""/) { + $new_buffer =~ s/^msgid "/msgstr[0] "/m; + $new_buffer =~ s/^msgid_plural "/msgstr[1] "/m; + $_ = <MAINPO>; # skip untranslated plural + } else { + $new_buffer =~ s/^msgid "/msgstr "/; + } + print OUTPO $new_buffer; } $buffer = ""; diff --git a/bin/make_emptyhomes_po b/bin/make_po index 10e840599..76dc4566b 100755 --- a/bin/make_emptyhomes_po +++ b/bin/make_po @@ -1,8 +1,8 @@ #!/usr/bin/perl -w use strict; -# Generates EmptyHomes version of .po file, which is a translation -# into a language the same as English, only "problem" becomes "empty property". +# Generates a version of .po file, which is a translation +# into a language the same as English, with replacement as specified in PoChange use POSIX; use FindBin; @@ -13,14 +13,16 @@ chdir("$FindBin::Bin/../locale"); mkdir("en_GB.UTF-8"); mkdir("en_GB.UTF-8/LC_MESSAGES"); +my $pofile = shift; + open(MAINPO, "FixMyStreet.po") or die ""; -open(EHAPO, ">FixMyStreet-EmptyHomes.po") or die ""; -open(NEWPO, ">en_GB.UTF-8/LC_MESSAGES/FixMyStreet-EmptyHomes.po") or die ""; +open(EHAPO, ">$pofile.po") or die ""; +open(NEWPO, ">en_GB.UTF-8/LC_MESSAGES/$pofile.po") or die ""; -print NEWPO "# AUTOMATICALLY GENERATED by make_emptyhomes_po, do not edit\n"; +print NEWPO "# AUTOMATICALLY GENERATED by make_po, do not edit\n"; print NEWPO "#\n"; -print EHAPO "# AUTOMATICALLY GENERATED by make_emptyhomes_po, do not edit\n"; +print EHAPO "# AUTOMATICALLY GENERATED by make_po, do not edit\n"; print EHAPO "#\n"; my $buffer = ""; @@ -30,7 +32,7 @@ while(<MAINPO>) { s/#, fuzzy/#/; } if (m/"Last-Translator: FULL NAME/) { - $_ = '"Last-Translator: mysociety/bin/make_emptyhomes_po\\n"'."\n"; + $_ = '"Last-Translator: mysociety/bin/make_po\\n"'."\n"; } if (m/"PO-Revision-Date: YEAR-MO-DA/) { my $time = POSIX::strftime("%Y-%m-%d %H:%M%z", localtime(time())); @@ -56,7 +58,7 @@ while(<MAINPO>) { } elsif ($start && (m/^msgstr ""/ || m/^msgstr\[0\] ""/)) { # start of translated text - translate English into Empty Homes language - $buffer = PoChange::fixmystreet_to_reportemptyhomes($buffer); + $buffer = PoChange::translate($pofile, $buffer); print EHAPO $buffer; diff --git a/bin/open311-populate-service-list b/bin/open311-populate-service-list index 9588e4360..232b0a6d4 100755 --- a/bin/open311-populate-service-list +++ b/bin/open311-populate-service-list @@ -4,11 +4,22 @@ use strict; use warnings; use FixMyStreet::App; use Open311::PopulateServiceList; +use Getopt::Long::Descriptive; +my ($opt, $usage) = describe_options( + '%c %o', + ['verbose|v', "print out all services as they are found"], + ['warn|w', "output warnings about any issues"], + ['help', "print usage message and exit" ], +); +print($usage->text), exit if $opt->help; my $council_list = FixMyStreet::App->model('DB::Open311conf')->search( { send_method => 'Open311' } ); -my $p = Open311::PopulateServiceList->new( council_list => $council_list ); +my $verbose = 0; +$verbose = 1 if $opt->warn; +$verbose = 2 if $opt->verbose; +my $p = Open311::PopulateServiceList->new( council_list => $council_list, verbose => $verbose ); $p->process_councils; diff --git a/bin/problem-creation-graph b/bin/problem-creation-graph index 6692ae724..b96d45540 100755 --- a/bin/problem-creation-graph +++ b/bin/problem-creation-graph @@ -26,12 +26,12 @@ else DATE="2007-02-01" fi -SOURCEA=/tmp/bci-creation-rate-graph-data-$RANDOM$RANDOM -SOURCEB=/tmp/bci-creation-rate-graph-data-$RANDOM$RANDOM -SOURCEC=/tmp/bci-creation-rate-graph-data-$RANDOM$RANDOM -SOURCED=/tmp/bci-creation-rate-graph-data-$RANDOM$RANDOM -SOURCEE=/tmp/bci-creation-rate-graph-data-$RANDOM$RANDOM -GPSCRIPT=/tmp/bci-creation-rate-graph-script-$RANDOM$RANDOM +SOURCEA=/tmp/fms-creation-rate-graph-data-$RANDOM$RANDOM +SOURCEB=/tmp/fms-creation-rate-graph-data-$RANDOM$RANDOM +SOURCEC=/tmp/fms-creation-rate-graph-data-$RANDOM$RANDOM +SOURCED=/tmp/fms-creation-rate-graph-data-$RANDOM$RANDOM +SOURCEE=/tmp/fms-creation-rate-graph-data-$RANDOM$RANDOM +GPSCRIPT=/tmp/fms-creation-rate-graph-script-$RANDOM$RANDOM # where status in ('draft') @@ -104,6 +104,6 @@ END #echo "gpscript $GPSCRIPT" export GDFONTPATH=/usr/share/fonts/truetype/ttf-bitstream-vera -gnuplot < $GPSCRIPT > fixmystreet/web/bci-live-creation$EXTENSION 2>/dev/null +gnuplot < $GPSCRIPT > fixmystreet/web/fms-live-creation$EXTENSION 2>/dev/null diff --git a/bin/problems-filed-graph b/bin/problems-filed-graph index 8addacd62..e5946b078 100755 --- a/bin/problems-filed-graph +++ b/bin/problems-filed-graph @@ -20,8 +20,8 @@ source fixmystreet/commonlib/shlib/deployfns read_conf fixmystreet/conf/general.yml -SOURCEO=/tmp/bci-report-rate-graph-data-nonwmc-$RANDOM$RANDOM -GPSCRIPT=/tmp/bci-report-rate-graph-script-$RANDOM$RANDOM +SOURCEO=/tmp/fms-report-rate-graph-data-nonwmc-$RANDOM$RANDOM +GPSCRIPT=/tmp/fms-report-rate-graph-script-$RANDOM$RANDOM echo "select date(created), count(*) @@ -57,5 +57,5 @@ END #echo "gpscript $GPSCRIPT" export GDFONTPATH=/usr/share/fonts/truetype/ttf-bitstream-vera -gnuplot < $GPSCRIPT > fixmystreet/web/bci-live-line$EXTENSION 2>/dev/null +gnuplot < $GPSCRIPT > fixmystreet/web/fms-live-line$EXTENSION 2>/dev/null diff --git a/bin/rotate-photos b/bin/rotate-photos index faf2748e6..31a60ff6c 100755 --- a/bin/rotate-photos +++ b/bin/rotate-photos @@ -16,6 +16,8 @@ use FindBin; use lib "$FindBin::Bin/../perllib"; use lib "$FindBin::Bin/../commonlib/perllib"; +use Digest::SHA1 qw(sha1_hex); + use Utils; use mySociety::Config; use mySociety::DBHandle qw(dbh select_all); @@ -36,17 +38,27 @@ my $r = select_all("select id, photo from problem where service='iPhone'"); foreach (@$r) { my $id = $_->{id}; my $photo = $_->{photo}; - my ($fh, $filename) = mySociety::TempFiles::named_tempfile('.jpeg'); - print $fh $photo; - close $fh; - my $out = `jhead -autorot $filename`; - if ($out) { - open(FP, $filename) or die $!; - $photo = join('', <FP>); - close FP; - Utils::workaround_pg_bytea("update problem set photo=? where id=?", 1, $photo, $id); - dbh()->commit(); + + if (length($photo) == 40) { + # If photo field contains a hash + my $filename = mySociety::Config::get('UPLOAD_DIR') . "$photo.jpeg"; + `jhead -autorot $filename`; + } else { + my ($fh, $filename) = mySociety::TempFiles::named_tempfile('.jpeg'); + print $fh $photo; + close $fh; + my $out = `jhead -autorot $filename`; + if ($out) { + open(FP, $filename) or die $!; + $photo = join('', <FP>); + close FP; + my $fileid = sha1_hex($photo); + rename $filename, mySociety::Config::get('UPLOAD_DIR') . "$fileid.jpeg"; + dbh()->do('UPDATE problem SET photo=? WHERE id=?', {}, $fileid, $id); + dbh()->commit(); + } else { + unlink $filename; + } } - unlink $filename; } diff --git a/bin/site-specific-install.sh b/bin/site-specific-install.sh new file mode 100644 index 000000000..6ebac7970 --- /dev/null +++ b/bin/site-specific-install.sh @@ -0,0 +1,53 @@ +#!/bin/sh + +PARENT_SCRIPT_URL=https://github.com/mysociety/commonlib/blob/master/bin/install-site.sh + +misuse() { + echo The variable $1 was not defined, and it should be. + echo This script should not be run directly - instead, please run: + echo $PARENT_SCRIPT_URL + exit 1 +} + +# Strictly speaking we don't need to check all of these, but it might +# catch some errors made when changing install-site.sh + +[ -z "$DIRECTORY" ] && misuse DIRECTORY +[ -z "$UNIX_USER" ] && misuse UNIX_USER +[ -z "$REPOSITORY" ] && misuse REPOSITORY +[ -z "$REPOSITORY_URL" ] && misuse REPOSITORY_URL +[ -z "$BRANCH" ] && misuse BRANCH +[ -z "$SITE" ] && misuse SITE +[ -z "$DEFAULT_SERVER" ] && misuse DEFAULT_SERVER +[ -z "$HOST" ] && misuse HOST +[ -z "$DISTRIBUTION" ] && misuse DISTRIBUTION +[ -z "$VERSION" ] && misuse VERSION + +install_nginx + +install_website_packages + +su -l -c "touch '$DIRECTORY/admin-htpasswd'" "$UNIX_USER" + +add_website_to_nginx + +add_postgresql_user + +su -l -c "$REPOSITORY/bin/install-as-user '$UNIX_USER' '$HOST' '$DIRECTORY'" "$UNIX_USER" + +install_sysvinit_script + +if [ $DEFAULT_SERVER = true ] && [ x != x$EC2_HOSTNAME ] +then + # If we're setting up as the default on an EC2 instance, + # make sure the ec2-rewrite-conf script is called from + # /etc/rc.local + overwrite_rc_local +fi + +# Tell the user what to do next: + +echo Installation complete - you should now be able to view the site at: +echo http://$HOST/ +echo Or you can run the tests by switching to the "'$UNIX_USER'" user and +echo running: $REPOSITORY/bin/cron-wrapper prove -r t |