aboutsummaryrefslogtreecommitdiffstats
path: root/bin
diff options
context:
space:
mode:
Diffstat (limited to 'bin')
-rwxr-xr-xbin/fetch-comments4
-rw-r--r--bin/fetch-comments-24hs5
-rwxr-xr-xbin/generate_cobrand_po40
-rwxr-xr-xbin/geocode79
-rwxr-xr-xbin/gettext-extract43
-rwxr-xr-xbin/gettext-merge21
-rwxr-xr-xbin/handlemail1
-rwxr-xr-xbin/handlemail-support62
-rwxr-xr-xbin/install-as-user108
-rwxr-xr-xbin/install_perl_modules10
-rwxr-xr-xbin/make_css22
-rwxr-xr-xbin/make_css_watch78
-rwxr-xr-xbin/make_po1
-rwxr-xr-xbin/merge_cobrand_po29
-rwxr-xr-xbin/open311-populate-service-list7
-rw-r--r--bin/open311-update-reports10
-rwxr-xr-xbin/oxfordshire/open311_service_request.cgi427
-rwxr-xr-xbin/oxfordshire/open311_service_request_update.cgi126
-rw-r--r--bin/oxfordshire/open311_services.pm150
-rwxr-xr-xbin/problem-creation-graph8
-rwxr-xr-xbin/rotate-photos2
-rwxr-xr-xbin/send-comments56
-rw-r--r--bin/site-specific-install.sh26
-rwxr-xr-xbin/update-all-reports47
-rwxr-xr-xbin/update-schema145
-rwxr-xr-xbin/update-send-questionnaire29
-rwxr-xr-xbin/update_po_header.bash33
-rwxr-xr-xbin/zerotb_import_clinic_list.pl115
-rwxr-xr-xbin/zurich-overdue-alert84
-rwxr-xr-xbin/zurich/convert_internal_notes_to_comments73
-rwxr-xr-xbin/zurich/geocode45
31 files changed, 1731 insertions, 155 deletions
diff --git a/bin/fetch-comments b/bin/fetch-comments
index ef099fcc9..a276433e4 100755
--- a/bin/fetch-comments
+++ b/bin/fetch-comments
@@ -1,4 +1,8 @@
#!/usr/bin/env perl
+#
+# This script utilises the Open311 extension explained at
+# https://github.com/mysociety/FixMyStreet/wiki/Open311-FMS---Proposed-differences-to-Open311
+# to fetch updates on service requests.
use strict;
use warnings;
diff --git a/bin/fetch-comments-24hs b/bin/fetch-comments-24hs
index b84f09ba7..602383d40 100644
--- a/bin/fetch-comments-24hs
+++ b/bin/fetch-comments-24hs
@@ -1,4 +1,9 @@
#!/usr/bin/env perl
+#
+# This script utilises the Open311 extension explained at
+# https://github.com/mysociety/FixMyStreet/wiki/Open311-FMS---Proposed-differences-to-Open311
+# to fetch updates on service requests from the past 24 hours, to check none
+# were missed.
use strict;
use warnings;
diff --git a/bin/generate_cobrand_po b/bin/generate_cobrand_po
new file mode 100755
index 000000000..1770de7c2
--- /dev/null
+++ b/bin/generate_cobrand_po
@@ -0,0 +1,40 @@
+#!/usr/bin/env perl
+use strict;
+
+=for instructions
+
+This script generates a po file that contains only the strings related to a cobrand.
+Its purpose is to avoid having lots of cobrand specific strings in the
+main FixMyStreet po file confusing translators. Once this is run the
+generated po file can be used to translate the strings. The po files
+with the translation should then be placed in the relevant language
+directories and merge_cobrand_po run which will merge these with
+the main FixMyStreet po files
+
+=cut
+
+my $cobrand = shift;
+
+die "Please provide a cobrand name\n" unless $cobrand;
+
+my $cobrand_lc = lc( $cobrand );
+
+my $PO = "locale/$cobrand.po";
+
+my $cobrand_module = "perllib/FixMyStreet/Cobrand/$cobrand.pm";
+my $web_templates = "templates/web/$cobrand_lc/";
+
+my $opts = '';
+if ( -e $cobrand_module ) {
+ $opts .= " $cobrand_module";
+}
+
+if ( -e -d $web_templates ) {
+ $opts .= " --directory $web_templates";
+}
+
+# first we get the strings from the cobrand
+system("xgettext.pl --gnu-gettext --verbose --output $PO --plugin perl=* --plugin tt2 $opts");
+
+# now replace the template header elements
+system("bin/update_po_header.bash $PO");
diff --git a/bin/geocode b/bin/geocode
new file mode 100755
index 000000000..254cf1578
--- /dev/null
+++ b/bin/geocode
@@ -0,0 +1,79 @@
+#!/usr/bin/perl
+
+=head1 NAME
+
+geocode - commandline tool to test geocoders
+
+=head1 SYNOPSIS
+
+ $ eval `perl setenv.pl`, or call with bin/cron-wrapper
+
+ $ bin/geocode --geocoder=Bing --cobrand=bromley "Glebe Rd"
+
+ # ... if your conf/general.yml supplies GEOCODER:
+ $ bin/geocode --cobrand=bromley "Glebe Rd"
+
+ # ... if you want to use config that you have in conf/general.bing.yml
+ $ bin/geocode --override-config=general.bing --cobrand=bromley "Glebe Rd"
+
+ ## ... output from geocoder
+
+=cut
+
+use strict;
+use warnings;
+require 5.8.0;
+
+use Data::Dumper;
+use Pod::Usage;
+use feature 'say';
+
+use Getopt::Long;
+
+my %options = ( help => sub { pod2usage(0) } );
+
+GetOptions \%options,
+ 'geocoder=s',
+ 'help|h',
+ 'cobrand=s',
+ 'override-config=s';
+
+my $s = join ' ', @ARGV
+ or pod2usage(0);
+
+pod2usage(0) unless $options{cobrand};
+
+local $ENV{FMS_OVERRIDE_CONFIG} = $options{'override-config'} if $options{'override-config'};
+
+eval 'use FixMyStreet';
+eval 'use FixMyStreet::App';
+eval 'use FixMyStreet::Cobrand';
+eval 'use FixMyStreet::Geocode';
+
+mySociety::Locale::gettext_domain( 'FixMyStreet' );
+
+my $geocoder_type = $options{geocoder} || do {
+ my $GEOCODER = FixMyStreet->config('GEOCODER');
+ ref $GEOCODER ? $GEOCODER->{type} : $GEOCODER;
+} or pod2usage(0);
+
+my $geocoder_name = "FixMyStreet::Geocode::${geocoder_type}";
+my $code_ref = $geocoder_name->can('string')
+ or die "$geocoder_name is not a valid geocoder?";
+
+my @allowed_cobrands = FixMyStreet::Cobrand->get_allowed_cobrands();
+
+my $cobrand_name = FixMyStreet::Cobrand->get_class_for_moniker($options{cobrand});
+my $cobrand = $cobrand_name->new();
+
+say "USING COBRAND $cobrand_name";
+if ($cobrand->moniker ne lc($options{cobrand})) {
+ say "!!! asked for $options{cobrand}";
+ say "!!! Check ALLOWED_COBRANDS setting in conf/general.yml (or supplied --override-config file)";
+ say Dumper(\@allowed_cobrands);
+}
+
+my $c = FixMyStreet::App->new();
+$c->stash->{cobrand} = $cobrand;
+
+say Dumper( $code_ref->( $s, $c ) );
diff --git a/bin/gettext-extract b/bin/gettext-extract
index e77cf9cb0..a38c17127 100755
--- a/bin/gettext-extract
+++ b/bin/gettext-extract
@@ -7,48 +7,25 @@
# Copyright (c) 2011 UK Citizens Online Democracy. All rights reserved.
# Email: matthew@mysociety.org; WWW: http://www.mysociety.org/
-if [ -e ../../locale ]
-then
- cd ../../
-else if [ -e ../locale ]
-then
- cd ../
-else if [ -e locale ]
-then
- cd .
-else
- echo "Please run with current directory fixmystreet/bin"
- exit 1
-fi
-fi
-fi
+cd "$(dirname $(readlink -f $BASH_SOURCE))/.."
# File to write to, clear it to start with
PO=locale/FixMyStreet.po
rm -f $PO
+# we don't want to extract strings from all the cobrand templates so list
+# the ones we care about
+find templates/web/default templates/web/fixmystreet templates/web/zurich templates/web/fiksgatami templates/web/emptyhomes templates/web/fixmybarangay -name '*.html' > template_list
+
# Extract from Perl
-xgettext.pl --gnu-gettext --verbose --output $PO --plugin perl=* --plugin tt2 --directory perllib --directory templates/web --directory db --directory bin
+xgettext.pl --gnu-gettext --verbose --output $PO --plugin perl=* --plugin tt2 --directory perllib -f template_list --directory db --directory bin
+
+# remove temporary list of templates
+rm template_list
# Fix headers
# no such thing as tempfile on OS X
-TEMP=`tempfile 2>/dev/null || mktemp /tmp/gettext-extract.XXXXXX`
-NOW=`date +"%Y-%m-%d %H:%M%z"`
-# strictly POSIX sed on e.g. OS X doesn't let you used \n in replacements so we do this
-nl=$'\n';
-cat $PO | sed "
- s/SOME DESCRIPTIVE TITLE/FixMyStreet original .po file, autogenerated by gettext-extract/;
- s/YEAR THE PACKAGE'S COPYRIGHT HOLDER/2011 UK Citizens Online Democracy/;
- s/PACKAGE package/main FixMyStreet code/;
- s/FIRST AUTHOR <EMAIL@ADDRESS>, YEAR./Matthew Somerville <matthew@mysociety.org>, 2011-06-03./;
-
- s/PACKAGE VERSION/1.0\\\n\"\\$nl\"Report-Msgid-Bugs-To: matthew@mysociety.org/;
- s/POT-Creation-Date: YEAR-MO-DA HO:MI+ZONE/POT-Creation-Date: $NOW/;
- s/LL@li.org/team@fixmystreet.com/;
- s/charset=CHARSET/charset=UTF-8/;
- s/8bit/8bit\\\n\"\\$nl\"Plural-Forms: nplurals=2; plural=n != 1;/;
-" >> $TEMP
-mv $TEMP $PO
+bin/update_po_header.bash $PO
echo "$( bin/gettext-nget-patch )" >> $PO
diff --git a/bin/gettext-merge b/bin/gettext-merge
index 84aa4fdb7..b84708ece 100755
--- a/bin/gettext-merge
+++ b/bin/gettext-merge
@@ -6,22 +6,7 @@
# Copyright (c) 2011 UK Citizens Online Democracy. All rights reserved.
# Email: matthew@mysociety.org; WWW: http://www.mysociety.org/
-# Yuck
-if [ -e ../../locale ]
-then
- cd ../../
-else if [ -e ../locale ]
-then
- cd ../
-else if [ -e locale ]
-then
- cd .
-else
- echo "Please run with current directory fixmystreet"
- exit 1
-fi
-fi
-fi
+cd "$(dirname $(readlink -f $BASH_SOURCE))/.."
for X in locale/*.UTF-8
do
@@ -31,12 +16,12 @@ do
echo $X
if [ -e EmptyHomes.po ]
then
- msgmerge -o New.po EmptyHomes.po ../../FixMyStreet-EmptyHomes.po
+ msgmerge --no-wrap -o New.po EmptyHomes.po ../../FixMyStreet-EmptyHomes.po
mv New.po EmptyHomes.po
fi
if [ -e FixMyStreet.po ]
then
- msgmerge -o New.po FixMyStreet.po ../../FixMyStreet.po
+ msgmerge --no-wrap -o New.po FixMyStreet.po ../../FixMyStreet.po
mv New.po FixMyStreet.po
fi
cd - >/dev/null
diff --git a/bin/handlemail b/bin/handlemail
index 8b8e03be9..8bc016241 100755
--- a/bin/handlemail
+++ b/bin/handlemail
@@ -57,6 +57,7 @@ my $mail = mySociety::Email::construct_email({
To => $data{return_path},
_template_ => $template,
_parameters_ => { },
+ _line_indent => '',
});
if (mySociety::EmailUtil::EMAIL_SUCCESS
diff --git a/bin/handlemail-support b/bin/handlemail-support
new file mode 100755
index 000000000..9808e447a
--- /dev/null
+++ b/bin/handlemail-support
@@ -0,0 +1,62 @@
+#!/usr/bin/perl -w
+#
+# handlemail-support:
+# Handle an individual incoming mail message.
+#
+# This script should be invoked through the .forward mechanism. It processes
+# emails to the support address to remove out of office and so on, before
+# forwarding on.
+#
+# Copyright (c) 2013 UK Citizens Online Democracy. All rights reserved.
+# Email: matthew@mysociety.org; WWW: http://www.mysociety.org/
+
+use strict;
+require 5.8.0;
+
+# Horrible boilerplate to set up appropriate library paths.
+use FindBin;
+use lib "$FindBin::Bin/../perllib";
+use lib "$FindBin::Bin/../commonlib/perllib";
+
+use mySociety::Config;
+BEGIN {
+ mySociety::Config::set_file("$FindBin::Bin/../conf/general");
+}
+use mySociety::EmailUtil;
+use mySociety::HandleMail;
+
+my %data = mySociety::HandleMail::get_message();
+exit 0 if is_ignorable($data{message});
+forward_on();
+
+# ---
+
+sub forward_on {
+ my ($l, $d) = split /\@/, mySociety::Config::get('CONTACT_EMAIL');
+ if (mySociety::EmailUtil::EMAIL_SUCCESS
+ != mySociety::EmailUtil::send_email(
+ join("\n", @{$data{lines}}) . "\n",
+ $data{return_path},
+ join('@', join('_deli', $l, 'very'), $d)
+ )) {
+ exit 75;
+ }
+ exit 0;
+}
+
+sub is_ignorable {
+ my $m = shift;
+ my $head = $m->head();
+ my ($from, $subject, $body) = ($head->get('From'), $head->get('Subject'), $m->body);
+ $body = join("\n", @$body);
+
+ open my $fp, "$FindBin::Bin/../../data/ignored-emails.csv" or exit 75;
+ while (<$fp>) {
+ chomp;
+ my ($f, $s, $b) = split /,/;
+ next unless $f || $s || $b;
+ return 1 unless ( $f && $from !~ /$f/ ) || ( $s && $subject !~ /$s/ ) || ( $b && $body !~ /$b/ );
+ }
+ return 0;
+}
+
diff --git a/bin/install-as-user b/bin/install-as-user
index 2656195f4..b05616702 100755
--- a/bin/install-as-user
+++ b/bin/install-as-user
@@ -1,7 +1,10 @@
#!/bin/sh
set -e
-set -x
+error_msg() { printf "\033[31m%s\033[0m\n" "$*"; }
+notice_msg() { printf "\033[33m%s\033[0m " "$*"; }
+done_msg() { printf "\033[32m%s\033[0m\n" "$*"; }
+DONE_MSG=$(done_msg done)
if [ $# -ne 3 ]
then
@@ -14,6 +17,15 @@ fi
UNIX_USER="$1"
HOST="$2"
DIRECTORY="$3"
+
+misuse() {
+ echo The variable $1 was not defined, and it should be.
+ echo This script should not be run directly.
+ exit 1
+}
+
+[ -z "$DEVELOPMENT_INSTALL" ] && misuse DEVELOPMENT_INSTALL
+
DB_NAME="fixmystreet"
# Check that the arguments we've been passed are sensible:
@@ -22,19 +34,19 @@ IP_ADDRESS_FOR_HOST="$(dig +short $HOST)"
if [ x = x"$IP_ADDRESS_FOR_HOST" ]
then
- echo "The hostname $HOST didn't resolve to an IP address"
+ error_msg "The hostname $HOST didn't resolve to an IP address"
exit 1
fi
if ! id "$UNIX_USER" 2> /dev/null > /dev/null
then
- echo "The user '$UNIX_USER' didn't exist."
+ error_msg "The user '$UNIX_USER' didn't exist."
exit 1
fi
if [ "$(whoami)" != "$UNIX_USER" ]
then
- echo "This script should be run by the user '$UNIX_USER'."
+ error_msg "This script should be run by the user '$UNIX_USER'."
exit 1
fi
@@ -44,51 +56,71 @@ LINK_DESTINATION="$HOME/fixmystreet"
ln -sfn "$REPOSITORY" $LINK_DESTINATION
cd "$REPOSITORY"
-# Add regularly scheduled tasks to cron:
-
-TEMPORARY_CRONTAB=$(mktemp)
-
-echo crontab file is $TEMPORARY_CRONTAB
-
-cp "$REPOSITORY"/conf/crontab.example "$TEMPORARY_CRONTAB"
-
-sed -i \
- -e 's,$FMS,'"$REPOSITORY,g" \
- -e 's,$LOCK_DIR,'"$DIRECTORY,g" \
- "$TEMPORARY_CRONTAB"
-
-crontab $TEMPORARY_CRONTAB
+if [ ! "$DEVELOPMENT_INSTALL" = true ]; then
+ echo -n "Adding crontab... "
+ # Add regularly scheduled tasks to cron:
+ TEMPORARY_CRONTAB=$(mktemp)
+ echo crontab file is $TEMPORARY_CRONTAB
+ cp "$REPOSITORY"/conf/crontab-example "$TEMPORARY_CRONTAB"
+ sed -i \
+ -e 's,$FMS,'"$REPOSITORY,g" \
+ -e 's,$LOCK_DIR,'"$DIRECTORY,g" \
+ "$TEMPORARY_CRONTAB"
+ crontab $TEMPORARY_CRONTAB
+ echo $DONE_MSG
+fi
# Install the compass gem locally - it's required for generating the
# CSS:
-
+echo "Setting up CSS... "
export GEM_HOME="$DIRECTORY/gems"
mkdir -p "$GEM_HOME"
export GEM_PATH=
export PATH="$GEM_HOME/bin:$PATH"
-gem install --no-ri --no-rdoc compass
+if ! grep -q 'Set up local gem directory for FixMyStreet' $HOME/.bashrc; then
+ cat >>$HOME/.bashrc <<EOBRC
+
+# Set up local gem directory for FixMyStreet
+export GEM_HOME="$DIRECTORY/gems"
+export GEM_PATH=
+export PATH="\$GEM_HOME/bin:\$PATH"
+EOBRC
+fi
+
+gem install --conservative --no-ri --no-rdoc sass -v 3.2.14
+gem install --conservative --no-ri --no-rdoc compass -v 0.12.2
# Use compass to generate the CSS, if it doesn't seem to already
# exist:
-
if [ ! -f web/cobrands/default/base.css ]
then
bin/make_css
fi
+echo $DONE_MSG
+
+# Write sensible values into the config file, if it doesn't already exist
+if [ ! -f conf/general.yml ]; then
+ echo -n "Setting up default conf/general.yml file... "
+ sed -r \
+ -e "s,^( *FMS_DB_HOST:).*,\\1 ''," \
+ -e "s,^( *FMS_DB_NAME:).*,\\1 '$DB_NAME'," \
+ -e "s,^( *FMS_DB_USER:).*,\\1 '$UNIX_USER'," \
+ -e "s,^( *BASE_URL:).*,\\1 'http://$HOST'," \
+ -e "s,^( *EMAIL_DOMAIN:).*,\\1 '$HOST'," \
+ -e "s,^( *CONTACT_EMAIL:).*,\\1 'help@$HOST'," \
+ -e "s,^( *DO_NOT_REPLY_EMAIL:).*,\\1 'help@$HOST'," \
+ conf/general.yml-example > conf/general.yml
+ echo $DONE_MSG
+fi
-# Write sensible values into the config file:
-
-sed -r \
- -e "s,^( *FMS_DB_HOST:).*,\\1 ''," \
- -e "s,^( *FMS_DB_NAME:).*,\\1 '$DB_NAME'," \
- -e "s,^( *FMS_DB_USER:).*,\\1 '$UNIX_USER'," \
- -e "s,^( *BASE_URL:).*,\\1 'http://$HOST'," \
- -e "s,^( *EMAIL_DOMAIN:).*,\\1 '$HOST'," \
- -e "s,^( *CONTACT_EMAIL:).*,\\1 'help@$HOST'," \
- conf/general.yml-example > conf/general.yml
+echo "Installing required Perl modules - this may take some time"
+cd "$REPOSITORY"
+bin/install_perl_modules
+echo $DONE_MSG
# Create the database if it doesn't exist:
+echo -n "Setting up database... "
if ! psql -l | egrep "^ *$DB_NAME *\|" > /dev/null
then
createdb --owner "$UNIX_USER" "$DB_NAME"
@@ -96,16 +128,16 @@ then
psql -U "$UNIX_USER" "$DB_NAME" < "$REPOSITORY"/db/schema.sql
psql -U "$UNIX_USER" "$DB_NAME" < "$REPOSITORY"/db/alert_types.sql
psql -U "$UNIX_USER" "$DB_NAME" < "$REPOSITORY"/db/generate_secret.sql
+else
+ bin/cron-wrapper update-schema --commit
fi
-
-# Install the required Perl modules - this may take a very long time:
-
-cd "$FMS_REPOSITORY"
-bin/install_perl_modules
+echo $DONE_MSG
# Generate po and mo files (these invocations taken from Kagee's script):
-
+echo "Creating locale .mo files"
bin/cron-wrapper bin/make_po FixMyStreet-EmptyHomes
bin/cron-wrapper bin/make_emptyhomes_welsh_po
-
commonlib/bin/gettext-makemo FixMyStreet
+echo $DONE_MSG
+
+bin/cron-wrapper update-all-reports
diff --git a/bin/install_perl_modules b/bin/install_perl_modules
index 2311ae5f1..883ced235 100755
--- a/bin/install_perl_modules
+++ b/bin/install_perl_modules
@@ -4,18 +4,18 @@ set -e
DIR="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd | sed -e 's/\/bin$//' )"
-$DIR/bin/cpanm -l $DIR/local Carton
+$DIR/bin/cpanm -l $DIR/local-carton Carton
-export PATH=$DIR/local/bin:$PATH
-export PERL5LIB=$DIR/local/lib/perl5
+export PATH=$DIR/local-carton/bin:$PATH
+export PERL5LIB=$DIR/local-carton/lib/perl5
-carton install --deployment
+carton install --deployment --without uk
if ! perl -MImage::Magick -e 'exit()' >/dev/null 2>&1
then
read -p "Image::Magick is not installed. Do you want to attempt to install it?" yn
case $yn in
- [Yy]* ) $DIR/local/bin/carton install Image::Magick;;
+ [Yy]* ) $DIR/local-carton/bin/carton install Image::Magick;;
* ) echo 'You will need to install it for FixMyStreet to work';;
esac
fi
diff --git a/bin/make_css b/bin/make_css
index a86fd4f0d..c48dda4d6 100755
--- a/bin/make_css
+++ b/bin/make_css
@@ -13,20 +13,12 @@
DIRECTORY=$(cd `dirname $0`/../web && pwd)
-# FixMyStreet uses compass
-NEWSTYLE=${1:-"fixmystreet bromley fixmybarangay barnet zurich default stevenage"}
-NEWSTYLE_REGEX=${NEWSTYLE// /\\|}
-for site in $NEWSTYLE; do
- compass compile --output-style compressed $DIRECTORY/cobrands/$site
-done
-
-# If given a command line argument, assume was a compass directory and exit
-if [ -n "$1" ]; then
- exit 0
-fi
+DIRS=${1:-`find $DIRECTORY -name "*.scss" -exec dirname {} \; | uniq`}
-# The rest are plain sass
-for scss in `find $DIRECTORY -name "*.scss" -exec dirname {} \; | uniq | grep -v "cobrands/\($NEWSTYLE_REGEX\)"`
-do
- sass --scss --update --style compressed $scss
+for dir in $DIRS; do
+ if [ -e "$dir/config.rb" ]; then
+ compass compile --output-style compressed $dir
+ else
+ sass --scss --update --style compressed $dir
+ fi
done
diff --git a/bin/make_css_watch b/bin/make_css_watch
new file mode 100755
index 000000000..d46ee8997
--- /dev/null
+++ b/bin/make_css_watch
@@ -0,0 +1,78 @@
+#!/usr/bin/env perl
+use strict;
+use warnings;
+use feature 'say';
+use File::ChangeNotify;
+use File::Find::Rule;
+use Path::Tiny;
+
+my @exts = qw/
+ scss
+/;
+
+my @dirs = qw/
+ web
+/;
+
+my $filter = do {
+ my $exts = join '|', @exts;
+ qr/\.(?:$exts)$/
+};
+
+my $watcher = File::ChangeNotify->instantiate_watcher(
+ directories => \@dirs,
+ filter => $filter,
+);
+
+sub title {
+ my $what = shift;
+ # TODO, check if xtitle is installed and if so, run following command:
+ # system 'xtitle', $what;
+}
+
+say sprintf "Watching [%s] for %s", (join ',' => @dirs), $filter;
+title 'watching';
+
+while ( my @events = $watcher->wait_for_events() ) {
+ my %seen;
+ my @update_dirs;
+ title 'updating';
+ for my $event (@events) {
+ my $file = path( $event->path );
+ say "$file was updated...";
+ my $dir = $file->dirname;
+ next if $seen{$dir}++;
+
+ if ($dir eq 'web/cobrands/sass/') {
+ # contains only partials, so we don't need to update
+ # this directory, but we *do* need to update everything else
+ push @update_dirs,
+ grep {
+ ! ($seen{$_}++)
+ }
+ map {
+ path($_)->dirname
+ }
+ File::Find::Rule->file->name($filter)->in( @dirs );
+ }
+ else {
+ push @update_dirs, $dir;
+ }
+ }
+ for my $dir (@update_dirs) {
+ if (-e "$dir/config.rb") {
+ system compass =>
+ 'compile',
+ '--output-style' => 'compressed',
+ $dir;
+ }
+ else {
+ system sass =>
+ '--scss',
+ '--update',
+ '--style' => 'compressed',
+ $dir;
+ }
+ }
+ title 'watching';
+}
diff --git a/bin/make_po b/bin/make_po
index 76dc4566b..ad1fc1a89 100755
--- a/bin/make_po
+++ b/bin/make_po
@@ -14,6 +14,7 @@ mkdir("en_GB.UTF-8");
mkdir("en_GB.UTF-8/LC_MESSAGES");
my $pofile = shift;
+die "Please supply a filename" unless $pofile;
open(MAINPO, "FixMyStreet.po") or die "";
open(EHAPO, ">$pofile.po") or die "";
diff --git a/bin/merge_cobrand_po b/bin/merge_cobrand_po
new file mode 100755
index 000000000..2785162cf
--- /dev/null
+++ b/bin/merge_cobrand_po
@@ -0,0 +1,29 @@
+#!/usr/bin/env perl
+use strict;
+
+=for instructions
+
+This script is used to merge cobrand po files with the main
+FixMyStreet po file. It should be run after generate_cobrand_po
+and once the cobrand po files with translations are placed in the
+language directories.
+
+It will then create an autoCobrand.po file for each language that
+has a Cobrand.po
+
+=cut
+
+my $cobrand = shift;
+
+die "Please provide a cobrand name\n" unless $cobrand;
+
+# for each language create a .po file with an existing translations
+for (glob( 'locale/*/LC_MESSAGES' ) ) {
+ my $fms = "$_/FixMyStreet.po";
+ my $cobrand_po = "$_/$cobrand.po";
+ my $out = "$_/auto$cobrand.po";
+ if ( -e $cobrand_po and -e $fms ) {
+ print "$_\n";
+ system("msgcat --no-wrap -o $out $fms $cobrand_po");
+ }
+}
diff --git a/bin/open311-populate-service-list b/bin/open311-populate-service-list
index 232b0a6d4..be1ace3b9 100755
--- a/bin/open311-populate-service-list
+++ b/bin/open311-populate-service-list
@@ -14,12 +14,13 @@ my ($opt, $usage) = describe_options(
);
print($usage->text), exit if $opt->help;
-my $council_list = FixMyStreet::App->model('DB::Open311conf')->search( {
+my $bodies = FixMyStreet::App->model('DB::Body')->search( {
+ id => { '!=', 2237 }, # XXX Until Oxfordshire does do so
send_method => 'Open311'
} );
my $verbose = 0;
$verbose = 1 if $opt->warn;
$verbose = 2 if $opt->verbose;
-my $p = Open311::PopulateServiceList->new( council_list => $council_list, verbose => $verbose );
+my $p = Open311::PopulateServiceList->new( bodies => $bodies, verbose => $verbose );
-$p->process_councils;
+$p->process_bodies;
diff --git a/bin/open311-update-reports b/bin/open311-update-reports
index 41c9c4546..3b77fef89 100644
--- a/bin/open311-update-reports
+++ b/bin/open311-update-reports
@@ -1,4 +1,10 @@
#!/usr/bin/perl
+#
+# This script utilises the standard Open311 way of getting updates on reports
+# (by fetching all reports for a body and looking for updates). If possible,
+# please use the extension explained at
+# https://github.com/mysociety/FixMyStreet/wiki/Open311-FMS---Proposed-differences-to-Open311
+# and the fetch-comments/send-comments scripts.
use strict;
use warnings;
@@ -13,9 +19,9 @@ my $system_user = FixMyStreet::App->model('DB::User')->find_or_create(
}
);
-my $council_list = FixMyStreet::App->model('DB::Open311conf');
+my $body_list = FixMyStreet::App->model('DB::Body');
my $update = Open311::GetUpdates->new(
- council_list => $council_list,
+ body_list => $body_list,
system_user => $system_user
)->get_updates;
diff --git a/bin/oxfordshire/open311_service_request.cgi b/bin/oxfordshire/open311_service_request.cgi
new file mode 100755
index 000000000..4496ff213
--- /dev/null
+++ b/bin/oxfordshire/open311_service_request.cgi
@@ -0,0 +1,427 @@
+#!/usr/bin/perl
+
+# script for absobring incoming Open311 service request POSTs and
+# passing them into Bentley EXOR backend via create_enquiry stored
+# procedure.
+#
+# mySociety: http://fixmystreet.org/
+#-----------------------------------------------------------------
+
+require 'open311_services.pm';
+use DBD::Oracle qw(:ora_types);
+### for local testing (no Oracle):
+### use constant { ORA_VARCHAR2=>1, ORA_DATE=>1, ORA_NUMBER=>1};
+
+my %PEM_BOUND_VAR_TYPES = get_pem_field_types();
+
+my $ERR_MSG = 'error'; # unique key in data hash
+
+# incoming (Open311, from FMS) field names
+# note: attribute[*] are being sent by FMS explicitly as attributes for Oxfordshire
+my %F = (
+ 'ACCOUNT_ID' => 'account_id',
+ 'ADDRESS_ID' => 'address_id',
+ 'ADDRESS_STRING' => 'address_string',
+ 'API_KEY' => 'api_key',
+ 'CLOSEST_ADDRESS' => 'attribute[closest_address]',
+ 'DESCRIPTION' => 'description',
+ 'DEVICE_ID' => 'device_id',
+ 'EASTING' => 'attribute[easting]',
+ 'EMAIL' => 'email',
+ 'FIRST_NAME' => 'first_name',
+ 'FMS_ID' => 'attribute[external_id]',
+ 'LAST_NAME' => 'last_name',
+ 'LAT' => 'lat',
+ 'LONG' => 'long',
+ 'MEDIA_URL' => 'media_url',
+ 'NORTHING' => 'attribute[northing]',
+ 'PHONE' => 'phone',
+ 'REQUESTED_DATETIME' => 'requested_datetime',
+ 'SERVICE_CODE' => 'service_code',
+ 'STATUS' => 'status',
+
+);
+
+my $req = new CGI;
+
+# normally, POST requests are inserting service requests
+# and GET requests are for returning service requests, although OCC aren't planning on
+# using that (it's part of the Open311 spec).
+# So actually the service discovery is more useful, so send in a 'services' param
+# to see that.
+#
+# But for testing the db connection, set $TEST_SERVICE_DISCOVERY so that
+# *all* requests simply do a service discovery by setting (possibly via the config file)
+
+if ($TEST_SERVICE_DISCOVERY) {
+ get_service_discovery($req); # to test
+}elsif ($ENV{'REQUEST_METHOD'} eq "POST") {
+ post_service_request($req);
+} elsif ($req -> param('services')) {
+ get_service_discovery($req);
+} elsif ($RUN_FAKE_INSERT_TEST) {
+ # allow a GET to make an insert, for testing (from the commandnd line!)
+ print "Running fake insert test... returned: " . get_FAKE_INSERT();
+ print "\nSee $OUT_FILENAME for data" if $TESTING_WRITE_TO_FILE;
+ print "\n";
+} else {
+ get_service_requests($req);
+}
+
+#----------------------------------------------------
+# post_service_request
+# accepts an incoming service request
+# If everything goes well, it puts it in the database and
+# returns the PEM ID to the caller
+#----------------------------------------------------
+sub post_service_request {
+ my $req = shift;
+ my %data;
+ my $pem_id = 0;
+
+ foreach (values %F) {
+ $data{$_} = $req -> param($_);
+ $data{$_} =~ s/^\s+|\s+$//g; # trim
+ }
+
+ error_and_exit(CODE_OR_ID_NOT_PROVIDED, "missing service code (Open311 requires one)")
+ unless $data{$F{SERVICE_CODE}};
+ error_and_exit(GENERAL_SERVICE_ERROR, "the service code you provided ($data{$F{SERVICE_CODE}}) was not recognised by this server")
+ unless service_exists($data{$F{SERVICE_CODE}});
+ error_and_exit(GENERAL_SERVICE_ERROR, "no address or long-lat provided")
+ unless ( (is_longlat($data{$F{LONG}}) && is_longlat($data{$F{LAT}})) || $data{$F{ADDRESS_STRING}} );
+
+ $pem_id = insert_into_pem(\%data);
+
+ if (! $pem_id) {
+ error_and_exit(FATAL_ERROR, $data{$ERR_MSG} || "failed to get PEM ID");
+ } else {
+ print <<XML;
+Content-type: text/xml
+
+<?xml version="1.0" encoding="utf-8"?>
+<service_requests>
+ <request>
+ <service_request_id>$pem_id</service_request_id>
+ </request>
+</service_requests>
+XML
+ }
+}
+
+#------------------------------------------------------------------
+# is_longlat
+# returns true if this looks like a long/lat value
+#------------------------------------------------------------------
+sub is_longlat {
+ return $_[0] =~ /^-?\d+\.\d+$/o? 1 : 0;
+}
+
+#------------------------------------------------------------------
+# service_exists
+# lookup the service code, to check that it exists
+# SELECT det_code, det_name FROM higatlas.doc_enquiry_types WHERE
+# det_dtp_code = 'REQS' AND det_dcl_code ='SERV' AND det_con_id=1
+# Actually, FMS is expected to be sending good codes because they
+# come from here anyway, and are expected to be stable. But could
+# cache and check... probably overkill since DB insert will probably
+# throw the error anyway.
+#------------------------------------------------------------------
+sub service_exists {
+ my $service_code = shift;
+ return 1;
+}
+
+#------------------------------------------------------------------
+# dump_to_file
+# args: ref to hash of data
+# for testing, log the incoming data into a local file
+# NB throws a fatal error
+#------------------------------------------------------------------
+sub dump_to_file {
+ my $h = shift; # href to data hash
+ if (open (OUTFILE, ">$OUT_FILENAME")) {
+ print OUTFILE "Data dump: " . gmtime() . "\n" . '-' x 64 . "\n\n";
+ foreach (sort keys %$h) {
+ print OUTFILE "$_ => " . $$h{$_} . "\n";
+ }
+ print OUTFILE "\n\n" . '-' x 64 . "\n[end]\n";
+ close OUTFILE;
+ $$h{$ERR_MSG} = "NB did not write to DB (see $OUT_FILENAME instead: switch off \$TESTING_WRITE_TO_FILE to stop this)";
+ } else {
+ $$h{$ERR_MSG} = "failed to write to outfile ($!)";
+ }
+ return 0; # test always throws an error so no risk of production confusion!
+}
+
+#------------------------------------------------------------------
+# insert_into_pem
+# args: hashref to data hash
+# returns PEM id of the new record (or passes an error message
+# into the data hash if no id is available)
+#------------------------------------------------------------------
+sub insert_into_pem {
+ my $h = shift; # href to data hash
+
+ my $pem_id;
+ my $error_value;
+ my $error_product;
+
+ # set specifc vars up where further processing on them might be needed:
+ my $undef = undef;
+ my $status = $$h{$F{STATUS}};
+ my $service_code = $$h{$F{SERVICE_CODE}};
+ my $description = $$h{$F{DESCRIPTION}};
+ my $media_url = $$h{$F{MEDIA_URL}};
+ if ($media_url) {
+ # don't put URL for full images into the database (because they're too big to see on a Blackberry)
+ $media_url =~ s/\.full(\.jpe?g)$/$1/;
+ $description .= ($STRIP_CONTROL_CHARS ne 'ruthless'? "\n\n":" ") . "Photo: $media_url";
+ }
+ my $location = $$h{$F{CLOSEST_ADDRESS}};
+ if ($location) {
+ # strip out everything apart from "Nearest" preamble
+ $location=~s/(Nearest road)[^:]+:/$1:/;
+ $location=~s/(Nearest postcode)[^:]+:(.*?)(\(\w+ away\))?\s*(\n|$)/$1: $2/;
+ }
+
+ my %bindings;
+ # comments here are suggested values
+ # field lengths are from OCC's Java portlet
+ # fixed values
+ $bindings{":ce_cat"} = 'REQS'; # or REQS ?
+ $bindings{":ce_class"} = 'SERV'; # 'FRML' ?
+ $bindings{":ce_contact_type"} = 'ENQUIRER'; # 'ENQUIRER'
+ $bindings{":ce_status_code"} = 'RE'; # RE=received (?)
+ $bindings{":ce_compl_user_type"}= 'USER'; # 'USER'
+
+ # ce_incident_datetime is *not* an optional param, but FMS isn't sending it at the moment
+ $bindings{":ce_incident_datetime"}=$$h{$F{REQUESTED_DATETIME}} || Time::Piece->new->strftime('%Y-%m-%d %H:%M');
+
+ # especially FMS-specific:
+ $bindings{":ce_source"} = "FMS"; # important, and specific to this script!
+ $bindings{":ce_doc_reference"} = $$h{$F{FMS_ID}}; # FMS id
+ $bindings{":ce_enquiry_type"} = $service_code;
+
+ # incoming data
+ $bindings{":ce_x"} = $$h{$F{EASTING}};
+ $bindings{":ce_y"} = $$h{$F{NORTHING}};
+ $bindings{":ce_forename"} = uc strip($$h{$F{FIRST_NAME}}, 30); # 'CLIFF'
+ $bindings{":ce_surname"} = uc strip($$h{$F{LAST_NAME}}, 30); # 'STEWART'
+ $bindings{":ce_work_phone"} = strip($$h{$F{PHONE}}, 25); # '0117 600 4200'
+ $bindings{":ce_email"} = uc strip($$h{$F{EMAIL}}, 50); # 'info@exor.co.uk'
+ $bindings{":ce_description"} = strip($description, 1970, $F{DESCRIPTION}); # 'Large Pothole'
+
+ # nearest address guesstimate
+ $bindings{":ce_location"} = strip($location, 254);
+
+ if ($TESTING_WRITE_TO_FILE) {
+ return dump_to_file(\%bindings);
+ }
+
+ # everything ready: now put it into the database
+ my $dbh = get_db_connection();
+
+ my $sth = $dbh->prepare(q#
+ BEGIN
+ PEM.create_enquiry(
+ ce_cat => :ce_cat,
+ ce_class => :ce_class,
+ ce_forename => :ce_forename,
+ ce_surname => :ce_surname,
+ ce_contact_type => :ce_contact_type,
+ ce_location => :ce_location,
+ ce_work_phone => :ce_work_phone,
+ ce_email => :ce_email,
+ ce_description => :ce_description,
+ ce_enquiry_type => :ce_enquiry_type,
+ ce_source => :ce_source,
+ ce_incident_datetime => to_Date(:ce_incident_datetime,'YYYY-MM-DD HH24:MI'),
+ ce_x => :ce_x,
+ ce_y => :ce_y,
+ ce_doc_reference => :ce_doc_reference,
+ ce_status_code => :ce_status_code,
+ ce_compl_user_type => :ce_compl_user_type,
+ error_value => :error_value,
+ error_product => :error_product,
+ ce_doc_id => :ce_doc_id);
+ END;
+#);
+
+ foreach my $name (sort keys %bindings) {
+ next if grep {$name eq $_} (':error_value', ':error_product', ':ce_doc_id'); # return values (see below)
+ $sth->bind_param(
+ $name,
+ $bindings{$name},
+ $PEM_BOUND_VAR_TYPES{$name} || ORA_VARCHAR2
+ );
+ }
+ # return values are bound explicitly here:
+ $sth->bind_param_inout(":error_value", \$error_value, 12); #> l_ERROR_VALUE # number
+ $sth->bind_param_inout(":error_product", \$error_product, 10); #> l_ERROR_PRODUCT (will always be 'DOC')
+ $sth->bind_param_inout(":ce_doc_id", \$pem_id, 12); #> l_ce_doc_id # number
+
+ # not used, but from the example docs, for reference
+ # $sth->bind_param(":ce_contact_title", $undef); # 'MR'
+ # $sth->bind_param(":ce_postcode", $undef); # 'BS11EJ' NB no spaces, upper case
+ # $sth->bind_param(":ce_building_no", $undef); # '1'
+ # $sth->bind_param(":ce_building_name", $undef); # 'CLIFTON HEIGHTS'
+ # $sth->bind_param(":ce_street", $undef); # 'HIGH STREET'
+ # $sth->bind_param(":ce_town", $undef); # 'BRSITOL'
+ # $sth->bind_param(":ce_enquiry_type", $undef); # 'CD' , ce_source => 'T'
+ # $sth->bind_param(":ce_cpr_id", $undef); # '5' (priority)
+ # $sth->bind_param(":ce_rse_he_id", $undef); #> nm3net.get_ne_id('1200D90970/09001','L')
+ # $sth->bind_param(":ce_compl_target", $undef); # '08-JAN-2004'
+ # $sth->bind_param(":ce_compl_corresp_date",$undef); # '02-JAN-2004'
+ # $sth->bind_param(":ce_compl_corresp_deliv_date", $undef); # '02-JAN-2004'
+ # $sth->bind_param(":ce_resp_of", $undef); # 'GBOWLER'
+ # $sth->bind_param(":ce_hct_vip", $undef); # 'CO'
+ # $sth->bind_param(":ce_hct_home_phone", $undef); # '0117 900 6201'
+ # $sth->bind_param(":ce_hct_mobile_phone", $undef); # '07111 1111111'
+ # $sth->bind_param(":ce_compl_remarks", $undef); # remarks (notes) max 254 char
+
+ $sth->execute();
+ $dbh->disconnect;
+
+ # if error, maybe need to look it up:
+ # error_value is the index HER_NO in table HIG_ERRORS, which has messages
+ # actually err_product not helpful (wil always be "DOC")
+ $$h{$ERR_MSG} = "$error_value $error_product" if ($error_value || $error_product);
+
+ return $pem_id;
+}
+
+#------------------------------------------------------------------
+# strip
+# args: data, max-length, field-name
+# Trims, strips control chars, truncates to max-length
+# Field-name only matters for description field
+#------------------------------------------------------------------
+sub strip {
+ my ($s, $max_len, $field_name) = @_;
+ if ($STRIP_CONTROL_CHARS) {
+ if ($STRIP_CONTROL_CHARS eq 'ruthless') {
+ $s =~ s/[[:cntrl:]]/ /g; # strip all control chars, simples
+ } elsif ($STRIP_CONTROL_CHARS eq 'desc') {
+ if ($field_name eq $F{DESCRIPTION}) {
+ $s =~ s/[^\t\n[:^cntrl:]]/ /g; # leave tabs and newlines
+ } else {
+ $s =~ s/[[:cntrl:]]/ /g; # strip all control chars, simples
+ }
+ } else {
+ $s =~ s/[^\t\n[:^cntrl:]]/ /g; # leave tabs and newlines
+ }
+ }
+ from_to($s, 'utf8', 'Windows-1252') if $ENCODE_TO_WIN1252;
+ return $max_len? substr($s, 0, $max_len) : $s;
+}
+
+#------------------------------------------------------------------
+# get_service_requests
+# Strictly speaking, Open311 would expect the GET request for service
+# requests to respond with all service requests (within a specified
+# period). But as we're not using that, do a service discovery
+# instead.
+#------------------------------------------------------------------
+sub get_service_requests {
+ # error_and_exit(BAD_METHOD, "sorry, currently only handling incoming Open311 service requests: use POST method");
+ get_service_discovery(); # for now
+}
+
+#------------------------------------------------------------------
+# get_FAKE_INSERT
+# for testing off command line, force the data
+#------------------------------------------------------------------
+sub get_FAKE_INSERT {
+ my %fake_data = (
+ $F{'DESCRIPTION'} => 'Testing, description: A acute (requires Latin-1): [á] '
+ . ' pound sign (requires WinLatin-1): [£] omega tonos (requires UTF-8): [ώ]',
+ $F{'EASTING'} => '45119',
+ $F{'EMAIL'} => 'email@example.com',
+ $F{'FIRST_NAME'} => 'Dave',
+ $F{'FMS_ID'} => '1012',
+ $F{'LAST_NAME'} => 'Test',
+ $F{'LAT'} => '51.756741605999',
+ $F{'LONG'} => '-1.2596387532192',
+ $F{'NORTHING'} => '206709',
+ $F{'SERVICE_CODE'} => 'OT',
+ $F{'MEDIA_URL'} => 'http://www.example.com/pothole.jpg',
+ $F{'CLOSEST_ADDRESS'} => <<TEXT
+Nearest road to the pin placed on the map (automatically generated by Bing Maps): St Giles, Oxford, OX1 3
+
+Nearest postcode to the pin placed on the map (automatically generated): OX1 2LA (46m away)
+TEXT
+ );
+ return insert_into_pem(\%fake_data)
+}
+
+#------------------------------------------------------------------
+# get_service_discovery
+# Although not really implementing this, use it as a test to read the
+# db and confirm connectivity.
+#------------------------------------------------------------------
+sub get_service_discovery {
+ my $dbh = get_db_connection();
+ my $ary_ref = $dbh->selectall_arrayref(qq(select det_code, det_name from higatlas.doc_enquiry_types where det_dtp_code = 'REQS' AND det_dcl_code='SERV' and det_con_id=1));
+ # rough and ready XML dump now (should use XML Simple to build/escape properly!)
+ my $xml = "";
+ foreach my $row(@{$ary_ref}) {
+ if (defined $row) {
+ my ($code, $name) = @$row;
+ $xml.= <<XML;
+ <service>
+ <service_code>$code</service_code>
+ <metadata>false</metadata>
+ <type>realtime</type>
+ <keywords/>
+ <group/>
+ <service_name>$name</service_name>
+ <description/>
+ </service>
+XML
+ }
+ }
+ print <<XML;
+Content-type: text/xml
+
+<?xml version="1.0" encoding="utf-8"?>
+<services>
+$xml
+</services>
+XML
+}
+
+#------------------------------------------------------------------
+# get_pem_field_types
+# return hash of types by field name: any not explicitly set here
+# can be defaulted to VARCHAR2
+#------------------------------------------------------------------
+sub get_pem_field_types {
+ return (
+ ':ce_incident_datetime' => ORA_DATE,
+ ':ce_x' => ORA_NUMBER,
+ ':ce_y' => ORA_NUMBER,
+ ':ce_date_expires' => ORA_DATE,
+ ':ce_issue_number' => ORA_NUMBER,
+ ':ce_status_date' => ORA_DATE,
+ ':ce_compl_ack_date' => ORA_DATE,
+ ':ce_compl_peo_date' => ORA_DATE,
+ ':ce_compl_target' => ORA_DATE,
+ ':ce_compl_complete' => ORA_DATE,
+ ':ce_compl_from' => ORA_DATE,
+ ':ce_compl_to' => ORA_DATE,
+ ':ce_compl_corresp_date' => ORA_DATE,
+ ':ce_compl_corresp_deliv_date' => ORA_DATE,
+ ':ce_compl_no_of_petitioners' => ORA_NUMBER,
+ ':ce_compl_est_cost' => ORA_NUMBER,
+ ':ce_compl_adv_cost' => ORA_NUMBER,
+ ':ce_compl_act_cost' => ORA_NUMBER,
+ ':ce_compl_follow_up1' => ORA_DATE,
+ ':ce_compl_follow_up2' => ORA_DATE,
+ ':ce_compl_follow_uo3' => ORA_DATE,
+ ':ce_date_time_arrived' => ORA_DATE,
+ ':error_value' => ORA_NUMBER,
+ ':ce_doc_id' => ORA_NUMBER,
+ )
+}
+
diff --git a/bin/oxfordshire/open311_service_request_update.cgi b/bin/oxfordshire/open311_service_request_update.cgi
new file mode 100755
index 000000000..87b14fb98
--- /dev/null
+++ b/bin/oxfordshire/open311_service_request_update.cgi
@@ -0,0 +1,126 @@
+#!/usr/bin/perl
+
+# script for querying the higatlas.fms_update table provided by
+# Bentley and offering them up as XML service request updates.
+# https://github.com/mysociety/fixmystreet/wiki/Open311-FMS---Proposed-differences-to-Open311
+#
+# mySociety: http://fixmystreet.org/
+#-----------------------------------------------------------------
+
+require 'open311_services.pm';
+
+# incoming query params
+my $CGI_VAR_START_DATE = 'start_date';
+my $CGI_VAR_END_DATE = 'end_date';
+my $CGI_VAR_NO_DEFAULT_DATE = 'force_no_default_date'; # for testing scratchy Oracle date stuff
+my $CGI_VAR_LIMIT = 'limit'; # for testing
+my $CGI_VAR_ANY_STATUS = 'any_status'; # for testing
+
+my $USE_ONLY_DATES = 0; # dates not times
+my $MAX_LIMIT = 1000;
+my $STATUS_CRITERIA = "(status='OPEN' OR status='CLOSED')";
+my $req = new CGI;
+
+get_service_request_updates($req);
+
+sub prepare_for_xml {
+ my $s = shift;
+ foreach ($s) {
+ from_to($_, 'utf8', 'Windows-1252') if $DECODE_FROM_WIN1252;
+ s/</&lt;/g; # numpty escaping pending XML Simple?
+ s/>/&gt;/g;
+ s/&/&amp;/g;
+ }
+ return $s;
+}
+
+#------------------------------------------------------------------
+# get_service_discovery
+# Although not really implementing this, use it as a test to read the
+# db and confirm connectivity.
+#
+# TABLE "HIGATLAS"."FMS_UPDATE"
+#
+# "ROW_ID" NUMBER(9,0) NOT NULL ENABLE,
+# "SERVICE_REQUEST_ID" NUMBER(9,0) NOT NULL ENABLE,
+# "UPDATED_TIMEDATE" DATE DEFAULT SYSDATE NOT NULL ENABLE,
+# "STATUS" VARCHAR2(10 BYTE) NOT NULL ENABLE,
+# "DESCRIPTION" VARCHAR2(254 BYTE) NOT NULL ENABLE,
+#
+# CONSTRAINT "FMS_UPDATE_PK" PRIMARY KEY ("ROW_ID")
+#------------------------------------------------------------------
+sub get_service_request_updates {
+ # by default, we only want last 24 hours
+ # also, limit to 1000 records
+
+ my $raw_start_date = $req -> param($CGI_VAR_START_DATE);
+ my $raw_end_date = $req -> param($CGI_VAR_END_DATE);
+ my $start_date = get_date_or_nothing( $raw_start_date, $USE_ONLY_DATES );
+ my $end_date = get_date_or_nothing( $raw_end_date, $USE_ONLY_DATES );
+
+ if (! $req -> param($CGI_VAR_NO_DEFAULT_DATE)) {
+ $start_date = get_date_or_nothing( $YESTERDAY, $USE_ONLY_DATES ) unless ($start_date or $end_date);
+ }
+
+ my $date_format = 'YYYY-MM-DD HH24:MI:SS'; # NB: hh24 (not hh)
+
+ $start_date = "updated_timedate >= to_date('$start_date', '$date_format')" if $start_date;
+ $end_date = "updated_timedate <= to_date('$end_date', '$date_format')" if $end_date;
+
+ my $where_clause = '';
+ my @criteria = ($start_date, $end_date);
+ push @criteria, $STATUS_CRITERIA unless $req -> param($CGI_VAR_ANY_STATUS);
+ $where_clause = join(' AND ', grep {$_} @criteria);
+ $where_clause = "WHERE $where_clause" if $where_clause;
+
+ my $sql = qq(SELECT row_id, service_request_id, to_char(updated_timedate, '$date_format'), status, description FROM higatlas.fms_update $where_clause ORDER BY updated_timedate DESC);
+
+ my $limit = $req -> param($CGI_VAR_LIMIT) =~ /^(\d{1,3})$/? $1 : $MAX_LIMIT;
+ $sql = "SELECT * FROM ($sql) WHERE ROWNUM <= $limit" if $limit;
+
+ my $debug_str;
+ my $ary_ref;
+
+ if ($TESTING_WRITE_TO_FILE) {
+ $ary_ref = [
+ [97, 1000, '2013-01-05', 'OPEN', 'report was opened'],
+ [99, 1000, '2013-01-06', 'CLOSED', 'report was closed']
+ ];
+ # only add debug now if config says we're testing
+ $debug_str = <<XML;
+ <!-- DEBUG: from: $raw_start_date => $start_date -->
+ <!-- DEBUG: to: $raw_end_date => $end_date -->
+ <!-- DEBUG: sql: $sql -->
+XML
+ } else {
+ my $dbh = get_db_connection();
+ $ary_ref = $dbh->selectall_arrayref($sql);
+ }
+
+ # rough and ready XML dump now (should use XML Simple to build/escape properly!)
+ my $xml = "";
+ foreach my $row(@{$ary_ref}) {
+ if (defined $row) {
+ my ($id, $service_req_id, $updated_at, $status, $desc) = map { prepare_for_xml($_) } @$row;
+ $updated_at=~s/(\d{4}-\d\d-\d\d) (\d\d:\d\d:\d\d)/${1}T${2}Z/; # for now assume OCC in Zulu time
+ $xml.= <<XML;
+ <request_update>
+ <update_id>$id</update_id>
+ <service_request_id>$service_req_id</service_request_id>
+ <status>$status</status>
+ <updated_datetime>$updated_at</updated_datetime>
+ <description>$desc</description>
+ </request_update>
+XML
+ }
+ }
+ print <<XML;
+Content-type: text/xml
+
+<?xml version="1.0" encoding="utf-8"?>
+<service_request_updates>
+$xml
+</service_request_updates>
+$debug_str
+XML
+}
diff --git a/bin/oxfordshire/open311_services.pm b/bin/oxfordshire/open311_services.pm
new file mode 100644
index 000000000..12d0754fa
--- /dev/null
+++ b/bin/oxfordshire/open311_services.pm
@@ -0,0 +1,150 @@
+#!/usr/bin/perl
+#
+# common stuff used by Oxfordshire Open311 glue scripts
+#
+# mySociety: http://fixmystreet.org/
+#-----------------------------------------------------------------
+
+use strict;
+use CGI;
+use Encode qw(from_to);
+use DBI;
+use Time::Piece;
+
+
+###################################################################
+# Config file: values in the config file override any values set
+# in the code below for the following things:
+#
+# host: SXX-SAN-FOO_BAR
+# sid: FOOBAR
+# port: 1531
+# username: foo
+# password: FooBar
+# testing: 0
+# encode-to-win1252: 1
+#
+# Absence of the config file fails silently in case you really are
+# using values directly set in this script.
+#------------------------------------------------------------------
+our $CONFIG_FILENAME = "/usr/local/etc/fixmystreet.config";
+
+use constant {
+ GENERAL_SERVICE_ERROR => 400,
+ CODE_OR_ID_NOT_FOUND => 404,
+ CODE_OR_ID_NOT_PROVIDED => 400,
+ BAD_METHOD => 405,
+ FATAL_ERROR => 500
+};
+
+our $DB_SERVER_NAME = 'FOO';
+our $DB_HOST = $DB_SERVER_NAME; # did this just in case we need to add more to the name (e.g, domain)
+our $DB_PORT = '1531';
+our $ORACLE_SID = 'FOOBAR';
+our $USERNAME = 'FIXMYSTREET';
+our $PASSWORD = 'XXX';
+our $STORED_PROC_NAME = 'PEM.create_enquiry';
+
+# NB can override these settings in the config file!
+
+# Strip control chars:
+# 'ruthless' removes everything (i.e. all POSIX control chars)
+# 'desc' removes everything, but keeps tabs and newlines in the 'description' field, where they matter
+# 'normal' keeps tabs and newlines
+our $STRIP_CONTROL_CHARS = 'ruthless';
+
+our $ENCODE_TO_WIN1252 = 1; # force encoding to Win-1252 for PEM data
+our $DECODE_FROM_WIN1252 = 1; # force encoding from Win-1252 for PEM data
+
+our $TESTING_WRITE_TO_FILE = 0; # write to file instead of DB or (get_service_request_update) don't really read the db
+our $OUT_FILENAME = "fms-test.txt"; # dump data here if TESTING_WRITE_TO_FILE is true
+our $TEST_SERVICE_DISCOVERY = 0; # switch to 1 to run service discovery, which confirms the DB connection at least
+our $RUN_FAKE_INSERT_TEST = 0; # command-line execution attempts insert with fake data (mimics a POST request)
+
+# Config file overrides existing values for these, if present:
+if ($CONFIG_FILENAME && open(CONF, $CONFIG_FILENAME)) {
+ while (<CONF>) {
+ next if /^#/;
+ if (/^\s*password:\s*(\S+)\s*$/i) {
+ $PASSWORD = $1;
+ } elsif (/^\s*sid:\s*(\S+)\s*$/i) {
+ $ORACLE_SID = $1;
+ } elsif (/^\s*username:\s*(\S+)\s*$/i) {
+ $USERNAME = $1;
+ } elsif (/^\s*port:\s*(\S+)\s*$/i) {
+ $DB_PORT = $1;
+ } elsif (/^\s*host:\s*(\S+)\s*$/i) {
+ $DB_HOST = $1;
+ } elsif (/^\s*testing:\s*(\S+)\s*$/i) {
+ $TESTING_WRITE_TO_FILE = $1;
+ } elsif (/^\s*test-service-discovery:\s*(\S+)\s*$/i) {
+ $TEST_SERVICE_DISCOVERY = $1;
+ } elsif (/^\s*strip-control-chars:\s*(\S+)\s*$/i) {
+ $STRIP_CONTROL_CHARS = lc $1;
+ } elsif (/^\s*encode-to-win1252:\s*(\S+)\s*$/i) {
+ $ENCODE_TO_WIN1252 = $1;
+ } elsif (/^\s*decode-from-win1252:\s*(\S+)\s*$/i) {
+ $DECODE_FROM_WIN1252 = $1;
+ } elsif (/^\s*run-fake-insert-test:\s*(\S+)\s*$/i) {
+ $RUN_FAKE_INSERT_TEST = $1;
+ }
+ }
+}
+
+our $YESTERDAY = localtime() - Time::Seconds::ONE_DAY; # yesterday
+$YESTERDAY = $YESTERDAY->strftime('%Y-%m-%d');
+
+#------------------------------------------------------------------
+# error_and_exit
+# args: HTTP status code, error message
+# Sends out the HTTP status code and message
+# and temrinates execution
+#------------------------------------------------------------------
+sub error_and_exit {
+ my ($status, $msg) = @_;
+ print "Status: $status $msg\n\n$msg\n";
+ exit;
+}
+
+
+#------------------------------------------------------------------
+# get_db_connection
+# no args: uses globals, possibly read from config
+# returns handle for the connection (otherwise terminates)
+#------------------------------------------------------------------
+sub get_db_connection {
+ return DBI->connect( "dbi:Oracle:host=$DB_HOST;sid=$ORACLE_SID;port=$DB_PORT", $USERNAME, $PASSWORD )
+ or error_and_exit(FATAL_ERROR, "failed to connect to database: " . $DBI::errstr, "");
+}
+
+#------------------------------------------------------------------
+# get_date_or_nothing {
+# parse date from incoming request, fail silently
+# expected format: 2003-02-15T13:50:05
+# These are coming from FMS for Oxford so don't expect to need
+# need to parse anyway
+#------------------------------------------------------------------
+sub get_date_or_nothing {
+ my $d = shift;
+ my $want_date_only = shift;
+ if ($d=~s/^(\d{4}-\d\d-\d\d)(T\d\d:\d\d(:\d\d)?)?.*/$1$2/) {
+ return $1 if $want_date_only;
+ $d="$1 00:00" unless $2; # no time provided
+ $d.=":00" unless $3; # no seconds
+ $d=~s/[TZ]/ /g;
+ # no point doing any parsing if regexp has done the work
+ # eval {
+ # $d=~s/(\d\d:\d\d).*/$1/; # bodge if we can't get DateTime installed
+ # $d = Time::Piece->strptime( $d, '%Y-%m-%dT%H:%M:%S');
+ # $d = $d->strftime('%Y-%m-%d %H:%M:%S');
+ # };
+ # return '' if $@;
+ } else {
+ return '';
+ }
+ return $d;
+}
+
+
+
+1;
diff --git a/bin/problem-creation-graph b/bin/problem-creation-graph
index b96d45540..2ee38783c 100755
--- a/bin/problem-creation-graph
+++ b/bin/problem-creation-graph
@@ -49,13 +49,13 @@ function grab_data {
grab_data "" $SOURCEA
grab_data "where state not in ('unconfirmed')" $SOURCEB
grab_data "where state not in ('unconfirmed', 'confirmed')" $SOURCEC
-grab_data "where state not in ('unconfirmed', 'confirmed', 'fixed')" $SOURCED
-grab_data "where state not in ('unconfirmed', 'confirmed', 'fixed', 'hidden')" $SOURCEE
+grab_data "where state not in ('unconfirmed', 'confirmed', 'fixed', 'fixed - council', 'fixed - user')" $SOURCED
+grab_data "where state not in ('unconfirmed', 'confirmed', 'fixed', 'fixed - council', 'fixed - user', 'hidden')" $SOURCEE
#state = 'unconfirmed'
#or state = 'confirmed'
-#or state = 'fixed'
+#or state in ('fixed', 'fixed - council', 'fixed - user')
#or state = 'hidden'
#or state = 'partial'
@@ -96,7 +96,7 @@ if [ -s $SOURCED ]; then
echo -n >>$GPSCRIPT " \"$SOURCED\" using 1:2 with impulses lt 6 lw 15 title \"hidden\","
fi
if [ -s $SOURCEE ]; then
-echo -n >>$GPSCRIPT " \"$SOURCEE\" using 1:2 with impulses lt 7 lw 15 title \"partial (and any other types)\","
+echo -n >>$GPSCRIPT " \"$SOURCEE\" using 1:2 with impulses lt 7 lw 15 title \"any other type\","
fi
cat >>$GPSCRIPT <<END
"< awk 'BEGIN { n = 0 } { n += \$2; print \$1, \$2, n; }' $SOURCEA" using 1:3 axes x1y2 with lines lt 2 title "cumulative total number of problems"
diff --git a/bin/rotate-photos b/bin/rotate-photos
index 31a60ff6c..7b8109d65 100755
--- a/bin/rotate-photos
+++ b/bin/rotate-photos
@@ -16,7 +16,7 @@ use FindBin;
use lib "$FindBin::Bin/../perllib";
use lib "$FindBin::Bin/../commonlib/perllib";
-use Digest::SHA1 qw(sha1_hex);
+use Digest::SHA qw(sha1_hex);
use Utils;
use mySociety::Config;
diff --git a/bin/send-comments b/bin/send-comments
index 4955cb2f3..3549113c9 100755
--- a/bin/send-comments
+++ b/bin/send-comments
@@ -1,7 +1,10 @@
#!/usr/bin/env perl
-# send-reports:
-# Send new problem reports to councils
+# send-comments:
+# Send comments/updates on reports to bodies
+# In Open311 parlance these are 'service request updates' and are sent using
+# mySociety's proposed extension to the Open311 Georeport v2 spec:
+# https://github.com/mysociety/fixmystreet/wiki/Open311-FMS---Proposed-differences-to-Open311
#
# Copyright (c) 2011 UK Citizens Online Democracy. All rights reserved.
# Email: matthew@mysociety.org. WWW: http://www.mysociety.org
@@ -23,24 +26,28 @@ use mySociety::EmailUtil;
use Open311;
-# maximum number of webservice attempts to send before not trying any more (XXX may be better in config?)
-use constant SEND_FAIL_RETRIES_CUTOFF => 3;
-
# send_method config values found in by-area config data, for selecting to appropriate method
use constant SEND_METHOD_EMAIL => 'email';
use constant SEND_METHOD_OPEN311 => 'Open311';
+use constant COUNCIL_ID_OXFORDSHIRE => 2237;
+
# Set up site, language etc.
my ($verbose, $nomail) = CronFns::options();
my $base_url = mySociety::Config::get('BASE_URL');
my $site = CronFns::site($base_url);
-my $councils = FixMyStreet::App->model('DB::Open311Conf')->search( {
+my $bodies = FixMyStreet::App->model('DB::Body')->search( {
send_method => SEND_METHOD_OPEN311,
send_comments => 1,
} );
-while ( my $council = $councils->next ) {
+while ( my $body = $bodies->next ) {
+
+ # Oxfordshire (OCC) is special:
+ # we do *receive* service_request_updates (aka comments) for OCC, but we never *send* them, so skip this pass
+ next if $body->areas->{+COUNCIL_ID_OXFORDSHIRE};
+
my $use_extended = 0;
my $comments = FixMyStreet::App->model('DB::Comment')->search( {
'me.whensent' => undef,
@@ -49,7 +56,7 @@ while ( my $council = $councils->next ) {
'me.confirmed' => { '!=' => undef },
'problem.whensent' => { '!=' => undef },
'problem.external_id' => { '!=' => undef },
- 'problem.council' => { -like => '%' . $council->area_id .'%' },
+ 'problem.bodies_str' => { -like => '%' . $body->id . '%' },
'problem.send_method_used' => 'Open311',
},
{
@@ -57,18 +64,25 @@ while ( my $council = $councils->next ) {
}
);
- if ( $council->area_id == 2482 ) {
+ if ( $body->areas->{2482} ) {
$use_extended = 1;
}
- my $o = Open311->new(
- endpoint => $council->endpoint,
- jurisdiction => $council->jurisdiction,
- api_key => $council->api_key,
+ my %open311_conf = (
+ endpoint => $body->endpoint,
+ jurisdiction => $body->jurisdiction,
+ api_key => $body->api_key,
use_extended_updates => $use_extended,
);
- if ( $council->area_id =~ /2482/ ) {
+
+ if ( $body->send_extended_statuses ) {
+ $open311_conf{extended_statuses} = 1;
+ }
+
+ my $o = Open311->new( %open311_conf );
+
+ if ( $body->areas->{2482} ) {
my $endpoints = $o->endpoints;
$endpoints->{update} = 'update.xml';
$endpoints->{service_request_updates} = 'update.xml';
@@ -78,11 +92,23 @@ while ( my $council = $councils->next ) {
while ( my $comment = $comments->next ) {
my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($comment->cobrand)->new();
+ # TODO actually this should be OK for any devolved endpoint if original Open311->can_be_devolved, presumably
+ if ( 0 ) { # Check can_be_devolved and do this properly if set
+ my $sender = $cobrand->get_body_sender( $body, $comment->problem->category );
+ my $config = $sender->{config};
+ $o = Open311->new(
+ endpoint => $config->endpoint,
+ jurisdiction => $config->jurisdiction,
+ api_key => $config->api_key,
+ use_extended_updates => 1, # FMB uses extended updates
+ );
+ }
+
if ( $comment->send_fail_count ) {
next if bromley_retry_timeout( $comment );
}
- if ( $council->area_id == 2482 ) {
+ if ( $body->areas->{2482} ) {
my $extra = $comment->extra;
if ( !$extra ) {
$extra = {};
diff --git a/bin/site-specific-install.sh b/bin/site-specific-install.sh
index 6ebac7970..3cc84007a 100644
--- a/bin/site-specific-install.sh
+++ b/bin/site-specific-install.sh
@@ -1,5 +1,8 @@
#!/bin/sh
+# Set this to the version we want to check out
+VERSION=${VERSION_OVERRIDE:-v1.3}
+
PARENT_SCRIPT_URL=https://github.com/mysociety/commonlib/blob/master/bin/install-site.sh
misuse() {
@@ -22,20 +25,33 @@ misuse() {
[ -z "$HOST" ] && misuse HOST
[ -z "$DISTRIBUTION" ] && misuse DISTRIBUTION
[ -z "$VERSION" ] && misuse VERSION
+[ -z "$DEVELOPMENT_INSTALL" ] && misuse DEVELOPMENT_INSTALL
+
+add_locale cy_GB
+add_locale nb_NO
+add_locale de_CH
+
+install_postfix
-install_nginx
+if [ ! "$DEVELOPMENT_INSTALL" = true ]; then
+ install_nginx
+ add_website_to_nginx
+ # Check out the current released version
+ su -l -c "cd '$REPOSITORY' && git checkout '$VERSION' && git submodule update" "$UNIX_USER"
+fi
install_website_packages
su -l -c "touch '$DIRECTORY/admin-htpasswd'" "$UNIX_USER"
-add_website_to_nginx
-
add_postgresql_user
-su -l -c "$REPOSITORY/bin/install-as-user '$UNIX_USER' '$HOST' '$DIRECTORY'" "$UNIX_USER"
+export DEVELOPMENT_INSTALL
+su -c "$REPOSITORY/bin/install-as-user '$UNIX_USER' '$HOST' '$DIRECTORY'" "$UNIX_USER"
-install_sysvinit_script
+if [ ! "$DEVELOPMENT_INSTALL" = true ]; then
+ install_sysvinit_script
+fi
if [ $DEFAULT_SERVER = true ] && [ x != x$EC2_HOSTNAME ]
then
diff --git a/bin/update-all-reports b/bin/update-all-reports
index 4c4838c77..f4d6113cb 100755
--- a/bin/update-all-reports
+++ b/bin/update-all-reports
@@ -18,50 +18,65 @@ use List::MoreUtils qw(zip);
my $fourweeks = 4*7*24*60*60;
+# Age problems from when they're confirmed, except on Zurich
+# where they appear as soon as they're created.
+my $age_column = 'confirmed';
+if ( FixMyStreet->config('BASE_URL') =~ /zurich|zueri/ ) {
+ $age_column = 'created';
+}
+
my $problems = FixMyStreet::App->model("DB::Problem")->search(
{
state => [ FixMyStreet::DB::Result::Problem->visible_states() ],
},
{
columns => [
- 'id', 'council', 'state', 'areas',
+ 'id', 'bodies_str', 'state', 'areas',
{ duration => { extract => "epoch from current_timestamp-lastupdate" } },
- { age => { extract => "epoch from current_timestamp-confirmed" } },
+ { age => { extract => "epoch from current_timestamp-$age_column" } },
]
}
);
$problems = $problems->cursor; # Raw DB cursor for speed
my ( %fixed, %open );
-my @cols = ( 'id', 'council', 'state', 'areas', 'duration', 'age' );
+my @cols = ( 'id', 'bodies_str', 'state', 'areas', 'duration', 'age' );
while ( my @problem = $problems->next ) {
my %problem = zip @cols, @problem;
- my @areas;
- if ( !$problem{council} ) {
- # Problem was not sent to any council, add to all areas
- @areas = grep { $_ } split( /,/, $problem{areas} );
- $problem{councils} = 0;
+ my @bodies;
+ if ( !$problem{bodies_str} ) {
+ # Problem was not sent to any bodies, add to all areas
+ @bodies = grep { $_ } split( /,/, $problem{areas} );
+ $problem{bodies} = 0;
} else {
- # Add to councils it was sent to
- (my $council = $problem{council}) =~ s/\|.*$//;
- @areas = split( /,/, $council );
- $problem{councils} = scalar @areas;
+ # Add to bodies it was sent to
+ (my $bodies = $problem{bodies_str}) =~ s/\|.*$//;
+ @bodies = split( /,/, $bodies );
+ $problem{bodies} = scalar @bodies;
}
- foreach my $council ( @areas ) {
+ foreach my $body ( @bodies ) {
my $duration_str = ( $problem{duration} > 2 * $fourweeks ) ? 'old' : 'new';
my $type = ( $problem{duration} > 2 * $fourweeks )
? 'unknown'
: ($problem{age} > $fourweeks ? 'older' : 'new');
- if (FixMyStreet::DB::Result::Problem->fixed_states()->{$problem{state}}) {
+ if (FixMyStreet::DB::Result::Problem->fixed_states()->{$problem{state}} || FixMyStreet::DB::Result::Problem->closed_states()->{$problem{state}}) {
# Fixed problems are either old or new
- $fixed{$council}{$duration_str}++ if FixMyStreet::DB::Result::Problem->fixed_states()->{$problem{state}};
+ $fixed{$body}{$duration_str}++;
} else {
# Open problems are either unknown, older, or new
- $open{$council}{$type}++ if $problem{state} eq 'confirmed';
+ $open{$body}{$type}++;
}
}
}
+if ( FixMyStreet->config('BASE_URL') =~ /emptyhomes/ ) {
+ my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker('emptyhomes')->new();
+ my $stats = $cobrand->old_site_stats;
+ foreach (keys %$stats) {
+ $open{$_}{unknown} += $stats->{$_};
+ }
+}
+
my $body = JSON->new->utf8(1)->encode( {
fixed => \%fixed,
open => \%open,
diff --git a/bin/update-schema b/bin/update-schema
new file mode 100755
index 000000000..be5bc50e7
--- /dev/null
+++ b/bin/update-schema
@@ -0,0 +1,145 @@
+#!/usr/bin/perl
+#
+# This script should hopefully work out at what state the database is and, if
+# the commit argument is provided, run the right schema files to bring it up to
+# date. Let us know if it doesn't work; as with any upgrade script, do take a
+# backup of your database before running.
+#
+# update-schema [--commit]
+
+use strict;
+use warnings;
+
+# Horrible boilerplate to set up appropriate library paths.
+use FindBin;
+use lib "$FindBin::Bin/../commonlib/perllib";
+
+use mySociety::Config;
+use mySociety::DBHandle qw(dbh);
+use mySociety::MaPit;
+
+mySociety::Config::set_file("$FindBin::Bin/../conf/general");
+my %args = (
+ Name => mySociety::Config::get('FMS_DB_NAME'),
+ User => mySociety::Config::get('FMS_DB_USER'),
+ Password => mySociety::Config::get('FMS_DB_PASS'),
+);
+$args{Host} = mySociety::Config::get('FMS_DB_HOST', undef) if mySociety::Config::get('FMS_DB_HOST');
+$args{Port} = mySociety::Config::get('FMS_DB_PORT', undef) if mySociety::Config::get('FMS_DB_PORT');
+mySociety::DBHandle::configure( %args );
+
+my $commit = 0;
+$commit = 1 if @ARGV && $ARGV[0] eq '--commit';
+
+my $nothing = 1;
+my $current_version = get_db_version();
+print "Current database version = $current_version\n";
+print "= Dry run =\n" unless $commit;
+
+for my $path (glob("$FindBin::Bin/../db/schema_*")) {
+ my ($name) = $path =~ /schema_(.*)\.sql$/;
+ next if $name le $current_version;
+ next if $name =~ /$current_version-/; # For number only match
+ print "* $name\n";
+ $nothing = 0;
+ next unless $commit;
+
+ open(FP, $path) or die $!;
+ my @statements;
+ my $s = '';
+ my $in_function = 0;
+ while(<FP>) {
+ next if /^--/; # Ignore comments
+ $s .= $_;
+ # Functions may have semicolons within them
+ $in_function = 1 if /create function/i;
+ $in_function = 0 if /language 'plpgsql'/i;
+ if ($s =~ /;/ && !$in_function) {
+ push @statements, $s;
+ $s = '';
+ }
+ }
+ close FP;
+
+ foreach my $st (@statements) {
+ dbh()->do($st);
+ }
+}
+
+if ( $commit && $current_version lt '0028' ) {
+ print "Bodies created, fetching names from mapit\n";
+ my $area_ids = dbh()->selectcol_arrayref('SELECT area_id FROM body_areas');
+ if ( @$area_ids ) {
+ my $areas = mySociety::MaPit::call('areas', $area_ids);
+ foreach (values %$areas) {
+ dbh()->do('UPDATE body SET name=? WHERE id=?', {}, $_->{name}, $_->{id});
+ }
+ dbh()->do('COMMIT');
+ }
+}
+
+print "Nothing to do\n" if $nothing;
+
+# ---
+
+# By querying the database schema, we can see where we're currently at
+# (assuming schema change files are never half-applied, which should be the case)
+sub get_db_version {
+ return '0031' if column_exists('body', 'external_url');
+ return '0030' if ! constraint_exists('admin_log_action_check');
+ return '0029' if column_exists('body', 'deleted');
+ return '0028' if table_exists('body');
+ return '0027' if column_exists('problem', 'subcategory');
+ return '0026' if column_exists('open311conf', 'send_extended_statuses');
+ return '0025' if column_like('alert_type', "ref='new_problems'", 'item_where', 'duplicate');
+ return '0024' if column_exists('contacts', 'non_public');
+ return '0023' if column_exists('open311conf', 'can_be_devolved');
+ return '0022' if column_exists('problem', 'interest_count');
+ return '0021' if column_exists('problem', 'external_source');
+ return '0020' if column_exists('open311conf', 'suppress_alerts');
+ return '0019' if column_exists('users', 'title');
+ return '0018' if column_exists('open311conf', 'comment_user_id');
+ return '0017' if column_exists('open311conf', 'send_comments');
+ return '0016' if column_exists('comment', 'send_fail_count');
+ return '0015-add_send_method_used_column_to_problem' if column_exists('problem', 'send_method_used');
+ return '0015-add_extra_to_comment' if column_exists('comment', 'extra');
+ return '0014' if column_exists('problem', 'send_fail_count');
+ return '0013-add_send_method_column_to_open311conf' if column_exists('open311conf', 'send_method');
+ return '0013-add_external_id_to_comment' if column_exists('comment', 'external_id');
+ return '0012' if column_exists('problem', 'geocode');
+ return '0011' if column_exists('contacts', 'extra');
+ return '0010' if table_exists('open311conf');
+ return '0009-update_alerts_problem_state_queries' if column_like('alert_type', "ref='new_problems'", 'item_where', 'investigating');
+ return '0009-add_extra_to_problem' if column_exists('problem', 'extra');
+ return '0008' if 0;
+ return '0007' if column_exists('comment', 'problem_state');
+ return '0006' if 0;
+ return '0005-add_council_user_flag' if column_exists('users', 'from_council');
+ return '0005-add_abuse_flags_to_users_and_reports' if column_exists('problem', 'flagged');
+ return '0000';
+}
+
+# Returns true if a table exists
+sub table_exists {
+ my $table = shift;
+ return dbh()->selectrow_array('select count(*) from pg_tables where tablename = ?', {}, $table);
+}
+
+# Returns true if a column of table exists
+sub column_exists {
+ my ( $table, $column ) = @_;
+ return dbh()->selectrow_array('select count(*) from pg_class, pg_attribute WHERE pg_class.relname=? AND pg_attribute.attname=? AND pg_class.oid=pg_attribute.attrelid AND pg_attribute.attnum > 0', {}, $table, $column);
+}
+
+# Returns true if a column of a row in a table contains some text
+sub column_like {
+ my ( $table, $where, $column, $contents ) = @_;
+ return dbh()->selectrow_array("select count(*) from $table WHERE $where AND $column LIKE ?", {}, "%$contents%");
+}
+
+# Returns true if a check constraint on a table exists
+sub constraint_exists {
+ my ( $constraint ) = @_;
+ return dbh()->selectrow_array('select count(*) from pg_constraint where conname = ?', {}, $constraint);
+}
+
diff --git a/bin/update-send-questionnaire b/bin/update-send-questionnaire
new file mode 100755
index 000000000..7a231b919
--- /dev/null
+++ b/bin/update-send-questionnaire
@@ -0,0 +1,29 @@
+#!/usr/bin/env perl
+
+=head1 DESCRIPTION
+
+Ad hoc script to update the send_questionnaire column on the
+reports in cobrands that don't send questionnaires at all.
+
+=cut
+
+use strict;
+use warnings;
+
+use FixMyStreet::App;
+
+my @cobrands;
+foreach my $cobrand ( FixMyStreet::Cobrand->available_cobrand_classes() ) {
+ next unless $cobrand->{class};
+ my $cls = $cobrand->{class}->new();
+ push @cobrands, $cls->moniker if !$cls->send_questionnaires();
+}
+
+my $problems = FixMyStreet::App->model('DB::Problem')->search({
+ cobrand => \@cobrands,
+ send_questionnaire => 1,
+});
+
+$problems->update( {
+ send_questionnaire => 0
+});
diff --git a/bin/update_po_header.bash b/bin/update_po_header.bash
new file mode 100755
index 000000000..2f37ad5d3
--- /dev/null
+++ b/bin/update_po_header.bash
@@ -0,0 +1,33 @@
+#!/bin/bash
+#
+# fixmystreet/bin/gettext-extract
+# Generate English language .po files from the source code and email templates,
+# for FixMyStreet. Writes the output to appropriate .po files in locale/.
+#
+# Copyright (c) 2011 UK Citizens Online Democracy. All rights reserved.
+# Email: matthew@mysociety.org; WWW: http://www.mysociety.org/
+
+cd "$(dirname $(readlink -f $BASH_SOURCE))/.."
+
+# File to write to, clear it to start with
+PO=$1
+
+# Fix headers
+# no such thing as tempfile on OS X
+TEMP=`tempfile 2>/dev/null || mktemp /tmp/gettext-extract.XXXXXX`
+NOW=`date +"%Y-%m-%d %H:%M%z"`
+# strictly POSIX sed on e.g. OS X doesn't let you used \n in replacements so we do this
+nl=$'\n';
+cat $PO | sed "
+ s/SOME DESCRIPTIVE TITLE/FixMyStreet original .po file, autogenerated by gettext-extract/;
+ s/YEAR THE PACKAGE'S COPYRIGHT HOLDER/2011 UK Citizens Online Democracy/;
+ s/PACKAGE package/main FixMyStreet code/;
+ s/FIRST AUTHOR <EMAIL@ADDRESS>, YEAR./Matthew Somerville <matthew@mysociety.org>, 2011-06-03./;
+
+ s/PACKAGE VERSION/1.0\\\n\"\\$nl\"Report-Msgid-Bugs-To: matthew@mysociety.org/;
+ s/POT-Creation-Date: YEAR-MO-DA HO:MI+ZONE/POT-Creation-Date: $NOW/;
+ s/LL@li.org/team@fixmystreet.com/;
+ s/charset=CHARSET/charset=UTF-8/;
+ s/8bit/8bit\\\n\"\\$nl\"Plural-Forms: nplurals=2; plural=n != 1;/;
+" >> $TEMP
+mv $TEMP $PO
diff --git a/bin/zerotb_import_clinic_list.pl b/bin/zerotb_import_clinic_list.pl
new file mode 100755
index 000000000..359a63925
--- /dev/null
+++ b/bin/zerotb_import_clinic_list.pl
@@ -0,0 +1,115 @@
+#!/usr/bin/perl
+
+use strict;
+use FixMyStreet::App;
+use Text::CSV;
+use Getopt::Long::Descriptive;
+
+use constant TITLE => 0;
+use constant DESC => 1;
+use constant LATLONG => 2;
+use constant EMAIL => 3;
+
+my ($opt, $usage) = describe_options(
+ '%c %o',
+ ['file|f=s', "path to csv file with list of clinics", { required => 1 } ],
+ ['email|e=s', "default email address for the updates to be sent to", { required => 1 } ],
+ ['verbose|v', "print out all services as they are found"],
+ ['help', "print usage message and exit" ],
+);
+print($usage->text), exit if $opt->help;
+
+my $csv = Text::CSV->new ( { binary => 1 } ) # should set binary attribute.
+ or die "Cannot use CSV: ".Text::CSV->error_diag ();
+open my $fh, "<:encoding(utf8)", $opt->file or die "Failed to open " . $opt->file . ": $!";
+
+my $clinic_user = FixMyStreet::App->model('DB::User')->find_or_create({
+ email => $opt->email
+});
+if ( not $clinic_user->in_storage ) {
+ $clinic_user->insert;
+}
+
+# throw away header line
+my $title_row = $csv->getline( $fh );
+
+while ( my $row = $csv->getline( $fh ) ) {
+ my $clinics = FixMyStreet::App->model('DB::Problem')->search({
+ title => $row->[TITLE]
+ });
+
+ my ($lat, $long) = split(',', $row->[LATLONG]);
+ my $p;
+ my $count = $clinics->count;
+ if ( $count == 0 ) {
+ $p = FixMyStreet::App->model('DB::Problem')->create({
+ title => $row->[TITLE],
+ latitude => $lat,
+ longitude => $long,
+ used_map => 1,
+ anonymous => 1,
+ state => 'unconfirmed',
+ name => '',
+ user => $clinic_user,
+ detail => '',
+ areas => '',
+ postcode => ''
+ });
+ } elsif ( $count == 1 ) {
+ $p = $clinics->first;
+ } else {
+ printf "Too many matches for: %s\n", $row->[TITLE];
+ next;
+ }
+ $p->detail( $row->[DESC] );
+ $p->latitude( $lat );
+ $p->longitude( $long );
+ $p->confirm;
+
+ if ( $p->in_storage ) {
+ printf( "Updating entry for %s\n", $row->[TITLE] ) if $opt->verbose;
+ $p->update;
+ } else {
+ printf( "Creating entry for %s\n", $row->[TITLE] ) if $opt->verbose;
+ $p->insert;
+ }
+ $p->discard_changes;
+
+ # disabling existing alerts in case email addresses have changed
+ my $existing = FixMyStreet::App->model('DB::Alert')->search({
+ alert_type => 'new_updates',
+ parameter => $p->id
+ });
+ $existing->update( { confirmed => 0 } );
+
+ if ( $row->[EMAIL] ) {
+ my $u = FixMyStreet::App->model('DB::User')->find_or_new({
+ email => $row->[EMAIL]
+ });
+ $u->insert unless $u->in_storage;
+ create_update_alert( $u, $p, $opt->verbose );
+ }
+
+ create_update_alert( $clinic_user, $p, $opt->verbose );
+}
+
+sub create_update_alert {
+ my ( $user, $p, $verbose ) = @_;
+ my $a = FixMyStreet::App->model('DB::Alert')->find_or_new({
+ alert_type => 'new_updates',
+ user => $user,
+ parameter => $p->id,
+ });
+
+ $a->confirmed(1);
+
+ if ( $a->in_storage ) {
+ printf( "Updating update alert for %s on %s\n", $user->email, $p->title )
+ if $verbose;
+ $a->update;
+ } else {
+ printf( "Creating update alert for %s on %s\n", $user->email, $p->title )
+ if $verbose;
+ $a->insert;
+ }
+}
diff --git a/bin/zurich-overdue-alert b/bin/zurich-overdue-alert
new file mode 100755
index 000000000..4e81b3263
--- /dev/null
+++ b/bin/zurich-overdue-alert
@@ -0,0 +1,84 @@
+#!/usr/bin/env perl
+
+# zurich-overdue-alert:
+# Send email alerts to administrators for overdue admin activities.
+#
+# Copyright (c) 2012 UK Citizens Online Democracy. All rights reserved.
+# Email: matthew@mysociety.org. WWW: http://www.mysociety.org
+
+use strict;
+use warnings;
+require 5.8.0;
+
+use DateTime;
+use CronFns;
+use FixMyStreet::App;
+
+my ($verbose, $nomail) = CronFns::options();
+
+# Only run on working days
+my $now = DateTime->now();
+exit if FixMyStreet::Cobrand::Zurich::is_public_holiday($now) or FixMyStreet::Cobrand::Zurich::is_weekend($now);
+
+my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker('zurich')->new();
+my %bodies = map { $_->id => $_ } FixMyStreet::App->model("DB::Body")->all;
+
+loop_through( 'alert-moderation-overdue.txt', 0, 1, [ 'unconfirmed' ] );
+loop_through( 'alert-overdue.txt', 1, 6, 'in progress' );
+loop_through( 'alert-overdue.txt', 0, 6, ['confirmed', 'planned'] );
+
+sub loop_through {
+ my ( $template, $include_parent, $days, $states ) = @_;
+
+ my $reports = FixMyStreet::App->model("DB::Problem")->search( {
+ state => $states,
+ created => { '<', FixMyStreet::Cobrand::Zurich::sub_days( $now, $days ) },
+ bodies_str => { '!=', undef },
+ } );
+
+ my %to_send = ();
+ while (my $row = $reports->next) {
+ $to_send{$row->bodies_str} .= '* ' . $row->id . ": '" . $row->title . "'\n\n";
+ }
+
+ my $template_path = FixMyStreet->path_to( "templates", "email", "zurich", $template )->stringify;
+ $template = Utils::read_file( $template_path );
+
+ foreach my $body_id (keys %to_send) {
+ send_alert( $template, $body_id, $to_send{$body_id}, $include_parent );
+ }
+}
+
+sub send_alert {
+ my ( $template, $body_id, $data, $include_parent ) = @_;
+
+ my $body = $bodies{$body_id};
+ my $body_email = $body->endpoint;
+
+ my $h = {
+ data => $data,
+ admin_url => $cobrand->admin_base_url,
+ };
+
+ my $env_to = [ $body_email ];
+ my $to = [ [ $body_email, $body->name ] ];
+ if ( $include_parent ) {
+ my $parent = $body->parent;
+ my $parent_email = $parent->endpoint;
+ push @$env_to, $parent_email;
+ push @$to, [ $parent_email, $parent->name ];
+ }
+
+ my $result = FixMyStreet::App->send_email_cron(
+ {
+ _template_ => $template,
+ _parameters_ => $h,
+ To => $to,
+ From => [ FixMyStreet->config('CONTACT_EMAIL'), FixMyStreet->config('CONTACT_NAME') ],
+ },
+ FixMyStreet->config('CONTACT_EMAIL'),
+ $env_to,
+ $nomail
+ );
+}
+
diff --git a/bin/zurich/convert_internal_notes_to_comments b/bin/zurich/convert_internal_notes_to_comments
new file mode 100755
index 000000000..ddf74851f
--- /dev/null
+++ b/bin/zurich/convert_internal_notes_to_comments
@@ -0,0 +1,73 @@
+#!/usr/bin/env perl
+
+=head1 DESCRIPTION
+
+This is a Zurich specific migration script.
+
+It converts the internal notes that used to be stored on the problem in the
+extra hash into comments instead.
+
+ ./bin/zurich/convert_internal_notes_to_comments user_id
+
+You need to select a user to have the internal notes be associated with, perhaps
+the superuser, or one created just for this purpose?
+
+=cut
+
+use strict;
+use warnings;
+
+use FixMyStreet::App;
+
+# Because it is not possible to determine the user that last edited the
+# internal_notes we need require a user to assign all the comments to.
+my $comment_user_id = $ARGV[0]
+ || die "Usage: $0 id_of_user_for_comments";
+my $comment_user = FixMyStreet::App #
+ ->model('DB::User') #
+ ->find($comment_user_id)
+ || die "Could not find user with id '$comment_user_id'";
+
+# We use now as the time for the internal note. This is not the time it was
+# actually created, but I don't think that can be extracted from the problem.
+my $comment_timestamp = DateTime->now();
+
+# The state of 'hidden' seems most appropriate for these internal_notes - as
+# they should not be shown to the public. Certainly more suitable than
+# 'unconfirmed' or 'confirmed'.
+my $comment_state = 'hidden';
+
+# Load all the comments, more reliable than trying to search on the contents of
+# the extra field.
+my $problems = FixMyStreet::App->model('DB::Problem')->search();
+
+while ( my $problem = $problems->next() ) {
+
+ my $problem_id = $problem->id;
+
+ # Extract the bits we are interested in. May not exist, in which case
+ # skip on.
+ my $extra = $problem->extra || {};
+ next unless exists $extra->{internal_notes};
+
+ # If there is something to save create a comment with the notes in them
+ if ( my $internal_notes = $extra->{internal_notes} ) {
+ print "Creating internal note comment for problem '$problem_id'\n";
+ $problem->add_to_comments(
+ {
+ text => $internal_notes,
+ created => $comment_timestamp,
+ user => $comment_user,
+ state => $comment_state,
+ mark_fixed => 0,
+ anonymous => 1,
+ extra => { is_internal_note => 1 },
+ }
+ );
+ }
+
+ # Remove the notes from extra and save back to the problem
+ delete $extra->{internal_notes};
+ $problem->update({ extra => $extra });
+}
+
diff --git a/bin/zurich/geocode b/bin/zurich/geocode
new file mode 100755
index 000000000..9482b27e6
--- /dev/null
+++ b/bin/zurich/geocode
@@ -0,0 +1,45 @@
+#!/usr/bin/perl
+
+=head1 NAME
+
+zurich/geocode - commandline tool to test the Zurich geocoder
+
+=head1 SYNOPSIS
+
+ # Firstly:
+ ## copy the GEOCODER config from a current Zurich conf to your conf/general.yml
+ $ eval `perl setenv.pl`
+
+ $ bin/zurich/geocode Magnus
+
+ # ... output from geocoder
+
+This can be used to test the results of, e.g.
+
+ https://www.zueriwieneu.ch/ajax/geocode?term=Magnus
+
+but without the caching which FixMyStreet applies, and passing on any 500
+errors from the server.
+
+=cut
+
+use strict;
+use warnings;
+require 5.8.0;
+
+
+use Data::Dumper;
+use feature 'say';
+
+use FixMyStreet;
+use FixMyStreet::App;
+use FixMyStreet::Geocode::Zurich;
+
+# TODO use FixMyStreet::override_config to get data from conf/general.yml.zurich if available
+my $geocoder = FixMyStreet->config('GEOCODER')
+ or die "No GEOCODER config -- please copy appropriate Zurich conf to conf/general.yml";
+
+my $c = FixMyStreet::App->new();
+my $s = join ' ', @ARGV;
+
+say Dumper( FixMyStreet::Geocode::Zurich::string( $s, $c ) );