diff options
Diffstat (limited to 'bin')
31 files changed, 1731 insertions, 155 deletions
diff --git a/bin/fetch-comments b/bin/fetch-comments index ef099fcc9..a276433e4 100755 --- a/bin/fetch-comments +++ b/bin/fetch-comments @@ -1,4 +1,8 @@ #!/usr/bin/env perl +# +# This script utilises the Open311 extension explained at +# https://github.com/mysociety/FixMyStreet/wiki/Open311-FMS---Proposed-differences-to-Open311 +# to fetch updates on service requests. use strict; use warnings; diff --git a/bin/fetch-comments-24hs b/bin/fetch-comments-24hs index b84f09ba7..602383d40 100644 --- a/bin/fetch-comments-24hs +++ b/bin/fetch-comments-24hs @@ -1,4 +1,9 @@ #!/usr/bin/env perl +# +# This script utilises the Open311 extension explained at +# https://github.com/mysociety/FixMyStreet/wiki/Open311-FMS---Proposed-differences-to-Open311 +# to fetch updates on service requests from the past 24 hours, to check none +# were missed. use strict; use warnings; diff --git a/bin/generate_cobrand_po b/bin/generate_cobrand_po new file mode 100755 index 000000000..1770de7c2 --- /dev/null +++ b/bin/generate_cobrand_po @@ -0,0 +1,40 @@ +#!/usr/bin/env perl +use strict; + +=for instructions + +This script generates a po file that contains only the strings related to a cobrand. +Its purpose is to avoid having lots of cobrand specific strings in the +main FixMyStreet po file confusing translators. Once this is run the +generated po file can be used to translate the strings. The po files +with the translation should then be placed in the relevant language +directories and merge_cobrand_po run which will merge these with +the main FixMyStreet po files + +=cut + +my $cobrand = shift; + +die "Please provide a cobrand name\n" unless $cobrand; + +my $cobrand_lc = lc( $cobrand ); + +my $PO = "locale/$cobrand.po"; + +my $cobrand_module = "perllib/FixMyStreet/Cobrand/$cobrand.pm"; +my $web_templates = "templates/web/$cobrand_lc/"; + +my $opts = ''; +if ( -e $cobrand_module ) { + $opts .= " $cobrand_module"; +} + +if ( -e -d $web_templates ) { + $opts .= " --directory $web_templates"; +} + +# first we get the strings from the cobrand +system("xgettext.pl --gnu-gettext --verbose --output $PO --plugin perl=* --plugin tt2 $opts"); + +# now replace the template header elements +system("bin/update_po_header.bash $PO"); diff --git a/bin/geocode b/bin/geocode new file mode 100755 index 000000000..254cf1578 --- /dev/null +++ b/bin/geocode @@ -0,0 +1,79 @@ +#!/usr/bin/perl + +=head1 NAME + +geocode - commandline tool to test geocoders + +=head1 SYNOPSIS + + $ eval `perl setenv.pl`, or call with bin/cron-wrapper + + $ bin/geocode --geocoder=Bing --cobrand=bromley "Glebe Rd" + + # ... if your conf/general.yml supplies GEOCODER: + $ bin/geocode --cobrand=bromley "Glebe Rd" + + # ... if you want to use config that you have in conf/general.bing.yml + $ bin/geocode --override-config=general.bing --cobrand=bromley "Glebe Rd" + + ## ... output from geocoder + +=cut + +use strict; +use warnings; +require 5.8.0; + +use Data::Dumper; +use Pod::Usage; +use feature 'say'; + +use Getopt::Long; + +my %options = ( help => sub { pod2usage(0) } ); + +GetOptions \%options, + 'geocoder=s', + 'help|h', + 'cobrand=s', + 'override-config=s'; + +my $s = join ' ', @ARGV + or pod2usage(0); + +pod2usage(0) unless $options{cobrand}; + +local $ENV{FMS_OVERRIDE_CONFIG} = $options{'override-config'} if $options{'override-config'}; + +eval 'use FixMyStreet'; +eval 'use FixMyStreet::App'; +eval 'use FixMyStreet::Cobrand'; +eval 'use FixMyStreet::Geocode'; + +mySociety::Locale::gettext_domain( 'FixMyStreet' ); + +my $geocoder_type = $options{geocoder} || do { + my $GEOCODER = FixMyStreet->config('GEOCODER'); + ref $GEOCODER ? $GEOCODER->{type} : $GEOCODER; +} or pod2usage(0); + +my $geocoder_name = "FixMyStreet::Geocode::${geocoder_type}"; +my $code_ref = $geocoder_name->can('string') + or die "$geocoder_name is not a valid geocoder?"; + +my @allowed_cobrands = FixMyStreet::Cobrand->get_allowed_cobrands(); + +my $cobrand_name = FixMyStreet::Cobrand->get_class_for_moniker($options{cobrand}); +my $cobrand = $cobrand_name->new(); + +say "USING COBRAND $cobrand_name"; +if ($cobrand->moniker ne lc($options{cobrand})) { + say "!!! asked for $options{cobrand}"; + say "!!! Check ALLOWED_COBRANDS setting in conf/general.yml (or supplied --override-config file)"; + say Dumper(\@allowed_cobrands); +} + +my $c = FixMyStreet::App->new(); +$c->stash->{cobrand} = $cobrand; + +say Dumper( $code_ref->( $s, $c ) ); diff --git a/bin/gettext-extract b/bin/gettext-extract index e77cf9cb0..a38c17127 100755 --- a/bin/gettext-extract +++ b/bin/gettext-extract @@ -7,48 +7,25 @@ # Copyright (c) 2011 UK Citizens Online Democracy. All rights reserved. # Email: matthew@mysociety.org; WWW: http://www.mysociety.org/ -if [ -e ../../locale ] -then - cd ../../ -else if [ -e ../locale ] -then - cd ../ -else if [ -e locale ] -then - cd . -else - echo "Please run with current directory fixmystreet/bin" - exit 1 -fi -fi -fi +cd "$(dirname $(readlink -f $BASH_SOURCE))/.." # File to write to, clear it to start with PO=locale/FixMyStreet.po rm -f $PO +# we don't want to extract strings from all the cobrand templates so list +# the ones we care about +find templates/web/default templates/web/fixmystreet templates/web/zurich templates/web/fiksgatami templates/web/emptyhomes templates/web/fixmybarangay -name '*.html' > template_list + # Extract from Perl -xgettext.pl --gnu-gettext --verbose --output $PO --plugin perl=* --plugin tt2 --directory perllib --directory templates/web --directory db --directory bin +xgettext.pl --gnu-gettext --verbose --output $PO --plugin perl=* --plugin tt2 --directory perllib -f template_list --directory db --directory bin + +# remove temporary list of templates +rm template_list # Fix headers # no such thing as tempfile on OS X -TEMP=`tempfile 2>/dev/null || mktemp /tmp/gettext-extract.XXXXXX` -NOW=`date +"%Y-%m-%d %H:%M%z"` -# strictly POSIX sed on e.g. OS X doesn't let you used \n in replacements so we do this -nl=$'\n'; -cat $PO | sed " - s/SOME DESCRIPTIVE TITLE/FixMyStreet original .po file, autogenerated by gettext-extract/; - s/YEAR THE PACKAGE'S COPYRIGHT HOLDER/2011 UK Citizens Online Democracy/; - s/PACKAGE package/main FixMyStreet code/; - s/FIRST AUTHOR <EMAIL@ADDRESS>, YEAR./Matthew Somerville <matthew@mysociety.org>, 2011-06-03./; - - s/PACKAGE VERSION/1.0\\\n\"\\$nl\"Report-Msgid-Bugs-To: matthew@mysociety.org/; - s/POT-Creation-Date: YEAR-MO-DA HO:MI+ZONE/POT-Creation-Date: $NOW/; - s/LL@li.org/team@fixmystreet.com/; - s/charset=CHARSET/charset=UTF-8/; - s/8bit/8bit\\\n\"\\$nl\"Plural-Forms: nplurals=2; plural=n != 1;/; -" >> $TEMP -mv $TEMP $PO +bin/update_po_header.bash $PO echo "$( bin/gettext-nget-patch )" >> $PO diff --git a/bin/gettext-merge b/bin/gettext-merge index 84aa4fdb7..b84708ece 100755 --- a/bin/gettext-merge +++ b/bin/gettext-merge @@ -6,22 +6,7 @@ # Copyright (c) 2011 UK Citizens Online Democracy. All rights reserved. # Email: matthew@mysociety.org; WWW: http://www.mysociety.org/ -# Yuck -if [ -e ../../locale ] -then - cd ../../ -else if [ -e ../locale ] -then - cd ../ -else if [ -e locale ] -then - cd . -else - echo "Please run with current directory fixmystreet" - exit 1 -fi -fi -fi +cd "$(dirname $(readlink -f $BASH_SOURCE))/.." for X in locale/*.UTF-8 do @@ -31,12 +16,12 @@ do echo $X if [ -e EmptyHomes.po ] then - msgmerge -o New.po EmptyHomes.po ../../FixMyStreet-EmptyHomes.po + msgmerge --no-wrap -o New.po EmptyHomes.po ../../FixMyStreet-EmptyHomes.po mv New.po EmptyHomes.po fi if [ -e FixMyStreet.po ] then - msgmerge -o New.po FixMyStreet.po ../../FixMyStreet.po + msgmerge --no-wrap -o New.po FixMyStreet.po ../../FixMyStreet.po mv New.po FixMyStreet.po fi cd - >/dev/null diff --git a/bin/handlemail b/bin/handlemail index 8b8e03be9..8bc016241 100755 --- a/bin/handlemail +++ b/bin/handlemail @@ -57,6 +57,7 @@ my $mail = mySociety::Email::construct_email({ To => $data{return_path}, _template_ => $template, _parameters_ => { }, + _line_indent => '', }); if (mySociety::EmailUtil::EMAIL_SUCCESS diff --git a/bin/handlemail-support b/bin/handlemail-support new file mode 100755 index 000000000..9808e447a --- /dev/null +++ b/bin/handlemail-support @@ -0,0 +1,62 @@ +#!/usr/bin/perl -w +# +# handlemail-support: +# Handle an individual incoming mail message. +# +# This script should be invoked through the .forward mechanism. It processes +# emails to the support address to remove out of office and so on, before +# forwarding on. +# +# Copyright (c) 2013 UK Citizens Online Democracy. All rights reserved. +# Email: matthew@mysociety.org; WWW: http://www.mysociety.org/ + +use strict; +require 5.8.0; + +# Horrible boilerplate to set up appropriate library paths. +use FindBin; +use lib "$FindBin::Bin/../perllib"; +use lib "$FindBin::Bin/../commonlib/perllib"; + +use mySociety::Config; +BEGIN { + mySociety::Config::set_file("$FindBin::Bin/../conf/general"); +} +use mySociety::EmailUtil; +use mySociety::HandleMail; + +my %data = mySociety::HandleMail::get_message(); +exit 0 if is_ignorable($data{message}); +forward_on(); + +# --- + +sub forward_on { + my ($l, $d) = split /\@/, mySociety::Config::get('CONTACT_EMAIL'); + if (mySociety::EmailUtil::EMAIL_SUCCESS + != mySociety::EmailUtil::send_email( + join("\n", @{$data{lines}}) . "\n", + $data{return_path}, + join('@', join('_deli', $l, 'very'), $d) + )) { + exit 75; + } + exit 0; +} + +sub is_ignorable { + my $m = shift; + my $head = $m->head(); + my ($from, $subject, $body) = ($head->get('From'), $head->get('Subject'), $m->body); + $body = join("\n", @$body); + + open my $fp, "$FindBin::Bin/../../data/ignored-emails.csv" or exit 75; + while (<$fp>) { + chomp; + my ($f, $s, $b) = split /,/; + next unless $f || $s || $b; + return 1 unless ( $f && $from !~ /$f/ ) || ( $s && $subject !~ /$s/ ) || ( $b && $body !~ /$b/ ); + } + return 0; +} + diff --git a/bin/install-as-user b/bin/install-as-user index 2656195f4..b05616702 100755 --- a/bin/install-as-user +++ b/bin/install-as-user @@ -1,7 +1,10 @@ #!/bin/sh set -e -set -x +error_msg() { printf "\033[31m%s\033[0m\n" "$*"; } +notice_msg() { printf "\033[33m%s\033[0m " "$*"; } +done_msg() { printf "\033[32m%s\033[0m\n" "$*"; } +DONE_MSG=$(done_msg done) if [ $# -ne 3 ] then @@ -14,6 +17,15 @@ fi UNIX_USER="$1" HOST="$2" DIRECTORY="$3" + +misuse() { + echo The variable $1 was not defined, and it should be. + echo This script should not be run directly. + exit 1 +} + +[ -z "$DEVELOPMENT_INSTALL" ] && misuse DEVELOPMENT_INSTALL + DB_NAME="fixmystreet" # Check that the arguments we've been passed are sensible: @@ -22,19 +34,19 @@ IP_ADDRESS_FOR_HOST="$(dig +short $HOST)" if [ x = x"$IP_ADDRESS_FOR_HOST" ] then - echo "The hostname $HOST didn't resolve to an IP address" + error_msg "The hostname $HOST didn't resolve to an IP address" exit 1 fi if ! id "$UNIX_USER" 2> /dev/null > /dev/null then - echo "The user '$UNIX_USER' didn't exist." + error_msg "The user '$UNIX_USER' didn't exist." exit 1 fi if [ "$(whoami)" != "$UNIX_USER" ] then - echo "This script should be run by the user '$UNIX_USER'." + error_msg "This script should be run by the user '$UNIX_USER'." exit 1 fi @@ -44,51 +56,71 @@ LINK_DESTINATION="$HOME/fixmystreet" ln -sfn "$REPOSITORY" $LINK_DESTINATION cd "$REPOSITORY" -# Add regularly scheduled tasks to cron: - -TEMPORARY_CRONTAB=$(mktemp) - -echo crontab file is $TEMPORARY_CRONTAB - -cp "$REPOSITORY"/conf/crontab.example "$TEMPORARY_CRONTAB" - -sed -i \ - -e 's,$FMS,'"$REPOSITORY,g" \ - -e 's,$LOCK_DIR,'"$DIRECTORY,g" \ - "$TEMPORARY_CRONTAB" - -crontab $TEMPORARY_CRONTAB +if [ ! "$DEVELOPMENT_INSTALL" = true ]; then + echo -n "Adding crontab... " + # Add regularly scheduled tasks to cron: + TEMPORARY_CRONTAB=$(mktemp) + echo crontab file is $TEMPORARY_CRONTAB + cp "$REPOSITORY"/conf/crontab-example "$TEMPORARY_CRONTAB" + sed -i \ + -e 's,$FMS,'"$REPOSITORY,g" \ + -e 's,$LOCK_DIR,'"$DIRECTORY,g" \ + "$TEMPORARY_CRONTAB" + crontab $TEMPORARY_CRONTAB + echo $DONE_MSG +fi # Install the compass gem locally - it's required for generating the # CSS: - +echo "Setting up CSS... " export GEM_HOME="$DIRECTORY/gems" mkdir -p "$GEM_HOME" export GEM_PATH= export PATH="$GEM_HOME/bin:$PATH" -gem install --no-ri --no-rdoc compass +if ! grep -q 'Set up local gem directory for FixMyStreet' $HOME/.bashrc; then + cat >>$HOME/.bashrc <<EOBRC + +# Set up local gem directory for FixMyStreet +export GEM_HOME="$DIRECTORY/gems" +export GEM_PATH= +export PATH="\$GEM_HOME/bin:\$PATH" +EOBRC +fi + +gem install --conservative --no-ri --no-rdoc sass -v 3.2.14 +gem install --conservative --no-ri --no-rdoc compass -v 0.12.2 # Use compass to generate the CSS, if it doesn't seem to already # exist: - if [ ! -f web/cobrands/default/base.css ] then bin/make_css fi +echo $DONE_MSG + +# Write sensible values into the config file, if it doesn't already exist +if [ ! -f conf/general.yml ]; then + echo -n "Setting up default conf/general.yml file... " + sed -r \ + -e "s,^( *FMS_DB_HOST:).*,\\1 ''," \ + -e "s,^( *FMS_DB_NAME:).*,\\1 '$DB_NAME'," \ + -e "s,^( *FMS_DB_USER:).*,\\1 '$UNIX_USER'," \ + -e "s,^( *BASE_URL:).*,\\1 'http://$HOST'," \ + -e "s,^( *EMAIL_DOMAIN:).*,\\1 '$HOST'," \ + -e "s,^( *CONTACT_EMAIL:).*,\\1 'help@$HOST'," \ + -e "s,^( *DO_NOT_REPLY_EMAIL:).*,\\1 'help@$HOST'," \ + conf/general.yml-example > conf/general.yml + echo $DONE_MSG +fi -# Write sensible values into the config file: - -sed -r \ - -e "s,^( *FMS_DB_HOST:).*,\\1 ''," \ - -e "s,^( *FMS_DB_NAME:).*,\\1 '$DB_NAME'," \ - -e "s,^( *FMS_DB_USER:).*,\\1 '$UNIX_USER'," \ - -e "s,^( *BASE_URL:).*,\\1 'http://$HOST'," \ - -e "s,^( *EMAIL_DOMAIN:).*,\\1 '$HOST'," \ - -e "s,^( *CONTACT_EMAIL:).*,\\1 'help@$HOST'," \ - conf/general.yml-example > conf/general.yml +echo "Installing required Perl modules - this may take some time" +cd "$REPOSITORY" +bin/install_perl_modules +echo $DONE_MSG # Create the database if it doesn't exist: +echo -n "Setting up database... " if ! psql -l | egrep "^ *$DB_NAME *\|" > /dev/null then createdb --owner "$UNIX_USER" "$DB_NAME" @@ -96,16 +128,16 @@ then psql -U "$UNIX_USER" "$DB_NAME" < "$REPOSITORY"/db/schema.sql psql -U "$UNIX_USER" "$DB_NAME" < "$REPOSITORY"/db/alert_types.sql psql -U "$UNIX_USER" "$DB_NAME" < "$REPOSITORY"/db/generate_secret.sql +else + bin/cron-wrapper update-schema --commit fi - -# Install the required Perl modules - this may take a very long time: - -cd "$FMS_REPOSITORY" -bin/install_perl_modules +echo $DONE_MSG # Generate po and mo files (these invocations taken from Kagee's script): - +echo "Creating locale .mo files" bin/cron-wrapper bin/make_po FixMyStreet-EmptyHomes bin/cron-wrapper bin/make_emptyhomes_welsh_po - commonlib/bin/gettext-makemo FixMyStreet +echo $DONE_MSG + +bin/cron-wrapper update-all-reports diff --git a/bin/install_perl_modules b/bin/install_perl_modules index 2311ae5f1..883ced235 100755 --- a/bin/install_perl_modules +++ b/bin/install_perl_modules @@ -4,18 +4,18 @@ set -e DIR="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd | sed -e 's/\/bin$//' )" -$DIR/bin/cpanm -l $DIR/local Carton +$DIR/bin/cpanm -l $DIR/local-carton Carton -export PATH=$DIR/local/bin:$PATH -export PERL5LIB=$DIR/local/lib/perl5 +export PATH=$DIR/local-carton/bin:$PATH +export PERL5LIB=$DIR/local-carton/lib/perl5 -carton install --deployment +carton install --deployment --without uk if ! perl -MImage::Magick -e 'exit()' >/dev/null 2>&1 then read -p "Image::Magick is not installed. Do you want to attempt to install it?" yn case $yn in - [Yy]* ) $DIR/local/bin/carton install Image::Magick;; + [Yy]* ) $DIR/local-carton/bin/carton install Image::Magick;; * ) echo 'You will need to install it for FixMyStreet to work';; esac fi diff --git a/bin/make_css b/bin/make_css index a86fd4f0d..c48dda4d6 100755 --- a/bin/make_css +++ b/bin/make_css @@ -13,20 +13,12 @@ DIRECTORY=$(cd `dirname $0`/../web && pwd) -# FixMyStreet uses compass -NEWSTYLE=${1:-"fixmystreet bromley fixmybarangay barnet zurich default stevenage"} -NEWSTYLE_REGEX=${NEWSTYLE// /\\|} -for site in $NEWSTYLE; do - compass compile --output-style compressed $DIRECTORY/cobrands/$site -done - -# If given a command line argument, assume was a compass directory and exit -if [ -n "$1" ]; then - exit 0 -fi +DIRS=${1:-`find $DIRECTORY -name "*.scss" -exec dirname {} \; | uniq`} -# The rest are plain sass -for scss in `find $DIRECTORY -name "*.scss" -exec dirname {} \; | uniq | grep -v "cobrands/\($NEWSTYLE_REGEX\)"` -do - sass --scss --update --style compressed $scss +for dir in $DIRS; do + if [ -e "$dir/config.rb" ]; then + compass compile --output-style compressed $dir + else + sass --scss --update --style compressed $dir + fi done diff --git a/bin/make_css_watch b/bin/make_css_watch new file mode 100755 index 000000000..d46ee8997 --- /dev/null +++ b/bin/make_css_watch @@ -0,0 +1,78 @@ +#!/usr/bin/env perl +use strict; +use warnings; +use feature 'say'; +use File::ChangeNotify; +use File::Find::Rule; +use Path::Tiny; + +my @exts = qw/ + scss +/; + +my @dirs = qw/ + web +/; + +my $filter = do { + my $exts = join '|', @exts; + qr/\.(?:$exts)$/ +}; + +my $watcher = File::ChangeNotify->instantiate_watcher( + directories => \@dirs, + filter => $filter, +); + +sub title { + my $what = shift; + # TODO, check if xtitle is installed and if so, run following command: + # system 'xtitle', $what; +} + +say sprintf "Watching [%s] for %s", (join ',' => @dirs), $filter; +title 'watching'; + +while ( my @events = $watcher->wait_for_events() ) { + my %seen; + my @update_dirs; + title 'updating'; + for my $event (@events) { + my $file = path( $event->path ); + say "$file was updated..."; + my $dir = $file->dirname; + next if $seen{$dir}++; + + if ($dir eq 'web/cobrands/sass/') { + # contains only partials, so we don't need to update + # this directory, but we *do* need to update everything else + push @update_dirs, + grep { + ! ($seen{$_}++) + } + map { + path($_)->dirname + } + File::Find::Rule->file->name($filter)->in( @dirs ); + } + else { + push @update_dirs, $dir; + } + } + for my $dir (@update_dirs) { + if (-e "$dir/config.rb") { + system compass => + 'compile', + '--output-style' => 'compressed', + $dir; + } + else { + system sass => + '--scss', + '--update', + '--style' => 'compressed', + $dir; + } + } + title 'watching'; +} diff --git a/bin/make_po b/bin/make_po index 76dc4566b..ad1fc1a89 100755 --- a/bin/make_po +++ b/bin/make_po @@ -14,6 +14,7 @@ mkdir("en_GB.UTF-8"); mkdir("en_GB.UTF-8/LC_MESSAGES"); my $pofile = shift; +die "Please supply a filename" unless $pofile; open(MAINPO, "FixMyStreet.po") or die ""; open(EHAPO, ">$pofile.po") or die ""; diff --git a/bin/merge_cobrand_po b/bin/merge_cobrand_po new file mode 100755 index 000000000..2785162cf --- /dev/null +++ b/bin/merge_cobrand_po @@ -0,0 +1,29 @@ +#!/usr/bin/env perl +use strict; + +=for instructions + +This script is used to merge cobrand po files with the main +FixMyStreet po file. It should be run after generate_cobrand_po +and once the cobrand po files with translations are placed in the +language directories. + +It will then create an autoCobrand.po file for each language that +has a Cobrand.po + +=cut + +my $cobrand = shift; + +die "Please provide a cobrand name\n" unless $cobrand; + +# for each language create a .po file with an existing translations +for (glob( 'locale/*/LC_MESSAGES' ) ) { + my $fms = "$_/FixMyStreet.po"; + my $cobrand_po = "$_/$cobrand.po"; + my $out = "$_/auto$cobrand.po"; + if ( -e $cobrand_po and -e $fms ) { + print "$_\n"; + system("msgcat --no-wrap -o $out $fms $cobrand_po"); + } +} diff --git a/bin/open311-populate-service-list b/bin/open311-populate-service-list index 232b0a6d4..be1ace3b9 100755 --- a/bin/open311-populate-service-list +++ b/bin/open311-populate-service-list @@ -14,12 +14,13 @@ my ($opt, $usage) = describe_options( ); print($usage->text), exit if $opt->help; -my $council_list = FixMyStreet::App->model('DB::Open311conf')->search( { +my $bodies = FixMyStreet::App->model('DB::Body')->search( { + id => { '!=', 2237 }, # XXX Until Oxfordshire does do so send_method => 'Open311' } ); my $verbose = 0; $verbose = 1 if $opt->warn; $verbose = 2 if $opt->verbose; -my $p = Open311::PopulateServiceList->new( council_list => $council_list, verbose => $verbose ); +my $p = Open311::PopulateServiceList->new( bodies => $bodies, verbose => $verbose ); -$p->process_councils; +$p->process_bodies; diff --git a/bin/open311-update-reports b/bin/open311-update-reports index 41c9c4546..3b77fef89 100644 --- a/bin/open311-update-reports +++ b/bin/open311-update-reports @@ -1,4 +1,10 @@ #!/usr/bin/perl +# +# This script utilises the standard Open311 way of getting updates on reports +# (by fetching all reports for a body and looking for updates). If possible, +# please use the extension explained at +# https://github.com/mysociety/FixMyStreet/wiki/Open311-FMS---Proposed-differences-to-Open311 +# and the fetch-comments/send-comments scripts. use strict; use warnings; @@ -13,9 +19,9 @@ my $system_user = FixMyStreet::App->model('DB::User')->find_or_create( } ); -my $council_list = FixMyStreet::App->model('DB::Open311conf'); +my $body_list = FixMyStreet::App->model('DB::Body'); my $update = Open311::GetUpdates->new( - council_list => $council_list, + body_list => $body_list, system_user => $system_user )->get_updates; diff --git a/bin/oxfordshire/open311_service_request.cgi b/bin/oxfordshire/open311_service_request.cgi new file mode 100755 index 000000000..4496ff213 --- /dev/null +++ b/bin/oxfordshire/open311_service_request.cgi @@ -0,0 +1,427 @@ +#!/usr/bin/perl + +# script for absobring incoming Open311 service request POSTs and +# passing them into Bentley EXOR backend via create_enquiry stored +# procedure. +# +# mySociety: http://fixmystreet.org/ +#----------------------------------------------------------------- + +require 'open311_services.pm'; +use DBD::Oracle qw(:ora_types); +### for local testing (no Oracle): +### use constant { ORA_VARCHAR2=>1, ORA_DATE=>1, ORA_NUMBER=>1}; + +my %PEM_BOUND_VAR_TYPES = get_pem_field_types(); + +my $ERR_MSG = 'error'; # unique key in data hash + +# incoming (Open311, from FMS) field names +# note: attribute[*] are being sent by FMS explicitly as attributes for Oxfordshire +my %F = ( + 'ACCOUNT_ID' => 'account_id', + 'ADDRESS_ID' => 'address_id', + 'ADDRESS_STRING' => 'address_string', + 'API_KEY' => 'api_key', + 'CLOSEST_ADDRESS' => 'attribute[closest_address]', + 'DESCRIPTION' => 'description', + 'DEVICE_ID' => 'device_id', + 'EASTING' => 'attribute[easting]', + 'EMAIL' => 'email', + 'FIRST_NAME' => 'first_name', + 'FMS_ID' => 'attribute[external_id]', + 'LAST_NAME' => 'last_name', + 'LAT' => 'lat', + 'LONG' => 'long', + 'MEDIA_URL' => 'media_url', + 'NORTHING' => 'attribute[northing]', + 'PHONE' => 'phone', + 'REQUESTED_DATETIME' => 'requested_datetime', + 'SERVICE_CODE' => 'service_code', + 'STATUS' => 'status', + +); + +my $req = new CGI; + +# normally, POST requests are inserting service requests +# and GET requests are for returning service requests, although OCC aren't planning on +# using that (it's part of the Open311 spec). +# So actually the service discovery is more useful, so send in a 'services' param +# to see that. +# +# But for testing the db connection, set $TEST_SERVICE_DISCOVERY so that +# *all* requests simply do a service discovery by setting (possibly via the config file) + +if ($TEST_SERVICE_DISCOVERY) { + get_service_discovery($req); # to test +}elsif ($ENV{'REQUEST_METHOD'} eq "POST") { + post_service_request($req); +} elsif ($req -> param('services')) { + get_service_discovery($req); +} elsif ($RUN_FAKE_INSERT_TEST) { + # allow a GET to make an insert, for testing (from the commandnd line!) + print "Running fake insert test... returned: " . get_FAKE_INSERT(); + print "\nSee $OUT_FILENAME for data" if $TESTING_WRITE_TO_FILE; + print "\n"; +} else { + get_service_requests($req); +} + +#---------------------------------------------------- +# post_service_request +# accepts an incoming service request +# If everything goes well, it puts it in the database and +# returns the PEM ID to the caller +#---------------------------------------------------- +sub post_service_request { + my $req = shift; + my %data; + my $pem_id = 0; + + foreach (values %F) { + $data{$_} = $req -> param($_); + $data{$_} =~ s/^\s+|\s+$//g; # trim + } + + error_and_exit(CODE_OR_ID_NOT_PROVIDED, "missing service code (Open311 requires one)") + unless $data{$F{SERVICE_CODE}}; + error_and_exit(GENERAL_SERVICE_ERROR, "the service code you provided ($data{$F{SERVICE_CODE}}) was not recognised by this server") + unless service_exists($data{$F{SERVICE_CODE}}); + error_and_exit(GENERAL_SERVICE_ERROR, "no address or long-lat provided") + unless ( (is_longlat($data{$F{LONG}}) && is_longlat($data{$F{LAT}})) || $data{$F{ADDRESS_STRING}} ); + + $pem_id = insert_into_pem(\%data); + + if (! $pem_id) { + error_and_exit(FATAL_ERROR, $data{$ERR_MSG} || "failed to get PEM ID"); + } else { + print <<XML; +Content-type: text/xml + +<?xml version="1.0" encoding="utf-8"?> +<service_requests> + <request> + <service_request_id>$pem_id</service_request_id> + </request> +</service_requests> +XML + } +} + +#------------------------------------------------------------------ +# is_longlat +# returns true if this looks like a long/lat value +#------------------------------------------------------------------ +sub is_longlat { + return $_[0] =~ /^-?\d+\.\d+$/o? 1 : 0; +} + +#------------------------------------------------------------------ +# service_exists +# lookup the service code, to check that it exists +# SELECT det_code, det_name FROM higatlas.doc_enquiry_types WHERE +# det_dtp_code = 'REQS' AND det_dcl_code ='SERV' AND det_con_id=1 +# Actually, FMS is expected to be sending good codes because they +# come from here anyway, and are expected to be stable. But could +# cache and check... probably overkill since DB insert will probably +# throw the error anyway. +#------------------------------------------------------------------ +sub service_exists { + my $service_code = shift; + return 1; +} + +#------------------------------------------------------------------ +# dump_to_file +# args: ref to hash of data +# for testing, log the incoming data into a local file +# NB throws a fatal error +#------------------------------------------------------------------ +sub dump_to_file { + my $h = shift; # href to data hash + if (open (OUTFILE, ">$OUT_FILENAME")) { + print OUTFILE "Data dump: " . gmtime() . "\n" . '-' x 64 . "\n\n"; + foreach (sort keys %$h) { + print OUTFILE "$_ => " . $$h{$_} . "\n"; + } + print OUTFILE "\n\n" . '-' x 64 . "\n[end]\n"; + close OUTFILE; + $$h{$ERR_MSG} = "NB did not write to DB (see $OUT_FILENAME instead: switch off \$TESTING_WRITE_TO_FILE to stop this)"; + } else { + $$h{$ERR_MSG} = "failed to write to outfile ($!)"; + } + return 0; # test always throws an error so no risk of production confusion! +} + +#------------------------------------------------------------------ +# insert_into_pem +# args: hashref to data hash +# returns PEM id of the new record (or passes an error message +# into the data hash if no id is available) +#------------------------------------------------------------------ +sub insert_into_pem { + my $h = shift; # href to data hash + + my $pem_id; + my $error_value; + my $error_product; + + # set specifc vars up where further processing on them might be needed: + my $undef = undef; + my $status = $$h{$F{STATUS}}; + my $service_code = $$h{$F{SERVICE_CODE}}; + my $description = $$h{$F{DESCRIPTION}}; + my $media_url = $$h{$F{MEDIA_URL}}; + if ($media_url) { + # don't put URL for full images into the database (because they're too big to see on a Blackberry) + $media_url =~ s/\.full(\.jpe?g)$/$1/; + $description .= ($STRIP_CONTROL_CHARS ne 'ruthless'? "\n\n":" ") . "Photo: $media_url"; + } + my $location = $$h{$F{CLOSEST_ADDRESS}}; + if ($location) { + # strip out everything apart from "Nearest" preamble + $location=~s/(Nearest road)[^:]+:/$1:/; + $location=~s/(Nearest postcode)[^:]+:(.*?)(\(\w+ away\))?\s*(\n|$)/$1: $2/; + } + + my %bindings; + # comments here are suggested values + # field lengths are from OCC's Java portlet + # fixed values + $bindings{":ce_cat"} = 'REQS'; # or REQS ? + $bindings{":ce_class"} = 'SERV'; # 'FRML' ? + $bindings{":ce_contact_type"} = 'ENQUIRER'; # 'ENQUIRER' + $bindings{":ce_status_code"} = 'RE'; # RE=received (?) + $bindings{":ce_compl_user_type"}= 'USER'; # 'USER' + + # ce_incident_datetime is *not* an optional param, but FMS isn't sending it at the moment + $bindings{":ce_incident_datetime"}=$$h{$F{REQUESTED_DATETIME}} || Time::Piece->new->strftime('%Y-%m-%d %H:%M'); + + # especially FMS-specific: + $bindings{":ce_source"} = "FMS"; # important, and specific to this script! + $bindings{":ce_doc_reference"} = $$h{$F{FMS_ID}}; # FMS id + $bindings{":ce_enquiry_type"} = $service_code; + + # incoming data + $bindings{":ce_x"} = $$h{$F{EASTING}}; + $bindings{":ce_y"} = $$h{$F{NORTHING}}; + $bindings{":ce_forename"} = uc strip($$h{$F{FIRST_NAME}}, 30); # 'CLIFF' + $bindings{":ce_surname"} = uc strip($$h{$F{LAST_NAME}}, 30); # 'STEWART' + $bindings{":ce_work_phone"} = strip($$h{$F{PHONE}}, 25); # '0117 600 4200' + $bindings{":ce_email"} = uc strip($$h{$F{EMAIL}}, 50); # 'info@exor.co.uk' + $bindings{":ce_description"} = strip($description, 1970, $F{DESCRIPTION}); # 'Large Pothole' + + # nearest address guesstimate + $bindings{":ce_location"} = strip($location, 254); + + if ($TESTING_WRITE_TO_FILE) { + return dump_to_file(\%bindings); + } + + # everything ready: now put it into the database + my $dbh = get_db_connection(); + + my $sth = $dbh->prepare(q# + BEGIN + PEM.create_enquiry( + ce_cat => :ce_cat, + ce_class => :ce_class, + ce_forename => :ce_forename, + ce_surname => :ce_surname, + ce_contact_type => :ce_contact_type, + ce_location => :ce_location, + ce_work_phone => :ce_work_phone, + ce_email => :ce_email, + ce_description => :ce_description, + ce_enquiry_type => :ce_enquiry_type, + ce_source => :ce_source, + ce_incident_datetime => to_Date(:ce_incident_datetime,'YYYY-MM-DD HH24:MI'), + ce_x => :ce_x, + ce_y => :ce_y, + ce_doc_reference => :ce_doc_reference, + ce_status_code => :ce_status_code, + ce_compl_user_type => :ce_compl_user_type, + error_value => :error_value, + error_product => :error_product, + ce_doc_id => :ce_doc_id); + END; +#); + + foreach my $name (sort keys %bindings) { + next if grep {$name eq $_} (':error_value', ':error_product', ':ce_doc_id'); # return values (see below) + $sth->bind_param( + $name, + $bindings{$name}, + $PEM_BOUND_VAR_TYPES{$name} || ORA_VARCHAR2 + ); + } + # return values are bound explicitly here: + $sth->bind_param_inout(":error_value", \$error_value, 12); #> l_ERROR_VALUE # number + $sth->bind_param_inout(":error_product", \$error_product, 10); #> l_ERROR_PRODUCT (will always be 'DOC') + $sth->bind_param_inout(":ce_doc_id", \$pem_id, 12); #> l_ce_doc_id # number + + # not used, but from the example docs, for reference + # $sth->bind_param(":ce_contact_title", $undef); # 'MR' + # $sth->bind_param(":ce_postcode", $undef); # 'BS11EJ' NB no spaces, upper case + # $sth->bind_param(":ce_building_no", $undef); # '1' + # $sth->bind_param(":ce_building_name", $undef); # 'CLIFTON HEIGHTS' + # $sth->bind_param(":ce_street", $undef); # 'HIGH STREET' + # $sth->bind_param(":ce_town", $undef); # 'BRSITOL' + # $sth->bind_param(":ce_enquiry_type", $undef); # 'CD' , ce_source => 'T' + # $sth->bind_param(":ce_cpr_id", $undef); # '5' (priority) + # $sth->bind_param(":ce_rse_he_id", $undef); #> nm3net.get_ne_id('1200D90970/09001','L') + # $sth->bind_param(":ce_compl_target", $undef); # '08-JAN-2004' + # $sth->bind_param(":ce_compl_corresp_date",$undef); # '02-JAN-2004' + # $sth->bind_param(":ce_compl_corresp_deliv_date", $undef); # '02-JAN-2004' + # $sth->bind_param(":ce_resp_of", $undef); # 'GBOWLER' + # $sth->bind_param(":ce_hct_vip", $undef); # 'CO' + # $sth->bind_param(":ce_hct_home_phone", $undef); # '0117 900 6201' + # $sth->bind_param(":ce_hct_mobile_phone", $undef); # '07111 1111111' + # $sth->bind_param(":ce_compl_remarks", $undef); # remarks (notes) max 254 char + + $sth->execute(); + $dbh->disconnect; + + # if error, maybe need to look it up: + # error_value is the index HER_NO in table HIG_ERRORS, which has messages + # actually err_product not helpful (wil always be "DOC") + $$h{$ERR_MSG} = "$error_value $error_product" if ($error_value || $error_product); + + return $pem_id; +} + +#------------------------------------------------------------------ +# strip +# args: data, max-length, field-name +# Trims, strips control chars, truncates to max-length +# Field-name only matters for description field +#------------------------------------------------------------------ +sub strip { + my ($s, $max_len, $field_name) = @_; + if ($STRIP_CONTROL_CHARS) { + if ($STRIP_CONTROL_CHARS eq 'ruthless') { + $s =~ s/[[:cntrl:]]/ /g; # strip all control chars, simples + } elsif ($STRIP_CONTROL_CHARS eq 'desc') { + if ($field_name eq $F{DESCRIPTION}) { + $s =~ s/[^\t\n[:^cntrl:]]/ /g; # leave tabs and newlines + } else { + $s =~ s/[[:cntrl:]]/ /g; # strip all control chars, simples + } + } else { + $s =~ s/[^\t\n[:^cntrl:]]/ /g; # leave tabs and newlines + } + } + from_to($s, 'utf8', 'Windows-1252') if $ENCODE_TO_WIN1252; + return $max_len? substr($s, 0, $max_len) : $s; +} + +#------------------------------------------------------------------ +# get_service_requests +# Strictly speaking, Open311 would expect the GET request for service +# requests to respond with all service requests (within a specified +# period). But as we're not using that, do a service discovery +# instead. +#------------------------------------------------------------------ +sub get_service_requests { + # error_and_exit(BAD_METHOD, "sorry, currently only handling incoming Open311 service requests: use POST method"); + get_service_discovery(); # for now +} + +#------------------------------------------------------------------ +# get_FAKE_INSERT +# for testing off command line, force the data +#------------------------------------------------------------------ +sub get_FAKE_INSERT { + my %fake_data = ( + $F{'DESCRIPTION'} => 'Testing, description: A acute (requires Latin-1): [á] ' + . ' pound sign (requires WinLatin-1): [£] omega tonos (requires UTF-8): [ώ]', + $F{'EASTING'} => '45119', + $F{'EMAIL'} => 'email@example.com', + $F{'FIRST_NAME'} => 'Dave', + $F{'FMS_ID'} => '1012', + $F{'LAST_NAME'} => 'Test', + $F{'LAT'} => '51.756741605999', + $F{'LONG'} => '-1.2596387532192', + $F{'NORTHING'} => '206709', + $F{'SERVICE_CODE'} => 'OT', + $F{'MEDIA_URL'} => 'http://www.example.com/pothole.jpg', + $F{'CLOSEST_ADDRESS'} => <<TEXT +Nearest road to the pin placed on the map (automatically generated by Bing Maps): St Giles, Oxford, OX1 3 + +Nearest postcode to the pin placed on the map (automatically generated): OX1 2LA (46m away) +TEXT + ); + return insert_into_pem(\%fake_data) +} + +#------------------------------------------------------------------ +# get_service_discovery +# Although not really implementing this, use it as a test to read the +# db and confirm connectivity. +#------------------------------------------------------------------ +sub get_service_discovery { + my $dbh = get_db_connection(); + my $ary_ref = $dbh->selectall_arrayref(qq(select det_code, det_name from higatlas.doc_enquiry_types where det_dtp_code = 'REQS' AND det_dcl_code='SERV' and det_con_id=1)); + # rough and ready XML dump now (should use XML Simple to build/escape properly!) + my $xml = ""; + foreach my $row(@{$ary_ref}) { + if (defined $row) { + my ($code, $name) = @$row; + $xml.= <<XML; + <service> + <service_code>$code</service_code> + <metadata>false</metadata> + <type>realtime</type> + <keywords/> + <group/> + <service_name>$name</service_name> + <description/> + </service> +XML + } + } + print <<XML; +Content-type: text/xml + +<?xml version="1.0" encoding="utf-8"?> +<services> +$xml +</services> +XML +} + +#------------------------------------------------------------------ +# get_pem_field_types +# return hash of types by field name: any not explicitly set here +# can be defaulted to VARCHAR2 +#------------------------------------------------------------------ +sub get_pem_field_types { + return ( + ':ce_incident_datetime' => ORA_DATE, + ':ce_x' => ORA_NUMBER, + ':ce_y' => ORA_NUMBER, + ':ce_date_expires' => ORA_DATE, + ':ce_issue_number' => ORA_NUMBER, + ':ce_status_date' => ORA_DATE, + ':ce_compl_ack_date' => ORA_DATE, + ':ce_compl_peo_date' => ORA_DATE, + ':ce_compl_target' => ORA_DATE, + ':ce_compl_complete' => ORA_DATE, + ':ce_compl_from' => ORA_DATE, + ':ce_compl_to' => ORA_DATE, + ':ce_compl_corresp_date' => ORA_DATE, + ':ce_compl_corresp_deliv_date' => ORA_DATE, + ':ce_compl_no_of_petitioners' => ORA_NUMBER, + ':ce_compl_est_cost' => ORA_NUMBER, + ':ce_compl_adv_cost' => ORA_NUMBER, + ':ce_compl_act_cost' => ORA_NUMBER, + ':ce_compl_follow_up1' => ORA_DATE, + ':ce_compl_follow_up2' => ORA_DATE, + ':ce_compl_follow_uo3' => ORA_DATE, + ':ce_date_time_arrived' => ORA_DATE, + ':error_value' => ORA_NUMBER, + ':ce_doc_id' => ORA_NUMBER, + ) +} + diff --git a/bin/oxfordshire/open311_service_request_update.cgi b/bin/oxfordshire/open311_service_request_update.cgi new file mode 100755 index 000000000..87b14fb98 --- /dev/null +++ b/bin/oxfordshire/open311_service_request_update.cgi @@ -0,0 +1,126 @@ +#!/usr/bin/perl + +# script for querying the higatlas.fms_update table provided by +# Bentley and offering them up as XML service request updates. +# https://github.com/mysociety/fixmystreet/wiki/Open311-FMS---Proposed-differences-to-Open311 +# +# mySociety: http://fixmystreet.org/ +#----------------------------------------------------------------- + +require 'open311_services.pm'; + +# incoming query params +my $CGI_VAR_START_DATE = 'start_date'; +my $CGI_VAR_END_DATE = 'end_date'; +my $CGI_VAR_NO_DEFAULT_DATE = 'force_no_default_date'; # for testing scratchy Oracle date stuff +my $CGI_VAR_LIMIT = 'limit'; # for testing +my $CGI_VAR_ANY_STATUS = 'any_status'; # for testing + +my $USE_ONLY_DATES = 0; # dates not times +my $MAX_LIMIT = 1000; +my $STATUS_CRITERIA = "(status='OPEN' OR status='CLOSED')"; +my $req = new CGI; + +get_service_request_updates($req); + +sub prepare_for_xml { + my $s = shift; + foreach ($s) { + from_to($_, 'utf8', 'Windows-1252') if $DECODE_FROM_WIN1252; + s/</</g; # numpty escaping pending XML Simple? + s/>/>/g; + s/&/&/g; + } + return $s; +} + +#------------------------------------------------------------------ +# get_service_discovery +# Although not really implementing this, use it as a test to read the +# db and confirm connectivity. +# +# TABLE "HIGATLAS"."FMS_UPDATE" +# +# "ROW_ID" NUMBER(9,0) NOT NULL ENABLE, +# "SERVICE_REQUEST_ID" NUMBER(9,0) NOT NULL ENABLE, +# "UPDATED_TIMEDATE" DATE DEFAULT SYSDATE NOT NULL ENABLE, +# "STATUS" VARCHAR2(10 BYTE) NOT NULL ENABLE, +# "DESCRIPTION" VARCHAR2(254 BYTE) NOT NULL ENABLE, +# +# CONSTRAINT "FMS_UPDATE_PK" PRIMARY KEY ("ROW_ID") +#------------------------------------------------------------------ +sub get_service_request_updates { + # by default, we only want last 24 hours + # also, limit to 1000 records + + my $raw_start_date = $req -> param($CGI_VAR_START_DATE); + my $raw_end_date = $req -> param($CGI_VAR_END_DATE); + my $start_date = get_date_or_nothing( $raw_start_date, $USE_ONLY_DATES ); + my $end_date = get_date_or_nothing( $raw_end_date, $USE_ONLY_DATES ); + + if (! $req -> param($CGI_VAR_NO_DEFAULT_DATE)) { + $start_date = get_date_or_nothing( $YESTERDAY, $USE_ONLY_DATES ) unless ($start_date or $end_date); + } + + my $date_format = 'YYYY-MM-DD HH24:MI:SS'; # NB: hh24 (not hh) + + $start_date = "updated_timedate >= to_date('$start_date', '$date_format')" if $start_date; + $end_date = "updated_timedate <= to_date('$end_date', '$date_format')" if $end_date; + + my $where_clause = ''; + my @criteria = ($start_date, $end_date); + push @criteria, $STATUS_CRITERIA unless $req -> param($CGI_VAR_ANY_STATUS); + $where_clause = join(' AND ', grep {$_} @criteria); + $where_clause = "WHERE $where_clause" if $where_clause; + + my $sql = qq(SELECT row_id, service_request_id, to_char(updated_timedate, '$date_format'), status, description FROM higatlas.fms_update $where_clause ORDER BY updated_timedate DESC); + + my $limit = $req -> param($CGI_VAR_LIMIT) =~ /^(\d{1,3})$/? $1 : $MAX_LIMIT; + $sql = "SELECT * FROM ($sql) WHERE ROWNUM <= $limit" if $limit; + + my $debug_str; + my $ary_ref; + + if ($TESTING_WRITE_TO_FILE) { + $ary_ref = [ + [97, 1000, '2013-01-05', 'OPEN', 'report was opened'], + [99, 1000, '2013-01-06', 'CLOSED', 'report was closed'] + ]; + # only add debug now if config says we're testing + $debug_str = <<XML; + <!-- DEBUG: from: $raw_start_date => $start_date --> + <!-- DEBUG: to: $raw_end_date => $end_date --> + <!-- DEBUG: sql: $sql --> +XML + } else { + my $dbh = get_db_connection(); + $ary_ref = $dbh->selectall_arrayref($sql); + } + + # rough and ready XML dump now (should use XML Simple to build/escape properly!) + my $xml = ""; + foreach my $row(@{$ary_ref}) { + if (defined $row) { + my ($id, $service_req_id, $updated_at, $status, $desc) = map { prepare_for_xml($_) } @$row; + $updated_at=~s/(\d{4}-\d\d-\d\d) (\d\d:\d\d:\d\d)/${1}T${2}Z/; # for now assume OCC in Zulu time + $xml.= <<XML; + <request_update> + <update_id>$id</update_id> + <service_request_id>$service_req_id</service_request_id> + <status>$status</status> + <updated_datetime>$updated_at</updated_datetime> + <description>$desc</description> + </request_update> +XML + } + } + print <<XML; +Content-type: text/xml + +<?xml version="1.0" encoding="utf-8"?> +<service_request_updates> +$xml +</service_request_updates> +$debug_str +XML +} diff --git a/bin/oxfordshire/open311_services.pm b/bin/oxfordshire/open311_services.pm new file mode 100644 index 000000000..12d0754fa --- /dev/null +++ b/bin/oxfordshire/open311_services.pm @@ -0,0 +1,150 @@ +#!/usr/bin/perl +# +# common stuff used by Oxfordshire Open311 glue scripts +# +# mySociety: http://fixmystreet.org/ +#----------------------------------------------------------------- + +use strict; +use CGI; +use Encode qw(from_to); +use DBI; +use Time::Piece; + + +################################################################### +# Config file: values in the config file override any values set +# in the code below for the following things: +# +# host: SXX-SAN-FOO_BAR +# sid: FOOBAR +# port: 1531 +# username: foo +# password: FooBar +# testing: 0 +# encode-to-win1252: 1 +# +# Absence of the config file fails silently in case you really are +# using values directly set in this script. +#------------------------------------------------------------------ +our $CONFIG_FILENAME = "/usr/local/etc/fixmystreet.config"; + +use constant { + GENERAL_SERVICE_ERROR => 400, + CODE_OR_ID_NOT_FOUND => 404, + CODE_OR_ID_NOT_PROVIDED => 400, + BAD_METHOD => 405, + FATAL_ERROR => 500 +}; + +our $DB_SERVER_NAME = 'FOO'; +our $DB_HOST = $DB_SERVER_NAME; # did this just in case we need to add more to the name (e.g, domain) +our $DB_PORT = '1531'; +our $ORACLE_SID = 'FOOBAR'; +our $USERNAME = 'FIXMYSTREET'; +our $PASSWORD = 'XXX'; +our $STORED_PROC_NAME = 'PEM.create_enquiry'; + +# NB can override these settings in the config file! + +# Strip control chars: +# 'ruthless' removes everything (i.e. all POSIX control chars) +# 'desc' removes everything, but keeps tabs and newlines in the 'description' field, where they matter +# 'normal' keeps tabs and newlines +our $STRIP_CONTROL_CHARS = 'ruthless'; + +our $ENCODE_TO_WIN1252 = 1; # force encoding to Win-1252 for PEM data +our $DECODE_FROM_WIN1252 = 1; # force encoding from Win-1252 for PEM data + +our $TESTING_WRITE_TO_FILE = 0; # write to file instead of DB or (get_service_request_update) don't really read the db +our $OUT_FILENAME = "fms-test.txt"; # dump data here if TESTING_WRITE_TO_FILE is true +our $TEST_SERVICE_DISCOVERY = 0; # switch to 1 to run service discovery, which confirms the DB connection at least +our $RUN_FAKE_INSERT_TEST = 0; # command-line execution attempts insert with fake data (mimics a POST request) + +# Config file overrides existing values for these, if present: +if ($CONFIG_FILENAME && open(CONF, $CONFIG_FILENAME)) { + while (<CONF>) { + next if /^#/; + if (/^\s*password:\s*(\S+)\s*$/i) { + $PASSWORD = $1; + } elsif (/^\s*sid:\s*(\S+)\s*$/i) { + $ORACLE_SID = $1; + } elsif (/^\s*username:\s*(\S+)\s*$/i) { + $USERNAME = $1; + } elsif (/^\s*port:\s*(\S+)\s*$/i) { + $DB_PORT = $1; + } elsif (/^\s*host:\s*(\S+)\s*$/i) { + $DB_HOST = $1; + } elsif (/^\s*testing:\s*(\S+)\s*$/i) { + $TESTING_WRITE_TO_FILE = $1; + } elsif (/^\s*test-service-discovery:\s*(\S+)\s*$/i) { + $TEST_SERVICE_DISCOVERY = $1; + } elsif (/^\s*strip-control-chars:\s*(\S+)\s*$/i) { + $STRIP_CONTROL_CHARS = lc $1; + } elsif (/^\s*encode-to-win1252:\s*(\S+)\s*$/i) { + $ENCODE_TO_WIN1252 = $1; + } elsif (/^\s*decode-from-win1252:\s*(\S+)\s*$/i) { + $DECODE_FROM_WIN1252 = $1; + } elsif (/^\s*run-fake-insert-test:\s*(\S+)\s*$/i) { + $RUN_FAKE_INSERT_TEST = $1; + } + } +} + +our $YESTERDAY = localtime() - Time::Seconds::ONE_DAY; # yesterday +$YESTERDAY = $YESTERDAY->strftime('%Y-%m-%d'); + +#------------------------------------------------------------------ +# error_and_exit +# args: HTTP status code, error message +# Sends out the HTTP status code and message +# and temrinates execution +#------------------------------------------------------------------ +sub error_and_exit { + my ($status, $msg) = @_; + print "Status: $status $msg\n\n$msg\n"; + exit; +} + + +#------------------------------------------------------------------ +# get_db_connection +# no args: uses globals, possibly read from config +# returns handle for the connection (otherwise terminates) +#------------------------------------------------------------------ +sub get_db_connection { + return DBI->connect( "dbi:Oracle:host=$DB_HOST;sid=$ORACLE_SID;port=$DB_PORT", $USERNAME, $PASSWORD ) + or error_and_exit(FATAL_ERROR, "failed to connect to database: " . $DBI::errstr, ""); +} + +#------------------------------------------------------------------ +# get_date_or_nothing { +# parse date from incoming request, fail silently +# expected format: 2003-02-15T13:50:05 +# These are coming from FMS for Oxford so don't expect to need +# need to parse anyway +#------------------------------------------------------------------ +sub get_date_or_nothing { + my $d = shift; + my $want_date_only = shift; + if ($d=~s/^(\d{4}-\d\d-\d\d)(T\d\d:\d\d(:\d\d)?)?.*/$1$2/) { + return $1 if $want_date_only; + $d="$1 00:00" unless $2; # no time provided + $d.=":00" unless $3; # no seconds + $d=~s/[TZ]/ /g; + # no point doing any parsing if regexp has done the work + # eval { + # $d=~s/(\d\d:\d\d).*/$1/; # bodge if we can't get DateTime installed + # $d = Time::Piece->strptime( $d, '%Y-%m-%dT%H:%M:%S'); + # $d = $d->strftime('%Y-%m-%d %H:%M:%S'); + # }; + # return '' if $@; + } else { + return ''; + } + return $d; +} + + + +1; diff --git a/bin/problem-creation-graph b/bin/problem-creation-graph index b96d45540..2ee38783c 100755 --- a/bin/problem-creation-graph +++ b/bin/problem-creation-graph @@ -49,13 +49,13 @@ function grab_data { grab_data "" $SOURCEA grab_data "where state not in ('unconfirmed')" $SOURCEB grab_data "where state not in ('unconfirmed', 'confirmed')" $SOURCEC -grab_data "where state not in ('unconfirmed', 'confirmed', 'fixed')" $SOURCED -grab_data "where state not in ('unconfirmed', 'confirmed', 'fixed', 'hidden')" $SOURCEE +grab_data "where state not in ('unconfirmed', 'confirmed', 'fixed', 'fixed - council', 'fixed - user')" $SOURCED +grab_data "where state not in ('unconfirmed', 'confirmed', 'fixed', 'fixed - council', 'fixed - user', 'hidden')" $SOURCEE #state = 'unconfirmed' #or state = 'confirmed' -#or state = 'fixed' +#or state in ('fixed', 'fixed - council', 'fixed - user') #or state = 'hidden' #or state = 'partial' @@ -96,7 +96,7 @@ if [ -s $SOURCED ]; then echo -n >>$GPSCRIPT " \"$SOURCED\" using 1:2 with impulses lt 6 lw 15 title \"hidden\"," fi if [ -s $SOURCEE ]; then -echo -n >>$GPSCRIPT " \"$SOURCEE\" using 1:2 with impulses lt 7 lw 15 title \"partial (and any other types)\"," +echo -n >>$GPSCRIPT " \"$SOURCEE\" using 1:2 with impulses lt 7 lw 15 title \"any other type\"," fi cat >>$GPSCRIPT <<END "< awk 'BEGIN { n = 0 } { n += \$2; print \$1, \$2, n; }' $SOURCEA" using 1:3 axes x1y2 with lines lt 2 title "cumulative total number of problems" diff --git a/bin/rotate-photos b/bin/rotate-photos index 31a60ff6c..7b8109d65 100755 --- a/bin/rotate-photos +++ b/bin/rotate-photos @@ -16,7 +16,7 @@ use FindBin; use lib "$FindBin::Bin/../perllib"; use lib "$FindBin::Bin/../commonlib/perllib"; -use Digest::SHA1 qw(sha1_hex); +use Digest::SHA qw(sha1_hex); use Utils; use mySociety::Config; diff --git a/bin/send-comments b/bin/send-comments index 4955cb2f3..3549113c9 100755 --- a/bin/send-comments +++ b/bin/send-comments @@ -1,7 +1,10 @@ #!/usr/bin/env perl -# send-reports: -# Send new problem reports to councils +# send-comments: +# Send comments/updates on reports to bodies +# In Open311 parlance these are 'service request updates' and are sent using +# mySociety's proposed extension to the Open311 Georeport v2 spec: +# https://github.com/mysociety/fixmystreet/wiki/Open311-FMS---Proposed-differences-to-Open311 # # Copyright (c) 2011 UK Citizens Online Democracy. All rights reserved. # Email: matthew@mysociety.org. WWW: http://www.mysociety.org @@ -23,24 +26,28 @@ use mySociety::EmailUtil; use Open311; -# maximum number of webservice attempts to send before not trying any more (XXX may be better in config?) -use constant SEND_FAIL_RETRIES_CUTOFF => 3; - # send_method config values found in by-area config data, for selecting to appropriate method use constant SEND_METHOD_EMAIL => 'email'; use constant SEND_METHOD_OPEN311 => 'Open311'; +use constant COUNCIL_ID_OXFORDSHIRE => 2237; + # Set up site, language etc. my ($verbose, $nomail) = CronFns::options(); my $base_url = mySociety::Config::get('BASE_URL'); my $site = CronFns::site($base_url); -my $councils = FixMyStreet::App->model('DB::Open311Conf')->search( { +my $bodies = FixMyStreet::App->model('DB::Body')->search( { send_method => SEND_METHOD_OPEN311, send_comments => 1, } ); -while ( my $council = $councils->next ) { +while ( my $body = $bodies->next ) { + + # Oxfordshire (OCC) is special: + # we do *receive* service_request_updates (aka comments) for OCC, but we never *send* them, so skip this pass + next if $body->areas->{+COUNCIL_ID_OXFORDSHIRE}; + my $use_extended = 0; my $comments = FixMyStreet::App->model('DB::Comment')->search( { 'me.whensent' => undef, @@ -49,7 +56,7 @@ while ( my $council = $councils->next ) { 'me.confirmed' => { '!=' => undef }, 'problem.whensent' => { '!=' => undef }, 'problem.external_id' => { '!=' => undef }, - 'problem.council' => { -like => '%' . $council->area_id .'%' }, + 'problem.bodies_str' => { -like => '%' . $body->id . '%' }, 'problem.send_method_used' => 'Open311', }, { @@ -57,18 +64,25 @@ while ( my $council = $councils->next ) { } ); - if ( $council->area_id == 2482 ) { + if ( $body->areas->{2482} ) { $use_extended = 1; } - my $o = Open311->new( - endpoint => $council->endpoint, - jurisdiction => $council->jurisdiction, - api_key => $council->api_key, + my %open311_conf = ( + endpoint => $body->endpoint, + jurisdiction => $body->jurisdiction, + api_key => $body->api_key, use_extended_updates => $use_extended, ); - if ( $council->area_id =~ /2482/ ) { + + if ( $body->send_extended_statuses ) { + $open311_conf{extended_statuses} = 1; + } + + my $o = Open311->new( %open311_conf ); + + if ( $body->areas->{2482} ) { my $endpoints = $o->endpoints; $endpoints->{update} = 'update.xml'; $endpoints->{service_request_updates} = 'update.xml'; @@ -78,11 +92,23 @@ while ( my $council = $councils->next ) { while ( my $comment = $comments->next ) { my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($comment->cobrand)->new(); + # TODO actually this should be OK for any devolved endpoint if original Open311->can_be_devolved, presumably + if ( 0 ) { # Check can_be_devolved and do this properly if set + my $sender = $cobrand->get_body_sender( $body, $comment->problem->category ); + my $config = $sender->{config}; + $o = Open311->new( + endpoint => $config->endpoint, + jurisdiction => $config->jurisdiction, + api_key => $config->api_key, + use_extended_updates => 1, # FMB uses extended updates + ); + } + if ( $comment->send_fail_count ) { next if bromley_retry_timeout( $comment ); } - if ( $council->area_id == 2482 ) { + if ( $body->areas->{2482} ) { my $extra = $comment->extra; if ( !$extra ) { $extra = {}; diff --git a/bin/site-specific-install.sh b/bin/site-specific-install.sh index 6ebac7970..3cc84007a 100644 --- a/bin/site-specific-install.sh +++ b/bin/site-specific-install.sh @@ -1,5 +1,8 @@ #!/bin/sh +# Set this to the version we want to check out +VERSION=${VERSION_OVERRIDE:-v1.3} + PARENT_SCRIPT_URL=https://github.com/mysociety/commonlib/blob/master/bin/install-site.sh misuse() { @@ -22,20 +25,33 @@ misuse() { [ -z "$HOST" ] && misuse HOST [ -z "$DISTRIBUTION" ] && misuse DISTRIBUTION [ -z "$VERSION" ] && misuse VERSION +[ -z "$DEVELOPMENT_INSTALL" ] && misuse DEVELOPMENT_INSTALL + +add_locale cy_GB +add_locale nb_NO +add_locale de_CH + +install_postfix -install_nginx +if [ ! "$DEVELOPMENT_INSTALL" = true ]; then + install_nginx + add_website_to_nginx + # Check out the current released version + su -l -c "cd '$REPOSITORY' && git checkout '$VERSION' && git submodule update" "$UNIX_USER" +fi install_website_packages su -l -c "touch '$DIRECTORY/admin-htpasswd'" "$UNIX_USER" -add_website_to_nginx - add_postgresql_user -su -l -c "$REPOSITORY/bin/install-as-user '$UNIX_USER' '$HOST' '$DIRECTORY'" "$UNIX_USER" +export DEVELOPMENT_INSTALL +su -c "$REPOSITORY/bin/install-as-user '$UNIX_USER' '$HOST' '$DIRECTORY'" "$UNIX_USER" -install_sysvinit_script +if [ ! "$DEVELOPMENT_INSTALL" = true ]; then + install_sysvinit_script +fi if [ $DEFAULT_SERVER = true ] && [ x != x$EC2_HOSTNAME ] then diff --git a/bin/update-all-reports b/bin/update-all-reports index 4c4838c77..f4d6113cb 100755 --- a/bin/update-all-reports +++ b/bin/update-all-reports @@ -18,50 +18,65 @@ use List::MoreUtils qw(zip); my $fourweeks = 4*7*24*60*60; +# Age problems from when they're confirmed, except on Zurich +# where they appear as soon as they're created. +my $age_column = 'confirmed'; +if ( FixMyStreet->config('BASE_URL') =~ /zurich|zueri/ ) { + $age_column = 'created'; +} + my $problems = FixMyStreet::App->model("DB::Problem")->search( { state => [ FixMyStreet::DB::Result::Problem->visible_states() ], }, { columns => [ - 'id', 'council', 'state', 'areas', + 'id', 'bodies_str', 'state', 'areas', { duration => { extract => "epoch from current_timestamp-lastupdate" } }, - { age => { extract => "epoch from current_timestamp-confirmed" } }, + { age => { extract => "epoch from current_timestamp-$age_column" } }, ] } ); $problems = $problems->cursor; # Raw DB cursor for speed my ( %fixed, %open ); -my @cols = ( 'id', 'council', 'state', 'areas', 'duration', 'age' ); +my @cols = ( 'id', 'bodies_str', 'state', 'areas', 'duration', 'age' ); while ( my @problem = $problems->next ) { my %problem = zip @cols, @problem; - my @areas; - if ( !$problem{council} ) { - # Problem was not sent to any council, add to all areas - @areas = grep { $_ } split( /,/, $problem{areas} ); - $problem{councils} = 0; + my @bodies; + if ( !$problem{bodies_str} ) { + # Problem was not sent to any bodies, add to all areas + @bodies = grep { $_ } split( /,/, $problem{areas} ); + $problem{bodies} = 0; } else { - # Add to councils it was sent to - (my $council = $problem{council}) =~ s/\|.*$//; - @areas = split( /,/, $council ); - $problem{councils} = scalar @areas; + # Add to bodies it was sent to + (my $bodies = $problem{bodies_str}) =~ s/\|.*$//; + @bodies = split( /,/, $bodies ); + $problem{bodies} = scalar @bodies; } - foreach my $council ( @areas ) { + foreach my $body ( @bodies ) { my $duration_str = ( $problem{duration} > 2 * $fourweeks ) ? 'old' : 'new'; my $type = ( $problem{duration} > 2 * $fourweeks ) ? 'unknown' : ($problem{age} > $fourweeks ? 'older' : 'new'); - if (FixMyStreet::DB::Result::Problem->fixed_states()->{$problem{state}}) { + if (FixMyStreet::DB::Result::Problem->fixed_states()->{$problem{state}} || FixMyStreet::DB::Result::Problem->closed_states()->{$problem{state}}) { # Fixed problems are either old or new - $fixed{$council}{$duration_str}++ if FixMyStreet::DB::Result::Problem->fixed_states()->{$problem{state}}; + $fixed{$body}{$duration_str}++; } else { # Open problems are either unknown, older, or new - $open{$council}{$type}++ if $problem{state} eq 'confirmed'; + $open{$body}{$type}++; } } } +if ( FixMyStreet->config('BASE_URL') =~ /emptyhomes/ ) { + my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker('emptyhomes')->new(); + my $stats = $cobrand->old_site_stats; + foreach (keys %$stats) { + $open{$_}{unknown} += $stats->{$_}; + } +} + my $body = JSON->new->utf8(1)->encode( { fixed => \%fixed, open => \%open, diff --git a/bin/update-schema b/bin/update-schema new file mode 100755 index 000000000..be5bc50e7 --- /dev/null +++ b/bin/update-schema @@ -0,0 +1,145 @@ +#!/usr/bin/perl +# +# This script should hopefully work out at what state the database is and, if +# the commit argument is provided, run the right schema files to bring it up to +# date. Let us know if it doesn't work; as with any upgrade script, do take a +# backup of your database before running. +# +# update-schema [--commit] + +use strict; +use warnings; + +# Horrible boilerplate to set up appropriate library paths. +use FindBin; +use lib "$FindBin::Bin/../commonlib/perllib"; + +use mySociety::Config; +use mySociety::DBHandle qw(dbh); +use mySociety::MaPit; + +mySociety::Config::set_file("$FindBin::Bin/../conf/general"); +my %args = ( + Name => mySociety::Config::get('FMS_DB_NAME'), + User => mySociety::Config::get('FMS_DB_USER'), + Password => mySociety::Config::get('FMS_DB_PASS'), +); +$args{Host} = mySociety::Config::get('FMS_DB_HOST', undef) if mySociety::Config::get('FMS_DB_HOST'); +$args{Port} = mySociety::Config::get('FMS_DB_PORT', undef) if mySociety::Config::get('FMS_DB_PORT'); +mySociety::DBHandle::configure( %args ); + +my $commit = 0; +$commit = 1 if @ARGV && $ARGV[0] eq '--commit'; + +my $nothing = 1; +my $current_version = get_db_version(); +print "Current database version = $current_version\n"; +print "= Dry run =\n" unless $commit; + +for my $path (glob("$FindBin::Bin/../db/schema_*")) { + my ($name) = $path =~ /schema_(.*)\.sql$/; + next if $name le $current_version; + next if $name =~ /$current_version-/; # For number only match + print "* $name\n"; + $nothing = 0; + next unless $commit; + + open(FP, $path) or die $!; + my @statements; + my $s = ''; + my $in_function = 0; + while(<FP>) { + next if /^--/; # Ignore comments + $s .= $_; + # Functions may have semicolons within them + $in_function = 1 if /create function/i; + $in_function = 0 if /language 'plpgsql'/i; + if ($s =~ /;/ && !$in_function) { + push @statements, $s; + $s = ''; + } + } + close FP; + + foreach my $st (@statements) { + dbh()->do($st); + } +} + +if ( $commit && $current_version lt '0028' ) { + print "Bodies created, fetching names from mapit\n"; + my $area_ids = dbh()->selectcol_arrayref('SELECT area_id FROM body_areas'); + if ( @$area_ids ) { + my $areas = mySociety::MaPit::call('areas', $area_ids); + foreach (values %$areas) { + dbh()->do('UPDATE body SET name=? WHERE id=?', {}, $_->{name}, $_->{id}); + } + dbh()->do('COMMIT'); + } +} + +print "Nothing to do\n" if $nothing; + +# --- + +# By querying the database schema, we can see where we're currently at +# (assuming schema change files are never half-applied, which should be the case) +sub get_db_version { + return '0031' if column_exists('body', 'external_url'); + return '0030' if ! constraint_exists('admin_log_action_check'); + return '0029' if column_exists('body', 'deleted'); + return '0028' if table_exists('body'); + return '0027' if column_exists('problem', 'subcategory'); + return '0026' if column_exists('open311conf', 'send_extended_statuses'); + return '0025' if column_like('alert_type', "ref='new_problems'", 'item_where', 'duplicate'); + return '0024' if column_exists('contacts', 'non_public'); + return '0023' if column_exists('open311conf', 'can_be_devolved'); + return '0022' if column_exists('problem', 'interest_count'); + return '0021' if column_exists('problem', 'external_source'); + return '0020' if column_exists('open311conf', 'suppress_alerts'); + return '0019' if column_exists('users', 'title'); + return '0018' if column_exists('open311conf', 'comment_user_id'); + return '0017' if column_exists('open311conf', 'send_comments'); + return '0016' if column_exists('comment', 'send_fail_count'); + return '0015-add_send_method_used_column_to_problem' if column_exists('problem', 'send_method_used'); + return '0015-add_extra_to_comment' if column_exists('comment', 'extra'); + return '0014' if column_exists('problem', 'send_fail_count'); + return '0013-add_send_method_column_to_open311conf' if column_exists('open311conf', 'send_method'); + return '0013-add_external_id_to_comment' if column_exists('comment', 'external_id'); + return '0012' if column_exists('problem', 'geocode'); + return '0011' if column_exists('contacts', 'extra'); + return '0010' if table_exists('open311conf'); + return '0009-update_alerts_problem_state_queries' if column_like('alert_type', "ref='new_problems'", 'item_where', 'investigating'); + return '0009-add_extra_to_problem' if column_exists('problem', 'extra'); + return '0008' if 0; + return '0007' if column_exists('comment', 'problem_state'); + return '0006' if 0; + return '0005-add_council_user_flag' if column_exists('users', 'from_council'); + return '0005-add_abuse_flags_to_users_and_reports' if column_exists('problem', 'flagged'); + return '0000'; +} + +# Returns true if a table exists +sub table_exists { + my $table = shift; + return dbh()->selectrow_array('select count(*) from pg_tables where tablename = ?', {}, $table); +} + +# Returns true if a column of table exists +sub column_exists { + my ( $table, $column ) = @_; + return dbh()->selectrow_array('select count(*) from pg_class, pg_attribute WHERE pg_class.relname=? AND pg_attribute.attname=? AND pg_class.oid=pg_attribute.attrelid AND pg_attribute.attnum > 0', {}, $table, $column); +} + +# Returns true if a column of a row in a table contains some text +sub column_like { + my ( $table, $where, $column, $contents ) = @_; + return dbh()->selectrow_array("select count(*) from $table WHERE $where AND $column LIKE ?", {}, "%$contents%"); +} + +# Returns true if a check constraint on a table exists +sub constraint_exists { + my ( $constraint ) = @_; + return dbh()->selectrow_array('select count(*) from pg_constraint where conname = ?', {}, $constraint); +} + diff --git a/bin/update-send-questionnaire b/bin/update-send-questionnaire new file mode 100755 index 000000000..7a231b919 --- /dev/null +++ b/bin/update-send-questionnaire @@ -0,0 +1,29 @@ +#!/usr/bin/env perl + +=head1 DESCRIPTION + +Ad hoc script to update the send_questionnaire column on the +reports in cobrands that don't send questionnaires at all. + +=cut + +use strict; +use warnings; + +use FixMyStreet::App; + +my @cobrands; +foreach my $cobrand ( FixMyStreet::Cobrand->available_cobrand_classes() ) { + next unless $cobrand->{class}; + my $cls = $cobrand->{class}->new(); + push @cobrands, $cls->moniker if !$cls->send_questionnaires(); +} + +my $problems = FixMyStreet::App->model('DB::Problem')->search({ + cobrand => \@cobrands, + send_questionnaire => 1, +}); + +$problems->update( { + send_questionnaire => 0 +}); diff --git a/bin/update_po_header.bash b/bin/update_po_header.bash new file mode 100755 index 000000000..2f37ad5d3 --- /dev/null +++ b/bin/update_po_header.bash @@ -0,0 +1,33 @@ +#!/bin/bash +# +# fixmystreet/bin/gettext-extract +# Generate English language .po files from the source code and email templates, +# for FixMyStreet. Writes the output to appropriate .po files in locale/. +# +# Copyright (c) 2011 UK Citizens Online Democracy. All rights reserved. +# Email: matthew@mysociety.org; WWW: http://www.mysociety.org/ + +cd "$(dirname $(readlink -f $BASH_SOURCE))/.." + +# File to write to, clear it to start with +PO=$1 + +# Fix headers +# no such thing as tempfile on OS X +TEMP=`tempfile 2>/dev/null || mktemp /tmp/gettext-extract.XXXXXX` +NOW=`date +"%Y-%m-%d %H:%M%z"` +# strictly POSIX sed on e.g. OS X doesn't let you used \n in replacements so we do this +nl=$'\n'; +cat $PO | sed " + s/SOME DESCRIPTIVE TITLE/FixMyStreet original .po file, autogenerated by gettext-extract/; + s/YEAR THE PACKAGE'S COPYRIGHT HOLDER/2011 UK Citizens Online Democracy/; + s/PACKAGE package/main FixMyStreet code/; + s/FIRST AUTHOR <EMAIL@ADDRESS>, YEAR./Matthew Somerville <matthew@mysociety.org>, 2011-06-03./; + + s/PACKAGE VERSION/1.0\\\n\"\\$nl\"Report-Msgid-Bugs-To: matthew@mysociety.org/; + s/POT-Creation-Date: YEAR-MO-DA HO:MI+ZONE/POT-Creation-Date: $NOW/; + s/LL@li.org/team@fixmystreet.com/; + s/charset=CHARSET/charset=UTF-8/; + s/8bit/8bit\\\n\"\\$nl\"Plural-Forms: nplurals=2; plural=n != 1;/; +" >> $TEMP +mv $TEMP $PO diff --git a/bin/zerotb_import_clinic_list.pl b/bin/zerotb_import_clinic_list.pl new file mode 100755 index 000000000..359a63925 --- /dev/null +++ b/bin/zerotb_import_clinic_list.pl @@ -0,0 +1,115 @@ +#!/usr/bin/perl + +use strict; +use FixMyStreet::App; +use Text::CSV; +use Getopt::Long::Descriptive; + +use constant TITLE => 0; +use constant DESC => 1; +use constant LATLONG => 2; +use constant EMAIL => 3; + +my ($opt, $usage) = describe_options( + '%c %o', + ['file|f=s', "path to csv file with list of clinics", { required => 1 } ], + ['email|e=s', "default email address for the updates to be sent to", { required => 1 } ], + ['verbose|v', "print out all services as they are found"], + ['help', "print usage message and exit" ], +); +print($usage->text), exit if $opt->help; + +my $csv = Text::CSV->new ( { binary => 1 } ) # should set binary attribute. + or die "Cannot use CSV: ".Text::CSV->error_diag (); +open my $fh, "<:encoding(utf8)", $opt->file or die "Failed to open " . $opt->file . ": $!"; + +my $clinic_user = FixMyStreet::App->model('DB::User')->find_or_create({ + email => $opt->email +}); +if ( not $clinic_user->in_storage ) { + $clinic_user->insert; +} + +# throw away header line +my $title_row = $csv->getline( $fh ); + +while ( my $row = $csv->getline( $fh ) ) { + my $clinics = FixMyStreet::App->model('DB::Problem')->search({ + title => $row->[TITLE] + }); + + my ($lat, $long) = split(',', $row->[LATLONG]); + my $p; + my $count = $clinics->count; + if ( $count == 0 ) { + $p = FixMyStreet::App->model('DB::Problem')->create({ + title => $row->[TITLE], + latitude => $lat, + longitude => $long, + used_map => 1, + anonymous => 1, + state => 'unconfirmed', + name => '', + user => $clinic_user, + detail => '', + areas => '', + postcode => '' + }); + } elsif ( $count == 1 ) { + $p = $clinics->first; + } else { + printf "Too many matches for: %s\n", $row->[TITLE]; + next; + } + $p->detail( $row->[DESC] ); + $p->latitude( $lat ); + $p->longitude( $long ); + $p->confirm; + + if ( $p->in_storage ) { + printf( "Updating entry for %s\n", $row->[TITLE] ) if $opt->verbose; + $p->update; + } else { + printf( "Creating entry for %s\n", $row->[TITLE] ) if $opt->verbose; + $p->insert; + } + $p->discard_changes; + + # disabling existing alerts in case email addresses have changed + my $existing = FixMyStreet::App->model('DB::Alert')->search({ + alert_type => 'new_updates', + parameter => $p->id + }); + $existing->update( { confirmed => 0 } ); + + if ( $row->[EMAIL] ) { + my $u = FixMyStreet::App->model('DB::User')->find_or_new({ + email => $row->[EMAIL] + }); + $u->insert unless $u->in_storage; + create_update_alert( $u, $p, $opt->verbose ); + } + + create_update_alert( $clinic_user, $p, $opt->verbose ); +} + +sub create_update_alert { + my ( $user, $p, $verbose ) = @_; + my $a = FixMyStreet::App->model('DB::Alert')->find_or_new({ + alert_type => 'new_updates', + user => $user, + parameter => $p->id, + }); + + $a->confirmed(1); + + if ( $a->in_storage ) { + printf( "Updating update alert for %s on %s\n", $user->email, $p->title ) + if $verbose; + $a->update; + } else { + printf( "Creating update alert for %s on %s\n", $user->email, $p->title ) + if $verbose; + $a->insert; + } +} diff --git a/bin/zurich-overdue-alert b/bin/zurich-overdue-alert new file mode 100755 index 000000000..4e81b3263 --- /dev/null +++ b/bin/zurich-overdue-alert @@ -0,0 +1,84 @@ +#!/usr/bin/env perl + +# zurich-overdue-alert: +# Send email alerts to administrators for overdue admin activities. +# +# Copyright (c) 2012 UK Citizens Online Democracy. All rights reserved. +# Email: matthew@mysociety.org. WWW: http://www.mysociety.org + +use strict; +use warnings; +require 5.8.0; + +use DateTime; +use CronFns; +use FixMyStreet::App; + +my ($verbose, $nomail) = CronFns::options(); + +# Only run on working days +my $now = DateTime->now(); +exit if FixMyStreet::Cobrand::Zurich::is_public_holiday($now) or FixMyStreet::Cobrand::Zurich::is_weekend($now); + +my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker('zurich')->new(); +my %bodies = map { $_->id => $_ } FixMyStreet::App->model("DB::Body")->all; + +loop_through( 'alert-moderation-overdue.txt', 0, 1, [ 'unconfirmed' ] ); +loop_through( 'alert-overdue.txt', 1, 6, 'in progress' ); +loop_through( 'alert-overdue.txt', 0, 6, ['confirmed', 'planned'] ); + +sub loop_through { + my ( $template, $include_parent, $days, $states ) = @_; + + my $reports = FixMyStreet::App->model("DB::Problem")->search( { + state => $states, + created => { '<', FixMyStreet::Cobrand::Zurich::sub_days( $now, $days ) }, + bodies_str => { '!=', undef }, + } ); + + my %to_send = (); + while (my $row = $reports->next) { + $to_send{$row->bodies_str} .= '* ' . $row->id . ": '" . $row->title . "'\n\n"; + } + + my $template_path = FixMyStreet->path_to( "templates", "email", "zurich", $template )->stringify; + $template = Utils::read_file( $template_path ); + + foreach my $body_id (keys %to_send) { + send_alert( $template, $body_id, $to_send{$body_id}, $include_parent ); + } +} + +sub send_alert { + my ( $template, $body_id, $data, $include_parent ) = @_; + + my $body = $bodies{$body_id}; + my $body_email = $body->endpoint; + + my $h = { + data => $data, + admin_url => $cobrand->admin_base_url, + }; + + my $env_to = [ $body_email ]; + my $to = [ [ $body_email, $body->name ] ]; + if ( $include_parent ) { + my $parent = $body->parent; + my $parent_email = $parent->endpoint; + push @$env_to, $parent_email; + push @$to, [ $parent_email, $parent->name ]; + } + + my $result = FixMyStreet::App->send_email_cron( + { + _template_ => $template, + _parameters_ => $h, + To => $to, + From => [ FixMyStreet->config('CONTACT_EMAIL'), FixMyStreet->config('CONTACT_NAME') ], + }, + FixMyStreet->config('CONTACT_EMAIL'), + $env_to, + $nomail + ); +} + diff --git a/bin/zurich/convert_internal_notes_to_comments b/bin/zurich/convert_internal_notes_to_comments new file mode 100755 index 000000000..ddf74851f --- /dev/null +++ b/bin/zurich/convert_internal_notes_to_comments @@ -0,0 +1,73 @@ +#!/usr/bin/env perl + +=head1 DESCRIPTION + +This is a Zurich specific migration script. + +It converts the internal notes that used to be stored on the problem in the +extra hash into comments instead. + + ./bin/zurich/convert_internal_notes_to_comments user_id + +You need to select a user to have the internal notes be associated with, perhaps +the superuser, or one created just for this purpose? + +=cut + +use strict; +use warnings; + +use FixMyStreet::App; + +# Because it is not possible to determine the user that last edited the +# internal_notes we need require a user to assign all the comments to. +my $comment_user_id = $ARGV[0] + || die "Usage: $0 id_of_user_for_comments"; +my $comment_user = FixMyStreet::App # + ->model('DB::User') # + ->find($comment_user_id) + || die "Could not find user with id '$comment_user_id'"; + +# We use now as the time for the internal note. This is not the time it was +# actually created, but I don't think that can be extracted from the problem. +my $comment_timestamp = DateTime->now(); + +# The state of 'hidden' seems most appropriate for these internal_notes - as +# they should not be shown to the public. Certainly more suitable than +# 'unconfirmed' or 'confirmed'. +my $comment_state = 'hidden'; + +# Load all the comments, more reliable than trying to search on the contents of +# the extra field. +my $problems = FixMyStreet::App->model('DB::Problem')->search(); + +while ( my $problem = $problems->next() ) { + + my $problem_id = $problem->id; + + # Extract the bits we are interested in. May not exist, in which case + # skip on. + my $extra = $problem->extra || {}; + next unless exists $extra->{internal_notes}; + + # If there is something to save create a comment with the notes in them + if ( my $internal_notes = $extra->{internal_notes} ) { + print "Creating internal note comment for problem '$problem_id'\n"; + $problem->add_to_comments( + { + text => $internal_notes, + created => $comment_timestamp, + user => $comment_user, + state => $comment_state, + mark_fixed => 0, + anonymous => 1, + extra => { is_internal_note => 1 }, + } + ); + } + + # Remove the notes from extra and save back to the problem + delete $extra->{internal_notes}; + $problem->update({ extra => $extra }); +} + diff --git a/bin/zurich/geocode b/bin/zurich/geocode new file mode 100755 index 000000000..9482b27e6 --- /dev/null +++ b/bin/zurich/geocode @@ -0,0 +1,45 @@ +#!/usr/bin/perl + +=head1 NAME + +zurich/geocode - commandline tool to test the Zurich geocoder + +=head1 SYNOPSIS + + # Firstly: + ## copy the GEOCODER config from a current Zurich conf to your conf/general.yml + $ eval `perl setenv.pl` + + $ bin/zurich/geocode Magnus + + # ... output from geocoder + +This can be used to test the results of, e.g. + + https://www.zueriwieneu.ch/ajax/geocode?term=Magnus + +but without the caching which FixMyStreet applies, and passing on any 500 +errors from the server. + +=cut + +use strict; +use warnings; +require 5.8.0; + + +use Data::Dumper; +use feature 'say'; + +use FixMyStreet; +use FixMyStreet::App; +use FixMyStreet::Geocode::Zurich; + +# TODO use FixMyStreet::override_config to get data from conf/general.yml.zurich if available +my $geocoder = FixMyStreet->config('GEOCODER') + or die "No GEOCODER config -- please copy appropriate Zurich conf to conf/general.yml"; + +my $c = FixMyStreet::App->new(); +my $s = join ' ', @ARGV; + +say Dumper( FixMyStreet::Geocode::Zurich::string( $s, $c ) ); |