diff options
Diffstat (limited to 'bin')
34 files changed, 2320 insertions, 254 deletions
diff --git a/bin/browser-tests b/bin/browser-tests index c663e56af..c7e5df27c 100755 --- a/bin/browser-tests +++ b/bin/browser-tests @@ -2,39 +2,58 @@ use strict; use warnings; +use lib '.'; # For the mock MapIt module + +use Getopt::Long ':config' => qw(pass_through auto_help); + +my ($run_server, $run_cypress, $vagrant, $wsl, $node, $config_file); +my ($cobrand, $coords, $area_id, $name, $mapit_url); BEGIN { + $config_file = 'conf/general.yml-example'; + $cobrand = [ 'borsetshire', 'fixmystreet', 'northamptonshire', 'bathnes', 'buckinghamshire', 'hounslow', 'isleofwight', 'peterborough', 'tfl' ]; + $coords = '51.532851,-2.284277'; + $area_id = 2608; + $name = 'Borsetshire'; + $mapit_url = 'https://mapit.uk/'; + $node = 'C:\Program Files\nodejs\node.exe'; + + GetOptions( + 'config=s' => \$config_file, + 'server' => \$run_server, + 'cypress' => \$run_cypress, + 'vagrant' => \$vagrant, + 'wsl=s' => \$wsl, + 'node=s' => \$node, + 'cobrand=s@' => \$cobrand, + 'coords=s' => \$coords, + 'area_id=s' => \$area_id, + 'name=s' => \$name, + 'mapit_url=s' => \$mapit_url, + ); + $cobrand = [ split(',', join(',', @$cobrand)) ]; + + if ($vagrant && $wsl) { + print 'You cannot use both --vagrant and --wsl'; + exit 1; + } + + if (!$run_server && !$run_cypress) { + # If asked for neither, run both + $run_server = $run_cypress = 1; + } + use File::Basename qw(dirname); use File::Spec; my $d = dirname(File::Spec->rel2abs($0)); - require "$d/../setenv.pl"; + if (!$vagrant && $run_server) { + require "$d/../setenv.pl"; + } } -use Getopt::Long ':config' => qw(pass_through auto_help); - -my ($run_server, $run_cypress, $vagrant); -my $config_file = 'conf/general.yml-example'; -my $cobrand = 'fixmystreet'; -my $coords = '51.532851,-2.284277'; -my $area_id = 2608; -my $name = 'Borsetshire'; -my $mapit_url = 'https://mapit.uk/'; - -GetOptions( - 'config=s' => \$config_file, - 'server' => \$run_server, - 'cypress' => \$run_cypress, - 'vagrant' => \$vagrant, - 'cobrand=s' => \$cobrand, - 'coords=s' => \$coords, - 'area_id=s' => \$area_id, - 'name=s' => \$name, - 'mapit_url=s' => \$mapit_url, -); - if ($vagrant) { # Test inception - system('vagrant ssh -c "cd fixmystreet && bin/browser-tests --server 2>/dev/null" &'); + system('vagrant ssh -- "cd fixmystreet && bin/browser-tests --server 2>/dev/null" &'); require IO::Socket; sub check_connection { @@ -52,15 +71,10 @@ if ($vagrant) { } print " done\n"; system('bin/browser-tests', '--cypress', @ARGV); - system('vagrant', 'ssh', '-c', 'kill $(cat /tmp/cypress-server.pid)'); + system('vagrant', 'ssh', '--', 'kill $(cat /tmp/cypress-server.pid)'); exit; } -if (!$run_server && !$run_cypress) { - # If asked for neither, run both - $run_server = $run_cypress = 1; -} - sub run { my $cmd = shift @ARGV; die "Must specify a cypress command\n" unless $cmd || !$run_cypress; @@ -71,15 +85,20 @@ sub run { my $config_out = FixMyStreet::TestAppProve->get_config({ config_file => $config_file, # Want this to be like .com - ALLOWED_COBRANDS => [ $cobrand ], + ALLOWED_COBRANDS => $cobrand, MAPIT_URL => $mapit_url, - BASE_URL => 'http://localhost:3001', + BASE_URL => 'http://fixmystreet.localhost:3001', + STAGING_FLAGS => { skip_checks => 1 }, + COBRAND_FEATURES => { + category_groups => { map { $_ => 1 } @$cobrand }, + suggest_duplicates => { map { $_ => 1 } @$cobrand }, + } }); $ENV{FMS_OVERRIDE_CONFIG} = $config_out; # Set up, and load in some data - system('bin/make_css', 'fixmystreet.com'); - system('bin/fixmystreet.com/fixture', '--nonrandom', '--coords', $coords, '--name', $name, '--area-id', $area_id, '--commit'); + system('bin/make_css', map { $_ eq 'fixmystreet' ? 'fixmystreet.com' : $_ } @$cobrand); + system('bin/fixmystreet.com/fixture', '--test_fixtures', '--nonrandom', '--coords', $coords, '--name', $name, '--area-id', $area_id, '--commit'); } my $pid; @@ -90,14 +109,16 @@ sub run { if (($run_cypress && !$run_server) || $pid) { # Parent, run the test runner (then kill the child) - my $exit = system("cypress", $cmd, '--config', 'pluginsFile=false,supportFile=false,blacklistHosts=[gaze.mysociety.org,*.openstreetmap.org]', '--project', '.cypress', '--env', "cobrand=$cobrand", @ARGV); + my @cypress = ('cypress'); + if ($wsl) { + @cypress = ('cmd.exe', '/c', $node, $wsl); + } + my $exit = system(@cypress, $cmd, '--config', 'pluginsFile=false,supportFile=false', '--project', '.cypress', @ARGV); kill 'TERM', $pid if $pid; exit $exit >> 8; } else { - use Test::MockModule; - my $c = Test::MockModule->new('FixMyStreet::Cobrand::FixMyStreet'); - $c->mock('enable_category_groups', sub { 1 }); # Child, run the server on port 3001 + require FixMyStreet; FixMyStreet->test_mode(1); # So email doesn't try to send local $ENV{FIXMYSTREET_APP_DEBUG} = 0; require Plack::Runner; @@ -125,10 +146,11 @@ browser-tests [running options] [fixture options] [cypress options] --server only run the test server, not cypress --cypress only run cypress, not the test server --vagrant run test server inside Vagrant, cypress outside + --wsl provide path to cypress node script, to run test server inside WSL, cypress outside --help this help message Fixture option: - --cobrand Cobrand to use, default 'fixmystreet' + --cobrand Cobrand(s) to use, default is fixmystreet,northamptonshire,bathnes,buckinghamshire,isleofwight,peterborough,tfl --coords Default co-ordinates for created reports --area_id Area ID to use for created body --name Name to use for created body @@ -143,6 +165,7 @@ server in the VM and Cypress outside of it. $ browser-tests run # run headlessly $ browser-tests run --record --key # record and upload a run $ browser-tests --vagrant run # run if you use Vagrant + $ browser-tests --wsl ..cypress run # run if you use WSL You need to have installed cypress already using npm, and it needs to be on your PATH. diff --git a/bin/createsuperuser b/bin/createsuperuser index 98b36aa36..7a4946228 100755 --- a/bin/createsuperuser +++ b/bin/createsuperuser @@ -30,4 +30,4 @@ BEGIN { use FixMyStreet; use FixMyStreet::Script::CreateSuperuser; -FixMyStreet::Script::CreateSuperuser::createsuperuser(); +exit FixMyStreet::Script::CreateSuperuser::createsuperuser(@ARGV); diff --git a/bin/export-import-data b/bin/export-import-data new file mode 100755 index 000000000..b73b4097e --- /dev/null +++ b/bin/export-import-data @@ -0,0 +1,153 @@ +#!/usr/bin/env perl + +use v5.14; +use warnings; + +BEGIN { + use File::Basename qw(dirname); + use File::Spec; + my $d = dirname(File::Spec->rel2abs($0)); + require "$d/../setenv.pl"; +} + +use FixMyStreet::DB; +use Getopt::Long::Descriptive; +use JSON::MaybeXS; +use Path::Tiny; + +my ($opt, $usage) = describe_options( + '%c', + [ 'name=s', "Name of body" ], + [ 'import=s', "File to import" ], + [ 'commit', "Actually commit changes to the database" ], + [ 'help', "print usage message and exit", { shortcircuit => 1 } ], +); +print($usage->text), exit if $opt->help; +$usage->die unless $opt->name; +die "Please specify a file to import\n" if $opt->import && (! -e $opt->import || -d $opt->import); + +my $J = JSON::MaybeXS->new(utf8 => 1, pretty => 1, canonical => 1); +my $body = FixMyStreet::DB->resultset("Body")->find({ name => $opt->name }) or die "Cannot find body " . $opt->name . "\n"; + +if ($opt->import) { + import($opt->import); +} else { + export(); +} + +sub export { + my %out; + + for ($body->response_templates->all) { + push @{$out{templates}}, { + title => $_->title, + text => $_->text, + state => $_->state, + categories => [ map { $_->category } $_->contacts->all ], + auto_response => $_->auto_response, + external_status_code => $_->external_status_code, + }; + } + + for ($body->roles->all) { + push @{$out{roles}}, { + permissions => $_->permissions, + name => $_->name, + }; + } + + for ($body->users->all) { + my $cats = $_->get_extra_metadata('categories'); + my @cat_names = map { $body->contacts->find({id => $_})->category } @$cats; + push @{$out{users}}, { + email => $_->email, + name => $_->name, + password => $_->password, + roles => [ map { $_->name } $_->roles->all ], + categories => \@cat_names, + areas => $_->area_ids || [], + }; + } + + print $J->encode(\%out); +} + +sub import { + my $file = shift; + + my $json = path($file)->slurp; + my $out = $J->decode($json); + + my $db = FixMyStreet::DB->schema->storage; + $db->txn_begin; + + foreach (@{$out->{templates}}) { + my $existing = $body->response_templates->search({ title => $_->{title} })->single; + if ($existing) { + warn "Template with title $_->{title} already exists, skipping"; + next; + } + my $template = $body->response_templates->new({ + title => $_->{title}, + text => $_->{text}, + state => $_->{state}, + auto_response => $_->{auto_response}, + external_status_code => $_->{external_status_code}, + }); + $template->insert; + foreach (@{$_->{categories}}) { + my $contact = $body->contacts->find({ category => $_ }) or die "Cannot find category $_ for template " . $template->title . "\n"; + $template->contact_response_templates->find_or_create({ + contact_id => $contact->id, + }); + } + } + + for my $r (@{$out->{roles}}) { + my $role = $body->roles->find_or_new({ + name => $r->{name}, + permissions => $r->{permissions}, + }); + if ($role->in_storage) { + warn "Role $r->{role} already exists; skipping"; + next; + } + $role->insert; + } + + for my $u (@{$out->{users}}) { + my $user = FixMyStreet::DB->resultset("User")->find_or_new({ email => $u->{email}, email_verified => 1 }); + if ($user->in_storage) { + warn "User $u->{email} already exists; skipping"; + next; + } + $user->from_body($body->id); + $user->name($u->{name}); + $user->password($u->{password}, 1); + $user->area_ids($u->{areas}); + + $user->insert; + + foreach my $role (@{$u->{roles}}) { + my $role = $body->roles->find({ name => $role }) or die "Couldn't find role $role for user $u->{email}\n"; + $user->user_roles->create({ + role_id => $role->id, + }); + } + + my @cat_ids; + for my $cat_name (@{$u->{categories}}) { + my $cat = $body->contacts->find({ category => $cat_name }) or die "Couldn't find category $cat_name for user $u->{email}\n"; + push @cat_ids, $cat->id; + } + $user->set_extra_metadata('categories', \@cat_ids); + $user->set_extra_metadata(last_password_change => time()); + $user->update; + } + + if ($opt->commit) { + $db->txn_commit; + } else { + $db->txn_rollback; + } +} diff --git a/bin/fetch b/bin/fetch new file mode 100755 index 000000000..5608cf195 --- /dev/null +++ b/bin/fetch @@ -0,0 +1,58 @@ +#!/usr/bin/env perl +# +# This script utilises Open311 as described at +# http://wiki.open311.org/GeoReport_v2/#get-service-requests +# and/or the Open311 extension explained at +# https://github.com/mysociety/FixMyStreet/wiki/Open311-FMS---Proposed-differences-to-Open311 +# to fetch service requests or updates on service requests. + +use strict; +use warnings; +use v5.14; + +BEGIN { + use File::Basename qw(dirname); + use File::Spec; + my $d = dirname(File::Spec->rel2abs($0)); + require "$d/../setenv.pl"; +} + +use DateTime; +use Getopt::Long::Descriptive; +use Open311::GetServiceRequests; +use Open311::GetServiceRequestUpdates; + +my ($opts, $usage) = describe_options( + '%c %o', + ['reports', 'fetch reports'], + ['updates', 'fetch updates'], + ['start|s:i', 'start time to use (hours before now), defaults to one (reports) or two (updates)' ], + ['end|e:i', 'end time to use (hours before now), defaults to zero' ], + ['body|b:s', 'body name to only fetch this body' ], + ['verbose|v', 'more verbose output'], + ['help|h', "print usage message and exit" ], +); +$usage->die if $opts->help; + +my %params = ( + verbose => $opts->verbose, + body => $opts->body, +); + +my $dt = DateTime->now(); +if ($opts->start) { + $params{start_date} = $dt->clone->add(hours => -$opts->start); +} +if ($opts->end) { + $params{end_date} = $dt->clone->add(hours => -$opts->end); +} + +if ($opts->reports) { + my $reports = Open311::GetServiceRequests->new(%params); + $reports->fetch; +} + +if ($opts->updates) { + my $updates = Open311::GetServiceRequestUpdates->new(%params); + $updates->fetch; +} diff --git a/bin/fetch-comments b/bin/fetch-comments index 55ba70d85..0fb30f236 100755 --- a/bin/fetch-comments +++ b/bin/fetch-comments @@ -1,25 +1,6 @@ -#!/usr/bin/env perl +#!/bin/bash # -# This script utilises the Open311 extension explained at -# https://github.com/mysociety/FixMyStreet/wiki/Open311-FMS---Proposed-differences-to-Open311 -# to fetch updates on service requests. +# Wrapper to call new script -use strict; -use warnings; -require 5.8.0; - -BEGIN { - use File::Basename qw(dirname); - use File::Spec; - my $d = dirname(File::Spec->rel2abs($0)); - require "$d/../setenv.pl"; -} - -use CronFns; -my ($verbose, $nomail) = CronFns::options(); - -use Open311::GetServiceRequestUpdates; - -my $updates = Open311::GetServiceRequestUpdates->new( verbose => $verbose ); - -$updates->fetch; +DIR="$(cd "$(dirname "$BASH_SOURCE")" && pwd -P)" +$DIR/fetch --updates diff --git a/bin/fetch-comments-24hs b/bin/fetch-comments-24hs index df208439e..86675be4c 100755 --- a/bin/fetch-comments-24hs +++ b/bin/fetch-comments-24hs @@ -1,37 +1,6 @@ -#!/usr/bin/env perl +#!/bin/bash # -# This script utilises the Open311 extension explained at -# https://github.com/mysociety/FixMyStreet/wiki/Open311-FMS---Proposed-differences-to-Open311 -# to fetch updates on service requests from the past 24 hours, to check none -# were missed. +# Wrapper to call new script -use strict; -use warnings; -require 5.8.0; - -BEGIN { - use File::Basename qw(dirname); - use File::Spec; - my $d = dirname(File::Spec->rel2abs($0)); - require "$d/../setenv.pl"; -} - -use DateTime; -use DateTime::Format::W3CDTF; - -use CronFns; -my ($verbose, $nomail) = CronFns::options(); - -use Open311::GetServiceRequestUpdates; - -my $dt = DateTime->now(); -my $dt_24hrs_ago = $dt->clone; -$dt_24hrs_ago->add( hours => -24 ); - -my $updates = Open311::GetServiceRequestUpdates->new( - verbose => 1, - start_date => DateTime::Format::W3CDTF->format_datetime( $dt_24hrs_ago ), - end_date => DateTime::Format::W3CDTF->format_datetime( $dt ) -); - -$updates->fetch; +DIR="$(cd "$(dirname "$BASH_SOURCE")" && pwd -P)" +$DIR/fetch --updates --start 24 diff --git a/bin/fetch-reports b/bin/fetch-reports index 20de10f4b..11bb0c44a 100755 --- a/bin/fetch-reports +++ b/bin/fetch-reports @@ -1,25 +1,6 @@ -#!/usr/bin/env perl +#!/bin/bash # -# This script utilises Open311 as described at -# http://wiki.open311.org/GeoReport_v2/#get-service-requests -# to fetch service requests. +# Wrapper to call new script -use strict; -use warnings; -require 5.8.0; - -BEGIN { - use File::Basename qw(dirname); - use File::Spec; - my $d = dirname(File::Spec->rel2abs($0)); - require "$d/../setenv.pl"; -} - -use CronFns; -my ($verbose, $nomail) = CronFns::options(); - -use Open311::GetServiceRequests; - -my $reports = Open311::GetServiceRequests->new( verbose => $verbose ); - -$reports->fetch; +DIR="$(cd "$(dirname "$BASH_SOURCE")" && pwd -P)" +$DIR/fetch --reports diff --git a/bin/fetch-reports-24hs b/bin/fetch-reports-24hs index ec0eabc2e..ffc1e9e72 100755 --- a/bin/fetch-reports-24hs +++ b/bin/fetch-reports-24hs @@ -1,35 +1,6 @@ -#!/usr/bin/env perl +#!/bin/bash # -# This script utilises Open311 as described at -# http://wiki.open311.org/GeoReport_v2/#get-service-requests -# to fetch service requests. +# Wrapper to call new script -use strict; -use warnings; -require 5.8.0; - -BEGIN { - use File::Basename qw(dirname); - use File::Spec; - my $d = dirname(File::Spec->rel2abs($0)); - require "$d/../setenv.pl"; -} - -use DateTime; - -use CronFns; -my ($verbose, $nomail) = CronFns::options(); - -use Open311::GetServiceRequests; - -my $dt = DateTime->now(); -my $dt_24hrs_ago = $dt->clone; -$dt_24hrs_ago->add( hours => -24 ); - -my $reports = Open311::GetServiceRequests->new( - verbose => 1, - start_date => $dt_24hrs_ago, - end_date => $dt -); - -$reports->fetch; +DIR="$(cd "$(dirname "$BASH_SOURCE")" && pwd -P)" +$DIR/fetch --reports --start 24 diff --git a/bin/fixmystreet.com/add_emergency_message b/bin/fixmystreet.com/add_emergency_message new file mode 100755 index 000000000..9264cc411 --- /dev/null +++ b/bin/fixmystreet.com/add_emergency_message @@ -0,0 +1,111 @@ +#!/usr/bin/env perl +# +# One off script to add emergency messages to a council + +use strict; +use warnings; +use v5.14; + +BEGIN { + use File::Basename qw(dirname); + use File::Spec; + my $d = dirname(File::Spec->rel2abs($0)); + require "$d/../../setenv.pl"; +} + +use Getopt::Long::Descriptive; +use Term::ANSIColor; +use FixMyStreet::DB; + +my ($opts, $usage) = describe_options( + '%c %o', + ['commit', 'whether to commit changes to the database' ], + ['body=s', 'name of body to attach question to', { required => 1 } ], + ['mode' => 'hidden' => { one_of => [ + ['questions' => 'add an emergency question'], + ['category' => 'add an emergency message'], + ], required => 1 }], + ['update' => 'only update existing messages'], + ['code=s', 'code to use for question'], + ['question=s', 'question to ask user first'], + ['yes=s', 'yes answer to question'], + ['no=s', 'no answer to question'], + ['message=s', 'message to be shown if form disabled', { required => 1 } ], + ['send_method=s', 'send method to restrict categories to' ], + ['group=s', 'group to restrict categories to' ], + ['help|h', "print usage message and exit" ], +); +$usage->die if $opts->help; + +if (!$opts->commit) { + say colored("*** DRY RUN ***", 'cyan'); +} + +my $field; + +if ( $opts->mode eq 'questions' ) { + die "question, code, yes and no required" + unless $opts->question && $opts->code && $opts->yes && $opts->no; + + $field = { + order => 0, + required => 'true', + protected => 'true', + code => $opts->code, + description => $opts->question, + datatype => 'singlevaluelist', + variable => 'true', + values => [ + { + key => 'yes', + name => $opts->yes, + disable => 1, + disable_message => $opts->message, + }, + { + key => 'no', + name => $opts->no, + } + ], + }; +} else { + $field = { + order => 0, + protected => 'true', + disable_form => 'true', + code => '_fms_disable_', + description => $opts->message, + variable => 'false', + }; +} + +my $body = FixMyStreet::DB->resultset("Body")->find({ name => $opts->body }); +unless ($body) { + say STDERR "Could not find body " . $opts->body; + exit 1; +} + +my $contacts = $body->contacts->not_deleted; +$contacts = $contacts->search({ send_method => $opts->send_method }) if $opts->send_method; +foreach my $category ($contacts->all) { + # Check if we're just adding the message to a certain group + if ($opts->group) { + next unless grep { $_ eq $opts->group } @{$category->groups}; + } + my $found = $category->get_extra_field(code => $field->{code}); + if ($found) { + say colored("Updating ", 'red') . $field->{code} . " message disable form on " . $category->category . ", " . $opts->body; + $category->update_extra_field($field); + } elsif (!$opts->update) { + say colored("Making ", 'green') . $field->{code} . " message disable form on " . $category->category . ", " . $opts->body; + $category->update_extra_field($field); + } + + if ($opts->commit) { + $category->update({ + editor => $0, + whenedited => \'current_timestamp', + note => $opts->code . ' extra field updated by script', + }); + } +} diff --git a/bin/fixmystreet.com/banes-close-reports b/bin/fixmystreet.com/banes-close-reports new file mode 100755 index 000000000..bba4c88e0 --- /dev/null +++ b/bin/fixmystreet.com/banes-close-reports @@ -0,0 +1,69 @@ +#!/usr/bin/env perl +# +# B&NES have sent a spreadsheet of reports that they want closing. This script +# accepts a comma separated list of report IDs, which will be extracted from +# the spreadsheet, marks those reports as "No further action" and leaves an +# update on them. + +use utf8; +use v5.14; +use warnings; + +BEGIN { + use File::Basename qw(dirname); + use File::Spec; + my $d = dirname(File::Spec->rel2abs($0)); + require "$d/../../setenv.pl"; +} + +use constant COUNCIL_NAME => 'Bath and North East Somerset Council'; +use constant CLOSURE_TEXT => <<'CLOSURE_TEXT'; +We’re sorry we've been unable to respond to your report. + +We’ve made a few changes recently to improve the way we respond to reports. We’re closing down any reports we think are now out of date, including this one, so that these changes can take effect. + +If you think this issue still requires attention, please report it again. +CLOSURE_TEXT + +use FixMyStreet::DB; +use FixMyStreet::Script::ArchiveOldEnquiries; +use Getopt::Long::Descriptive; + +my ($opts, $usage) = describe_options( + '%c %o', + ['report-ids=s', "comma-separated list of report IDs to close", { required => 1 } ], + ['commit|c', "actually close reports and add comments. Omitting this flag will do a dry-run"], + ['help|h', "print usage message and exit" ], +); +print($usage->text), exit if $opts->help; + +my $body = FixMyStreet::DB->resultset("Body")->search({ name => COUNCIL_NAME })->first; +die "Could not find body" unless $body; + +my @report_ids = split(',', $opts->{report_ids}); + +my $q = FixMyStreet::DB->resultset("Problem")->search({ + id => \@report_ids, + state => [ FixMyStreet::DB::Result::Problem->open_states() ], +}); + +# Provide some variables to the archiving script +FixMyStreet::Script::ArchiveOldEnquiries::update_options({ + user => $body->comment_user->id, + user_name => $body->comment_user->name, + closure_text => CLOSURE_TEXT, + retain_alerts => 1, + commit => $opts->commit, + closed_state => 'unable to fix', +}); + +if ($opts->{commit}) { + if ($q->count > scalar(@report_ids)) { + die "Found more reports than expected, bailing"; + } + printf("Closing %d old reports: ", $q->count); + FixMyStreet::Script::ArchiveOldEnquiries::close_problems($q); + printf("done.\n"); +} else { + printf("Would close %d old reports. Run with --commit to actually close reports.\n", $q->count); +} diff --git a/bin/fixmystreet.com/buckinghamshire-flytipping b/bin/fixmystreet.com/buckinghamshire-flytipping index 27548be88..a312f9fbe 100755 --- a/bin/fixmystreet.com/buckinghamshire-flytipping +++ b/bin/fixmystreet.com/buckinghamshire-flytipping @@ -14,8 +14,8 @@ BEGIN { require "$d/../../setenv.pl"; } -use constant BUCKS_AREA_ID => 2217; -use constant DISTRICT_IDS => (2255, 2256, 2257, 2258); +use constant BUCKS_NAME => 'Buckinghamshire Council'; +use constant EX_DISTRICTS => ['Aylesbury Vale District Council', 'Chiltern District Council', 'South Bucks District Council', 'Wycombe District Council']; use constant TIME_OPEN => '3 weeks'; use constant TIME_OPEN_ALERT => '6 weeks'; @@ -30,10 +30,10 @@ my ($opts, $usage) = describe_options( ); print($usage->text), exit if $opts->help; -my $body = FixMyStreet::DB->resultset("Body")->for_areas(BUCKS_AREA_ID)->first; +my $body = FixMyStreet::DB->resultset("Body")->search({ name => BUCKS_NAME })->first; die "Could not find Bucks body" unless $body; -my @districts = FixMyStreet::DB->resultset("Body")->for_areas(DISTRICT_IDS)->all; +my @districts = FixMyStreet::DB->resultset("Body")->search({ name => EX_DISTRICTS })->all; my @district_ids = map { $_->id } @districts; die "Did not find all districts" unless @district_ids == 4; @@ -57,12 +57,22 @@ sub find_problems { $time_param = { '<', \$to }; } - # Fetch all Flytipping problems sent only to districts, between $from and $to - my $q = FixMyStreet::DB->resultset("Problem")->search( - \[ "? @> regexp_split_to_array(bodies_str, ',')", [ {} => \@district_ids ] ] - )->search({ + # Fetch all Flytipping problems made off-road (i.e. those that previously + # would been sent only to districts) any any older ones which actually were + # sent to districts, between $from and $to + my $q = FixMyStreet::DB->resultset("Problem")->search([ + # Reports sent to district, made before the unitary switchover + -and => [ + \[ "? @> regexp_split_to_array(bodies_str, ',')", [ {} => \@district_ids ] ], + category => 'Flytipping' + ], + # Reports sent to Bucks Council after unitary switchover + { + bodies_str => $body->id, + category => 'Flytipping (off-road)' + } + ])->search({ state => [ FixMyStreet::DB::Result::Problem->open_states() ], - category => 'Flytipping', confirmed => $time_param, }); diff --git a/bin/fixmystreet.com/fixture b/bin/fixmystreet.com/fixture index 622bae018..1062eb16b 100755 --- a/bin/fixmystreet.com/fixture +++ b/bin/fixmystreet.com/fixture @@ -7,6 +7,7 @@ use strict; use warnings; use v5.14; +use utf8; BEGIN { use File::Basename qw(dirname); @@ -30,6 +31,7 @@ my ($opt, $usage) = describe_options( [ 'empty', "Empty all tables of the database first" ], [ 'commit', "Actually commit changes to the database" ], [ 'nonrandom', "Output things in a fixed manner, for testing" ], + [ 'test_fixtures', "Generate test specific fixtures" ], [ 'coords=s', "Co-ordinates to use instead of example postcode" ], [ 'help', "print usage message and exit", { shortcircuit => 1 } ], ); @@ -89,6 +91,150 @@ for my $cat ('Dropped Kerbs', 'Skips') { $child_cat->update; } +if ($opt->test_fixtures) { + my $bodies; + foreach ( + { area_id => 2234, categories => ['Shelter Damaged', 'Very Urgent'], name => 'Northamptonshire County Council' }, + { area_id => 2217, categories => ['Flytipping', 'Roads', 'Parks'], name => 'Buckinghamshire Council' }, + { area_id => 2257, categories => ['Flytipping', 'Graffiti'], name => 'Chiltern District Council' }, + { area_id => 2397, categories => [ 'Graffiti' ], name => 'Northampton Borough Council' }, + { area_id => 2483, categories => [ 'Potholes', 'Other' ], name => 'Hounslow Borough Council' }, + { area_id => 2636, categories => [ 'Potholes', 'Private', 'Extra' ], name => 'Isle of Wight Council' }, + { area_id => 2566, categories => [ 'Fallen branch' ], name => 'Peterborough City Council' }, + { area_id => 2498, categories => [ 'Incorrect timetable', 'Glass broken', 'Mobile Crane Operation' ], name => 'TfL' }, + ) { + $bodies->{$_->{area_id}} = FixMyStreet::DB::Factory::Body->find_or_create($_); + my $cats = join(', ', @{$_->{categories}}); + say "Created body $_->{name} for MapIt area ID $_->{area_id}, categories $cats"; + } + + my $child_cat = FixMyStreet::DB->resultset("Contact")->find({ + body => $bodies->{2234}, + category => 'Very Urgent', + }); + $child_cat->set_extra_fields({ + code => 'emergency', + datatype => 'string', + description => 'Please call us instead, it is very urgent.', + order => 1, + variable => 'false', + disable_form => 'true', + }); + $child_cat->update; + + $child_cat = FixMyStreet::DB->resultset("Contact")->find({ + body => $bodies->{2217}, + category => 'Flytipping', + }); + $child_cat->set_extra_fields({ + code => 'road-placement', + datatype => 'singlevaluelist', + description => 'Is the fly-tip located on', + order => 100, + required => 'true', + variable => 'true', + values => [ + { key => 'road', name => 'The road' }, + { key => 'off-road', name => 'Off the road/on a verge' }, + ], + }); + $child_cat->update; + + $child_cat = FixMyStreet::DB->resultset("Contact")->find({ + body => $bodies->{2217}, + category => 'Parks', + }); + $child_cat->set_extra_fields({ + code => 'site_code', + datatype => 'string', + description => 'site code', + order => 100, + required => 'false', + variable => 'true', + automated => 'hidden_field', + }); + $child_cat->update; + + $child_cat = FixMyStreet::DB->resultset("Contact")->find({ + body => $bodies->{2483}, + category => 'Other', + }); + $child_cat->update({ + non_public => 1 + }); + + $child_cat = FixMyStreet::DB->resultset("Contact")->find({ + body => $bodies->{2636}, + category => 'Potholes', + }); + $child_cat->update({ + send_method => 'Triage' + }); + $child_cat = FixMyStreet::DB->resultset("Contact")->find({ + body => $bodies->{2636}, + category => 'Private', + }); + $child_cat->update({ + non_public => 1, + send_method => 'Triage' + }); + $child_cat = FixMyStreet::DB->resultset("Contact")->find({ + body => $bodies->{2636}, + category => 'Extra', + }); + $child_cat->update({ + send_method => 'Triage' + }); + $child_cat->set_extra_fields({ + code => 'extra', + datatype => 'string', + order => 1, + variable => 'true', + }); + $child_cat->update; + + $child_cat = FixMyStreet::DB->resultset("Contact")->find({ + body => $bodies->{2566}, + category => 'Fallen branch', + }); + $child_cat->set_extra_fields( + { + code => 'emergency', + datatype => 'singlevaluelist', + description => 'Is it blocking a footpath or a highway?', + order => 0, + variable => 'true', + required => 'true', + values => [ + { key => 'yes', name => 'Yes', disable => 1, disable_message => 'Please phone customer services to report this problem.' }, + { key => 'no', name => 'No' }, + ] + }, + { + code => 'private_land', + datatype => 'singlevaluelist', + description => 'Is this problem on private land?', + order => 0, + variable => 'true', + required => 'true', + values => [ + { key => 'yes', name => 'Yes', disable => 1, disable_message => 'The council do not have powers to address issues on private land.' }, + { key => 'no', name => 'No' }, + ] + } + ); + $child_cat->update; + + FixMyStreet::DB->resultset('BodyArea')->create({ body_id => $bodies->{2498}->id, area_id => 2457 }); + FixMyStreet::DB->resultset('BodyArea')->create({ body_id => $bodies->{2498}->id, area_id => 2483 }); + $child_cat = FixMyStreet::DB->resultset("Contact")->find({ body => $bodies->{2498}, category => 'Incorrect timetable' }); + $child_cat->set_extra_metadata(group => ['Bus Stops and Shelters']); + $child_cat->update; + $child_cat = FixMyStreet::DB->resultset("Contact")->find({ body => $bodies->{2498}, category => 'Glass broken' }); + $child_cat->set_extra_metadata(group => ['Bus Stops and Shelters']); + $child_cat->update; +} + FixMyStreet::DB::Factory::ResponseTemplate->create({ body => $body, title => 'Generic', text => 'Thank you for your report, we will be in touch with an update soon.' }); @@ -156,7 +302,7 @@ if ($opt->coords) { my $cobrand = 'default'; foreach (FixMyStreet::Cobrand->available_cobrand_classes) { my $sub = $_->{class} && $_->{class}->can('council_area_id'); - if ($sub && &$sub == $opt->area_id) { + if ($sub && &$sub($_->{class}) == $opt->area_id) { $cobrand = $_->{class}->moniker; last; } diff --git a/bin/fixmystreet.com/one-off-move-js-messages-to-db b/bin/fixmystreet.com/one-off-move-js-messages-to-db new file mode 100644 index 000000000..bdbcd0a8c --- /dev/null +++ b/bin/fixmystreet.com/one-off-move-js-messages-to-db @@ -0,0 +1,128 @@ +#!/usr/bin/env perl +# +# One off script to transfer the hardcoded JS messages to the database + +use strict; +use warnings; +use v5.14; + +BEGIN { + use File::Basename qw(dirname); + use File::Spec; + my $d = dirname(File::Spec->rel2abs($0)); + require "$d/../../setenv.pl"; +} + +use FixMyStreet::DB; + +use Getopt::Long; + +my $commit; +GetOptions( + 'commit' => \$commit, +); + +if (!$commit) { + say "*** DRY RUN ***"; +} + +my @messages = ( + { + body => 'Oxfordshire County Council', + category => 'Countryside Paths / Public Rights of Way (usually not tarmac)', + message => 'Please report problems with rights of way using <a href="https://publicrightsofway.oxfordshire.gov.uk/web/standardmap.aspx">this page</a>.' + }, + { + body => 'Buckinghamshire Council', + category => 'Rights of Way', + message => 'If you wish to report an issue on a Public Right of Way, please use <a href="https://www.buckscc.gov.uk/services/environment/public-rights-of-way/report-a-rights-of-way-issue/">this service</a>.' + }, + { + body => 'Northamptonshire County Council', + category => 'Street lighting', + message => 'Street lighting in Northamptonshire is maintained by Balfour Beatty on behalf of the County Council under a Street Lighting Private Finance Initiative (PFI) contract. Please view our <b><a href="https://www3.northamptonshire.gov.uk/councilservices/northamptonshire-highways/roads-and-streets/Pages/street-lighting.aspx">Street Lighting</a></b> page to report any issues.' + }, +); + +my %bristol = ( + "Abandoned vehicles" => "https://www.bristol.gov.uk/streets-travel/abandoned-vehicles", + "Flytipping" => "https://www.bristol.gov.uk/streets-travel/flytipping", + "Flyposting" => "https://www.bristol.gov.uk/streets-travel/flyposting", + "Graffiti" => "https://www.bristol.gov.uk/streets-travel/graffiti", + "Dog fouling" => "https://www.bristol.gov.uk/streets-travel/dog-fouling", + "Street cleaning" => "https://www.bristol.gov.uk/streets-travel/street-that-needs-cleaning", +); +foreach (keys %bristol) { + push @messages, { + body => 'Bristol City Council', + category => $_, + message => "If you wish to report an issue with $_, please use <a href=\"$bristol{$_}\">this service</a>." + }; +} + +MESSAGE: foreach my $msg (@messages) { + my $body = FixMyStreet::DB->resultset("Body")->find({ name => $msg->{body} }); + unless ($body) { + say STDERR "Could not find body $msg->{body}"; + next; + } + my $category = FixMyStreet::DB->resultset("Contact")->find({ body_id => $body->id, category => $msg->{category} }); + unless ($category) { + say STDERR "Could not find $msg->{category}, $msg->{body}"; + next; + } + my $extra_fields = $category->get_extra_fields; + foreach (@$extra_fields) { + if ($_->{code} eq 'stopper-message') { + say "Stopper message already present for $msg->{category}, $msg->{body}"; + next MESSAGE; + } + } + unshift @$extra_fields, { + code => 'stopper-message', + order => -1, + datatype => 'string', + required => 'true', + protected => 'true', + variable => 'false', + disable_form => 'true', + description => $msg->{message}, + datatype_description => '', + }; + $category->set_extra_fields(@$extra_fields); + say "Adding stopper message to $msg->{category}, $msg->{body}"; + if ($commit) { + $category->update; + } +} + +my $northants = FixMyStreet::DB->resultset("Body")->find({ name => 'Northamptonshire County Council' }); +if ($northants) { + my @northants_contacts = $northants->contacts->all; + my $found_total = 0; + foreach my $category (@northants_contacts) { + my $extra_fields = $category->get_extra_fields; + my $found = 0; + foreach (@$extra_fields) { + next unless $_->{code} eq 'emergency'; + $found_total++; + if (!$_->{disable_form} || $_->{disable_form} eq 'false') { + $_->{disable_form} = 'true'; + $_->{protected} = 'true'; + $found = 1; + } + } + if ($found) { + $category->set_extra_fields(@$extra_fields); + say "Making emergency message disable form on " . $category->category . ", Northamptonshire"; + if ($commit) { + $category->update; + } + } + } + if (!$found_total) { + say STDERR "No emergency messages found for Northamptonshire"; + } +} else { + say STDERR "Could not find Northamptonshire"; +} diff --git a/bin/fixmystreet.com/one-off-status-update b/bin/fixmystreet.com/one-off-status-update new file mode 100755 index 000000000..e9ba66c75 --- /dev/null +++ b/bin/fixmystreet.com/one-off-status-update @@ -0,0 +1,214 @@ +#!/usr/bin/env perl + +=head1 NAME + +one-off-status-update + +=head1 SUMMARY + +one-off-status-update [--commit] path/to/file.csv + +=head1 DESCRIPTION + +This is a one off script to update the status of NCC reports affected +by a bug in the NCC update process. It relies on a csv file containing +a mapping of report id to Alloy status and uses this to update the +FixMyStreet status. + +It takes a single command line argument which is the path to the csv +file containing the reports to update, along with details of the current +alloy state which it uses to determine what state to update them too. + +Only reports that do not match the state in the spreadsheet will be updated. +Reports that are closed/fixed on FixMyStreet but open in the spreadsheet +will also be ignored. It will also skip reports that it cannot determine +the state for. + +Any update made will set the appropriate response template. + +It will not update the database unless the `--commit` flag is used. + +=head1 EXPECTED CSV HEADERS + +RESOURCE_ID,INSPECTION_NUMBER,Street_Doctor_No,Reason_for_Closure,Enquiry_TASK_STATUS,Enquiry_Type,Summary,Reported_DateTime,Response_to_Customer,DEFECT_REPORTED_DATE,DEFECT_STATUS_NO,DEFECT_STATUS,DEFECT_NUMBER,REMEDIED_DATE,TEAM_DESCRIPTION,User_Name,Category,Stage,Stage_Description,blank,blank2 + + +Of these the following are used: + +=over + +=item * Street_Doctor_No - FixMyStreet id + + +=item * Enquiry_TASK_STATUS - Alloy status + +=item * Reason_for_Closure - Alloy reason for closure + +=item * Response_to_Customer - Alloy text to add to update on FixMyStreet + +=item * DEFECT_STATUS - Alloy defect status + +=back + +=cut + +use strict; +use warnings; +use v5.14; + +BEGIN { + use File::Basename qw(dirname); + use File::Spec; + my $d = dirname(File::Spec->rel2abs($0)); + require "$d/../../setenv.pl"; +} + +use FixMyStreet::DB; +use Text::CSV; + +use Getopt::Long; + +my ($commit, $verbose); +GetOptions( + 'commit' => \$commit, + 'verbose' => \$verbose, +); + +if (!$commit) { + say "*** DRY RUN ***"; +} + +my %enq_to_fms_map = ( + 'Issued to Inspector' => 'investigating', + 'In Progress' => 'investigating', + 'Completed' => 'closed', +); + +my %rfc_to_fms_map = ( + 'No Action Necessary' => 'unable to fix', + 'Outside NCC Control' => 'not responsible', + 'Highways to Monitor. No Action' => 'unable to fix', + 'Work Instructed' => 'action scheduled', +); + +my %defect_to_fms_map = ( + 'Remedied' => 'fixed - council', + 'Order Raised' => 'action scheduled', + 'Rejected' => 'closed', + 'Found and Forwarded' => 'closed', +); + +my $file = shift; + +my $csv = Text::CSV->new; +open my $fh, "<:encoding(utf-8)", $file or die "Failed to open $file: $!\n"; +my @cols = @{$csv->getline($fh)}; +$csv->column_names (@cols); + +my ($checked, $updated) = (0,0); +my $northants = FixMyStreet::DB->resultset("Body")->find({ name => 'Northamptonshire County Council' }); +if ($northants) { + my $comment_user = $northants->comment_user; + while (my $report = $csv->getline_hr( $fh ) ) { + $checked++; + my $id = $report->{Street_Doctor_No}; + next unless $id; + my $p = FixMyStreet::DB->resultset("Problem")->find({ id => $id }); + unless ($p) { + warn "Could not find matching report with id $id\n"; + next; + } + + # we don't want to update hidden reports + next if $p->state eq 'hidden'; + + my $new_state = get_state( $report ); + + unless ( $new_state ) { + warn "skipping $id as can't determine new state\n" if $verbose; + next; + } + + if ($p->state eq $new_state) { + warn "skipping $id as has correct state\n" if $verbose; + next; + } + + if ( ($p->is_fixed || $p->is_closed) + && $p->open_states->{$new_state} ) { + warn "skipping $id as already closed and would re-open\n" if $verbose; + next; + } + + my $text = $report->{Response_to_Customer}; + # do not add a second response to customer if it's already there. + if ($text) { + # sometimes there are extra spaces on the end in the database + # there's also quite a few occurrences of the text having minor corrections + # to the thank you which we don't want to send out an update for potentially + # weeks later + my $search_text = $text; + $search_text =~ s/\sthank you[.\s]*$//i; + my $search = { text => { ilike => "$search_text%" } }; + my $c = $p->comments->search($search); + $text = '' if $c->count; + } + + if (!$text) { + if (my $t = $p->response_templates->search({ + auto_response => 1, + 'me.state' => $new_state + })->first) { + $text = $t->text; + } + } + + if ($commit) { + warn "updating $id to $new_state\n" if $verbose; + $updated++; + $p->update({ + state => $new_state, + lastupdate => \'current_timestamp', + }); + + my $comment = FixMyStreet::DB->resultset('Comment')->new({ + problem => $p, + user => $comment_user, + name => $comment_user->name, + text => $text, + problem_state => $new_state, + external_id => -1, + mark_fixed => 0, + mark_open => 0, + anonymous => 0, + confirmed => \'current_timestamp', + created => \'current_timestamp', + state => 'confirmed', + }); + $comment->set_extra_metadata(one_off_script_update => 1); + $comment->insert; + } + } +} else { + say STDERR "Could not find Northamptonshire"; +} + +say "$checked reports looked at, $updated updated"; + +sub get_state { + my $report = shift; + + my $fms_state = $enq_to_fms_map{$report->{Enquiry_TASK_STATUS}}; + + if ( $report->{Reason_for_Closure} ) { + $fms_state = $rfc_to_fms_map{$report->{Reason_for_Closure}}; + } + + if ( $report->{DEFECT_STATUS} ) { + $fms_state = $defect_to_fms_map{$report->{DEFECT_STATUS}}; + + say STDERR $report->{Street_Doctor_No} . " has an unmapped defect status: " . $report->{DEFECT_STATUS} unless $fms_state; + } + + return $fms_state; +} diff --git a/bin/fixmystreet.com/setup_island_roads_triage_cats b/bin/fixmystreet.com/setup_island_roads_triage_cats new file mode 100755 index 000000000..b747fb99b --- /dev/null +++ b/bin/fixmystreet.com/setup_island_roads_triage_cats @@ -0,0 +1,133 @@ +#!/usr/bin/env perl + +=head1 NAME + +setup_island_roads_triage_cats - create/update triage contacts for Island Roads cobrand + +=head1 DESCRIPTION + +This script creates a set of contacts for Island Road to be used as part of their triage +system. The contacts created all have the `Triage` send_method and so will be displayed +to non staff users. + +It also adds the relevant extra fields to the contacts to enable asset details to be +added to a report created in the category. + +If a contact with the same category already exists that does not have a send_method of +`Triage` then the script will emit a warning. Any existing contacts with the `Triage` +send_method will have their state reset to `confirmed`. + +=cut + +use v5.14; +use warnings; + +BEGIN { + use File::Basename qw(dirname); + use File::Spec; + my $d = dirname(File::Spec->rel2abs($0)); + require "$d/../../setenv.pl"; +} + + +my @cat_list = ( + "Dog Fouling", + "Manholes", + "Trees & Hedges", + "Pavements/footpaths", + "Drainage", + "Car Parking", + "Street Lighting", + "Bus Stops", + "Flyposting", + "Potholes", + "Street Cleaning", + "Bridges & Walls", + "Traffic Lights", + "Street Furniture", + "Roads/Highways", + "Road Traffic Signs & Markings", + "Grass Verges & Weeds", + "Flytipping", + "Graffiti", + "Street Nameplates", + "Abandoned Vehicles" +); + +use FixMyStreet::DB; +use FixMyStreet::Cobrand::IsleOfWight; + +my $iow = FixMyStreet::DB->resultset("Body")->search({ + name => "Isle of Wight Council" +}); + +my $cobrand = FixMyStreet::Cobrand::IsleOfWight->new; + +if ($iow->count != 1) { + die "Could not find IoW body\n"; +} + +$iow = $iow->first; + +for my $cat (@cat_list) { + + my $existing = FixMyStreet::DB->resultset("Contact")->search({ + category => $cat, body_id => $iow->id + })->first; + + if ( $existing ) { + if (!$existing->send_method || $existing->send_method ne 'Triage') { + warn "$cat is not a Triage category\n"; + next; + } + # make sure category is not deleted + $existing->state('confirmed'); + } else { + $existing = FixMyStreet::DB->resultset('Contact')->create({ + name => $cat, + body => $iow, + send_method => 'Triage', + state => 'confirmed', + editor => $0, + note => 'created automatically by script', + }); + } + + my $extra_fields = $existing->get_extra_fields; + my @meta = grep { $_->{code} ne 'central_asset_id' && $_->{code} ne 'site_code'} @$extra_fields; + + push @meta, + { + code => 'central_asset_id', + datatype => 'string', + description => 'central asset id', + order => 100, + required => 'false', + variable => 'true', + automated => 'hidden_field', + }, + { + code => 'asset_details', + datatype => 'string', + description => 'asset details', + order => 100, + required => 'false', + variable => 'true', + automated => 'hidden_field', + }, + { + code => 'site_code', + datatype => 'string', + description => 'site code', + order => 100, + required => 'false', + variable => 'true', + automated => 'hidden_field', + }; + + $cobrand->call_hook( + open311_contact_meta_override => {}, $existing, \@meta); + + $existing->set_extra_fields(@meta); + $existing->update; +} diff --git a/bin/fixmystreet.com/update_council_user_permissions b/bin/fixmystreet.com/update_council_user_permissions new file mode 100755 index 000000000..0463db8b7 --- /dev/null +++ b/bin/fixmystreet.com/update_council_user_permissions @@ -0,0 +1,70 @@ +#!/usr/bin/env perl +# +# script to add/remove a list of permissions to all staff belonging to a body. + +use strict; +use warnings; +use v5.14; + +BEGIN { + use File::Basename qw(dirname); + use File::Spec; + my $d = dirname(File::Spec->rel2abs($0)); + require "$d/../../setenv.pl"; +} + +use FixMyStreet::DB; + +use Getopt::Long::Descriptive; + +my ($opts, $usage) = describe_options( + '%c %o', + ['commit', 'whether to commit changes to the database'], + ['permissions=s', 'comma seperated list of permissions to add/delete', { required => 1 }], + ['council=s', 'name of council to update', { required => 1 }], + ['mode' => 'hidden' => { one_of => [ + ['add', 'add permissions to council\'s users'], + ['remove', 'remove permissions from council\'s users'], + ], required => 1 }], + ['help|h', 'print usage message and exit'] +); +$usage->die if $opts->help; + +if (!$opts->commit) { + say "*** DRY RUN ***"; +} + +my $body = FixMyStreet::DB->resultset("Body")->find({ name => $opts->council}); + +if ($body) { + my @permissions_list = map { Utils::trim_text($_) } split(',', $opts->permissions); + + my $staff = FixMyStreet::DB->resultset("User")->search({ from_body => $body->id }); + for my $user ( $staff->all ) { + my $permissions = $user->user_body_permissions->search({ + body_id => $body->id, + permission_type => { in => \@permissions_list} + }); + if ( $opts->mode eq 'remove') { + next unless $permissions->count; + if ($opts->commit) { + $permissions->delete; + } + } elsif ( $opts->mode eq 'add' ) { + my %existing = map { $_->permission_type => 1 } $permissions->all; + my @permissions_to_add = grep { !$existing{$_} } @permissions_list; + next unless @permissions_to_add; + if ($opts->commit) { + for my $permission ( @permissions_to_add ) { + $user->user_body_permissions->create({ + body_id => $body->id, + permission_type => $permission + }); + } + } + } + say "updated permissions for user id " . $user->id; + } +} else { + say STDERR "Could not find " . $opts->council; +} diff --git a/bin/handlemail b/bin/handlemail index ade5d42d3..6903439c9 100755 --- a/bin/handlemail +++ b/bin/handlemail @@ -22,6 +22,7 @@ BEGIN { use Getopt::Long; use Path::Tiny; use FixMyStreet; +use FixMyStreet::Cobrand; use FixMyStreet::DB; use FixMyStreet::Email; use mySociety::HandleMail; @@ -31,7 +32,7 @@ use mySociety::SystemMisc qw(print_log); # messages being generated (only in response to non-bounce input, obviously). mySociety::SystemMisc::log_to_stderr(0); -my $cobrand = "default"; +my $cobrand; # Where to forward mail that should be looked at by a person, # such as a permanent bounce for a report. @@ -180,20 +181,44 @@ sub handle_non_bounce_to_null_address { # Send an automatic response print_log('info', "Received non-bounce to null address, auto-replying"); + my ( $cobrand, $from_addr, $from_name ) = get_config_for_autoresponse(); + my $template = path(FixMyStreet->path_to("templates", "email", $cobrand, 'reply-autoresponse'))->slurp_utf8; # We generate this as a bounce. my ($rp) = $data{return_path} =~ /^\s*<(.*)>\s*$/; my $mail = FixMyStreet::Email::construct_email({ 'Auto-Submitted' => 'auto-replied', - From => [ FixMyStreet->config('CONTACT_EMAIL'), - FixMyStreet->config('CONTACT_NAME') ], + From => [ $from_addr, $from_name ], To => $rp, _body_ => $template, }); send_mail($mail, $rp); } +# Based on the address the incoming message was sent to, we might want to +# use a cobrand's own reply-autoresponse template. +sub get_config_for_autoresponse { + # cobrand might have been set from command line, so prefer that if so. + if ( defined $cobrand ) { + return ( $cobrand, FixMyStreet->config('CONTACT_EMAIL'), FixMyStreet->config('CONTACT_NAME') ); + } + + # Try and find a matching email address in the COBRAND_FEATURES config + my $recipient = mySociety::HandleMail::get_bounce_recipient($data{message})->address; + my $features = FixMyStreet->config('COBRAND_FEATURES') || {}; + my $cobrands = $features->{do_not_reply_email} || {}; + for my $moniker ( keys %$cobrands ) { + if ( $cobrands->{$moniker} eq $recipient ) { + my $cb = FixMyStreet::Cobrand->get_class_for_moniker($moniker)->new(); + return ( $moniker, $cb->contact_email, $cb->contact_name ); + } + } + + # No match found, so use default cobrand + return ( "default", FixMyStreet->config('CONTACT_EMAIL'), FixMyStreet->config('CONTACT_NAME') ); +} + sub forward_on_to { my $recipient = shift; my $text = join("\n", @lines) . "\n"; diff --git a/bin/highwaysengland/augment-junctions-database b/bin/highwaysengland/augment-junctions-database new file mode 100755 index 000000000..537556d6a --- /dev/null +++ b/bin/highwaysengland/augment-junctions-database @@ -0,0 +1,90 @@ +#!/usr/bin/env perl + +=head1 NAME + +make-junctions-database + +=head1 USAGE + +augment-junctions-database path/to/data.csv + +=head1 DESCRIPTION + +Adds to the SQLite file created by `make-junctions-database`. This requires that +script to have been run and the initial database created. + +It expects a CSV with a header row: road,junction,point_x,point_y +and the data in those columns to be: + +* road - name of road, eg A1 +* junction - description of the point +* point_x - easting +* point_y - northing + +The junction field can be either + +* a junction number, e.g J12 +* a junction number and a description, e.g J12 Town Name +* a junction/point descrption, e.g Town Name + +Where the function field is a junction number and a description two rows will be created, +one for the junction number and one for the description. + +The resulting database is used by the L<HighwaysEngland> package to lookup junctions. + +=cut + +use strict; +use warnings; + +BEGIN { + use File::Basename qw(dirname); + use File::Spec; + my $d = dirname(File::Spec->rel2abs($0)); + require "$d/../../setenv.pl"; +} + +use DBI; +use Text::CSV; +use FixMyStreet; +use HighwaysEngland; + +my $db = DBI->connect('dbi:SQLite:dbname='. HighwaysEngland::database_file); + +my $q_junction_chk = $db->prepare('SELECT road, name FROM junction WHERE road = ? AND name = ?'); +my $q_junction = $db->prepare('INSERT INTO junction (road, name, easting, northing) VALUES (?, ?, ?, ?)'); + +my $csv = Text::CSV->new ({ binary => 1, auto_diag => 1 }); +die "Usage: $0 <csv_file>\n" unless @ARGV; +open my $fh, "<:encoding(utf8)", $ARGV[0] or die "$ARGV[0]: $!"; +$csv->header($fh); + +my %all; +my $c = 0; +while (my $row = $csv->getline_hr($fh)) { + my $junction = uc $row->{junction}; + my $alt_name; + if ($junction =~ /^(J\d+)\s*(.*)$/) { + ($junction, $alt_name) = ($1, $2); + } + + my $road = uc $row->{road}; + next unless $road =~ /^[AM]/; + + # strip extra precision + (my $e = $row->{point_x}) =~ s/\.\d+$//; + (my $n = $row->{point_y}) =~ s/\.\d+$//; + + if ($alt_name) { + $q_junction_chk->execute($road, $alt_name) or warn $db->errstr . " $road $alt_name"; + unless ($q_junction_chk->fetchrow_hashref) { + $q_junction->execute($road, $alt_name, $e, $n) or warn $db->errstr . " $road $alt_name"; + } + } + + $q_junction_chk->execute($road, $junction) or warn $db->errstr . " $road $junction"; + next if $q_junction_chk->fetchrow_hashref; + $q_junction->execute($road, $junction, $e, $n) or warn $db->errstr . " $road $junction"; +} + +close $fh; diff --git a/bin/highwaysengland/update_areas_covered b/bin/highwaysengland/update_areas_covered new file mode 100755 index 000000000..3affe3adb --- /dev/null +++ b/bin/highwaysengland/update_areas_covered @@ -0,0 +1,51 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use feature qw/ say /; + +BEGIN { + use File::Basename qw(dirname); + use File::Spec; + my $d = dirname(File::Spec->rel2abs($0)); + require "$d/../../setenv.pl"; +} + +use FixMyStreet; +use FixMyStreet::DB; +use FixMyStreet::MapIt; +use Term::ANSIColor; +use Getopt::Long::Descriptive; + +my ($opt, $usage) = describe_options( + '%c %o', + [ 'commit', "Actually commit changes to the database" ], + [ 'help', "print usage message and exit", { shortcircuit => 1 } ], +); +print($usage->text), exit if $opt->help; + +my $areas = FixMyStreet::MapIt::call( + 'areas/CTY,UTA,MTD,LBO' +); + +my $db; +END { + if ($db) { + $opt->commit ? $db->txn_commit : $db->txn_rollback; + } +} + +$db = FixMyStreet::DB->schema->storage; +$db->txn_begin; +if (!$opt->commit) { + say colored("NOT COMMITTING TO DATABASE", 'cyan'); +} + +my $body = FixMyStreet::DB->resultset('Body')->find({ name => 'Highways England' }); +die "Highways England not found\n" unless $body; + +for my $area ( values %{ $areas } ) { + next unless $area->{country} eq 'E'; + + FixMyStreet::DB->resultset('BodyArea')->find_or_create( { body => $body, area_id => $area->{id} } ); +} diff --git a/bin/import_categories b/bin/import_categories new file mode 100644 index 000000000..490e2187f --- /dev/null +++ b/bin/import_categories @@ -0,0 +1,112 @@ +#!/usr/bin/env perl + +use warnings; +use v5.14; +use utf8; + +BEGIN { + use File::Basename qw(dirname); + use File::Spec; + my $d = dirname(File::Spec->rel2abs($0)); + require "$d/../setenv.pl"; +} + +use Term::ANSIColor; +use FixMyStreet; +use FixMyStreet::DB; +use Getopt::Long::Descriptive; +use JSON::MaybeXS; +use Path::Tiny; + +my ($opt, $usage) = describe_options( + '%c %o', + [ 'body=s', "Name of body to add categories to" ], + [ 'commit', "Actually commit changes to the database" ], + [ 'delete', "Delete all existing body categories first" ], + [ 'help', "print usage message and exit", { shortcircuit => 1 } ], +); +print($usage->text), exit if $opt->help; + +die "Usage: $0 <path/to/categories.json>" unless $opt->body; +die "Usage: $0 <path/to/categories.json>" unless -f $ARGV[0]; + +my $db; +END { + if ($db) { + $opt->commit ? $db->txn_commit : $db->txn_rollback; + } +} + +$db = FixMyStreet::DB->schema->storage; +$db->txn_begin; +if (!$opt->commit) { + say colored("NOT COMMITTING TO DATABASE", 'cyan'); +} + +my $config = decode_json(path($ARGV[0])->slurp_utf8); + +my $body = FixMyStreet::DB->resultset('Body')->find({ name => $opt->body }); + +$body->contacts->delete_all if $opt->delete; + +die "Couldn't find body" unless $body; + +my $groups = $config->{groups}; +if ($groups) { + for my $group (keys %$groups) { + my $cats = $groups->{$group}; + make_categories($cats, $group); + say "Created $group group"; + } +} else { + my $categories = $config->{categories}; + make_categories($categories); + say "Created non group categories"; +} + +sub make_categories { + my ($cats, $group) = @_; + for my $cat (@$cats) { + my $child_cat = FixMyStreet::DB->resultset("Contact")->find_or_new({ + body => $body, + category => $cat->{category} + }); + $child_cat->email($cat->{email}); + $child_cat->state('confirmed'); + $child_cat->editor($0); + $child_cat->whenedited(\'current_timestamp'); + $child_cat->note($child_cat->in_storage ? 'Updated by import_categories' : 'Created by import_categories'); + say colored("WARNING", 'red') . " " . $child_cat->category . " already exists" if $child_cat->in_storage and $child_cat->category ne 'Other (TfL)'; + $child_cat->extra(undef) if $child_cat->in_storage; + + if ($group) { + my $groups = $child_cat->groups; + my %groups = map { $_ => 1} @$groups; + $groups{$group} = 1; + my @groups = keys %groups; + $child_cat->set_extra_metadata(group => \@groups); + } + + if ($cat->{disable}) { + $child_cat->update_extra_field({ + code => "_fms_disable_", + disable_form => "true", + variable => "false", + protected => "true", + description => $cat->{disable} eq 1 ? $config->{disabled_message} : $cat->{disable}, + order => 0, + }); + } + $child_cat->set_extra_fields(@{ $cat->{extra_fields} }) if $cat->{extra_fields}; + if (my $asset_field = $cat->{asset_field}) { + my ($description, $code) = @$asset_field; + $child_cat->update_extra_field({ + code => $code, + description => $description, + automated => "hidden_field", + order => 1, + }); + } + $child_cat->in_storage ? $child_cat->update : $child_cat->insert; + } +} diff --git a/bin/install_packages b/bin/install_packages new file mode 100755 index 000000000..7b4bfd8e1 --- /dev/null +++ b/bin/install_packages @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +PACKAGE_FILE=conf/packages + +[ -n "$1" ] && PACKAGE_FILE="conf/packages.${1}" + +apt-get update + +grep -v ^# $PACKAGE_FILE | grep -v ^$ | xargs apt-get install -qq -y + diff --git a/bin/make-junctions-database b/bin/make-junctions-database new file mode 100755 index 000000000..f7a12c586 --- /dev/null +++ b/bin/make-junctions-database @@ -0,0 +1,116 @@ +#!/usr/bin/env perl + +=head1 NAME + +make-junctions-database + +=head1 USAGE + +make-junctions-database path/to/markerposts.csv + +=head1 DESCRIPTION + +Creates a SQLite database of Highways England junctions to facilitate looking +up locations by junction name, e.g. "M60, Junction 2" or "M6 323.5". + + +1. Download the database of all marker posts from +https://www.whatdotheyknow.com/request/positions_of_driver_location_sig +The filename is "Gazetteer All Mposts only.zip". + +2. Unzip and export the XLSX file as a CSV (using in2csv from csvkit, for example). + +3. Run this script to build the database. + +The resulting database is used by the L<HighwaysEngland> package to lookup junctions. + +=cut + +use strict; +use warnings; + +BEGIN { + use File::Basename qw(dirname); + use File::Spec; + my $d = dirname(File::Spec->rel2abs($0)); + require "$d/../setenv.pl"; +} + +use DBI; +use Text::CSV; +use FixMyStreet; +use HighwaysEngland; + +my $db = DBI->connect('dbi:SQLite:dbname='. HighwaysEngland::database_file); + +$db->do(<<EOF) or die $db->errstr; +CREATE TABLE IF NOT EXISTS junction ( + name TEXT NOT NULL, + road TEXT NOT NULL, + easting INTEGER NOT NULL, + northing INTEGER NOT NULL, + PRIMARY KEY (name, road) +); +EOF + +$db->do(<<EOF) or die $db->errstr; +CREATE TABLE IF NOT EXISTS sign ( + road TEXT NOT NULL, + distance TEXT NOT NULL, + side TEXT NOT NULL, + easting INTEGER NOT NULL, + northing INTEGER NOT NULL +); +EOF + +$db->do(<<EOF) or die $db->errstr; +CREATE INDEX IF NOT EXISTS sign_idx ON sign (road, distance); +EOF + +my $q_sign = $db->prepare('INSERT INTO sign (road, distance, side, easting, northing) VALUES (?, ?, ?, ?, ?)'); +my $q_junction = $db->prepare('INSERT INTO junction (road, name, easting, northing) VALUES (?, ?, ?, ?)'); + +my $csv = Text::CSV->new ({ binary => 1, auto_diag => 1 }); +die "Usage: $0 <csv_file>\n" unless @ARGV; +open my $fh, "<:encoding(utf8)", $ARGV[0] or die "$ARGV[0]: $!"; +$csv->header($fh); + +my %all; +my $c = 0; +while (my $row = $csv->getline_hr($fh)) { + my $marker = $row->{bd}; + $marker =~ /P(\d+)\/(\d+)([ABJMKL])/ or next; + + my $kms = "$1.$2"; + my $letter = $3; + + my $road = $row->{dd}; + next unless $road =~ /^[AM]/; + + my $e = $row->{easting}; + my $n = $row->{northing}; + my $name = $row->{tn}; + + $q_sign->execute($road, $kms, $letter, $e, $n) or warn $db->errstr . " $road $kms $letter"; + print '.' unless $c++ % 1000; + + next unless $name =~ /ASIDE|BSIDE/; + $name =~ s/ ASIDE| BSIDE//; + push @{$all{$road}{$name}}, $row; +} + +close $fh; + +for my $road (sort keys %all) { + foreach my $junction (sort keys %{$all{$road}}) { + my $tot_e = 0; my $tot_n = 0; my $n = 0; + foreach (@{$all{$road}{$junction}}) { + $tot_e += $_->{easting}; + $tot_n += $_->{northing}; + $n++; + } + my $avg_e = int($tot_e/$n+0.5); + my $avg_n = int($tot_n/$n+0.5); + $q_junction->execute($road, $junction, $avg_e, $avg_n); + } +} diff --git a/bin/northamptonshire/update-emergency-message b/bin/northamptonshire/update-emergency-message new file mode 100755 index 000000000..7248e9159 --- /dev/null +++ b/bin/northamptonshire/update-emergency-message @@ -0,0 +1,59 @@ +#!/usr/bin/env perl + +# update the emergency message on NCC categories + +use strict; +use warnings; +use v5.14; + +BEGIN { + use File::Basename qw(dirname); + use File::Spec; + my $d = dirname(File::Spec->rel2abs($0)); + require "$d/../../setenv.pl"; +} + +use FixMyStreet::DB; + +use Getopt::Long; + +my ($commit, $message); +GetOptions( + 'commit' => \$commit, + 'message=s' => \$message, +); + +if (!$commit) { + say "*** DRY RUN ***"; +} + +my $northants = FixMyStreet::DB->resultset("Body")->find({ name => 'Northamptonshire County Council' }); +if ($northants) { + my @northants_contacts = $northants->contacts->all; + my $found_total = 0; + foreach my $category (@northants_contacts) { + my $extra_fields = $category->get_extra_fields; + my $found = 0; + foreach (@$extra_fields) { + next unless $_->{code} eq 'emergency'; + $found_total++; + $_->{code} = '_fms_disable_'; + $_->{description} = $message; + $_->{protected} = 'true'; + $_->{disable_form} = 'true'; + $found = 1; + } + if ($found) { + $category->set_extra_fields(@$extra_fields); + say "Updating emergency message on " . $category->category . ", Northamptonshire"; + if ($commit) { + $category->update; + } + } + } + if (!$found_total) { + say STDERR "No emergency messages found for Northamptonshire"; + } +} else { + say STDERR "Could not find Northamptonshire"; +} diff --git a/bin/open311-populate-service-list b/bin/open311-populate-service-list index 9c05055c6..d81df9321 100755 --- a/bin/open311-populate-service-list +++ b/bin/open311-populate-service-list @@ -16,17 +16,22 @@ use Getopt::Long::Descriptive; my ($opt, $usage) = describe_options( '%c %o', + ['body|b:s', "body name to only fetch this body"], ['verbose|v', "print out all services as they are found"], ['warn|w', "output warnings about any issues"], ['help', "print usage message and exit" ], ); -print($usage->text), exit if $opt->help; +$usage->die if $opt->help; my $bodies = FixMyStreet::DB->resultset('Body')->search( { # Until Oxfordshire does name => { -not_in => [ 'Oxfordshire County Council' ] }, send_method => 'Open311' } ); +if ($opt->body) { + $bodies = $bodies->search({ name => $opt->body }); +} + my $verbose = 0; $verbose = 1 if $opt->warn; $verbose = 2 if $opt->verbose; diff --git a/bin/open311-update-reports b/bin/open311-update-reports index 378f8e8bc..2d384b813 100755 --- a/bin/open311-update-reports +++ b/bin/open311-update-reports @@ -4,7 +4,7 @@ # (by fetching all reports for a body and looking for updates). If possible, # please use the extension explained at # https://github.com/mysociety/FixMyStreet/wiki/Open311-FMS---Proposed-differences-to-Open311 -# and the fetch-comments/send-comments scripts. +# and the fetch/send-comments scripts. use strict; use warnings; diff --git a/bin/oxfordshire/send-rdi-emails b/bin/oxfordshire/send-rdi-emails deleted file mode 100755 index 9cc3e5502..000000000 --- a/bin/oxfordshire/send-rdi-emails +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env perl - -use strict; -use warnings; -use v5.14; - -BEGIN { - use File::Basename qw(dirname); - use File::Spec; - my $d = dirname(File::Spec->rel2abs($0)); - require "$d/../../setenv.pl"; -} - -use DateTime; -use Try::Tiny; -use FixMyStreet; -use FixMyStreet::Cobrand; -use FixMyStreet::Email; -use FixMyStreet::Integrations::ExorRDI; - -my $end_date = DateTime->now( time_zone => FixMyStreet->time_zone || FixMyStreet->local_time_zone ) - ->truncate(to => 'hour')->set_hour(16); -my $start_date = $end_date->clone->subtract(days => 1); -my $inspection_date = $end_date; # All inspections are considered to have happened on the same date. - -my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker('oxfordshire')->new; -$cobrand->set_lang_and_domain('en-gb', 1); -my @inspectors = $cobrand->users->search({ - 'user_body_permissions.permission_type' => 'report_inspect' -}, { - join => 'user_body_permissions', - distinct => 1, -})->all; - -foreach my $inspector (@inspectors) { - my $params = { - start_date => $start_date, - end_date => $end_date, - inspection_date => $inspection_date, - user => $inspector, - mark_as_processed => 1, - }; - my $email_params = { - start_date => $start_date, - end_date => $end_date, - user => $inspector, - staging_site => FixMyStreet->config('STAGING_SITE'), - }; - my $rdi = FixMyStreet::Integrations::ExorRDI->new($params); - try { - my $hdrs = { - To => join('', 'fms', '_', 'admin', '@', $cobrand->admin_user_domain), - _attachments_ => [ { - body_str => $rdi->construct, - attributes => { - filename => $rdi->filename, - charset => 'utf-8', - encoding => 'quoted-printable', - content_type => 'text/csv', - name => $rdi->filename, - } - } ], - }; - - my $result = FixMyStreet::Email::send_cron( - FixMyStreet::DB->schema, - "rdi.txt", $email_params, $hdrs, - undef, 0, $cobrand, - ); - if ($result) { - say "Failed to send inspection for " . $inspector->id; - } - } catch { - die $_ unless $_ =~ /FixMyStreet::Integrations::ExorRDI::Error/; - # Nothing to report, continue - } -} - -1; diff --git a/bin/process-inactive-reports b/bin/process-inactive-reports index d2c030c2c..f1e9af2eb 100755 --- a/bin/process-inactive-reports +++ b/bin/process-inactive-reports @@ -15,9 +15,9 @@ use FixMyStreet::Script::Inactive; use Pod::Usage; my %h; -GetOptions(\%h, 'anonymize=i', 'close=i', 'verbose|v', 'help|h', 'dry-run|n'); +GetOptions(\%h, 'anonymize=i', 'close=i', 'delete=i', 'cobrand=s', 'verbose|v', 'help|h', 'dry-run|n'); pod2usage(0) if $h{help}; -pod2usage(1) unless $h{anonymize} || $h{close}; +pod2usage(1) unless $h{anonymize} || $h{close} || $h{delete}; FixMyStreet::Script::Inactive->new(%h)->reports; @@ -29,11 +29,13 @@ process-inactive-reports - deal with anonymizing inactive non-open reports =head1 SYNOPSIS -process-inactive-reports [--anonymize N] [--close N] +process-inactive-reports [--anonymize N] [--close N] [--delete N] [--cobrand COBRAND] Options: --anonymize Anonymize non-open reports (and related) inactive longer than this time (months) --close Close comments on non-open reports inactive longer than this time (months) + --delete Delete non-open reports inactive longer than this time (months) + --cobrand Only act upon reports made on this cobrand --dry-run Don't actually anonymize anything or send any emails --verbose Output as to which reports are being affected --help This help message diff --git a/bin/send-daemon b/bin/send-daemon new file mode 100755 index 000000000..dee9e949f --- /dev/null +++ b/bin/send-daemon @@ -0,0 +1,147 @@ +#!/usr/bin/env perl +# +# send-daemon +# FixMyStreet daemon for sending reports and updates. + +use strict; +use warnings; +use v5.14; + +BEGIN { + use File::Basename qw(dirname); + use File::Spec; + my $d = dirname(File::Spec->rel2abs($0)); + require "$d/../setenv.pl"; +} + +use Getopt::Long::Descriptive; +use Parallel::ForkManager; +use CronFns; +use FixMyStreet; +use FixMyStreet::DB; +use FixMyStreet::Script::Reports; +use FixMyStreet::Queue::Item::Report; +use Open311::PostServiceRequestUpdates; + +my ($opts, $usage) = describe_options( + '%c %o', + ['verbose|v+', 'more verbose output'], + ['nomail', 'do not send any email, print instead'], + ['debug', 'always try and send reports (no back-off skipping)'], + ['help|h', "print usage message and exit" ], + [], + ['Send a USR1 signal to the parent to cycle through verbose levels.'], +); +$usage->die if $opts->help; +my $verbose = $opts->verbose || 0; + +my $site = CronFns::site(FixMyStreet->config('BASE_URL')); +my $states = [ FixMyStreet::DB::Result::Problem::open_states() ]; +$states = [ 'submitted', 'confirmed', 'in progress', 'feedback pending', 'external', 'wish' ] if $site eq 'zurich'; + +my $db = FixMyStreet::DB->schema->storage; + +my %children; + +my $exit = 0; +$SIG{TERM} = $SIG{INT} = sub { $exit = 1; }; + +my $changeverboselevel = 0; +$SIG{USR1} = sub { + kill 'USR1', keys %children; + ++$changeverboselevel; +}; + +my $procs = FixMyStreet->config('QUEUE_DAEMON_PROCESSES') || 4; +my $pm = Parallel::ForkManager->new($procs); + +$pm->run_on_start(sub { + my $pid = shift; + $children{$pid} = time(); +}); +$pm->run_on_finish(sub { + my $pid = shift; + if ($children{$pid} > time() - 10) { + # It didn't live very long, let's wait a bit + sleep(5); + } + delete $children{$pid}; +}); + +# The parent loop +while (!$exit) { + while (keys %children < $procs) { + $pm->start and next; + srand; + $SIG{USR1} = sub { ++$changeverboselevel; }; + while (!$exit) { + $0 = "fmsd (running queue)"; + $db->txn_do(\&look_for_report); + $db->txn_do(\&look_for_update); + $0 = "fmsd"; + sleep(5 + rand(10)); + } + $pm->finish; + } + + if (!keys %children) { # Very high load, something wrong + sleep(10); + next; + } + + $pm->wait_for_available_procs; +} + +sub look_for_report { + my $params = FixMyStreet::Script::Reports::construct_query($opts->debug); + my $unsent = FixMyStreet::DB->resultset('Problem')->search($params, { + for => \'UPDATE SKIP LOCKED', + rows => 1, + } )->single or return; + + print_log('debug', "Trying to send report " . $unsent->id); + my $item = FixMyStreet::Queue::Item::Report->new( + report => $unsent, + verbose => $verbose, + nomail => $opts->nomail, + ); + $item->process; +} + +sub look_for_update { + my $updates = Open311::PostServiceRequestUpdates->new( + verbose => $verbose, + ); + + my $bodies = $updates->fetch_bodies; + my $params = $updates->construct_query($opts->debug); + my $comment = FixMyStreet::DB->resultset('Comment') + ->to_body([ keys %$bodies ]) + ->search($params, { for => \'UPDATE SKIP LOCKED', rows => 1 }) + ->single or return; + + print_log('debug', "Trying to send update " . $comment->id); + + my ($body) = grep { $bodies->{$_} } @{$comment->problem->bodies_str_ids}; + $body = $bodies->{$body}; + + $updates->construct_open311($body); + $updates->process_update($body, $comment); +} + +sub print_log { + my $prio = shift; + + if ($changeverboselevel) { + $verbose = ($verbose + $changeverboselevel) % 3; + STDERR->print("fmsd: info: verbose level now $verbose\n"); + $changeverboselevel = 0; + } + + if ($verbose < 2) { + return if ($prio eq 'noise'); + return if ($verbose < 1 && $prio eq 'debug'); + return if ($verbose < 0 && $prio eq 'info'); + } + STDERR->print("[fmsd] [$prio] ", join("", @_), "\n"); +} diff --git a/bin/send-reports b/bin/send-reports index 81be691e7..9651c271c 100755 --- a/bin/send-reports +++ b/bin/send-reports @@ -17,6 +17,7 @@ BEGIN { require "$d/../setenv.pl"; } +use Getopt::Long::Descriptive; use CronFns; use FixMyStreet; @@ -25,4 +26,13 @@ use FixMyStreet::Script::Reports; my $site = CronFns::site(FixMyStreet->config('BASE_URL')); CronFns::language($site); -FixMyStreet::Script::Reports::send(); +my ($opts, $usage) = describe_options( + '%c %o', + ['verbose|v', 'more verbose output'], + ['nomail', 'do not send any email, print instead'], + ['debug', 'always try and send reports (no back-off skipping)'], + ['help|h', "print usage message and exit" ], +); +$usage->die if $opts->help; + +FixMyStreet::Script::Reports::send($opts->verbose, $opts->nomail, $opts->debug); diff --git a/bin/send-reports-failure-summary b/bin/send-reports-failure-summary new file mode 100755 index 000000000..c7a5b4086 --- /dev/null +++ b/bin/send-reports-failure-summary @@ -0,0 +1,20 @@ +#!/usr/bin/env perl + +# send-reports-failure-summary: +# Prints a summary of report sending failures + +use strict; +use warnings; +use v5.14; + +BEGIN { + use File::Basename qw(dirname); + use File::Spec; + my $d = dirname(File::Spec->rel2abs($0)); + require "$d/../setenv.pl"; +} + +use FixMyStreet::Script::Reports; + +my $manager = FixMyStreet::Script::Reports->new; +$manager->end_summary_failures; diff --git a/bin/site-specific-install.sh b/bin/site-specific-install.sh index f0b78302f..af52dbdbc 100644 --- a/bin/site-specific-install.sh +++ b/bin/site-specific-install.sh @@ -1,7 +1,7 @@ #!/bin/sh # Set this to the version we want to check out -VERSION=${VERSION_OVERRIDE:-v2.6} +VERSION=${VERSION_OVERRIDE:-v3.0.1} PARENT_SCRIPT_URL=https://github.com/mysociety/commonlib/blob/master/bin/install-site.sh @@ -31,7 +31,7 @@ misuse() { [ -z "$INSTALL_POSTFIX" ] && misuse INSTALL_POSTFIX add_locale cy_GB -add_locale nb_NO +add_locale sv_SE add_locale de_CH if [ $INSTALL_POSTFIX = true ]; then diff --git a/bin/tfl/import_categories b/bin/tfl/import_categories new file mode 100755 index 000000000..bb48ddc4e --- /dev/null +++ b/bin/tfl/import_categories @@ -0,0 +1,116 @@ +#!/usr/bin/env perl + +use warnings; +use v5.14; +use utf8; + +BEGIN { + use File::Basename qw(dirname); + use File::Spec; + my $d = dirname(File::Spec->rel2abs($0)); + require "$d/../../setenv.pl"; +} + +use Term::ANSIColor; +use FixMyStreet; +use FixMyStreet::DB; +use Getopt::Long::Descriptive; +use JSON::MaybeXS; +use Path::Tiny; + +my ($opt, $usage) = describe_options( + '%c %o', + [ 'commit', "Actually commit changes to the database" ], + [ 'delete', "Delete all existing TfL categories first" ], + [ 'help', "print usage message and exit", { shortcircuit => 1 } ], +); +print($usage->text), exit if $opt->help; + +die "Usage: $0 <path/to/categories.json>" unless -f $ARGV[0]; + +my $db; +END { + if ($db) { + $opt->commit ? $db->txn_commit : $db->txn_rollback; + } +} + +$db = FixMyStreet::DB->schema->storage; +$db->txn_begin; +if (!$opt->commit) { + say colored("NOT COMMITTING TO DATABASE", 'cyan'); +} + +my $config = decode_json(path($ARGV[0])->slurp_utf8); + +my $body = FixMyStreet::DB->resultset('Body')->find({ name => 'TfL' }); + +$body->contacts->delete_all if $opt->delete; + +die "Couldn't find TfL body" unless $body; + +my $groups = $config->{groups}; +for my $group (keys %$groups) { + my $cats = $groups->{$group}; + for my $cat (@$cats) { + $cat->{category} = 'Other (TfL)' if $cat->{category} eq 'Other'; + my $child_cat = FixMyStreet::DB->resultset("Contact")->find_or_new({ + body => $body, + category => $cat->{category} + }); + $child_cat->email($cat->{email}); + $child_cat->state('confirmed'); + $child_cat->editor($0); + $child_cat->whenedited(\'current_timestamp'); + $child_cat->note($child_cat->in_storage ? 'Updated by import_categories' : 'Created by import_categories'); + say colored("WARNING", 'red') . " " . $child_cat->category . " already exists" if $child_cat->in_storage and $child_cat->category ne 'Other (TfL)'; + my $groups = $child_cat->groups; + my %groups = map { $_ => 1} @$groups; + $groups{$group} = 1; + my @groups = keys %groups; + $child_cat->extra(undef) if $child_cat->in_storage; + $child_cat->set_extra_metadata(group => \@groups); + $child_cat->set_extra_metadata(display_name => 'Other') if $child_cat->category eq 'Other (TfL)'; + if ($cat->{disable}) { + $child_cat->update_extra_field({ + code => "_fms_disable_", + disable_form => "true", + variable => "false", + protected => "true", + description => $cat->{disable} eq 1 ? $config->{disabled_message} : $cat->{disable}, + order => 0, + }); + } + $child_cat->set_extra_fields(@{ $cat->{extra_fields} }) if $cat->{extra_fields}; + if (my $asset_field = $cat->{asset_field}) { + my ($description, $code) = @$asset_field; + $child_cat->update_extra_field({ + code => $code, + description => $description, + automated => "hidden_field", + order => 1, + }); + } + # Add the safety critical hidden field + $child_cat->update_extra_field({ + code => "safety_critical", + description => "Safety critical", + automated => "hidden_field", + order => 1, + datatype => "singlevaluelist", + values => [ + { + name => "Yes", + key => "yes" + }, + { + name => "No", + key => "no" + } + ] + }); + $child_cat->in_storage ? $child_cat->update : $child_cat->insert; + } + + say "Created $group group"; +} diff --git a/bin/update-schema b/bin/update-schema index 9aff9ec5b..8f31085c9 100755 --- a/bin/update-schema +++ b/bin/update-schema @@ -212,6 +212,12 @@ else { # (assuming schema change files are never half-applied, which should be the case) sub get_db_version { return 'EMPTY' if ! table_exists('problem'); + return '0072' if constraint_contains('contacts_state_check', 'staff'); + return '0071' if table_exists('manifest_theme'); + return '0070' if column_like('alert_type', "ref='new_problems'", 'head_title', '{{SITE_NAME}}'); + return '0069' if constraint_contains('admin_log_object_type_check', 'template'); + return '0068' if column_exists('users', 'oidc_ids'); + return '0067' if table_exists('roles'); return '0066' if column_exists('users', 'area_ids'); return '0065' if constraint_contains('admin_log_object_type_check', 'moderation'); return '0064' if index_exists('moderation_original_data_problem_id_comment_id_idx'); diff --git a/bin/westminster/fixture b/bin/westminster/fixture new file mode 100755 index 000000000..4f7a86afb --- /dev/null +++ b/bin/westminster/fixture @@ -0,0 +1,356 @@ +#!/usr/bin/env perl + +use warnings; +use v5.14; +use utf8; + +BEGIN { + use File::Basename qw(dirname); + use File::Spec; + my $d = dirname(File::Spec->rel2abs($0)); + require "$d/../../setenv.pl"; +} + +use FixMyStreet; +use FixMyStreet::DB::Factories; +use Getopt::Long::Descriptive; + +my ($opt, $usage) = describe_options( + '%c %o', + [ 'empty', "Empty all tables of the database first" ], + [ 'commit', "Actually commit changes to the database" ], + [ 'help', "print usage message and exit", { shortcircuit => 1 } ], +); +print($usage->text), exit if $opt->help; + +FixMyStreet::DB::Factories->setup($opt); + +# Westminster City Council https://mapit.mysociety.org/area/2504.html +my $area_id = 2504; + +my $group_drainage = [ + 'Drainage or Surface water', +]; +my $group_flyposting = [ + 'Flyposting', + 'Stickers', + 'Graffiti', +]; +my $group_noise = [ + 'Aircraft', + 'Basement construction', + 'Birds', + 'Building site', + 'Burglar or fire alarms', + 'Buskers', + 'Car alarm', + 'Crossrail or Tideway construction', + 'Dogs barking or animal noise', + 'Noise from a business', + 'Noise from a home', + 'Noise in the street', + 'Underground tube trains or stations', +]; + +# Despite appearances, this will create a new Body every time, +# because it uses the `key_field` of `id` (see Factories.pm) to +# find existing bodies, and we’re not providing an `id` here. +# I guess it makes sense, because you might want a single MapIt +# area to be covered by multiple administrative bodies. +my $body = FixMyStreet::DB::Factory::Body->find_or_create({ + area_id => $area_id, + categories => [ @$group_drainage, @$group_flyposting, @$group_noise ], +}); + +say "Found/created body " . $body->name . " for MapIt area ID " . $area_id; + +my $yes_no_list = [ + { 'name' => 'Yes', 'key' => 'yes' }, + { 'name' => 'No', 'key' => 'no' }, +]; + +for my $cat (@$group_drainage) { + my $child_cat = FixMyStreet::DB->resultset("Contact")->find({ + body => $body, + category => $cat + }); + $child_cat->set_extra_fields( + { + description => 'Note: Please report dangerous issues by telephone on 0207 641 2000.', + hint => 'For example where there is a danger of causing serious harm to the public or significant damage to property, including a missing drain cover or immediate flooding to a property.', + code => 'danger', + required => 'false', + variable => 'false', # set 'true' to show data input + datatype => 'string', + }, + { + description => 'What is the problem with the drain?', + code => 'drain_problem_type', + required => 'false', + variable => 'true', + datatype => 'singlevaluelist', + values => [ + { 'name' => 'Burst water main', 'key' => 'burst' }, + { 'name' => 'Blocked drain', 'key' => 'blocked' }, + { 'name' => 'Flooding from a blocked drain', 'key' => 'flooded' }, + { 'name' => 'Smelly drain', 'key' => 'smelly' }, + { 'name' => 'Broken drain cover', 'key' => 'broken' }, + { 'name' => 'Missing drain cover', 'key' => 'missing' }, + { 'name' => 'Rats, flies or cockroaches', 'key' => 'vermin' }, + ], + }, + { + description => 'What type of customer are you?', + code => 'customer_type', + required => 'false', + variable => 'true', + datatype => 'singlevaluelist', + values => [ + { 'name' => 'Member of the public', 'key' => 'public' }, + { 'name' => 'Company', 'key' => 'company' }, + { 'name' => 'Police', 'key' => 'police' }, + { 'name' => 'Emergency services', 'key' => 'emergency' }, + { 'name' => 'Council officer', 'key' => 'officer' }, + { 'name' => 'Council contractor', 'key' => 'contractor' }, + ], + }, + ); + $child_cat->update; +} + +say "Created drainage category"; + +for my $cat (@$group_flyposting) { + my $child_cat = FixMyStreet::DB->resultset("Contact")->find({ + body => $body, + category => $cat + }); + $child_cat->set_extra_metadata( group => 'Graffiti or Flyposting' ); + $child_cat->set_extra_fields( + { + description => 'Surface type', + code => 'surface_type', + required => 'false', + variable => 'true', # set 'false' to hide data input + datatype => 'singlevaluelist', + values => [ + { 'name' => 'Brick', 'key' => 'brick' }, + { 'name' => 'Concrete', 'key' => 'concrete' }, + { 'name' => 'Glass', 'key' => 'glass' }, + { 'name' => 'Metal', 'key' => 'metal' }, + { 'name' => 'Plastic', 'key' => 'plastic' }, + { 'name' => 'Stone', 'key' => 'stone' }, + { 'name' => 'Tarmac', 'key' => 'tarmac' }, + { 'name' => 'Other / not known', 'key' => 'other' }, + ], + }, + { + description => 'Size', + code => 'size', + required => 'false', + variable => 'true', + datatype => 'singlevaluelist', + values => [ + { 'name' => 'less than 1m sq', 'key' => 'small' }, + { 'name' => '1m sq to 3m sq', 'key' => 'medium' }, + { 'name' => '3m sq +', 'key' => 'large' }, + ], + }, + { + description => 'Is it offensive?', + code => 'offensive', + required => 'false', + variable => 'true', + datatype => 'singlevaluelist', + values => $yes_no_list, + }, + ); + $child_cat->update; +} + +say "Created flyposting categories"; + +my $happening_now = { + description => 'Is the noise happening now?', + code => 'now', + required => 'false', + variable => 'true', + datatype => 'singlevaluelist', + values => $yes_no_list, +}; + +for my $cat (@$group_noise) { + my $child_cat = FixMyStreet::DB->resultset("Contact")->find({ + body => $body, + category => $cat + }); + $child_cat->set_extra_metadata( group => 'Noise' ); + + if ( $cat eq 'Aircraft' ) { + $child_cat->set_extra_fields( + { + description => 'Please contact the Civil Aviation Authority, or Ministry of Defence if it’s a military aircraft.', + code => 'go_away', + required => 'false', + variable => 'false', # set 'true' to show data input + datatype => 'string', + }, + ); + + } elsif ( $cat eq 'Basement construction') { + $child_cat->set_extra_fields( + { + description => 'Note: Building work is only allowed from 8am to 6pm on weekdays, and 8am to 1pm on Saturdays. Building work that is extremely noisy, such as demolition, is not allowed in residential areas at weekends. Builders need special permission from the council to work outside these times.', + code => 'building_permits', + required => 'false', + variable => 'false', # set 'true' to show data input + datatype => 'string', + }, + { + description => 'What is the name of the person responsible?', + code => 'responsible', + required => 'false', + variable => 'true', + datatype => 'string', + }, + $happening_now, + ); + + } elsif ( $cat eq 'Birds') { + $child_cat->set_extra_fields( + $happening_now, + ); + + } elsif ( $cat eq 'Building site') { + $child_cat->set_extra_fields( + { + description => 'What is the name of the business making the noise?', + code => 'responsible', + required => 'false', + variable => 'true', + datatype => 'string', + }, + $happening_now, + ); + + } elsif ( $cat eq 'Burglar or fire alarms') { + $child_cat->set_extra_fields( + { + description => 'What is the name of the person or business making the noise?', + code => 'responsible', + required => 'false', + variable => 'true', + datatype => 'string', + }, + $happening_now, + ); + + } elsif ( $cat eq 'Busking') { + $child_cat->set_extra_fields( + { + description => 'Note: Busking is not licensed in Westminster, nor is it illegal. If a busker is behaving unreasonably the Council can help.', + code => 'danger', + required => 'false', + variable => 'false', # set 'true' to show data input + datatype => 'string', + }, + $happening_now, + ); + + } elsif ( $cat eq 'Car alarm') { + $child_cat->set_extra_fields( + { + description => 'Vehicle registration', + code => 'vehicle_registration', + required => 'false', + variable => 'true', + datatype => 'string', + }, + $happening_now, + ); + + } elsif ( $cat eq 'Crossrail or Tideway construction') { + $child_cat->set_extra_fields( + { + description => 'Please contact Crossrail or Tideway directly.', + code => 'go_away', + required => 'false', + variable => 'false', # set 'true' to show data input + datatype => 'string', + }, + ); + + } elsif ( $cat eq 'Dogs barking or animal noise') { + $child_cat->set_extra_fields( + { + description => 'Note: Animal cruelty should be reported to the RSPCA.', + code => 'rspca', + required => 'false', + variable => 'false', # set 'true' to show data input + datatype => 'string', + }, + { + description => 'What is the name of the person or business making the noise?', + code => 'responsible', + required => 'false', + variable => 'true', + datatype => 'string', + }, + $happening_now, + ); + + } elsif ( $cat eq 'Noise from a business') { + $child_cat->set_extra_fields( + { + description => 'What is the name of the business making the noise?', + code => 'responsible', + required => 'false', + variable => 'true', + datatype => 'string', + }, + $happening_now, + ); + + } elsif ( $cat eq 'Noise from a home') { + $child_cat->set_extra_fields( + { + description => 'What is the name of the person responsible?', + code => 'responsible', + required => 'false', + variable => 'true', + datatype => 'string', + }, + $happening_now, + ); + + } elsif ( $cat eq 'Noise in the street') { + $child_cat->set_extra_fields( + $happening_now, + ); + + } elsif ( $cat eq 'Underground tube trains or stations') { + $child_cat->set_extra_fields( + { + description => 'TFL are a Statutory Undertaker and you should contact them first to investigate your complaint. If they have failed to resolve your complaint please continue with this form, including your TFL reference number, and we will look into this for you further.', + code => 'tfl', + required => 'false', + variable => 'false', # set 'true' to show data input + datatype => 'string', + }, + $happening_now, + ); + + } + + $child_cat->update; +} + +say "Created noise categories"; + +foreach ( + { name => 'Wizard of Oz', email_verified => 1, email => 'admin@example.org', is_superuser => 't' }, + { name => 'Norma User', email_verified => 1, email => 'user@example.org' }, +) { + FixMyStreet::DB::Factory::User->find_or_create($_); + say "Found/created user $_->{email}"; +} |