diff options
| -rwxr-xr-x | bin/oxfordshire/send-rdi-emails | 79 | ||||
| -rw-r--r-- | perllib/FixMyStreet/App/Controller/Admin/ExorDefects.pm | 81 | ||||
| -rw-r--r-- | perllib/FixMyStreet/Cobrand/Oxfordshire.pm | 8 | ||||
| -rw-r--r-- | perllib/FixMyStreet/Integrations/ExorRDI.pm | 250 | ||||
| -rw-r--r-- | t/cobrand/oxfordshire.t | 140 | ||||
| -rw-r--r-- | templates/web/base/admin/exordefects/index.html | 39 |
6 files changed, 0 insertions, 597 deletions
diff --git a/bin/oxfordshire/send-rdi-emails b/bin/oxfordshire/send-rdi-emails deleted file mode 100755 index 9cc3e5502..000000000 --- a/bin/oxfordshire/send-rdi-emails +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env perl - -use strict; -use warnings; -use v5.14; - -BEGIN { - use File::Basename qw(dirname); - use File::Spec; - my $d = dirname(File::Spec->rel2abs($0)); - require "$d/../../setenv.pl"; -} - -use DateTime; -use Try::Tiny; -use FixMyStreet; -use FixMyStreet::Cobrand; -use FixMyStreet::Email; -use FixMyStreet::Integrations::ExorRDI; - -my $end_date = DateTime->now( time_zone => FixMyStreet->time_zone || FixMyStreet->local_time_zone ) - ->truncate(to => 'hour')->set_hour(16); -my $start_date = $end_date->clone->subtract(days => 1); -my $inspection_date = $end_date; # All inspections are considered to have happened on the same date. - -my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker('oxfordshire')->new; -$cobrand->set_lang_and_domain('en-gb', 1); -my @inspectors = $cobrand->users->search({ - 'user_body_permissions.permission_type' => 'report_inspect' -}, { - join => 'user_body_permissions', - distinct => 1, -})->all; - -foreach my $inspector (@inspectors) { - my $params = { - start_date => $start_date, - end_date => $end_date, - inspection_date => $inspection_date, - user => $inspector, - mark_as_processed => 1, - }; - my $email_params = { - start_date => $start_date, - end_date => $end_date, - user => $inspector, - staging_site => FixMyStreet->config('STAGING_SITE'), - }; - my $rdi = FixMyStreet::Integrations::ExorRDI->new($params); - try { - my $hdrs = { - To => join('', 'fms', '_', 'admin', '@', $cobrand->admin_user_domain), - _attachments_ => [ { - body_str => $rdi->construct, - attributes => { - filename => $rdi->filename, - charset => 'utf-8', - encoding => 'quoted-printable', - content_type => 'text/csv', - name => $rdi->filename, - } - } ], - }; - - my $result = FixMyStreet::Email::send_cron( - FixMyStreet::DB->schema, - "rdi.txt", $email_params, $hdrs, - undef, 0, $cobrand, - ); - if ($result) { - say "Failed to send inspection for " . $inspector->id; - } - } catch { - die $_ unless $_ =~ /FixMyStreet::Integrations::ExorRDI::Error/; - # Nothing to report, continue - } -} - -1; diff --git a/perllib/FixMyStreet/App/Controller/Admin/ExorDefects.pm b/perllib/FixMyStreet/App/Controller/Admin/ExorDefects.pm deleted file mode 100644 index 0026acb9c..000000000 --- a/perllib/FixMyStreet/App/Controller/Admin/ExorDefects.pm +++ /dev/null @@ -1,81 +0,0 @@ -package FixMyStreet::App::Controller::Admin::ExorDefects; -use Moose; -use namespace::autoclean; - -use DateTime; -use Try::Tiny; -use FixMyStreet::Integrations::ExorRDI; -use FixMyStreet::DateRange; - -BEGIN { extends 'Catalyst::Controller'; } - - -sub index : Path : Args(0) { - my ( $self, $c ) = @_; - - foreach (qw(error_message start_date end_date user_id)) { - if ( defined $c->flash->{$_} ) { - $c->stash->{$_} = $c->flash->{$_}; - } - } - - my @inspectors = $c->cobrand->users->search({ - 'user_body_permissions.permission_type' => 'report_inspect' - }, { - join => 'user_body_permissions', - distinct => 1, - } - )->all; - $c->stash->{inspectors} = \@inspectors; - - # Default start/end date is today - my $now = DateTime->now( time_zone => - FixMyStreet->time_zone || FixMyStreet->local_time_zone ); - $c->stash->{start_date} ||= $now; - $c->stash->{end_date} ||= $now; - -} - -sub download : Path('download') : Args(0) { - my ( $self, $c ) = @_; - - if ( !$c->cobrand->can('exor_rdi_link_id') ) { - # This only works on the Oxfordshire cobrand currently. - $c->detach( '/page_error_404_not_found', [] ); - } - - my $range = FixMyStreet::DateRange->new( - start_date => $c->get_param('start_date'), - end_date => $c->get_param('end_date'), - parser => DateTime::Format::Strptime->new( pattern => '%Y-%m-%d' ), - ); - - my $params = { - start_date => $range->start, - inspection_date => $range->start, - end_date => $range->end, - user => $c->get_param('user_id'), - mark_as_processed => 0, - }; - my $rdi = FixMyStreet::Integrations::ExorRDI->new($params); - - try { - my $out = $rdi->construct; - $c->res->content_type('text/csv; charset=utf-8'); - $c->res->header('content-disposition' => "attachment; filename=" . $rdi->filename); - $c->res->body( $out ); - } catch { - die $_ unless $_ =~ /FixMyStreet::Integrations::ExorRDI::Error/; - if ($params->{user}) { - $c->flash->{error_message} = _("No inspections by that inspector in the selected date range."); - } else { - $c->flash->{error_message} = _("No inspections in the selected date range."); - } - $c->flash->{start_date} = $params->{start_date}; - $c->flash->{end_date} = $params->{end_date}; - $c->flash->{user_id} = $params->{user}; - $c->res->redirect( $c->uri_for( '' ) ); - }; -} - -1; diff --git a/perllib/FixMyStreet/Cobrand/Oxfordshire.pm b/perllib/FixMyStreet/Cobrand/Oxfordshire.pm index ea6f14904..50b71bb69 100644 --- a/perllib/FixMyStreet/Cobrand/Oxfordshire.pm +++ b/perllib/FixMyStreet/Cobrand/Oxfordshire.pm @@ -164,11 +164,6 @@ sub admin_pages { my $pages = $self->next::method(); - # Oxfordshire have a custom admin page for downloading reports in an Exor- - # friendly format which anyone with report_instruct permission can use. - if ( $user->has_body_permission_to('report_instruct') ) { - $pages->{exordefects} = [ ('Download Exor RDI'), 10 ]; - } if ( $user->has_body_permission_to('defect_type_edit') ) { $pages->{defecttypes} = [ ('Defect Types'), 11 ]; $pages->{defecttype_edit} = [ undef, undef ]; @@ -184,9 +179,6 @@ sub defect_types { }; } -sub exor_rdi_link_id { 1989169 } -sub exor_rdi_link_length { 50 } - sub reputation_increment_states { return { 'action scheduled' => 1, diff --git a/perllib/FixMyStreet/Integrations/ExorRDI.pm b/perllib/FixMyStreet/Integrations/ExorRDI.pm deleted file mode 100644 index ce59df9be..000000000 --- a/perllib/FixMyStreet/Integrations/ExorRDI.pm +++ /dev/null @@ -1,250 +0,0 @@ -package FixMyStreet::Integrations::ExorRDI::Error; - -use Moo; -with 'Throwable'; - -has message => (is => 'ro'); - -package FixMyStreet::Integrations::ExorRDI::CSV; - -use parent 'Text::CSV'; - -sub add_row { - my ($self, $data, @data) = @_; - $self->combine(@data); - push @$data, $self->string; -} - -package FixMyStreet::Integrations::ExorRDI; - -use DateTime; -use Moo; -use Scalar::Util 'blessed'; -use FixMyStreet::DB; -use namespace::clean; - -has [qw(start_date end_date inspection_date mark_as_processed)] => ( - is => 'ro', - required => 1, -); - -has user => ( - is => 'ro', - coerce => sub { - return $_[0] if blessed($_[0]) && $_[0]->isa('FixMyStreet::DB::Result::User'); - FixMyStreet::DB->resultset('User')->find( { id => $_[0] } ) - if $_[0]; - }, -); - -sub construct { - my $self = shift; - - my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker('oxfordshire')->new; - my $dtf = $cobrand->problems->result_source->storage->datetime_parser; - my $now = DateTime->now( - time_zone => FixMyStreet->time_zone || FixMyStreet->local_time_zone - ); - - my $tmo = $cobrand->traffic_management_options; - my %tm_lookup = map { $tmo->[$_] => $_ + 1 } 0..$#$tmo; - - my $missed_cutoff = $now - DateTime::Duration->new( hours => 24 ); - my %params = ( - -and => [ - state => [ 'action scheduled' ], - external_id => { '!=' => undef }, - -or => [ - -and => [ - 'admin_log_entries.action' => 'inspected', - 'admin_log_entries.whenedited' => { '>=', $dtf->format_datetime($self->start_date) }, - 'admin_log_entries.whenedited' => { '<=', $dtf->format_datetime($self->end_date) }, - ], - -and => [ - extra => { -not_like => '%rdi_processed%' }, - 'admin_log_entries.action' => 'inspected', - 'admin_log_entries.whenedited' => { '<=', $dtf->format_datetime($missed_cutoff) }, - ] - ] - ] - ); - - $params{'admin_log_entries.user_id'} = $self->user->id if $self->user; - - my $problems = $cobrand->problems->search( - \%params, - { - join => 'admin_log_entries', - distinct => 1, - } - ); - FixMyStreet::Integrations::ExorRDI::Error->throw unless $problems->count; - - # A single RDI file might contain inspections from multiple inspectors, so - # we need to group inspections by inspector within G records. - my $inspectors = {}; - my $inspector_initials = {}; - while ( my $report = $problems->next ) { - my $user = $report->inspection_log_entry->user; - $inspectors->{$user->id} ||= []; - push @{ $inspectors->{$user->id} }, $report; - unless ( $inspector_initials->{$user->id} ) { - $inspector_initials->{$user->id} = $user->get_extra_metadata('initials'); - } - } - - my $csv = FixMyStreet::Integrations::ExorRDI::CSV->new({ binary => 1, eol => "" }); - - my $p_count = 0; - my $link_id = $cobrand->exor_rdi_link_id; - - # RDI first line is always the same - my $body = []; - $csv->add_row($body, "1", "1.8", "1.0.0.0", "ENHN", ""); - - my $i = 0; - foreach my $inspector_id (keys %$inspectors) { - my $inspections = $inspectors->{$inspector_id}; - my $initials = $inspector_initials->{$inspector_id} || "XX"; - - my %body_by_activity_code; - foreach my $report (@$inspections) { - my ($eastings, $northings) = $report->local_coords; - - my $location = "${eastings}E ${northings}N"; - $location = "[DID NOT USE MAP] $location" unless $report->used_map; - my $closest_address = $cobrand->find_closest($report); - if (%$closest_address) { - $location .= " Nearest road: $closest_address->{name}." if $closest_address->{name}; - $location .= " Nearest postcode: $closest_address->{postcode}{postcode}." if $closest_address->{postcode}; - } - - my $traffic_information = $report->get_extra_metadata('traffic_information') || 'none'; - my $description = sprintf("%s %s %s %s", - $report->external_id || "", - $initials, - 'TM' . ($tm_lookup{$traffic_information} || '0'), - $report->get_extra_metadata('detailed_information') || ""); - # Maximum length of 180 characters total - $description = substr($description, 0, 180); - my $activity_code = $report->defect_type ? - $report->defect_type->get_extra_metadata('activity_code') - : 'MC'; - $body_by_activity_code{$activity_code} ||= []; - - $csv->add_row($body_by_activity_code{$activity_code}, - "I", # beginning of defect record - $activity_code, # activity code - minor carriageway, also FC (footway) - "", # empty field, can also be A (seen on MC) or B (seen on FC) - sprintf("%03d", ++$i), # randomised sequence number - $location, # defect location field, which we don't capture from inspectors - $report->inspection_log_entry->whenedited->strftime("%H%M"), # defect time raised - "","","","","","","", # empty fields - "TM $traffic_information", - $description, # defect description - ); - - my $defect_type = $report->defect_type ? - $report->defect_type->get_extra_metadata('defect_code') - : 'SFP2'; - $csv->add_row($body_by_activity_code{$activity_code}, - "J", # georeferencing record - $defect_type, # defect type - SFP2: sweep and fill <1m2, POT2 also seen - $report->response_priority ? - $report->response_priority->external_id : - "2", # priority of defect - "","", # empty fields - $eastings, # eastings - $northings, # northings - "","","","","" # empty fields - ); - - my $m_row_activity_code = $activity_code; - $m_row_activity_code .= 'I' if length $activity_code == 1; - - $csv->add_row($body_by_activity_code{$activity_code}, - "M", # bill of quantities record - "resolve", # permanent repair - "","", # empty fields - "/C$m_row_activity_code", # /C + activity code + perhaps an "I" - "", "" # empty fields - ); - } - - foreach my $activity_code (sort keys %body_by_activity_code) { - $csv->add_row($body, - "G", # start of an area/sequence - $link_id, # area/link id, fixed value for our purposes - "","", # must be empty - $initials, # inspector initials - $self->inspection_date->strftime("%y%m%d"), # date of inspection yymmdd - "1600", # time of inspection hhmm, set to static value for now - "D", # inspection variant, should always be D - "INS", # inspection type, always INS - "N", # Area of the county - north (N) or south (S) - "", "", "", "" # empty fields - ); - - $csv->add_row($body, - "H", # initial inspection type - $activity_code # e.g. MC = minor carriageway - ); - - # List of I/J/M entries from above - push @$body, @{$body_by_activity_code{$activity_code}}; - - # end this group of defects with a P record - $csv->add_row($body, - "P", # end of area/sequence - 0, # always 0 - 999999, # charging code, always 999999 in OCC - ); - $p_count++; - } - } - - # end the RDI file with an X record - my $record_count = $i; - $csv->add_row($body, - "X", # end of inspection record - $p_count, - $p_count, - $record_count, # number of I records - $record_count, # number of J records - 0, 0, 0, # always zero - $record_count, # number of M records - 0, # always zero - $p_count, - 0, 0, 0 # error counts, always zero - ); - - if ($self->mark_as_processed) { - # Mark all these problems are having been included in an RDI - $problems->reset; - while ( my $report = $problems->next ) { - $report->set_extra_metadata('rdi_processed' => $now->strftime( '%Y-%m-%d %H:%M' )); - $report->update; - } - } - - # The RDI format is very weird CSV - each line must be wrapped in - # double quotes. - return join "", map { "\"$_\"\r\n" } @$body; -} - -has filename => ( - is => 'lazy', - default => sub { - my $self = shift; - my $start = $self->inspection_date->strftime("%Y%m%d"); - my $end = $self->end_date->strftime("%Y%m%d"); - my $filename = sprintf("exor_defects-%s-%s.rdi", $start, $end); - if ( $self->user ) { - my $initials = $self->user->get_extra_metadata("initials") || ""; - $filename = sprintf("exor_defects-%s-%s-%s.rdi", $start, $end, $initials); - } - return $filename; - }, -); - -1; diff --git a/t/cobrand/oxfordshire.t b/t/cobrand/oxfordshire.t index 2625aa0d5..b8422a28b 100644 --- a/t/cobrand/oxfordshire.t +++ b/t/cobrand/oxfordshire.t @@ -1,6 +1,4 @@ - use Test::MockModule; -use FixMyStreet::Integrations::ExorRDI; use FixMyStreet::TestMech; my $mech = FixMyStreet::TestMech->new; @@ -47,145 +45,8 @@ subtest 'check /around?ajax defaults to open reports only' => sub { } }; -my $superuser = $mech->create_user_ok('superuser@example.com', name => 'Super User', is_superuser => 1); -my $inspector = $mech->create_user_ok('inspector@example.com', name => 'Inspector'); -$inspector->user_body_permissions->create({ body => $oxon, permission_type => 'report_inspect' }); - my @problems = FixMyStreet::DB->resultset('Problem')->search({}, { rows => 3 })->all; -subtest 'Exor RDI download appears on Oxfordshire cobrand admin' => sub { - FixMyStreet::override_config { - ALLOWED_COBRANDS => [ { 'oxfordshire' => '.' } ], - }, sub { - $mech->log_in_ok( $superuser->email ); - $mech->get_ok('/admin'); - $mech->content_contains("Download Exor RDI"); - } -}; - -subtest "Exor RDI download doesn't appear outside of Oxfordshire cobrand admin" => sub { - FixMyStreet::override_config { - ALLOWED_COBRANDS => [ { 'fixmystreet' => '.' } ], - }, sub { - $mech->log_in_ok( $superuser->email ); - $mech->get_ok('/admin'); - $mech->content_lacks("Download Exor RDI"); - } -}; - -subtest 'Exor file looks okay' => sub { - FixMyStreet::override_config { - ALLOWED_COBRANDS => [ 'oxfordshire' ], - MAPIT_URL => 'http://mapit.uk/', - }, sub { - $mech->log_in_ok( $superuser->email ); - $mech->get_ok('/admin/exordefects'); - $mech->submit_form_ok( { with_fields => { - start_date => '2017-05-05', - end_date => '2017-05-05', - user_id => $inspector->id, - } }, 'submit download'); - $mech->content_contains("No inspections by that inspector in the selected date range"); - - my $dt = FixMyStreet::DB->resultset('DefectType')->create({ - body => $oxon, - name => 'Footpath', - description => 'Footpath stuff', - }); - $dt->set_extra_metadata(activity_code => 'FC'); - $dt->set_extra_metadata(defect_code => 'SFP1'); - $dt->update; - my $dt2 = FixMyStreet::DB->resultset('DefectType')->create({ - body => $oxon, - name => 'Accidental sign damage', - description => 'Accidental sign damage', - }); - $dt2->set_extra_metadata(activity_code => 'S'); - $dt2->set_extra_metadata(defect_code => 'ACC2'); - $dt2->update; - my $i = 123; - foreach my $problem (@problems) { - $problem->update({ state => 'action scheduled', external_id => $i }); - $problem->update({ defect_type => $dt }) if $i == 123; - $problem->set_extra_metadata(traffic_information => 'Signs and Cones') if $i == 124; - $problem->update({ defect_type => $dt2 }) if $i == 124; - FixMyStreet::DB->resultset('AdminLog')->create({ - admin_user => $inspector->name, - user => $inspector, - object_type => 'problem', - action => 'inspected', - object_id => $problem->id, - whenedited => DateTime->new(year => 2017, month => 5, day => 5, hour => 12), - }); - $i++; - } - $mech->submit_form_ok( { with_fields => { - start_date => '2017-05-05', - end_date => '2017-05-05', - user_id => $inspector->id, - } }, 'submit download'); - (my $rdi = $mech->content) =~ s/\r\n/\n/g; - $rdi =~ s/(I,[FMS]C?,,)\d+/$1XXX/g; # Remove unique ID figures, unknown order - is $rdi, <<EOF, "RDI file matches expected"; -"1,1.8,1.0.0.0,ENHN," -"G,1989169,,,XX,170505,1600,D,INS,N,,,," -"H,FC" -"I,FC,,XXX,"434970E 209683N Nearest postcode: OX28 4DS.",1200,,,,,,,,"TM none","123 XX TM0 "" -"J,SFP1,2,,,434970,209683,,,,," -"M,resolve,,,/CFC,," -"P,0,999999" -"G,1989169,,,XX,170505,1600,D,INS,N,,,," -"H,MC" -"I,MC,,XXX,"434970E 209683N Nearest postcode: OX28 4DS.",1200,,,,,,,,"TM none","125 XX TM0 "" -"J,SFP2,2,,,434970,209683,,,,," -"M,resolve,,,/CMC,," -"P,0,999999" -"G,1989169,,,XX,170505,1600,D,INS,N,,,," -"H,S" -"I,S,,XXX,"434970E 209683N Nearest postcode: OX28 4DS.",1200,,,,,,,,"TM Signs and Cones","124 XX TM1 "" -"J,ACC2,2,,,434970,209683,,,,," -"M,resolve,,,/CSI,," -"P,0,999999" -"X,3,3,3,3,0,0,0,3,0,3,0,0,0" -EOF - foreach my $problem (@problems) { - $problem->discard_changes; - is $problem->get_extra_metadata('rdi_processed'), undef, "Problem was not logged as sent in RDI"; - } - - } -}; - -subtest 'Reports are marked as inspected correctly' => sub { - FixMyStreet::override_config { - ALLOWED_COBRANDS => [ 'oxfordshire' ], - MAPIT_URL => 'http://mapit.uk/', - }, sub { - my $date = DateTime->new(year => 2017, month => 5, day => 5, hour => 12); - - my $now = DateTime->now( - time_zone => FixMyStreet->time_zone || FixMyStreet->local_time_zone - ); - my $datetime = Test::MockModule->new('DateTime'); - $datetime->mock('now', sub { $now }); - - my $params = { - start_date => $date, - end_date => $date, - inspection_date => $date, - user => $inspector, - mark_as_processed => 1, - }; - my $rdi = FixMyStreet::Integrations::ExorRDI->new($params); - $rdi->construct; - - foreach my $problem (@problems) { - $problem->discard_changes; - is $problem->get_extra_metadata('rdi_processed'), $now->strftime( '%Y-%m-%d %H:%M' ), "Problem was logged as sent in RDI"; - } - }; -}; - subtest 'can use customer reference to search for reports' => sub { FixMyStreet::override_config { ALLOWED_COBRANDS => [ 'oxfordshire' ], @@ -195,7 +56,6 @@ subtest 'can use customer reference to search for reports' => sub { $problem->set_extra_metadata( customer_reference => 'ENQ12456' ); $problem->update; - $mech->log_out_ok; $mech->get_ok('/around?pc=ENQ12456'); is $mech->uri->path, '/report/' . $problem->id, 'redirects to report'; }; diff --git a/templates/web/base/admin/exordefects/index.html b/templates/web/base/admin/exordefects/index.html deleted file mode 100644 index 65b2aa486..000000000 --- a/templates/web/base/admin/exordefects/index.html +++ /dev/null @@ -1,39 +0,0 @@ -[% INCLUDE 'admin/header.html' title=('Download Exor RDI') -%] - -[% IF error_message %] - <h2>Error</h2> - <p>[% error_message %]</p> -[% END %] - -<form method="get" action="[% c.uri_for('download') %]" enctype="application/x-www-form-urlencoded" accept-charset="utf-8"> - <div class="filters"> - <p> - <label for="start_date">[% ('Start Date:') %]</label><input type="date" class="form-control" - name="start_date" id="start_date" - value="[% start_date ? start_date.strftime( '%Y-%m-%d') : '' | html %]" /> - </p> - - <p> - <label for="end_date">[% ('End Date:') %]</label><input type="date" class="form-control" - name="end_date" id="end_date" size="5" - value="[% end_date ? end_date.strftime( '%Y-%m-%d') : '' | html %]" /> - </p> - - <p> - <label for="user_id">[% ('Inspector:') %]</label> - <select class="form-control" id='user_id' name='user_id'> - <option value=''>[% ('All inspectors') %]</option> - [% FOR inspector IN inspectors %] - <option value="[% inspector.id %]" [% 'selected' IF user_id == inspector.id %]>[% inspector.name %] ([% inspector.get_extra_metadata('initials') %])</option> - [% END %] - </select> - </p> - </div> - - <p> - <input type="submit" class="btn" size="30" value="Download RDI file" /> - </p> -</form> - - -[% INCLUDE 'admin/footer.html' %] |