diff options
Diffstat (limited to 'perllib')
| -rw-r--r-- | perllib/FixMyStreet/App/Controller/Report/New.pm | 21 | ||||
| -rw-r--r-- | perllib/FixMyStreet/Cobrand/EastSussex.pm | 41 | ||||
| -rw-r--r-- | perllib/FixMyStreet/Cobrand/Oxfordshire.pm | 24 | ||||
| -rw-r--r-- | perllib/FixMyStreet/Cobrand/Peterborough.pm | 13 | ||||
| -rw-r--r-- | perllib/FixMyStreet/DB/Result/Problem.pm | 2 | ||||
| -rw-r--r-- | perllib/FixMyStreet/DB/ResultSet/Problem.pm | 2 | ||||
| -rw-r--r-- | perllib/FixMyStreet/Queue/Item/Report.pm | 24 | ||||
| -rw-r--r-- | perllib/FixMyStreet/Script/Reports.pm | 50 | ||||
| -rwxr-xr-x | perllib/Open311/PostServiceRequestUpdates.pm | 113 |
9 files changed, 202 insertions, 88 deletions
diff --git a/perllib/FixMyStreet/App/Controller/Report/New.pm b/perllib/FixMyStreet/App/Controller/Report/New.pm index 516752b89..69d20171a 100644 --- a/perllib/FixMyStreet/App/Controller/Report/New.pm +++ b/perllib/FixMyStreet/App/Controller/Report/New.pm @@ -358,8 +358,7 @@ sub report_import : Path('/import') { # If this is not a POST then just print out instructions for using page return unless $c->req->method eq 'POST'; - # anything else we return is plain text - $c->res->content_type('text/plain; charset=utf-8'); + my $format = $c->get_param('web') ? 'web' : 'text'; my %input = map { $_ => $c->get_param($_) || '' } ( @@ -412,8 +411,14 @@ sub report_import : Path('/import') { # if we have errors then we should bail out if (@errors) { - my $body = join '', map { "ERROR:$_\n" } @errors; - $c->res->body($body); + if ($format eq 'web') { + $c->stash->{input} = \%input; + $c->stash->{errors} = \@errors; + } else { + my $body = join '', map { "ERROR:$_\n" } @errors; + $c->res->content_type('text/plain; charset=utf-8'); + $c->res->body($body); + } return; } @@ -469,13 +474,13 @@ sub report_import : Path('/import') { $c->send_email( 'partial.txt', { to => $report->user->email, } ); - if ( $c->get_param('web') ) { - $c->res->content_type('text/html; charset=utf-8'); + if ($format eq 'web') { $c->stash->{template} = 'email_sent.html'; $c->stash->{email_type} = 'problem'; - return 1; + } else { + $c->res->content_type('text/plain; charset=utf-8'); + $c->res->body('SUCCESS'); } - $c->res->body('SUCCESS'); return 1; } diff --git a/perllib/FixMyStreet/Cobrand/EastSussex.pm b/perllib/FixMyStreet/Cobrand/EastSussex.pm new file mode 100644 index 000000000..c113a0986 --- /dev/null +++ b/perllib/FixMyStreet/Cobrand/EastSussex.pm @@ -0,0 +1,41 @@ +package FixMyStreet::Cobrand::EastSussex; +use parent 'FixMyStreet::Cobrand::UK'; + +use strict; +use warnings; + +sub council_area_id { return 2224; } + +sub open311_pre_send { + my ($self, $row, $open311) = @_; + + my $contact = $row->category_row; + my $fields = $contact->get_extra_fields; + for my $field ( @$fields ) { + if ($field->{variable} && !$field->{automated}) { + my $text = $row->detail; + my $q = $row->get_extra_field_value( $field->{code} ) || ''; + $text .= "\n\n" . $field->{description} . "\n" . $q; + $row->detail($text); + } + } +} + +sub open311_post_send { + my ($self, $row, $h, $contact) = @_; + + my $fields = $contact->get_extra_fields; + my $text = $row->detail; + my $added = ''; + for my $field ( @$fields ) { + if ($field->{variable} && !$field->{automated}) { + my $q = $row->get_extra_field_value( $field->{code} ) || ''; + $added .= "\n\n" . $field->{description} . "\n" . $q; + } + } + + $text =~ s/\Q$added\E//; + $row->detail($text); +} + +1; diff --git a/perllib/FixMyStreet/Cobrand/Oxfordshire.pm b/perllib/FixMyStreet/Cobrand/Oxfordshire.pm index 12714185d..b110731e6 100644 --- a/perllib/FixMyStreet/Cobrand/Oxfordshire.pm +++ b/perllib/FixMyStreet/Cobrand/Oxfordshire.pm @@ -142,8 +142,13 @@ sub should_skip_sending_update { my ($self, $update ) = @_; # Oxfordshire stores the external id of the problem as a customer reference - # in metadata - return 1 if !$update->problem->get_extra_metadata('customer_reference'); + # in metadata, it arrives in a fetched update (but give up if it never does, + # or the update is for an old pre-ref report) + my $customer_ref = $update->problem->get_extra_metadata('customer_reference'); + my $diff = time() - $update->confirmed->epoch; + return 1 if !$customer_ref && $diff > 60*60*24; + return 'WAIT' if !$customer_ref; + return 0; } sub on_map_default_status { return 'open'; } @@ -197,4 +202,19 @@ sub available_permissions { return $perms; } +sub dashboard_export_problems_add_columns { + my $self = shift; + my $c = $self->{c}; + + push @{$c->stash->{csv}->{headers}}, "HIAMS Ref"; + push @{$c->stash->{csv}->{columns}}, "customer_reference"; + + $c->stash->{csv}->{extra_data} = sub { + my $ref = shift->get_extra_metadata('customer_reference') || ''; + return { + customer_reference => $ref, + }; + }; +} + 1; diff --git a/perllib/FixMyStreet/Cobrand/Peterborough.pm b/perllib/FixMyStreet/Cobrand/Peterborough.pm index 882bef7eb..0ddaeacb6 100644 --- a/perllib/FixMyStreet/Cobrand/Peterborough.pm +++ b/perllib/FixMyStreet/Cobrand/Peterborough.pm @@ -40,6 +40,19 @@ sub geocoder_munge_results { sub admin_user_domain { "peterborough.gov.uk" } +around open311_extra_data => sub { + my ($orig, $self, $row, $h, $extra) = @_; + + my $open311_only = $self->$orig($row, $h, $extra); + foreach (@$open311_only) { + if ($_->{name} eq 'description') { + my ($ref) = grep { $_->{name} =~ /pcc-Skanska-csc-ref/i } @{$row->get_extra_fields}; + $_->{value} .= "\n\nSkanska CSC ref: $ref->{value}" if $ref; + } + } + return $open311_only; +}; + # remove categories which are informational only sub open311_pre_send { my ($self, $row, $open311) = @_; diff --git a/perllib/FixMyStreet/DB/Result/Problem.pm b/perllib/FixMyStreet/DB/Result/Problem.pm index 489c43090..37563d327 100644 --- a/perllib/FixMyStreet/DB/Result/Problem.pm +++ b/perllib/FixMyStreet/DB/Result/Problem.pm @@ -800,7 +800,7 @@ sub defect_types { # Note: this only makes sense when called on a problem that has been sent! sub can_display_external_id { my $self = shift; - if ($self->external_id && $self->to_body_named('Oxfordshire|Lincolnshire|Isle of Wight')) { + if ($self->external_id && $self->to_body_named('Oxfordshire|Lincolnshire|Isle of Wight|East Sussex')) { return 1; } return 0; diff --git a/perllib/FixMyStreet/DB/ResultSet/Problem.pm b/perllib/FixMyStreet/DB/ResultSet/Problem.pm index 3e48170d8..359d5224a 100644 --- a/perllib/FixMyStreet/DB/ResultSet/Problem.pm +++ b/perllib/FixMyStreet/DB/ResultSet/Problem.pm @@ -159,7 +159,7 @@ sub _recent { $probs = [ $rs->search({ id => [ map { $_->id } @$probs ], %$query, - })->all ]; + }, $attrs)->all ]; } else { $probs = [ $rs->search( $query, $attrs )->all ]; Memcached::set($key, $probs, _cache_timeout()); diff --git a/perllib/FixMyStreet/Queue/Item/Report.pm b/perllib/FixMyStreet/Queue/Item/Report.pm index 4d0d62752..e38987838 100644 --- a/perllib/FixMyStreet/Queue/Item/Report.pm +++ b/perllib/FixMyStreet/Queue/Item/Report.pm @@ -181,7 +181,7 @@ sub _create_reporters { } $reporters{ $sender } ||= $sender->new(); - $self->log("OK, adding recipient body " . $body->id . ":" . $body->name . ", " . $sender_info->{method}); + $self->log("Adding recipient body " . $body->id . ":" . $body->name . ", " . $sender_info->{method}); push @dear, $body->name; $reporters{ $sender }->add_body( $body, $sender_info->{config} ); } @@ -218,21 +218,23 @@ sub _send { my $result = -1; for my $sender ( keys %{$self->reporters} ) { - $self->log("sending using " . $sender); + $self->log("Sending using " . $sender); $sender = $self->reporters->{$sender}; my $res = $sender->send( $self->report, $self->h ); $result *= $res; $self->report->add_send_method($sender) if !$res; - if ( $sender->unconfirmed_data) { - foreach my $e (keys %{ $sender->unconfirmed_data } ) { - foreach my $c (keys %{ $sender->unconfirmed_data->{$e} }) { - $self->manager->unconfirmed_data->{$e}{$c}{count} += $sender->unconfirmed_data->{$e}{$c}{count}; - $self->manager->unconfirmed_data->{$e}{$c}{note} = $sender->unconfirmed_data->{$e}{$c}{note}; + if ( $self->manager ) { + if ($sender->unconfirmed_data) { + foreach my $e (keys %{ $sender->unconfirmed_data } ) { + foreach my $c (keys %{ $sender->unconfirmed_data->{$e} }) { + $self->manager->unconfirmed_data->{$e}{$c}{count} += $sender->unconfirmed_data->{$e}{$c}{count}; + $self->manager->unconfirmed_data->{$e}{$c}{note} = $sender->unconfirmed_data->{$e}{$c}{note}; + } } } + $self->manager->test_data->{test_req_used} = $sender->open311_test_req_used + if FixMyStreet->test_mode && $sender->can('open311_test_req_used'); } - $self->manager->test_data->{test_req_used} = $sender->open311_test_req_used - if FixMyStreet->test_mode && $sender->can('open311_test_req_used'); } return $result; @@ -251,7 +253,7 @@ sub _post_send { $self->h->{sent_confirm_id_ref} = $self->report->$send_confirmation_email; $self->_send_report_sent_email; } - $self->log("send successful: OK"); + $self->log("Send successful"); } else { my @errors; for my $sender ( keys %{$self->reporters} ) { @@ -260,7 +262,7 @@ sub _post_send { } } $self->report->update_send_failed( join( '|', @errors ) ); - $self->log("send FAILED: " . join( '|', @errors )); + $self->log("Send failed"); } } diff --git a/perllib/FixMyStreet/Script/Reports.pm b/perllib/FixMyStreet/Script/Reports.pm index 3e9b2d693..3d5afe216 100644 --- a/perllib/FixMyStreet/Script/Reports.pm +++ b/perllib/FixMyStreet/Script/Reports.pm @@ -20,6 +20,36 @@ sub send { verbose => $verbose, ); + my $params = construct_query($debug); + my $db = FixMyStreet::DB->schema->storage; + + $db->txn_do(sub { + my $unsent = FixMyStreet::DB->resultset('Problem')->search($params, { + for => \'UPDATE SKIP LOCKED', + }); + + $manager->log("starting to loop through unsent problem reports..."); + my $unsent_count = 0; + while (my $row = $unsent->next) { + $unsent_count++; + my $item = FixMyStreet::Queue::Item::Report->new( + report => $row, + manager => $manager, + verbose => $verbose, + nomail => $nomail, + ); + $item->process; + } + + $manager->end_line($unsent_count); + $manager->end_summary_unconfirmed; + }); + + return $manager->test_data; +} + +sub construct_query { + my ($debug) = @_; my $site = CronFns::site(FixMyStreet->config('BASE_URL')); my $states = [ FixMyStreet::DB::Result::Problem::open_states() ]; $states = [ 'submitted', 'confirmed', 'in progress', 'feedback pending', 'external', 'wish' ] if $site eq 'zurich'; @@ -55,25 +85,7 @@ sub send { ]; } - my $unsent = FixMyStreet::DB->resultset('Problem')->search($params); - - $manager->log("starting to loop through unsent problem reports..."); - my $unsent_count = 0; - while (my $row = $unsent->next) { - $unsent_count++; - my $item = FixMyStreet::Queue::Item::Report->new( - report => $row, - manager => $manager, - verbose => $verbose, - nomail => $nomail, - ); - $item->process; - } - - $manager->end_line($unsent_count); - $manager->end_summary_unconfirmed; - - return $manager->test_data; + return $params; } sub end_line { diff --git a/perllib/Open311/PostServiceRequestUpdates.pm b/perllib/Open311/PostServiceRequestUpdates.pm index 14bebfcb7..fadd063da 100755 --- a/perllib/Open311/PostServiceRequestUpdates.pm +++ b/perllib/Open311/PostServiceRequestUpdates.pm @@ -19,17 +19,31 @@ has current_open311 => ( is => 'rw' ); sub send { my $self = shift; + my $bodies = $self->fetch_bodies; + foreach my $body (values %$bodies) { + $self->construct_open311($body); + $self->process_body($body); + } +} + +sub fetch_bodies { my $bodies = FixMyStreet::DB->resultset('Body')->search( { send_method => SEND_METHOD_OPEN311, send_comments => 1, - } ); - + }, { prefetch => 'body_areas' } ); + my %bodies; while ( my $body = $bodies->next ) { my $cobrand = $body->get_cobrand_handler; next if $cobrand && $cobrand->call_hook('open311_post_update_skip'); - $self->current_open311(Open311->new($self->open311_params($body))); - $self->process_body($body); + $bodies{$body->id} = $body; } + return \%bodies; +} + +sub construct_open311 { + my ($self, $body) = @_; + my $o = Open311->new($self->open311_params($body)); + $self->current_open311($o); } sub open311_params { @@ -53,41 +67,57 @@ sub open311_params { sub process_body { my ($self, $body) = @_; - my $comments = FixMyStreet::DB->resultset('Comment')->to_body($body)->search( { - 'me.whensent' => undef, - 'me.external_id' => undef, - 'me.state' => 'confirmed', - 'me.confirmed' => { '!=' => undef }, - 'problem.whensent' => { '!=' => undef }, - 'problem.external_id' => { '!=' => undef }, - 'problem.send_method_used' => { -like => '%Open311%' }, - }, - { + my $params = $self->construct_query($self->verbose); + + my $db = FixMyStreet::DB->schema->storage; + $db->txn_do(sub { + my $comments = FixMyStreet::DB->resultset('Comment')->to_body($body)->search($params, { + for => \'UPDATE SKIP LOCKED', order_by => [ 'confirmed', 'id' ], - } - ); + }); - while ( my $comment = $comments->next ) { - my $cobrand = $body->get_cobrand_handler || $comment->get_cobrand_logged; - - # Some cobrands (e.g. Buckinghamshire) don't want to receive updates - # from anyone except the original problem reporter. - if ($cobrand->call_hook(should_skip_sending_update => $comment)) { - unless (defined $comment->get_extra_metadata('cobrand_skipped_sending')) { - $comment->set_extra_metadata(cobrand_skipped_sending => 1); - $comment->update; - } - next; + while ( my $comment = $comments->next ) { + $self->process_update($body, $comment); } + }); +} - next if !$self->verbose && $comment->send_fail_count && retry_timeout($comment); - - $self->process_update($body, $comment, $cobrand); +sub construct_query { + my ($self, $debug) = @_; + my $params = { + 'me.whensent' => undef, + 'me.external_id' => undef, + 'me.state' => 'confirmed', + 'me.confirmed' => { '!=' => undef }, + 'me.extra' => [ undef, { -not_like => '%cobrand_skipped_sending%' } ], + 'problem.whensent' => { '!=' => undef }, + 'problem.external_id' => { '!=' => undef }, + 'problem.send_method_used' => { -like => '%Open311%' }, + }; + if (!$debug) { + $params->{'-or'} = [ + 'me.send_fail_count' => 0, + 'me.send_fail_timestamp' => { '<', \"current_timestamp - '30 minutes'::interval" }, + ]; } + return $params; } sub process_update { - my ($self, $body, $comment, $cobrand) = @_; + my ($self, $body, $comment) = @_; + + my $cobrand = $body->get_cobrand_handler || $comment->get_cobrand_logged; + + # Some cobrands (e.g. Buckinghamshire) don't want to receive updates + # from anyone except the original problem reporter. + if (my $skip = $cobrand->call_hook(should_skip_sending_update => $comment)) { + if ($skip ne 'WAIT' && !defined $comment->get_extra_metadata('cobrand_skipped_sending')) { + $comment->set_extra_metadata(cobrand_skipped_sending => 1); + $comment->update; + } + $self->log($comment, 'Skipping'); + return; + } my $o = $self->current_open311; @@ -100,30 +130,21 @@ sub process_update { external_id => $id, whensent => \'current_timestamp', } ); + $self->log($comment, 'Send successful'); } else { $comment->update( { send_fail_count => $comment->send_fail_count + 1, send_fail_timestamp => \'current_timestamp', send_fail_reason => "Failed to post over Open311\n\n" . $o->error, } ); - - if ( $self->verbose && $o->error ) { - warn $o->error; - } + $self->log($comment, 'Send failed'); } } -sub retry_timeout { - my $row = shift; - - my $tz = FixMyStreet->local_time_zone; - my $now = DateTime->now( time_zone => $tz ); - my $diff = $now - $row->send_fail_timestamp; - if ( $diff->in_units( 'minutes' ) < 30 ) { - return 1; - } - - return 0; +sub log { + my ($self, $comment, $msg) = @_; + return unless $self->verbose; + STDERR->print("[fmsd] [" . $comment->id . "] $msg\n"); } 1; |