diff options
| -rw-r--r-- | CHANGELOG.md | 2 | ||||
| -rwxr-xr-x | bin/send-daemon | 23 | ||||
| -rw-r--r-- | perllib/FixMyStreet/Cobrand/Oxfordshire.pm | 9 | ||||
| -rwxr-xr-x | perllib/Open311/PostServiceRequestUpdates.pm | 113 | ||||
| -rw-r--r-- | t/cobrand/northamptonshire.t | 1 |
5 files changed, 99 insertions, 49 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index c8d98b8bc..cefc72f00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ - Add script to export/import body data. - Add fetch script that does combined job of fetch-comments and fetch-reports. - Show error page when submitting with web param to /import. - - Add a daemon option for sending reports. + - Add a daemon option for sending reports and updates. - Open311 improvements: - match response templates on external status code over state - Add flag to protect category/group names from Open311 overwrite. diff --git a/bin/send-daemon b/bin/send-daemon index 6df5aa17f..dee9e949f 100755 --- a/bin/send-daemon +++ b/bin/send-daemon @@ -21,6 +21,7 @@ use FixMyStreet; use FixMyStreet::DB; use FixMyStreet::Script::Reports; use FixMyStreet::Queue::Item::Report; +use Open311::PostServiceRequestUpdates; my ($opts, $usage) = describe_options( '%c %o', @@ -76,6 +77,7 @@ while (!$exit) { while (!$exit) { $0 = "fmsd (running queue)"; $db->txn_do(\&look_for_report); + $db->txn_do(\&look_for_update); $0 = "fmsd"; sleep(5 + rand(10)); } @@ -106,6 +108,27 @@ sub look_for_report { $item->process; } +sub look_for_update { + my $updates = Open311::PostServiceRequestUpdates->new( + verbose => $verbose, + ); + + my $bodies = $updates->fetch_bodies; + my $params = $updates->construct_query($opts->debug); + my $comment = FixMyStreet::DB->resultset('Comment') + ->to_body([ keys %$bodies ]) + ->search($params, { for => \'UPDATE SKIP LOCKED', rows => 1 }) + ->single or return; + + print_log('debug', "Trying to send update " . $comment->id); + + my ($body) = grep { $bodies->{$_} } @{$comment->problem->bodies_str_ids}; + $body = $bodies->{$body}; + + $updates->construct_open311($body); + $updates->process_update($body, $comment); +} + sub print_log { my $prio = shift; diff --git a/perllib/FixMyStreet/Cobrand/Oxfordshire.pm b/perllib/FixMyStreet/Cobrand/Oxfordshire.pm index 12714185d..d81ecea74 100644 --- a/perllib/FixMyStreet/Cobrand/Oxfordshire.pm +++ b/perllib/FixMyStreet/Cobrand/Oxfordshire.pm @@ -142,8 +142,13 @@ sub should_skip_sending_update { my ($self, $update ) = @_; # Oxfordshire stores the external id of the problem as a customer reference - # in metadata - return 1 if !$update->problem->get_extra_metadata('customer_reference'); + # in metadata, it arrives in a fetched update (but give up if it never does, + # or the update is for an old pre-ref report) + my $customer_ref = $update->problem->get_extra_metadata('customer_reference'); + my $diff = time() - $update->confirmed->epoch; + return 1 if !$customer_ref && $diff > 60*60*24; + return 'WAIT' if !$customer_ref; + return 0; } sub on_map_default_status { return 'open'; } diff --git a/perllib/Open311/PostServiceRequestUpdates.pm b/perllib/Open311/PostServiceRequestUpdates.pm index 14bebfcb7..fadd063da 100755 --- a/perllib/Open311/PostServiceRequestUpdates.pm +++ b/perllib/Open311/PostServiceRequestUpdates.pm @@ -19,17 +19,31 @@ has current_open311 => ( is => 'rw' ); sub send { my $self = shift; + my $bodies = $self->fetch_bodies; + foreach my $body (values %$bodies) { + $self->construct_open311($body); + $self->process_body($body); + } +} + +sub fetch_bodies { my $bodies = FixMyStreet::DB->resultset('Body')->search( { send_method => SEND_METHOD_OPEN311, send_comments => 1, - } ); - + }, { prefetch => 'body_areas' } ); + my %bodies; while ( my $body = $bodies->next ) { my $cobrand = $body->get_cobrand_handler; next if $cobrand && $cobrand->call_hook('open311_post_update_skip'); - $self->current_open311(Open311->new($self->open311_params($body))); - $self->process_body($body); + $bodies{$body->id} = $body; } + return \%bodies; +} + +sub construct_open311 { + my ($self, $body) = @_; + my $o = Open311->new($self->open311_params($body)); + $self->current_open311($o); } sub open311_params { @@ -53,41 +67,57 @@ sub open311_params { sub process_body { my ($self, $body) = @_; - my $comments = FixMyStreet::DB->resultset('Comment')->to_body($body)->search( { - 'me.whensent' => undef, - 'me.external_id' => undef, - 'me.state' => 'confirmed', - 'me.confirmed' => { '!=' => undef }, - 'problem.whensent' => { '!=' => undef }, - 'problem.external_id' => { '!=' => undef }, - 'problem.send_method_used' => { -like => '%Open311%' }, - }, - { + my $params = $self->construct_query($self->verbose); + + my $db = FixMyStreet::DB->schema->storage; + $db->txn_do(sub { + my $comments = FixMyStreet::DB->resultset('Comment')->to_body($body)->search($params, { + for => \'UPDATE SKIP LOCKED', order_by => [ 'confirmed', 'id' ], - } - ); + }); - while ( my $comment = $comments->next ) { - my $cobrand = $body->get_cobrand_handler || $comment->get_cobrand_logged; - - # Some cobrands (e.g. Buckinghamshire) don't want to receive updates - # from anyone except the original problem reporter. - if ($cobrand->call_hook(should_skip_sending_update => $comment)) { - unless (defined $comment->get_extra_metadata('cobrand_skipped_sending')) { - $comment->set_extra_metadata(cobrand_skipped_sending => 1); - $comment->update; - } - next; + while ( my $comment = $comments->next ) { + $self->process_update($body, $comment); } + }); +} - next if !$self->verbose && $comment->send_fail_count && retry_timeout($comment); - - $self->process_update($body, $comment, $cobrand); +sub construct_query { + my ($self, $debug) = @_; + my $params = { + 'me.whensent' => undef, + 'me.external_id' => undef, + 'me.state' => 'confirmed', + 'me.confirmed' => { '!=' => undef }, + 'me.extra' => [ undef, { -not_like => '%cobrand_skipped_sending%' } ], + 'problem.whensent' => { '!=' => undef }, + 'problem.external_id' => { '!=' => undef }, + 'problem.send_method_used' => { -like => '%Open311%' }, + }; + if (!$debug) { + $params->{'-or'} = [ + 'me.send_fail_count' => 0, + 'me.send_fail_timestamp' => { '<', \"current_timestamp - '30 minutes'::interval" }, + ]; } + return $params; } sub process_update { - my ($self, $body, $comment, $cobrand) = @_; + my ($self, $body, $comment) = @_; + + my $cobrand = $body->get_cobrand_handler || $comment->get_cobrand_logged; + + # Some cobrands (e.g. Buckinghamshire) don't want to receive updates + # from anyone except the original problem reporter. + if (my $skip = $cobrand->call_hook(should_skip_sending_update => $comment)) { + if ($skip ne 'WAIT' && !defined $comment->get_extra_metadata('cobrand_skipped_sending')) { + $comment->set_extra_metadata(cobrand_skipped_sending => 1); + $comment->update; + } + $self->log($comment, 'Skipping'); + return; + } my $o = $self->current_open311; @@ -100,30 +130,21 @@ sub process_update { external_id => $id, whensent => \'current_timestamp', } ); + $self->log($comment, 'Send successful'); } else { $comment->update( { send_fail_count => $comment->send_fail_count + 1, send_fail_timestamp => \'current_timestamp', send_fail_reason => "Failed to post over Open311\n\n" . $o->error, } ); - - if ( $self->verbose && $o->error ) { - warn $o->error; - } + $self->log($comment, 'Send failed'); } } -sub retry_timeout { - my $row = shift; - - my $tz = FixMyStreet->local_time_zone; - my $now = DateTime->now( time_zone => $tz ); - my $diff = $now - $row->send_fail_timestamp; - if ( $diff->in_units( 'minutes' ) < 30 ) { - return 1; - } - - return 0; +sub log { + my ($self, $comment, $msg) = @_; + return unless $self->verbose; + STDERR->print("[fmsd] [" . $comment->id . "] $msg\n"); } 1; diff --git a/t/cobrand/northamptonshire.t b/t/cobrand/northamptonshire.t index 70df10340..57fe319a9 100644 --- a/t/cobrand/northamptonshire.t +++ b/t/cobrand/northamptonshire.t @@ -83,6 +83,7 @@ subtest 'Check updates not sent for defects' => sub { }; $report->update({ user => $user }); +$comment->update({ extra => undef }); subtest 'check updates sent for non defects' => sub { FixMyStreet::override_config { ALLOWED_COBRANDS => [ { northamptonshire => '.' } ], |