From c7dbb65e2d01e37f276af3db0372123366b3a1a1 Mon Sep 17 00:00:00 2001 From: Matthew Somerville Date: Mon, 22 Jun 2020 17:20:10 +0100 Subject: Rewrite open311-update-reports to share code. Make GetUpdates and GetServiceRequestUpdates share a common base; spot all visible states. --- perllib/Open311/GetServiceRequestUpdates.pm | 280 +--------------------------- 1 file changed, 8 insertions(+), 272 deletions(-) (limited to 'perllib/Open311/GetServiceRequestUpdates.pm') diff --git a/perllib/Open311/GetServiceRequestUpdates.pm b/perllib/Open311/GetServiceRequestUpdates.pm index 9fa81ac9e..b4f3f4430 100644 --- a/perllib/Open311/GetServiceRequestUpdates.pm +++ b/perllib/Open311/GetServiceRequestUpdates.pm @@ -1,92 +1,17 @@ package Open311::GetServiceRequestUpdates; use Moo; -use Open311; -use Parallel::ForkManager; -use FixMyStreet::DB; -use FixMyStreet::App::Model::PhotoSet; +extends 'Open311::UpdatesBase'; + use DateTime::Format::W3CDTF; -has system_user => ( is => 'rw' ); +has '+send_comments_flag' => ( default => 1 ); has start_date => ( is => 'ro', default => sub { undef } ); has end_date => ( is => 'ro', default => sub { undef } ); -has body => ( is => 'ro', default => sub { undef } ); -has suppress_alerts => ( is => 'rw', default => 0 ); -has verbose => ( is => 'ro', default => 0 ); -has schema => ( is =>'ro', lazy => 1, default => sub { FixMyStreet::DB->schema->connect } ); -has blank_updates_permitted => ( is => 'rw', default => 0 ); - -has current_body => ( is => 'rw' ); -has current_open311 => ( is => 'rw' ); Readonly::Scalar my $AREA_ID_BROMLEY => 2482; Readonly::Scalar my $AREA_ID_OXFORDSHIRE => 2237; -sub fetch { - my ($self, $open311) = @_; - - my $bodies = $self->schema->resultset('Body')->search( - { - send_method => 'Open311', - send_comments => 1, - comment_user_id => { '!=', undef }, - endpoint => { '!=', '' }, - } - ); - - if ( $self->body ) { - $bodies = $bodies->search( { name => $self->body } ); - } - - my $procs_min = FixMyStreet->config('FETCH_COMMENTS_PROCESSES_MIN') || 0; - my $procs_max = FixMyStreet->config('FETCH_COMMENTS_PROCESSES_MAX'); - my $procs_timeout = FixMyStreet->config('FETCH_COMMENTS_PROCESS_TIMEOUT'); - - my $pm = Parallel::ForkManager->new(FixMyStreet->test_mode ? 0 : $procs_min); - - if ($procs_max && $procs_timeout) { - my %workers; - $pm->run_on_wait(sub { - while (my ($pid, $started_at) = each %workers) { - next unless time() - $started_at > $procs_timeout; - next if $pm->max_procs == $procs_max; - $pm->set_max_procs($pm->max_procs + 1); - delete $workers{$pid}; # Only want to increase once per long-running thing - } - }, 1); - $pm->run_on_start(sub { my $pid = shift; $workers{$pid} = time(); }); - $pm->run_on_finish(sub { my $pid = shift; delete $workers{$pid}; }); - } - - while ( my $body = $bodies->next ) { - $pm->start and next; - - $self->current_body( $body ); - - my %open311_conf = ( - endpoint => $body->endpoint, - api_key => $body->api_key, - jurisdiction => $body->jurisdiction, - extended_statuses => $body->send_extended_statuses, - ); - - my $cobrand = $body->get_cobrand_handler; - $cobrand->call_hook(open311_config_updates => \%open311_conf) - if $cobrand; - - $self->current_open311( $open311 || Open311->new(%open311_conf) ); - - $self->suppress_alerts( $body->suppress_alerts ); - $self->blank_updates_permitted( $body->blank_updates_permitted ); - $self->system_user( $body->comment_user ); - $self->process_body(); - - $pm->finish; - } - - $pm->wait_all_children; -} - sub parse_dates { my $self = shift; my $body = $self->current_body; @@ -141,201 +66,12 @@ sub process_body { return 1; } -sub check_date { - my ($self, $request, @args) = @_; - - my $comment_time = eval { - DateTime::Format::W3CDTF->parse_datetime( $request->{updated_datetime} || "" ) - ->set_time_zone(FixMyStreet->local_time_zone); - }; - return if $@; - my $updated = DateTime::Format::W3CDTF->format_datetime($comment_time->clone->set_time_zone('UTC')); - return if @args && ($updated lt $args[0] || $updated gt $args[1]); - $request->{comment_time} = $comment_time; - return 1; -} - -sub find_problem { - my ($self, $request, @args) = @_; - - $self->check_date($request, @args) or return; - - my $body = $self->current_body; - my $request_id = $request->{service_request_id}; - - # If there's no request id then we can't work out - # what problem it belongs to so just skip - return unless $request_id || $request->{fixmystreet_id}; - - my $problem; - my $criteria = { - external_id => $request_id, - }; - - # in some cases we only have the FMS id and not the request id so use that - if ( $request->{fixmystreet_id} ) { - unless ( $request->{fixmystreet_id} =~ /^\d+$/ ) { - warn "skipping bad fixmystreet id in updates for " . $body->name . ": [" . $request->{fixmystreet_id} . "], external id is $request_id\n"; - return; - } - - $criteria = { - id => $request->{fixmystreet_id}, - }; - } - - $problem = $self->schema->resultset('Problem')->to_body($body)->search( $criteria ); +sub _find_problem { + my ($self, $criteria) = @_; + my $problem = $self->schema->resultset('Problem') + ->to_body($self->current_body) + ->search( $criteria ); return $problem->first; } -sub process_update { - my ($self, $request, $p) = @_; - my $open311 = $self->current_open311; - my $body = $self->current_body; - - my $state = $open311->map_state( $request->{status} ); - my $old_state = $p->state; - my $external_status_code = $request->{external_status_code} || ''; - my $customer_reference = $request->{customer_reference} || ''; - my $old_external_status_code = $p->get_extra_metadata('external_status_code') || ''; - my $comment = $self->schema->resultset('Comment')->new( - { - problem => $p, - user => $self->system_user, - external_id => $request->{update_id}, - text => $self->comment_text_for_request( - $request, $p, $state, $old_state, - $external_status_code, $old_external_status_code - ), - mark_fixed => 0, - mark_open => 0, - anonymous => 0, - name => $self->system_user->name, - confirmed => $request->{comment_time}, - created => $request->{comment_time}, - state => 'confirmed', - } - ); - - # Some Open311 services, e.g. Confirm via open311-adapter, provide - # a more fine-grained status code that we use within FMS for - # response templates. - if ( $external_status_code ) { - $comment->set_extra_metadata(external_status_code => $external_status_code); - $p->set_extra_metadata(external_status_code => $external_status_code); - } else { - $p->set_extra_metadata(external_status_code => ''); - } - - # if the customer reference to display in the report metadata is - # not the same as the external_id - if ( $customer_reference ) { - $p->set_extra_metadata( customer_reference => $customer_reference ); - } - - $open311->add_media($request->{media_url}, $comment) - if $request->{media_url}; - - # don't update state unless it's an allowed state - if ( FixMyStreet::DB::Result::Problem->visible_states()->{$state} && - # For Oxfordshire, don't allow changes back to Open from other open states - !( $body->areas->{$AREA_ID_OXFORDSHIRE} && $state eq 'confirmed' && $p->is_open ) && - # Don't let it change between the (same in the front end) fixed states - !( $p->is_fixed && FixMyStreet::DB::Result::Problem->fixed_states()->{$state} ) ) { - - $comment->problem_state($state); - - # we only want to update the problem state if that makes sense. We never want to unhide a problem. - # If the update is older than the last update then we also do not want to update the state. This - # is largely to avoid the situation where we miss some updates, make more updates and then catch - # the updates when we fetch the last 24 hours of updates. The exception to this is the first - # comment. This is to catch automated updates which happen faster than we get the external_id - # back from the endpoint and hence have an created time before the lastupdate. - if ( $p->is_visible && $p->state ne $state && - ( $comment->created >= $p->lastupdate || $p->comments->count == 0 ) ) { - $p->state($state); - } - } - - # If nothing to show (no text, photo, or state change), don't show this update - $comment->state('hidden') unless $comment->text || $comment->photo - || ($comment->problem_state && $state ne $old_state); - - my $cobrand = $body->get_cobrand_handler; - $cobrand->call_hook(open311_get_update_munging => $comment) - if $cobrand; - - # As comment->created has been looked at above, its time zone has been shifted - # to TIME_ZONE (if set). We therefore need to set it back to local before - # insertion. We also then need a clone, otherwise the setting of lastupdate - # will *also* reshift comment->created's time zone to TIME_ZONE. - my $created = $comment->created->set_time_zone(FixMyStreet->local_time_zone); - $p->lastupdate($created->clone); - $p->update; - $comment->insert(); - - if ( $self->suppress_alerts ) { - my @alerts = $self->schema->resultset('Alert')->search( { - alert_type => 'new_updates', - parameter => $p->id, - confirmed => 1, - user_id => $p->user->id, - } ); - - for my $alert (@alerts) { - my $alerts_sent = $self->schema->resultset('AlertSent')->find_or_create( { - alert_id => $alert->id, - parameter => $comment->id, - } ); - } - } - - return $comment; -} - -sub comment_text_for_request { - my ($self, $request, $problem, $state, $old_state, - $ext_code, $old_ext_code) = @_; - - # Response templates are only triggered if the state/external status has changed. - # And treat any fixed state as fixed. - my $state_changed = $state ne $old_state - && !( $problem->is_fixed && FixMyStreet::DB::Result::Problem->fixed_states()->{$state} ); - my $ext_code_changed = $ext_code ne $old_ext_code; - my $template; - if ($state_changed || $ext_code_changed) { - my $order; - my $state_params = { - 'me.state' => $state - }; - if ($ext_code) { - $state_params->{'me.external_status_code'} = $ext_code; - # make sure that empty string/nulls come last. - $order = { order_by => \"me.external_status_code DESC NULLS LAST" }; - }; - - if (my $t = $problem->response_templates->search({ - auto_response => 1, - -or => $state_params, - }, $order )->first) { - $template = $t->text; - } - } - - my $desc = $request->{description} || ''; - if ($desc && (!$template || $template !~ /\{\{description}}/)) { - return $desc; - } - - if ($template) { - $template =~ s/\{\{description}}/$desc/; - return $template; - } - - return "" if $self->blank_updates_permitted; - - print STDERR "Couldn't determine update text for $request->{update_id} (report " . $problem->id . ")\n"; - return ""; -} - 1; -- cgit v1.2.3