diff options
Diffstat (limited to 'perllib/Open311')
| -rw-r--r-- | perllib/Open311/GetServiceRequestUpdates.pm | 76 | ||||
| -rw-r--r-- | perllib/Open311/GetServiceRequests.pm | 105 | ||||
| -rw-r--r-- | perllib/Open311/PopulateServiceList.pm | 103 | ||||
| -rwxr-xr-x | perllib/Open311/PostServiceRequestUpdates.pm | 127 |
4 files changed, 276 insertions, 135 deletions
diff --git a/perllib/Open311/GetServiceRequestUpdates.pm b/perllib/Open311/GetServiceRequestUpdates.pm index b4d7c6347..3b436aaa7 100644 --- a/perllib/Open311/GetServiceRequestUpdates.pm +++ b/perllib/Open311/GetServiceRequestUpdates.pm @@ -9,6 +9,7 @@ use DateTime::Format::W3CDTF; has system_user => ( is => 'rw' ); has start_date => ( is => 'ro', default => sub { undef } ); has end_date => ( is => 'ro', default => sub { undef } ); +has body => ( is => 'ro', default => sub { undef } ); has suppress_alerts => ( is => 'rw', default => 0 ); has verbose => ( is => 'ro', default => 0 ); has schema => ( is =>'ro', lazy => 1, default => sub { FixMyStreet::DB->schema->connect } ); @@ -18,7 +19,7 @@ Readonly::Scalar my $AREA_ID_BROMLEY => 2482; Readonly::Scalar my $AREA_ID_OXFORDSHIRE => 2237; sub fetch { - my $self = shift; + my ($self, $open311) = @_; my $bodies = $self->schema->resultset('Body')->search( { @@ -29,25 +30,24 @@ sub fetch { } ); + if ( $self->body ) { + $bodies = $bodies->search( { name => $self->body } ); + } + while ( my $body = $bodies->next ) { - my $o = Open311->new( - endpoint => $body->endpoint, - api_key => $body->api_key, + my %open311_conf = ( + endpoint => $body->endpoint, + api_key => $body->api_key, jurisdiction => $body->jurisdiction, + extended_statuses => $body->send_extended_statuses, ); - # custom endpoint URLs because these councils have non-standard paths - if ( $body->areas->{$AREA_ID_BROMLEY} ) { - my $endpoints = $o->endpoints; - $endpoints->{update} = 'update.xml'; - $endpoints->{service_request_updates} = 'update.xml'; - $o->endpoints( $endpoints ); - } elsif ( $body->areas->{$AREA_ID_OXFORDSHIRE} ) { - my $endpoints = $o->endpoints; - $endpoints->{service_request_updates} = 'open311_service_request_update.cgi'; - $o->endpoints( $endpoints ); - } + my $cobrand = $body->get_cobrand_handler; + $cobrand->call_hook(open311_config_updates => \%open311_conf) + if $cobrand; + + my $o = $open311 || Open311->new(%open311_conf); $self->suppress_alerts( $body->suppress_alerts ); $self->blank_updates_permitted( $body->blank_updates_permitted ); @@ -69,6 +69,8 @@ sub update_comments { # default to asking for last 2 hours worth if not Bromley } elsif ( ! $body->areas->{$AREA_ID_BROMLEY} ) { my $end_dt = DateTime->now(); + # Oxfordshire uses local time and not UTC for dates + FixMyStreet->set_time_zone($end_dt) if ( $body->areas->{$AREA_ID_OXFORDSHIRE} ); my $start_dt = $end_dt->clone; $start_dt->add( hours => -2 ); @@ -89,19 +91,35 @@ sub update_comments { # If there's no request id then we can't work out # what problem it belongs to so just skip - next unless $request_id; + next unless $request_id || $request->{fixmystreet_id}; my $comment_time = eval { - DateTime::Format::W3CDTF->parse_datetime( $request->{updated_datetime} || "" ); + DateTime::Format::W3CDTF->parse_datetime( $request->{updated_datetime} || "" ) + ->set_time_zone(FixMyStreet->local_time_zone); }; next if $@; my $updated = DateTime::Format::W3CDTF->format_datetime($comment_time->clone->set_time_zone('UTC')); next if @args && ($updated lt $args[0] || $updated gt $args[1]); my $problem; + my $match_field = 'external_id'; my $criteria = { external_id => $request_id, }; + + # in some cases we only have the FMS id and not the request id so use that + if ( $request->{fixmystreet_id} ) { + unless ( $request->{fixmystreet_id} =~ /^\d+$/ ) { + warn "skipping bad fixmystreet id in updates for " . $body->name . ": [" . $request->{fixmystreet_id} . "], external id is $request_id\n"; + next; + } + + $criteria = { + id => $request->{fixmystreet_id}, + }; + $match_field = 'fixmystreet id'; + } + $problem = $self->schema->resultset('Problem')->to_body($body)->search( $criteria ); if (my $p = $problem->first) { @@ -112,6 +130,7 @@ sub update_comments { my $state = $open311->map_state( $request->{status} ); my $old_state = $p->state; my $external_status_code = $request->{external_status_code} || ''; + my $customer_reference = $request->{customer_reference} || ''; my $old_external_status_code = $p->get_extra_metadata('external_status_code') || ''; my $comment = $self->schema->resultset('Comment')->new( { @@ -140,6 +159,12 @@ sub update_comments { $p->set_extra_metadata(external_status_code => $external_status_code); } + # if the customer reference to display in the report metadata is + # not the same as the external_id + if ( $customer_reference ) { + $p->set_extra_metadata( customer_reference => $customer_reference ); + } + $open311->add_media($request->{media_url}, $comment) if $request->{media_url}; @@ -166,7 +191,12 @@ sub update_comments { $comment->state('hidden') unless $comment->text || $comment->photo || ($comment->problem_state && $state ne $old_state); - $p->lastupdate( $comment->created ); + # As comment->created has been looked at above, its time zone has been shifted + # to TIME_ZONE (if set). We therefore need to set it back to local before + # insertion. We also then need a clone, otherwise the setting of lastupdate + # will *also* reshift comment->created's time zone to TIME_ZONE. + my $created = $comment->created->set_time_zone(FixMyStreet->local_time_zone); + $p->lastupdate($created->clone); $p->update; $comment->insert(); @@ -186,6 +216,10 @@ sub update_comments { } } } + # we get lots of comments that are not related to FMS issues from Lewisham so ignore those otherwise + # way too many warnings. + } elsif (FixMyStreet->config('STAGING_SITE') and $body->name !~ /Lewisham/) { + warn "Failed to match comment to problem with $match_field $request_id for " . $body->name . "\n"; } } @@ -198,8 +232,10 @@ sub comment_text_for_request { return $request->{description} if $request->{description}; - # Response templates are only triggered if the state/external status has changed - my $state_changed = $state ne $old_state; + # Response templates are only triggered if the state/external status has changed. + # And treat any fixed state as fixed. + my $state_changed = $state ne $old_state + && !( $problem->is_fixed && FixMyStreet::DB::Result::Problem->fixed_states()->{$state} ); my $ext_code_changed = $ext_code ne $old_ext_code; if ($state_changed || $ext_code_changed) { my $state_params = { diff --git a/perllib/Open311/GetServiceRequests.pm b/perllib/Open311/GetServiceRequests.pm index 2d15347fd..194d8d296 100644 --- a/perllib/Open311/GetServiceRequests.pm +++ b/perllib/Open311/GetServiceRequests.pm @@ -3,6 +3,7 @@ package Open311::GetServiceRequests; use Moo; use Open311; use FixMyStreet::DB; +use FixMyStreet::MapIt; use FixMyStreet::App::Model::PhotoSet; use DateTime::Format::W3CDTF; @@ -89,18 +90,13 @@ sub create_problems { } my $request_id = $request->{service_request_id}; - my %params; - $params{generation} = mySociety::Config::get('MAPIT_GENERATION') - if mySociety::Config::get('MAPIT_GENERATION'); - my ($latitude, $longitude) = ( $request->{lat}, $request->{long} ); ($latitude, $longitude) = Utils::convert_en_to_latlon_truncated( $longitude, $latitude ) if $self->convert_latlong; my $all_areas = - mySociety::MaPit::call( 'point', - "4326/$longitude,$latitude", %params ); + FixMyStreet::MapIt::call('point', "4326/$longitude,$latitude"); # skip if it doesn't look like it's for this body my @areas = grep { $all_areas->{$_->area_id} } $body->body_areas; @@ -113,76 +109,85 @@ sub create_problems { my $updated_time = eval { DateTime::Format::W3CDTF->parse_datetime( $request->{updated_datetime} || "" - )->set_time_zone( - FixMyStreet->time_zone || FixMyStreet->local_time_zone - ); + )->set_time_zone(FixMyStreet->local_time_zone); }; if ($@) { warn "Not creating problem $request_id for @{[$body->name]}, bad update time" if $self->verbose; next; } - my $updated = DateTime::Format::W3CDTF->format_datetime( $updated_time->clone->set_time_zone('UTC') ); - if ($args->{start_date} && $args->{end_date} && ($updated lt $args->{start_date} || $updated gt $args->{end_date}) ) { - warn "Problem id $request_id for @{[$body->name]} has an invalid time, not creating: " - . "$updated either less than $args->{start_date} or greater than $args->{end_date}" - if $self->verbose; - next; - } my $created_time = eval { DateTime::Format::W3CDTF->parse_datetime( $request->{requested_datetime} || "" - )->set_time_zone( - FixMyStreet->time_zone || FixMyStreet->local_time_zone - ); + )->set_time_zone(FixMyStreet->local_time_zone); }; $created_time = $updated_time if $@; + # Updated time must not be before created time, check and adjust as necessary. + # (This has happened with some fetched reports, oddly.) + $updated_time = $created_time if $updated_time lt $created_time; + my $problems; my $criteria = { external_id => $request_id, }; - $problems = $self->schema->resultset('Problem')->to_body($body)->search( $criteria ); + + # Skip if this problem already exists (e.g. it may have originated from FMS and is being mirrored back!) + next if $self->schema->resultset('Problem')->to_body($body)->search( $criteria )->count; + + if ($args->{start_date} && $args->{end_date} && ($updated lt $args->{start_date} || $updated gt $args->{end_date}) ) { + warn "Problem id $request_id for @{[$body->name]} has an invalid time, not creating: " + . "$updated either less than $args->{start_date} or greater than $args->{end_date}" + if $self->verbose; + next; + } + + if ( my $cobrand = $body->get_cobrand_handler ) { + my $filtered = $cobrand->call_hook('filter_report_description', $request->{description}); + $request->{description} = $filtered if defined $filtered; + } my @contacts = grep { $request->{service_code} eq $_->email } $contacts->all; my $contact = $contacts[0] ? $contacts[0]->category : 'Other'; my $state = $open311->map_state($request->{status}); - unless (my $p = $problems->first) { - my $problem = $self->schema->resultset('Problem')->new( - { - user => $self->system_user, - external_id => $request_id, - detail => $request->{description}, - title => $request->{title} || $request->{service_name} . ' problem', - anonymous => 0, - name => $self->system_user->name, - confirmed => $created_time, - created => $created_time, - lastupdate => $updated_time, - whensent => $created_time, - state => $state, - postcode => '', - used_map => 1, - latitude => $latitude, - longitude => $longitude, - areas => ',' . $body->id . ',', - bodies_str => $body->id, - send_method_used => 'Open311', - category => $contact, - } - ); - - $open311->add_media($request->{media_url}, $problem) - if $request->{media_url}; - - $problem->insert(); - } + my $non_public = $request->{non_public} ? 1 : 0; + + my $problem = $self->schema->resultset('Problem')->new( + { + user => $self->system_user, + external_id => $request_id, + detail => $request->{description} || $request->{service_name} . ' problem', + title => $request->{title} || $request->{service_name} . ' problem', + anonymous => 0, + name => $self->system_user->name, + confirmed => $created_time, + created => $created_time, + lastupdate => $updated_time, + whensent => $created_time, + state => $state, + postcode => '', + used_map => 1, + latitude => $latitude, + longitude => $longitude, + areas => ',' . $body->id . ',', + bodies_str => $body->id, + send_method_used => 'Open311', + category => $contact, + send_questionnaire => 0, + non_public => $non_public, + } + ); + + $open311->add_media($request->{media_url}, $problem) + if $request->{media_url}; + + $problem->insert(); } return 1; diff --git a/perllib/Open311/PopulateServiceList.pm b/perllib/Open311/PopulateServiceList.pm index f10bdf7fb..d506111ec 100644 --- a/perllib/Open311/PopulateServiceList.pm +++ b/perllib/Open311/PopulateServiceList.pm @@ -8,7 +8,11 @@ has found_contacts => ( is => 'rw', default => sub { [] } ); has verbose => ( is => 'ro', default => 0 ); has schema => ( is => 'ro', lazy => 1, default => sub { FixMyStreet::DB->schema->connect } ); -has _current_body => ( is => 'rw' ); +has _current_body => ( is => 'rw', trigger => sub { + my ($self, $body) = @_; + $self->_current_body_cobrand($body->get_cobrand_handler); +} ); +has _current_body_cobrand => ( is => 'rw' ); has _current_open311 => ( is => 'rw' ); has _current_service => ( is => 'rw' ); @@ -160,10 +164,7 @@ sub _handle_existing_contact { $contact->update; } - if (my $group = $self->_current_service->{group}) { - $contact->set_extra_metadata(group => $group); - $contact->update; - } + $self->_set_contact_group($contact); push @{ $self->found_contacts }, $self->_current_service->{service_code}; } @@ -188,12 +189,6 @@ sub _create_contact { ); }; - if (my $group = $self->_current_service->{group}) { - $contact->set_extra_metadata(group => $group); - $contact->update; - } - - if ( $@ ) { warn "Failed to create contact for service code " . $self->_current_service->{service_code} . " for body @{[$self->_current_body->id]}: $@\n" if $self->verbose >= 1; @@ -205,6 +200,8 @@ sub _create_contact { $self->_add_meta_to_contact( $contact ); } + $self->_set_contact_group($contact); + if ( $contact ) { push @{ $self->found_contacts }, $self->_current_service->{service_code}; print "created contact for service code " . $self->_current_service->{service_code} . " for body @{[$self->_current_body->id]}\n" if $self->verbose >= 2; @@ -236,65 +233,15 @@ sub _add_meta_to_contact { # Some Open311 endpoints, such as Bromley and Warwickshire send <metadata> # for attributes which we *don't* want to display to the user (e.g. as - # fields in "category_extras" - $self->_add_meta_to_contact_cobrand_overrides($contact, \@meta); + # fields in "category_extras"), or need additional attributes adding not + # returned by the server for whatever reason. + $self->_current_body_cobrand && $self->_current_body_cobrand->call_hook( + open311_contact_meta_override => $self->_current_service, $contact, \@meta); $contact->set_extra_fields(@meta); $contact->update; } -sub _add_meta_to_contact_cobrand_overrides { - my ( $self, $contact, $meta ) = @_; - - if ($self->_current_body->name eq 'Bromley Council') { - $contact->set_extra_metadata( id_field => 'service_request_id_ext'); - # Lights we want to store feature ID, PROW on all categories. - push @$meta, { - code => 'prow_reference', - datatype => 'string', - description => 'Right of way reference', - order => 101, - required => 'false', - variable => 'true', - automated => 'hidden_field', - }; - push @$meta, { - code => 'feature_id', - datatype => 'string', - description => 'Feature ID', - order => 100, - required => 'false', - variable => 'true', - automated => 'hidden_field', - } if $self->_current_service->{service_code} eq 'LIGHTS'; - } elsif ($self->_current_body->name eq 'Warwickshire County Council') { - $contact->set_extra_metadata( id_field => 'external_id'); - } - - my %override = ( - #2482 - 'Bromley Council' => [qw( - requested_datetime - report_url - title - last_name - email - report_title - public_anonymity_required - email_alerts_requested - ) ], - #2243, - 'Warwickshire County Council' => [qw( - closest_address - ) ], - ); - - if (my $override = $override{ $self->_current_body->name }) { - my %ignore = map { $_ => 1 } @{ $override }; - @$meta = grep { ! $ignore{ $_->{ code } } } @$meta; - } -} - sub _normalize_service_name { my $self = shift; @@ -310,6 +257,32 @@ sub _normalize_service_name { return $service_name; } +sub _set_contact_group { + my ($self, $contact) = @_; + + my $groups_enabled = $self->_current_body_cobrand && $self->_current_body_cobrand->call_hook('enable_category_groups'); + my $old_group = $contact->get_extra_metadata('group') || ''; + my $new_group = $groups_enabled ? $self->_current_service->{group} || '' : ''; + + if ($old_group ne $new_group) { + if ($new_group) { + $contact->set_extra_metadata(group => $new_group); + $contact->update({ + editor => $0, + whenedited => \'current_timestamp', + note => 'group updated automatically by script', + }); + } else { + $contact->unset_extra_metadata('group'); + $contact->update({ + editor => $0, + whenedited => \'current_timestamp', + note => 'group removed automatically by script', + }); + } + } +} + sub _delete_contacts_not_in_service_list { my $self = shift; diff --git a/perllib/Open311/PostServiceRequestUpdates.pm b/perllib/Open311/PostServiceRequestUpdates.pm new file mode 100755 index 000000000..1f080b168 --- /dev/null +++ b/perllib/Open311/PostServiceRequestUpdates.pm @@ -0,0 +1,127 @@ +package Open311::PostServiceRequestUpdates; + +use strict; +use warnings; +use v5.14; + +use DateTime; +use Moo; +use FixMyStreet; +use FixMyStreet::Cobrand; +use FixMyStreet::DB; +use Open311; + +use constant SEND_METHOD_OPEN311 => 'Open311'; + +has verbose => ( is => 'ro', default => 0 ); + +sub send { + my $self = shift; + + my $bodies = FixMyStreet::DB->resultset('Body')->search( { + send_method => SEND_METHOD_OPEN311, + send_comments => 1, + } ); + + while ( my $body = $bodies->next ) { + my $cobrand = $body->get_cobrand_handler; + next if $cobrand && $cobrand->call_hook('open311_post_update_skip'); + $self->process_body($body); + } +} + +sub open311_params { + my ($self, $body) = @_; + + my %open311_conf = ( + endpoint => $body->endpoint, + jurisdiction => $body->jurisdiction, + api_key => $body->api_key, + extended_statuses => $body->send_extended_statuses, + fixmystreet_body => $body, + ); + + my $cobrand = $body->get_cobrand_handler; + $cobrand->call_hook(open311_config_updates => \%open311_conf) + if $cobrand; + + return %open311_conf; +} + +sub process_body { + my ($self, $body) = @_; + + my $o = Open311->new( $self->open311_params($body) ); + + my $comments = FixMyStreet::DB->resultset('Comment')->to_body($body)->search( { + 'me.whensent' => undef, + 'me.external_id' => undef, + 'me.state' => 'confirmed', + 'me.confirmed' => { '!=' => undef }, + 'problem.whensent' => { '!=' => undef }, + 'problem.external_id' => { '!=' => undef }, + 'problem.send_method_used' => { -like => '%Open311%' }, + }, + { + order_by => [ 'confirmed', 'id' ], + } + ); + + while ( my $comment = $comments->next ) { + my $cobrand = $body->get_cobrand_handler || $comment->get_cobrand_logged; + + # Some cobrands (e.g. Buckinghamshire) don't want to receive updates + # from anyone except the original problem reporter. + if ($cobrand->call_hook(should_skip_sending_update => $comment)) { + unless (defined $comment->get_extra_metadata('cobrand_skipped_sending')) { + $comment->set_extra_metadata(cobrand_skipped_sending => 1); + $comment->update; + } + next; + } + + next if !$self->verbose && $comment->send_fail_count && retry_timeout($comment); + + $self->process_update($body, $o, $comment, $cobrand); + } +} + +sub process_update { + my ($self, $body, $o, $comment, $cobrand) = @_; + + $cobrand->call_hook(open311_pre_send => $comment, $o); + + my $id = $o->post_service_request_update( $comment ); + + if ( $id ) { + $comment->update( { + external_id => $id, + whensent => \'current_timestamp', + } ); + } else { + $comment->update( { + send_fail_count => $comment->send_fail_count + 1, + send_fail_timestamp => \'current_timestamp', + send_fail_reason => "Failed to post over Open311\n\n" . $o->error, + } ); + + if ( $self->verbose && $o->error ) { + warn $o->error; + } + } +} + +sub retry_timeout { + my $row = shift; + + my $tz = FixMyStreet->local_time_zone; + my $now = DateTime->now( time_zone => $tz ); + my $diff = $now - $row->send_fail_timestamp; + if ( $diff->in_units( 'minutes' ) < 30 ) { + return 1; + } + + return 0; +} + +1; |