diff options
Diffstat (limited to 'perllib/Open311')
| -rw-r--r-- | perllib/Open311/GetServiceRequestUpdates.pm | 110 | ||||
| -rw-r--r-- | perllib/Open311/GetServiceRequests.pm | 191 | ||||
| -rw-r--r-- | perllib/Open311/GetUpdates.pm | 2 | ||||
| -rw-r--r-- | perllib/Open311/PopulateServiceList.pm | 61 |
4 files changed, 293 insertions, 71 deletions
diff --git a/perllib/Open311/GetServiceRequestUpdates.pm b/perllib/Open311/GetServiceRequestUpdates.pm index 2620b176a..b4d7c6347 100644 --- a/perllib/Open311/GetServiceRequestUpdates.pm +++ b/perllib/Open311/GetServiceRequestUpdates.pm @@ -12,6 +12,7 @@ has end_date => ( is => 'ro', default => sub { undef } ); has suppress_alerts => ( is => 'rw', default => 0 ); has verbose => ( is => 'ro', default => 0 ); has schema => ( is =>'ro', lazy => 1, default => sub { FixMyStreet::DB->schema->connect } ); +has blank_updates_permitted => ( is => 'rw', default => 0 ); Readonly::Scalar my $AREA_ID_BROMLEY => 2482; Readonly::Scalar my $AREA_ID_OXFORDSHIRE => 2237; @@ -49,6 +50,7 @@ sub fetch { } $self->suppress_alerts( $body->suppress_alerts ); + $self->blank_updates_permitted( $body->blank_updates_permitted ); $self->system_user( $body->comment_user ); $self->update_comments( $o, $body ); } @@ -107,13 +109,19 @@ sub update_comments { my $c = $p->comments->search( { external_id => $request->{update_id} } ); if ( !$c->first ) { - my $state = $self->map_state( $request->{status} ); + my $state = $open311->map_state( $request->{status} ); + my $old_state = $p->state; + my $external_status_code = $request->{external_status_code} || ''; + my $old_external_status_code = $p->get_extra_metadata('external_status_code') || ''; my $comment = $self->schema->resultset('Comment')->new( { problem => $p, user => $self->system_user, external_id => $request->{update_id}, - text => $self->comment_text_for_request($request, $p, $state), + text => $self->comment_text_for_request( + $request, $p, $state, $old_state, + $external_status_code, $old_external_status_code + ), mark_fixed => 0, mark_open => 0, anonymous => 0, @@ -124,35 +132,40 @@ sub update_comments { } ); - if ($request->{media_url}) { - my $ua = LWP::UserAgent->new; - my $res = $ua->get($request->{media_url}); - if ( $res->is_success && $res->content_type eq 'image/jpeg' ) { - my $photoset = FixMyStreet::App::Model::PhotoSet->new({ - data_items => [ $res->decoded_content ], - }); - $comment->photo($photoset->data); - } + # Some Open311 services, e.g. Confirm via open311-adapter, provide + # a more fine-grained status code that we use within FMS for + # response templates. + if ( $external_status_code ) { + $comment->set_extra_metadata(external_status_code => $external_status_code); + $p->set_extra_metadata(external_status_code => $external_status_code); } - # if the comment is older than the last update - # do not change the status of the problem as it's - # tricky to determine the right thing to do. - if ( $comment->created > $p->lastupdate ) { - # don't update state unless it's an allowed state and it's - # actually changing the state of the problem - if ( FixMyStreet::DB::Result::Problem->visible_states()->{$state} && $p->state ne $state && - # For Oxfordshire, don't allow changes back to Open from other open states - !( $body->areas->{$AREA_ID_OXFORDSHIRE} && $state eq 'confirmed' && $p->is_open ) && - # Don't let it change between the (same in the front end) fixed states - !( $p->is_fixed && FixMyStreet::DB::Result::Problem->fixed_states()->{$state} ) ) { - if ($p->is_visible) { - $p->state($state); - } - $comment->problem_state($state); + $open311->add_media($request->{media_url}, $comment) + if $request->{media_url}; + + # don't update state unless it's an allowed state + if ( FixMyStreet::DB::Result::Problem->visible_states()->{$state} && + # For Oxfordshire, don't allow changes back to Open from other open states + !( $body->areas->{$AREA_ID_OXFORDSHIRE} && $state eq 'confirmed' && $p->is_open ) && + # Don't let it change between the (same in the front end) fixed states + !( $p->is_fixed && FixMyStreet::DB::Result::Problem->fixed_states()->{$state} ) ) { + + $comment->problem_state($state); + + # if the comment is older than the last update do not + # change the status of the problem as it's tricky to + # determine the right thing to do. Allow the same time in + # case report/update created at same time (in external + # system). Only do this if the report is currently visible. + if ( $comment->created >= $p->lastupdate && $p->state ne $state && $p->is_visible ) { + $p->state($state); } } + # If nothing to show (no text, photo, or state change), don't show this update + $comment->state('hidden') unless $comment->text || $comment->photo + || ($comment->problem_state && $state ne $old_state); + $p->lastupdate( $comment->created ); $p->update; $comment->insert(); @@ -180,37 +193,34 @@ sub update_comments { } sub comment_text_for_request { - my ($self, $request, $problem, $state) = @_; + my ($self, $request, $problem, $state, $old_state, + $ext_code, $old_ext_code) = @_; return $request->{description} if $request->{description}; - if (my $template = $problem->response_templates->search({ - auto_response => 1, - 'me.state' => $state - })->first) { - return $template->text; + # Response templates are only triggered if the state/external status has changed + my $state_changed = $state ne $old_state; + my $ext_code_changed = $ext_code ne $old_ext_code; + if ($state_changed || $ext_code_changed) { + my $state_params = { + 'me.state' => $state + }; + if ($ext_code) { + $state_params->{'me.external_status_code'} = $ext_code; + }; + + if (my $template = $problem->response_templates->search({ + auto_response => 1, + -or => $state_params, + })->first) { + return $template->text; + } } + return "" if $self->blank_updates_permitted; + print STDERR "Couldn't determine update text for $request->{update_id} (report " . $problem->id . ")\n"; return ""; } -sub map_state { - my $self = shift; - my $incoming_state = shift; - - $incoming_state = lc($incoming_state); - $incoming_state =~ s/_/ /g; - - my %state_map = ( - fixed => 'fixed - council', - 'not councils responsibility' => 'not responsible', - 'no further action' => 'unable to fix', - open => 'confirmed', - closed => 'fixed - council' - ); - - return $state_map{$incoming_state} || $incoming_state; -} - 1; diff --git a/perllib/Open311/GetServiceRequests.pm b/perllib/Open311/GetServiceRequests.pm new file mode 100644 index 000000000..2d15347fd --- /dev/null +++ b/perllib/Open311/GetServiceRequests.pm @@ -0,0 +1,191 @@ +package Open311::GetServiceRequests; + +use Moo; +use Open311; +use FixMyStreet::DB; +use FixMyStreet::App::Model::PhotoSet; +use DateTime::Format::W3CDTF; + +has system_user => ( is => 'rw' ); +has start_date => ( is => 'ro', default => sub { undef } ); +has end_date => ( is => 'ro', default => sub { undef } ); +has fetch_all => ( is => 'rw', default => 0 ); +has verbose => ( is => 'ro', default => 0 ); +has schema => ( is =>'ro', lazy => 1, default => sub { FixMyStreet::DB->schema->connect } ); +has convert_latlong => ( is => 'rw', default => 0 ); + +sub fetch { + my $self = shift; + + my $bodies = $self->schema->resultset('Body')->search( + { + send_method => 'Open311', + fetch_problems => 1, + comment_user_id => { '!=', undef }, + endpoint => { '!=', '' }, + } + ); + + while ( my $body = $bodies->next ) { + my $o = $self->create_open311_object( $body ); + + $self->system_user( $body->comment_user ); + $self->convert_latlong( $body->convert_latlong ); + $self->fetch_all( $body->get_extra_metadata('fetch_all_problems') ); + $self->create_problems( $o, $body ); + } +} + +# this is so we can test +sub create_open311_object { + my ($self, $body) = @_; + + my $o = Open311->new( + endpoint => $body->endpoint, + api_key => $body->api_key, + jurisdiction => $body->jurisdiction, + ); + + return $o; +} + +sub create_problems { + my ( $self, $open311, $body ) = @_; + + my $args = {}; + + if ( $self->start_date || $self->end_date ) { + return 0 unless $self->start_date && $self->end_date; + + $args->{start_date} = DateTime::Format::W3CDTF->format_datetime( $self->start_date ); + $args->{end_date} = DateTime::Format::W3CDTF->format_datetime( $self->end_date ); + } elsif ( !$self->fetch_all ) { + my $end_dt = DateTime->now(); + my $start_dt = $end_dt->clone; + $start_dt->add( hours => -1 ); + + $args->{start_date} = DateTime::Format::W3CDTF->format_datetime( $start_dt ); + $args->{end_date} = DateTime::Format::W3CDTF->format_datetime( $end_dt ); + } + + my $requests = $open311->get_service_requests( $args ); + + unless ( $open311->success ) { + warn "Failed to fetch ServiceRequest Updates for " . $body->name . ":\n" . $open311->error + if $self->verbose; + return 0; + } + + my $contacts = $self->schema->resultset('Contact') + ->active + ->search( { body_id => $body->id } ); + + for my $request (@{$requests->{request}}) { + # no point importing if we can't put it on the map + unless ($request->{service_request_id} && $request->{lat} && $request->{long}) { + warn "Not creating request '$request->{description}' for @{[$body->name]} as missing one of id, lat or long" + if $self->verbose; + next; + } + my $request_id = $request->{service_request_id}; + + my %params; + $params{generation} = mySociety::Config::get('MAPIT_GENERATION') + if mySociety::Config::get('MAPIT_GENERATION'); + + my ($latitude, $longitude) = ( $request->{lat}, $request->{long} ); + + ($latitude, $longitude) = Utils::convert_en_to_latlon_truncated( $longitude, $latitude ) + if $self->convert_latlong; + + my $all_areas = + mySociety::MaPit::call( 'point', + "4326/$longitude,$latitude", %params ); + + # skip if it doesn't look like it's for this body + my @areas = grep { $all_areas->{$_->area_id} } $body->body_areas; + unless (@areas) { + warn "Not creating request id $request_id for @{[$body->name]} as outside body area" + if $self->verbose; + next; + } + + my $updated_time = eval { + DateTime::Format::W3CDTF->parse_datetime( + $request->{updated_datetime} || "" + )->set_time_zone( + FixMyStreet->time_zone || FixMyStreet->local_time_zone + ); + }; + if ($@) { + warn "Not creating problem $request_id for @{[$body->name]}, bad update time" + if $self->verbose; + next; + } + + my $updated = DateTime::Format::W3CDTF->format_datetime( + $updated_time->clone->set_time_zone('UTC') + ); + if ($args->{start_date} && $args->{end_date} && ($updated lt $args->{start_date} || $updated gt $args->{end_date}) ) { + warn "Problem id $request_id for @{[$body->name]} has an invalid time, not creating: " + . "$updated either less than $args->{start_date} or greater than $args->{end_date}" + if $self->verbose; + next; + } + + my $created_time = eval { + DateTime::Format::W3CDTF->parse_datetime( + $request->{requested_datetime} || "" + )->set_time_zone( + FixMyStreet->time_zone || FixMyStreet->local_time_zone + ); + }; + $created_time = $updated_time if $@; + + my $problems; + my $criteria = { + external_id => $request_id, + }; + $problems = $self->schema->resultset('Problem')->to_body($body)->search( $criteria ); + + my @contacts = grep { $request->{service_code} eq $_->email } $contacts->all; + my $contact = $contacts[0] ? $contacts[0]->category : 'Other'; + + my $state = $open311->map_state($request->{status}); + + unless (my $p = $problems->first) { + my $problem = $self->schema->resultset('Problem')->new( + { + user => $self->system_user, + external_id => $request_id, + detail => $request->{description}, + title => $request->{title} || $request->{service_name} . ' problem', + anonymous => 0, + name => $self->system_user->name, + confirmed => $created_time, + created => $created_time, + lastupdate => $updated_time, + whensent => $created_time, + state => $state, + postcode => '', + used_map => 1, + latitude => $latitude, + longitude => $longitude, + areas => ',' . $body->id . ',', + bodies_str => $body->id, + send_method_used => 'Open311', + category => $contact, + } + ); + + $open311->add_media($request->{media_url}, $problem) + if $request->{media_url}; + + $problem->insert(); + } + } + + return 1; +} + +1; diff --git a/perllib/Open311/GetUpdates.pm b/perllib/Open311/GetUpdates.pm index 1b1e339e3..f62acf4a8 100644 --- a/perllib/Open311/GetUpdates.pm +++ b/perllib/Open311/GetUpdates.pm @@ -41,7 +41,7 @@ sub get_updates { sub update_reports { my ( $self, $report_ids, $open311, $body ) = @_; - my $service_requests = $open311->get_service_requests( $report_ids ); + my $service_requests = $open311->get_service_requests( { report_ids => $report_ids } ); my $requests = $service_requests->{request}; for my $request (@$requests) { diff --git a/perllib/Open311/PopulateServiceList.pm b/perllib/Open311/PopulateServiceList.pm index 30d888eb4..f10bdf7fb 100644 --- a/perllib/Open311/PopulateServiceList.pm +++ b/perllib/Open311/PopulateServiceList.pm @@ -237,9 +237,36 @@ sub _add_meta_to_contact { # Some Open311 endpoints, such as Bromley and Warwickshire send <metadata> # for attributes which we *don't* want to display to the user (e.g. as # fields in "category_extras" + $self->_add_meta_to_contact_cobrand_overrides($contact, \@meta); + + $contact->set_extra_fields(@meta); + $contact->update; +} + +sub _add_meta_to_contact_cobrand_overrides { + my ( $self, $contact, $meta ) = @_; if ($self->_current_body->name eq 'Bromley Council') { $contact->set_extra_metadata( id_field => 'service_request_id_ext'); + # Lights we want to store feature ID, PROW on all categories. + push @$meta, { + code => 'prow_reference', + datatype => 'string', + description => 'Right of way reference', + order => 101, + required => 'false', + variable => 'true', + automated => 'hidden_field', + }; + push @$meta, { + code => 'feature_id', + datatype => 'string', + description => 'Feature ID', + order => 100, + required => 'false', + variable => 'true', + automated => 'hidden_field', + } if $self->_current_service->{service_code} eq 'LIGHTS'; } elsif ($self->_current_body->name eq 'Warwickshire County Council') { $contact->set_extra_metadata( id_field => 'external_id'); } @@ -256,7 +283,7 @@ sub _add_meta_to_contact { public_anonymity_required email_alerts_requested ) ], - #2243, + #2243, 'Warwickshire County Council' => [qw( closest_address ) ], @@ -264,11 +291,8 @@ sub _add_meta_to_contact { if (my $override = $override{ $self->_current_body->name }) { my %ignore = map { $_ => 1 } @{ $override }; - @meta = grep { ! $ignore{ $_->{ code } } } @meta; + @$meta = grep { ! $ignore{ $_->{ code } } } @$meta; } - - $contact->set_extra_fields(@meta); - $contact->update; } sub _normalize_service_name { @@ -296,25 +320,16 @@ sub _delete_contacts_not_in_service_list { } ); - # for Warwickshire/Bristol, which are mixed Open311 and email, don't delete - # the email addresses - if ($self->_current_body->name eq 'Warwickshire County Council' || - $self->_current_body->name eq 'Bristol City Council') { + if ($self->_current_body->can_be_devolved) { + # If the body has can_be_devolved switched on, it's most likely a + # combination of Open311/email, so ignore any email addresses. $found_contacts = $found_contacts->search( - { - email => { -not_like => '%@%' } - } - ); - } elsif ($self->_current_body->name eq 'East Hertfordshire District Council') { - # For EHDC we need to leave the 'Other' category alone or reports made - # in this category will be sent only to Hertfordshire County Council. - $found_contacts = $found_contacts->search( - { - category => { '!=' => 'Other' } - } + { email => { -not_like => '%@%' } } ); } + $found_contacts = $self->_delete_contacts_not_in_service_list_cobrand_overrides($found_contacts); + $found_contacts->update( { state => 'deleted', @@ -325,4 +340,10 @@ sub _delete_contacts_not_in_service_list { ); } +sub _delete_contacts_not_in_service_list_cobrand_overrides { + my ( $self, $found_contacts ) = @_; + + return $found_contacts; +} + 1; |