diff options
| -rwxr-xr-x | bin/open311-populate-service-list | 121 | ||||
| -rw-r--r-- | perllib/Open311/PopulateServiceList.pm | 205 | ||||
| -rw-r--r-- | t/open311/populate-service-list.t | 148 |
3 files changed, 356 insertions, 118 deletions
diff --git a/bin/open311-populate-service-list b/bin/open311-populate-service-list index cb22d6dad..36e04f5a6 100755 --- a/bin/open311-populate-service-list +++ b/bin/open311-populate-service-list @@ -2,126 +2,11 @@ use strict; use warnings; -use LWP::Simple; -use XML::Simple; use FixMyStreet::App; -use Open311; +use Open311::PopulateServiceList; -use Data::Dumper; my $council_list = FixMyStreet::App->model('DB::Open311conf'); +my $p = Open311::PopulateServiceList->new( council_list => $council_list ); -while ( my $council = $council_list->next ) { - - my $open311 = Open311->new( - endpoint => $council->endpoint, - jurisdiction => $council->jurisdiction, - api_key => $council->api_key - ); - - # west berks end point not standard - if ( $council->area_id == 2619 ) { - $open311->endpoints( - { - services => 'Services', - requests => 'Requests' - } - ); - } - - my $list = $open311->get_service_list; - - my @found_contacts; - - # print Dumper $list; - - foreach my $service ( @{ $list->{service} } ) { - print $service->{service_code} . ': ' . $service->{service_name} . "\n"; - my $contacts = FixMyStreet::App->model( 'DB::Contact')->search( - { - area_id => $council->area_id, - -OR => [ - email => $service->{service_code}, - category => $service->{service_name} - ] - } - ); - - my $contact = $contacts->first; - - # remove trailing whitespace as it upsets db queries - # to look up contact details when creating problem - my $service_name = $service->{service_name}; - $service_name =~ s/\s+$//; - - # FIXME - handle change of service name or service code - if ( $contact ) { - - print $council->area_id . " already has a contact for service code " . $service->{service_code} . "\n"; - push @found_contacts, $service->{service_code}; - - if ( $contact->deleted ) { - $contact->update( - { - category => $service_name, - email => $service->{service_code}, - confirmed => 1, - deleted => 0, - editor => $0, - whenedited => \'ms_current_timestamp()', - note => 'automatically undeleted by script', - } - ); - } - } else { - my $contact = FixMyStreet::App->model( 'DB::Contact')->create( - { - email => $service->{service_code}, - area_id => $council->area_id, - category => $service_name, - confirmed => 1, - deleted => 0, - editor => $0, - whenedited => \'ms_current_timestamp()', - note => 'created automatically by script', - } - ); - - if ( lc( $service->{metadata} ) eq 'true' ) { - print "Fetching meta data for $service->{service_code}\n"; - my $meta_data = $open311->get_service_meta_info( $service->{service_code} ); - - # turn the data into something a bit more friendly to use - my @meta = - # remove trailing colon as we add this when we display so we don't want 2 - map { $_->{description} =~ s/:\s*//; $_ } - # there is a display order and we only want to sort once - sort { $a->{order} <=> $b->{order} } - @{ $meta_data->{attributes}->{attribute} }; - - $contact->extra( \@meta ); - $contact->update; - } - - push @found_contacts, $service->{service_code}; - print "created contact for service code " . $service->{service_code} . " for council @{[$council->area_id]}\n"; - } - } - - my $found_contacts = FixMyStreet::App->model( 'DB::Contact')->search( - { - email => { -not_in => \@found_contacts }, - area_id => $council->area_id, - deleted => 0, - } - ); - - $found_contacts->update( - { - deleted => 1, - editor => $0, - whenedited => \'ms_current_timestamp()', - note => 'automatically marked as deleted by script' - } - ); -} +$p->process_councils; diff --git a/perllib/Open311/PopulateServiceList.pm b/perllib/Open311/PopulateServiceList.pm new file mode 100644 index 000000000..8a5a14a92 --- /dev/null +++ b/perllib/Open311/PopulateServiceList.pm @@ -0,0 +1,205 @@ +package Open311::PopulateServiceList; + +use Moose; +use LWP::Simple; +use XML::Simple; +use FixMyStreet::App; +use Open311; + +has council_list => ( is => 'ro' ); +has found_contacts => ( is => 'rw', default => sub { [] } ); + +has _current_council => ( is => 'rw' ); +has _current_open311 => ( is => 'rw' ); +has _current_service => ( is => 'rw' ); + +my $council_list = FixMyStreet::App->model('DB::Open311conf'); + +sub process_councils { + my $self = shift; + + while ( my $council = $self->council_list->next ) { + $self->_current_council( $council ); + $self->process_council; + } +} + +sub process_council { + my $self = shift; + + my $open311 = Open311->new( + endpoint => $self->_current_council->endpoint, + jurisdiction => $self->_current_council->jurisdiction, + api_key => $self->_current_council->api_key + ); + + $self->_current_open311( $open311 ); + $self->_check_endpoints; + + my $list = $open311->get_service_list; + $self->process_services( $list ); +} + + + +sub _check_endpoints { + my $self = shift; + + # west berks end point not standard + if ( $self->_current_council->area_id == 2619 ) { + $self->_current_open311->endpoints( + { + services => 'Services', + requests => 'Requests' + } + ); + } +} + + +sub process_services { + my $self = shift; + my $list = shift; + + foreach my $service ( @{ $list->{service} } ) { + $self->_current_service( $service ); + $self->process_service; + } +} + +sub process_service { + my $self = shift; + + print $self->_current_service->{service_code} . ': ' . $self->_current_service->{service_name} . "\n"; + my $contacts = FixMyStreet::App->model( 'DB::Contact')->search( + { + area_id => $self->_current_council->area_id, + -OR => [ + email => $self->_current_service->{service_code}, + category => $self->_current_service->{service_name} + ] + } + ); + + my $contact = $contacts->first; + + + if ( $contact ) { + $self->_handle_existing_contact( $contact ); + } else { + $self->_create_contact; + } +} + +# FIXME - handle change of service name or service code +sub _handle_existing_contact { + my ( $self, $contact ) = @_; + + my $service_name = $self->_normalize_service_name; + + print $self->_current_council->area_id . " already has a contact for service code " . $self->_current_service->{service_code} . "\n"; + push @{ $self->found_contacts }, $self->_current_service->{service_code}; + + if ( $contact->deleted ) { + $contact->update( + { + category => $service_name, + email => $self->_current_service->{service_code}, + confirmed => 1, + deleted => 0, + editor => $0, + whenedited => \'ms_current_timestamp()', + note => 'automatically undeleted by script', + } + ); + } +} + +sub _create_contact { + my $self = shift; + + my $service_name = $self->_normalize_service_name; + + # FIXME - don't die if existing open311 category with same name + my $contact; + eval { + $contact = FixMyStreet::App->model( 'DB::Contact')->create( + { + email => $self->_current_service->{service_code}, + area_id => $self->_current_council->area_id, + category => $service_name, + confirmed => 1, + deleted => 0, + editor => $0, + whenedited => \'ms_current_timestamp()', + note => 'created automatically by script', + } + ); + }; + + if ( $@ ) { + warn "Failed to create contact for service code " . $self->_current_service->{service_code} . " for council @{[$self->_current_council->area_id]}: $@\n"; + return; + } + + if ( $contact and lc( $self->_current_service->{metadata} ) eq 'true' ) { + $self->_add_meta_to_contact( $contact ); + } + + if ( $contact ) { + push @{ $self->found_contacts }, $self->_current_service->{service_code}; + print "created contact for service code " . $self->_current_service->{service_code} . " for council @{[$self->_current_council->area_id]}\n"; + } +} + +sub _add_contact_to_meta { + my ( $self, $contact ) = @_; + + print "Fetching meta data for $self->_current_service->{service_code}\n"; + my $meta_data = $self->_current_open311->get_service_meta_info( $self->_current_service->{service_code} ); + + # turn the data into something a bit more friendly to use + my @meta = + # remove trailing colon as we add this when we display so we don't want 2 + map { $_->{description} =~ s/:\s*//; $_ } + # there is a display order and we only want to sort once + sort { $a->{order} <=> $b->{order} } + @{ $meta_data->{attributes}->{attribute} }; + + $contact->extra( \@meta ); + $contact->update; +} + +sub _normalize_service_name { + my $self = shift; + + # remove trailing whitespace as it upsets db queries + # to look up contact details when creating problem + my $service_name = $self->_current_service->{service_name}; + $service_name =~ s/\s+$//; + + return $service_name; +} + +sub _delete_contacts_not_in_service_list { + my $self = shift; + + my $found_contacts = FixMyStreet::App->model( 'DB::Contact')->search( + { + email => { -not_in => $self->found_contacts }, + area_id => $self->_current_council->area_id, + deleted => 0, + } + ); + + $found_contacts->update( + { + deleted => 1, + editor => $0, + whenedited => \'ms_current_timestamp()', + note => 'automatically marked as deleted by script' + } + ); +} + +1; diff --git a/t/open311/populate-service-list.t b/t/open311/populate-service-list.t new file mode 100644 index 000000000..c7a9fe960 --- /dev/null +++ b/t/open311/populate-service-list.t @@ -0,0 +1,148 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use Test::More; + +use FixMyStreet::App; + +use FindBin; +use lib "$FindBin::Bin/../perllib"; +use lib "$FindBin::Bin/../commonlib/perllib"; + +use_ok( 'Open311::PopulateServiceList' ); +use_ok( 'Open311' ); + + +my $processor = Open311::PopulateServiceList->new( council_list => [] ); +ok $processor, 'created object'; + +subtest 'check basic functionality' => sub { + FixMyStreet::App->model('DB::Contact')->search( { area_id => 1 } )->delete(); + + my $xml = qq{<?xml version="1.0" encoding="utf-8"?> +<services> + <service> + <service_code>001</service_code> + <service_name>Cans left out 24x7</service_name> + <description>Garbage or recycling cans that have been left out for more than 24 hours after collection. Violators will be cited.</description> + <metadata>false</metadata> + <type>realtime</type> + <keywords>lorem, ipsum, dolor</keywords> + <group>sanitation</group> + </service> + <service> + <service_code>002</service_code> + <metadata>false</metadata> + <type>realtime</type> + <keywords>lorem, ipsum, dolor</keywords> + <group>street</group> + <service_name>Construction plate shifted</service_name> + <description>Metal construction plate covering the street or sidewalk has been moved.</description> + </service> + <service> + <service_code>003</service_code> + <metadata>false</metadata> + <type>realtime</type> + <keywords>lorem, ipsum, dolor</keywords> + <group>street</group> + <service_name>Curb or curb ramp defect</service_name> + <description>Sidewalk curb or ramp has problems such as cracking, missing pieces, holes, and/or chipped curb.</description> + </service> +</services> +}; + + my $simple = XML::Simple->new(); + my $obj; + + eval { + $obj = $simple->XMLin( $xml ); + }; + + my $council = FixMyStreet::App->model('DB::Open311Conf')->new( { + area_id => 1 + } ); + + my $processor = Open311::PopulateServiceList->new( council_list => [] ); + $processor->_current_council( $council ); + $processor->process_services( $obj ); + + my $contact_count = FixMyStreet::App->model('DB::Contact')->search( { area_id => 1 } )->count(); + is $contact_count, 3, 'correct number of contacts'; +}; + +subtest 'check duplicate service name issues error' => sub { + FixMyStreet::App->model('DB::Contact')->search( { area_id => 1 } )->delete(); + + my $contact = FixMyStreet::App->model('DB::Contact')->create( + { + area_id => 1, + email => '009', + category => 'Cans left out 24x7', + confirmed => 1, + deleted => 0, + editor => $0, + whenedited => \'ms_current_timestamp()', + note => 'test contact', + } + ); + + ok $contact, 'contact created'; + + my $xml = qq{<?xml version="1.0" encoding="utf-8"?> +<services> + <service> + <service_code>001</service_code> + <service_name>Cans left out 24x7</service_name> + <description>Garbage or recycling cans that have been left out for more than 24 hours after collection. Violators will be cited.</description> + <metadata>false</metadata> + <type>realtime</type> + <keywords>lorem, ipsum, dolor</keywords> + <group>sanitation</group> + </service> + <service> + <service_code>002</service_code> + <metadata>false</metadata> + <type>realtime</type> + <keywords>lorem, ipsum, dolor</keywords> + <group>street</group> + <service_name>Construction plate shifted</service_name> + <description>Metal construction plate covering the street or sidewalk has been moved.</description> + </service> + <service> + <service_code>003</service_code> + <metadata>false</metadata> + <type>realtime</type> + <keywords>lorem, ipsum, dolor</keywords> + <group>street</group> + <service_name>Curb or curb ramp defect</service_name> + <description>Sidewalk curb or ramp has problems such as cracking, missing pieces, holes, and/or chipped curb.</description> + </service> +</services> +}; + + my $simple = XML::Simple->new(); + my $obj; + + eval { + $obj = $simple->XMLin( $xml ); + }; + + my $council = FixMyStreet::App->model('DB::Open311Conf')->new( { + area_id => 1 + } ); + + my $processor = Open311::PopulateServiceList->new( council_list => [] ); + $processor->_current_council( $council ); + $processor->process_services( $obj ); + + $contact->discard_changes; + is $contact->email, '009', 'email unchanged'; + is $contact->confirmed, 1, 'contact still confirmed'; + is $contact->deleted, 0, 'contact still not deleted'; + + my $contact_count = FixMyStreet::App->model('DB::Contact')->search( { area_id => 1 } )->count(); + is $contact_count, 3, 'correct number of contacts'; +}; + +done_testing(); |