rewrite populate service list to enable testing and add some tests

3 files changed, 356 insertions, 118 deletions

diff --git a/bin/open311-populate-service-list b/bin/open311-populate-service-list
index cb22d6dad..36e04f5a6 100755
--- a/bin/open311-populate-service-list
+++ b/bin/open311-populate-service-list
@@ -2,126 +2,11 @@
 
 use strict;
 use warnings;
-use LWP::Simple;
-use XML::Simple;
 use FixMyStreet::App;
-use Open311;
+use Open311::PopulateServiceList;
 
-use Data::Dumper;
 
 my $council_list = FixMyStreet::App->model('DB::Open311conf');
+my $p = Open311::PopulateServiceList->new( council_list => $council_list );
 
-while ( my $council = $council_list->next ) {
-
-    my $open311 = Open311->new(
-        endpoint => $council->endpoint,
-        jurisdiction => $council->jurisdiction,
-        api_key => $council->api_key
-    );
-
-    # west berks end point not standard
-    if ( $council->area_id == 2619 ) {
-        $open311->endpoints(
-            {
-                services => 'Services',
-                requests => 'Requests'
-            }
-        );
-    }
-
-    my $list = $open311->get_service_list;
-
-    my @found_contacts;
-
-    # print Dumper $list;
-
-    foreach my $service ( @{ $list->{service} } ) {
-        print $service->{service_code} . ': ' . $service->{service_name} .  "\n";
-        my $contacts = FixMyStreet::App->model( 'DB::Contact')->search(
-            {
-                area_id => $council->area_id,
-                -OR => [
-                    email => $service->{service_code},
-                    category => $service->{service_name}
-                ]
-            }
-        );
-
-        my $contact = $contacts->first;
-
-        # remove trailing whitespace as it upsets db queries
-        # to look up contact details when creating problem
-        my $service_name = $service->{service_name};
-        $service_name =~ s/\s+$//;
-
-        # FIXME - handle change of service name or service code
-        if ( $contact ) {
-
-            print $council->area_id . " already has a contact for service code " . $service->{service_code} . "\n";
-            push @found_contacts, $service->{service_code};
-
-            if ( $contact->deleted ) {
-                $contact->update(
-                    {
-                        category => $service_name,
-                        email => $service->{service_code},
-                        confirmed => 1,
-                        deleted => 0,
-                        editor => $0,
-                        whenedited => \'ms_current_timestamp()',
-                        note => 'automatically undeleted by script',
-                    }
-                );
-            }
-        } else {
-            my $contact = FixMyStreet::App->model( 'DB::Contact')->create(
-                {
-                    email => $service->{service_code},
-                    area_id => $council->area_id,
-                    category => $service_name,
-                    confirmed => 1,
-                    deleted => 0,
-                    editor => $0,
-                    whenedited => \'ms_current_timestamp()',
-                    note => 'created automatically by script',
-                }
-            );
-
-            if ( lc( $service->{metadata} ) eq 'true' ) {
-                print "Fetching meta data for $service->{service_code}\n";
-                my $meta_data = $open311->get_service_meta_info( $service->{service_code} );
-
-                # turn the data into something a bit more friendly to use
-                my @meta =
-                    # remove trailing colon as we add this when we display so we don't want 2
-                    map { $_->{description} =~ s/:\s*//; $_ }
-                    # there is a display order and we only want to sort once
-                    sort { $a->{order} <=> $b->{order} }
-                    @{ $meta_data->{attributes}->{attribute} };
-
-                $contact->extra( \@meta );
-                $contact->update;
-            }
-
-            push @found_contacts, $service->{service_code};
-            print "created contact for service code " . $service->{service_code} . " for council @{[$council->area_id]}\n";
-        }
-    }
-
-    my $found_contacts = FixMyStreet::App->model( 'DB::Contact')->search(
-        {
-            email => { -not_in => \@found_contacts },
-            area_id => $council->area_id,
-            deleted => 0,
-        }
-    );
-
-    $found_contacts->update(
-        {
-            deleted => 1,
-            editor  => $0,
-            whenedited => \'ms_current_timestamp()',
-            note => 'automatically marked as deleted by script'
-        }
-    );
-}
+$p->process_councils;
diff --git a/perllib/Open311/PopulateServiceList.pm b/perllib/Open311/PopulateServiceList.pm
new file mode 100644
index 000000000..8a5a14a92
--- /dev/null
+++ b/perllib/Open311/PopulateServiceList.pm
@@ -0,0 +1,205 @@
+package Open311::PopulateServiceList;
+
+use Moose;
+use LWP::Simple;
+use XML::Simple;
+use FixMyStreet::App;
+use Open311;
+
+has council_list => ( is => 'ro' );
+has found_contacts => ( is => 'rw', default => sub { [] } );
+
+has _current_council => ( is => 'rw' );
+has _current_open311 => ( is => 'rw' );
+has _current_service => ( is => 'rw' );
+
+my $council_list = FixMyStreet::App->model('DB::Open311conf');
+
+sub process_councils {
+    my $self = shift;
+
+    while ( my $council = $self->council_list->next ) {
+        $self->_current_council( $council );
+        $self->process_council;
+    }
+}
+
+sub process_council {
+    my $self = shift;
+
+    my $open311 = Open311->new(
+        endpoint => $self->_current_council->endpoint,
+        jurisdiction => $self->_current_council->jurisdiction,
+        api_key => $self->_current_council->api_key
+    );
+
+    $self->_current_open311( $open311 );
+    $self->_check_endpoints;
+
+    my $list = $open311->get_service_list;
+    $self->process_services( $list );
+}
+
+
+
+sub _check_endpoints {
+    my $self = shift;
+
+    # west berks end point not standard
+    if ( $self->_current_council->area_id == 2619 ) {
+        $self->_current_open311->endpoints(
+            {
+                services => 'Services',
+                requests => 'Requests'
+            }
+        );
+    }
+}
+
+
+sub process_services {
+    my $self = shift;
+    my $list = shift;
+
+    foreach my $service ( @{ $list->{service} } ) {
+        $self->_current_service( $service );
+        $self->process_service;
+    }
+}
+
+sub process_service {
+    my $self = shift;
+
+    print $self->_current_service->{service_code} . ': ' . $self->_current_service->{service_name} .  "\n";
+    my $contacts = FixMyStreet::App->model( 'DB::Contact')->search(
+        {
+            area_id => $self->_current_council->area_id,
+            -OR => [
+                email => $self->_current_service->{service_code},
+                category => $self->_current_service->{service_name}
+            ]
+        }
+    );
+
+    my $contact = $contacts->first;
+
+
+    if ( $contact ) {
+        $self->_handle_existing_contact( $contact );
+    } else {
+        $self->_create_contact;
+    }
+}
+
+# FIXME - handle change of service name or service code
+sub _handle_existing_contact {
+    my ( $self, $contact ) = @_;
+
+    my $service_name = $self->_normalize_service_name;
+
+    print $self->_current_council->area_id . " already has a contact for service code " . $self->_current_service->{service_code} . "\n";
+    push @{ $self->found_contacts }, $self->_current_service->{service_code};
+
+    if ( $contact->deleted ) {
+        $contact->update(
+            {
+                category => $service_name,
+                email => $self->_current_service->{service_code},
+                confirmed => 1,
+                deleted => 0,
+                editor => $0,
+                whenedited => \'ms_current_timestamp()',
+                note => 'automatically undeleted by script',
+            }
+        );
+    }
+}
+
+sub _create_contact {
+    my $self = shift;
+
+    my $service_name = $self->_normalize_service_name;
+
+    # FIXME - don't die if existing open311 category with same name
+    my $contact;
+    eval {
+        $contact = FixMyStreet::App->model( 'DB::Contact')->create(
+            {
+                email => $self->_current_service->{service_code},
+                area_id => $self->_current_council->area_id,
+                category => $service_name,
+                confirmed => 1,
+                deleted => 0,
+                editor => $0,
+                whenedited => \'ms_current_timestamp()',
+                note => 'created automatically by script',
+            }
+        );
+    };
+
+    if ( $@ ) {
+        warn "Failed to create contact for service code " . $self->_current_service->{service_code} . " for council @{[$self->_current_council->area_id]}: $@\n";
+        return;
+    }
+
+    if ( $contact and lc( $self->_current_service->{metadata} ) eq 'true' ) {
+        $self->_add_meta_to_contact( $contact );
+    }
+
+    if ( $contact ) {
+        push @{ $self->found_contacts }, $self->_current_service->{service_code};
+        print "created contact for service code " . $self->_current_service->{service_code} . " for council @{[$self->_current_council->area_id]}\n";
+    }
+}
+
+sub _add_contact_to_meta {
+    my ( $self, $contact ) = @_;
+
+    print "Fetching meta data for $self->_current_service->{service_code}\n";
+    my $meta_data = $self->_current_open311->get_service_meta_info( $self->_current_service->{service_code} );
+
+    # turn the data into something a bit more friendly to use
+    my @meta =
+        # remove trailing colon as we add this when we display so we don't want 2
+        map { $_->{description} =~ s/:\s*//; $_ }
+        # there is a display order and we only want to sort once
+        sort { $a->{order} <=> $b->{order} }
+        @{ $meta_data->{attributes}->{attribute} };
+
+    $contact->extra( \@meta );
+    $contact->update;
+}
+
+sub _normalize_service_name {
+    my $self = shift;
+
+    # remove trailing whitespace as it upsets db queries
+    # to look up contact details when creating problem
+    my $service_name = $self->_current_service->{service_name};
+    $service_name =~ s/\s+$//;
+
+    return $service_name;
+}
+
+sub _delete_contacts_not_in_service_list {
+    my $self = shift;
+
+    my $found_contacts = FixMyStreet::App->model( 'DB::Contact')->search(
+        {
+            email => { -not_in => $self->found_contacts },
+            area_id => $self->_current_council->area_id,
+            deleted => 0,
+        }
+    );
+
+    $found_contacts->update(
+        {
+            deleted => 1,
+            editor  => $0,
+            whenedited => \'ms_current_timestamp()',
+            note => 'automatically marked as deleted by script'
+        }
+    );
+}
+
+1;
diff --git a/t/open311/populate-service-list.t b/t/open311/populate-service-list.t
new file mode 100644
index 000000000..c7a9fe960
--- /dev/null
+++ b/t/open311/populate-service-list.t
@@ -0,0 +1,148 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+use Test::More;
+
+use FixMyStreet::App;
+
+use FindBin;
+use lib "$FindBin::Bin/../perllib";
+use lib "$FindBin::Bin/../commonlib/perllib";
+
+use_ok( 'Open311::PopulateServiceList' );
+use_ok( 'Open311' );
+
+
+my $processor = Open311::PopulateServiceList->new( council_list => [] );
+ok $processor, 'created object';
+
+subtest 'check basic functionality' => sub {
+    FixMyStreet::App->model('DB::Contact')->search( { area_id => 1 } )->delete();
+
+    my $xml = qq{<?xml version="1.0" encoding="utf-8"?>
+<services>
+	<service>
+		<service_code>001</service_code>
+		<service_name>Cans left out 24x7</service_name>
+		<description>Garbage or recycling cans that have been left out for more than 24 hours after collection. Violators will be cited.</description>
+		<metadata>false</metadata>
+		<type>realtime</type>
+		<keywords>lorem, ipsum, dolor</keywords>
+		<group>sanitation</group>
+	</service>
+	<service>
+		<service_code>002</service_code>
+		<metadata>false</metadata>
+		<type>realtime</type>
+		<keywords>lorem, ipsum, dolor</keywords>
+		<group>street</group>
+		<service_name>Construction plate shifted</service_name>
+		<description>Metal construction plate covering the street or sidewalk has been moved.</description>
+	</service>
+	<service>
+		<service_code>003</service_code>
+		<metadata>false</metadata>
+		<type>realtime</type>
+		<keywords>lorem, ipsum, dolor</keywords>
+		<group>street</group>
+		<service_name>Curb or curb ramp defect</service_name>
+		<description>Sidewalk curb or ramp has problems such as cracking, missing pieces, holes, and/or chipped curb.</description>
+	</service>
+</services>
+};
+
+    my $simple = XML::Simple->new();
+    my $obj;
+
+    eval {
+        $obj = $simple->XMLin( $xml );
+    };
+
+    my $council = FixMyStreet::App->model('DB::Open311Conf')->new( {
+        area_id => 1
+    } );
+
+    my $processor = Open311::PopulateServiceList->new( council_list => [] );
+    $processor->_current_council( $council );
+    $processor->process_services( $obj );
+
+    my $contact_count = FixMyStreet::App->model('DB::Contact')->search( { area_id => 1 } )->count();
+    is $contact_count, 3, 'correct number of contacts';
+};
+
+subtest 'check duplicate service name issues error' => sub {
+    FixMyStreet::App->model('DB::Contact')->search( { area_id => 1 } )->delete();
+
+    my $contact = FixMyStreet::App->model('DB::Contact')->create(
+        {
+            area_id => 1,
+            email =>   '009',
+            category => 'Cans left out 24x7',
+            confirmed => 1,
+            deleted => 0,
+            editor => $0,
+            whenedited => \'ms_current_timestamp()',
+            note => 'test contact',
+        }
+    );
+
+    ok $contact, 'contact created';
+
+    my $xml = qq{<?xml version="1.0" encoding="utf-8"?>
+<services>
+	<service>
+		<service_code>001</service_code>
+		<service_name>Cans left out 24x7</service_name>
+		<description>Garbage or recycling cans that have been left out for more than 24 hours after collection. Violators will be cited.</description>
+		<metadata>false</metadata>
+		<type>realtime</type>
+		<keywords>lorem, ipsum, dolor</keywords>
+		<group>sanitation</group>
+	</service>
+	<service>
+		<service_code>002</service_code>
+		<metadata>false</metadata>
+		<type>realtime</type>
+		<keywords>lorem, ipsum, dolor</keywords>
+		<group>street</group>
+		<service_name>Construction plate shifted</service_name>
+		<description>Metal construction plate covering the street or sidewalk has been moved.</description>
+	</service>
+	<service>
+		<service_code>003</service_code>
+		<metadata>false</metadata>
+		<type>realtime</type>
+		<keywords>lorem, ipsum, dolor</keywords>
+		<group>street</group>
+		<service_name>Curb or curb ramp defect</service_name>
+		<description>Sidewalk curb or ramp has problems such as cracking, missing pieces, holes, and/or chipped curb.</description>
+	</service>
+</services>
+};
+
+    my $simple = XML::Simple->new();
+    my $obj;
+
+    eval {
+        $obj = $simple->XMLin( $xml );
+    };
+
+    my $council = FixMyStreet::App->model('DB::Open311Conf')->new( {
+        area_id => 1
+    } );
+
+    my $processor = Open311::PopulateServiceList->new( council_list => [] );
+    $processor->_current_council( $council );
+    $processor->process_services( $obj );
+
+    $contact->discard_changes;
+    is $contact->email, '009', 'email unchanged';
+    is $contact->confirmed, 1, 'contact still confirmed';
+    is $contact->deleted, 0, 'contact still not deleted';
+
+    my $contact_count = FixMyStreet::App->model('DB::Contact')->search( { area_id => 1 } )->count();
+    is $contact_count, 3, 'correct number of contacts';
+};
+
+done_testing();