Merge remote branch 'origin/zurich'

Conflicts:
	bin/open311-populate-service-list
	bin/send-comments
	bin/update-all-reports
	conf/crontab.ugly
	db/schema.sql
	perllib/FixMyStreet/App/Controller/Admin.pm
	perllib/FixMyStreet/App/Controller/Report/New.pm
	perllib/FixMyStreet/App/Controller/Reports.pm
	perllib/FixMyStreet/Cobrand/Default.pm
	perllib/FixMyStreet/Cobrand/LichfieldDC.pm
	perllib/FixMyStreet/DB/Result/Open311conf.pm
	perllib/FixMyStreet/DB/Result/Problem.pm
	perllib/FixMyStreet/DB/ResultSet/Problem.pm
	perllib/FixMyStreet/SendReport.pm
	perllib/FixMyStreet/SendReport/Email.pm
	perllib/FixMyStreet/SendReport/Open311.pm
	perllib/Open311/GetServiceRequestUpdates.pm
	perllib/Open311/PopulateServiceList.pm
	t/app/controller/report_new.t
	t/app/controller/rss.t
	templates/web/bromley/report/display.html
	templates/web/default/admin/council_contacts.html
	templates/web/default/common_header_tags.html
	templates/web/default/dashboard/index.html
	templates/web/default/front/stats.html
	templates/web/default/report/_main.html
	templates/web/default/report/update-form.html
	templates/web/emptyhomes/index.html
	templates/web/emptyhomes/report/display.html
	templates/web/emptyhomes/report/new/councils_text_all.html
	templates/web/emptyhomes/reports/body.html
	templates/web/emptyhomes/reports/index.html
	templates/web/fixmystreet/report/new/fill_in_details_form.html
	templates/web/fixmystreet/report/update-form.html
	web/cobrands/fixmystreet/fixmystreet.js
	web/js/fixmystreet.js

3 files changed, 46 insertions, 127 deletions

diff --git a/perllib/Open311/GetServiceRequestUpdates.pm b/perllib/Open311/GetServiceRequestUpdates.pm
index a97c4c379..e4416f792 100644
--- a/perllib/Open311/GetServiceRequestUpdates.pm
+++ b/perllib/Open311/GetServiceRequestUpdates.pm
@@ -5,7 +5,6 @@ use Open311;
 use FixMyStreet::App;
 use DateTime::Format::W3CDTF;
 
-has council_list => ( is => 'ro' );
 has system_user => ( is => 'rw' );
 has start_date => ( is => 'ro', default => undef );
 has end_date => ( is => 'ro', default => undef );
@@ -18,7 +17,7 @@ Readonly::Scalar my $AREA_ID_OXFORDSHIRE => 2237;
 sub fetch {
     my $self = shift;
 
-    my $councils = FixMyStreet::App->model('DB::Open311Conf')->search(
+    my $bodies = FixMyStreet::App->model('DB::Body')->search(
         {
             send_method     => 'Open311',
             send_comments   => 1,
@@ -27,34 +26,34 @@ sub fetch {
         }
     );
 
-    while ( my $council = $councils->next ) {
+    while ( my $body = $bodies->next ) {
 
         my $o = Open311->new(
-            endpoint     => $council->endpoint,
-            api_key      => $council->api_key,
-            jurisdiction => $council->jurisdiction,
+            endpoint     => $body->endpoint,
+            api_key      => $body->api_key,
+            jurisdiction => $body->jurisdiction,
         );
 
         # custom endpoint URLs because these councils have non-standard paths
-        if ( $council->area_id =~ /\b$AREA_ID_BROMLEY\b/o ) {
+        if ( $body->areas->{$AREA_ID_BROMLEY} ) {
             my $endpoints = $o->endpoints;
             $endpoints->{update} = 'update.xml';
             $endpoints->{service_request_updates} = 'update.xml';
             $o->endpoints( $endpoints );
-        } elsif ($council->area_id =~/\b$AREA_ID_OXFORDSHIRE\b/o) {
+        } elsif ( $body->areas->{$AREA_ID_OXFORDSHIRE} ) {
             my $endpoints = $o->endpoints;
             $endpoints->{service_request_updates} = 'open311_service_request_update.cgi';
             $o->endpoints( $endpoints );
         }
 
-        $self->suppress_alerts( $council->suppress_alerts );
-        $self->system_user( $council->comment_user );
-        $self->update_comments( $o, { areaid => $council->area_id }, );
+        $self->suppress_alerts( $body->suppress_alerts );
+        $self->system_user( $body->comment_user );
+        $self->update_comments( $o, { areas => $body->areas }, );
     }
 }
 
 sub update_comments {
-    my ( $self, $open311, $council_details ) = @_;
+    my ( $self, $open311, $body_details ) = @_;
 
     my @args = ();
 
@@ -64,7 +63,7 @@ sub update_comments {
         push @args, $self->start_date;
         push @args, $self->end_date;
     # default to asking for last 2 hours worth if not Bromley
-    } elsif ( $council_details->{areaid} != $AREA_ID_BROMLEY ) {
+    } elsif ( ! $body_details->{areas}->{$AREA_ID_BROMLEY} ) {
         my $end_dt = DateTime->now();
         my $start_dt = $end_dt->clone;
         $start_dt->add( hours => -2 );
@@ -76,7 +75,7 @@ sub update_comments {
     my $requests = $open311->get_service_request_updates( @args );
 
     unless ( $open311->success ) {
-        warn "Failed to fetch ServiceRequest Updates for " . $council_details->{areaid} . ":\n" . $open311->error
+        warn "Failed to fetch ServiceRequest Updates for " . join(",", keys %{$body_details->{areas}}) . ":\n" . $open311->error
             if $self->verbose;
         return 0;
     }
@@ -93,7 +92,8 @@ sub update_comments {
         if ($open311->jurisdiction =~ /^fixmybarangay_(dps|dpwh|depw)$/i) { # use jurisdiction (not area_id) for FMB bodies
             $criteria->{ external_body } = uc $1;
         } else {
-            $criteria->{ council } = { like => '%' . $council_details->{areaid} . '%' };
+            # XXX This assumes that areas will actually only be one area.
+            $criteria->{ bodies_str } = { like => '%' . join(",", keys %{$body_details->{areas}}) . '%' };
         }
         $problem = FixMyStreet::App->model('DB::Problem')->search( $criteria );
 
diff --git a/perllib/Open311/GetUpdates.pm b/perllib/Open311/GetUpdates.pm
deleted file mode 100644
index 5d5291d47..000000000
--- a/perllib/Open311/GetUpdates.pm
+++ /dev/null
@@ -1,82 +0,0 @@
-package Open311::GetUpdates;
-
-use Moose;
-use Open311;
-use FixMyStreet::App;
-
-has council_list => ( is => 'ro' );
-has system_user => ( is => 'ro' );
-
-sub get_updates {
-    my $self = shift;
-
-    while ( my $council = $self->council_list->next ) {
-        my $open311 = Open311->new(
-            endpoint     => $council->endpoint,
-            jurisdiction => $council->jurisdiction,
-            api_key      => $council->api_key
-        );
-
-        my $area_id = $council->area_id;
-
-        my $council_details = mySociety::MaPit::call( 'area', $area_id );
-
-        my $reports = FixMyStreet::App->model('DB::Problem')->search(
-            {
-                council => { like => "\%$area_id\%" },
-                state => { 'IN', [qw/confirmed fixed/] },
-                -and => [
-                    external_id => { '!=', undef },
-                    external_id => { '!=', '' },
-                ],
-            }
-        );
-
-        my @report_ids = ();
-        while ( my $report = $reports->next ) {
-            push @report_ids, $report->external_id;
-        }
-
-        next unless @report_ids;
-
-        $self->update_reports( \@report_ids, $open311, $council_details );
-    }
-}
-
-sub update_reports {
-    my ( $self, $report_ids, $open311, $council_details ) = @_;
-
-    my $service_requests = $open311->get_service_requests( $report_ids );
-
-    my $requests;
-
-    # XML::Simple is a bit inconsistent in how it structures
-    # things depending on the number of children an element has :(
-    if ( ref $service_requests->{request} eq 'ARRAY' ) {
-        $requests = $service_requests->{request};
-    }
-    else {
-        $requests = [ $service_requests->{request} ];
-    }
-
-    for my $request (@$requests) {
-        # if it's a ref that means it's an empty element
-        # however, if there's no updated date then we can't
-        # tell if it's newer that what we have so we should skip it
-        next if ref $request->{updated_datetime} || ! exists $request->{updated_datetime};
-
-        my $request_id = $request->{service_request_id};
-
-        my $problem =
-          FixMyStreet::App->model('DB::Problem')
-          ->search( { external_id => $request_id, } );
-
-        if (my $p = $problem->first) {
-            $p->update_from_open311_service_request( $request, $council_details, $self->system_user );
-        }
-    }
-
-    return 1;
-}
-
-1;
diff --git a/perllib/Open311/PopulateServiceList.pm b/perllib/Open311/PopulateServiceList.pm
index 9331bb5b2..57ef90ecb 100644
--- a/perllib/Open311/PopulateServiceList.pm
+++ b/perllib/Open311/PopulateServiceList.pm
@@ -6,34 +6,34 @@ use XML::Simple;
 use FixMyStreet::App;
 use Open311;
 
-has council_list => ( is => 'ro' );
+has bodies => ( is => 'ro' );
 has found_contacts => ( is => 'rw', default => sub { [] } );
 has verbose => ( is => 'ro', default => 0 );
 
-has _current_council => ( is => 'rw' );
+has _current_body => ( is => 'rw' );
 has _current_open311 => ( is => 'rw' );
 has _current_service => ( is => 'rw' );
 
-my $council_list = FixMyStreet::App->model('DB::Open311conf');
+my $bodies = FixMyStreet::App->model('DB::Body');
 
-sub process_councils {
+sub process_bodies {
     my $self = shift;
 
-    while ( my $council = $self->council_list->next ) {
-        next unless $council->endpoint;
-        next unless lc($council->send_method) eq 'open311';
-        next if $council->jurisdiction =~ /^fixmybarangay_\w+$/; # FMB depts. not using service discovery yet
-        $self->_current_council( $council );
-        $self->process_council;
+    while ( my $body = $self->bodies->next ) {
+        next unless $body->endpoint;
+        next unless lc($body->send_method) eq 'open311';
+        next if $body->jurisdiction =~ /^fixmybarangay_\w+$/; # FMB depts. not using service discovery yet
+        $self->_current_body( $body );
+        $self->process_body;
     }
 }
 
-sub process_council {
+sub process_body {
     my $self = shift;
     my $open311 = Open311->new(
-        endpoint => $self->_current_council->endpoint,
-        jurisdiction => $self->_current_council->jurisdiction,
-        api_key => $self->_current_council->api_key
+        endpoint => $self->_current_body->endpoint,
+        jurisdiction => $self->_current_body->jurisdiction,
+        api_key => $self->_current_body->api_key
     );
 
     $self->_current_open311( $open311 );
@@ -41,8 +41,9 @@ sub process_council {
 
     my $list = $open311->get_service_list;
     unless ( $list ) {
-        my $id = $self->_current_council->area_id;
-        warn "Council $id - http://mapit.mysociety.org/area/$id.html - did not return a service list\n"
+        my $id = $self->_current_body->id;
+        my $areas = join( ",", keys %{$self->_current_body->areas} );
+        warn "Body $id for areas $areas - http://mapit.mysociety.org/areas/$areas.html - did not return a service list\n"
             if $self->verbose >= 1;
         return;
     }
@@ -55,7 +56,7 @@ sub _check_endpoints {
     my $self = shift;
 
     # west berks end point not standard
-    if ( $self->_current_council->area_id == 2619 ) {
+    if ( $self->_current_body->areas->{2619} ) {
         $self->_current_open311->endpoints(
             {
                 services => 'Services',
@@ -81,14 +82,14 @@ sub process_services {
 sub process_service {
     my $self = shift;
 
-    my $category = $self->_current_council->area_id == 2218 ? 
-                    $self->_current_service->{description} : 
+    my $category = $self->_current_body->areas->{2218} ?
+                    $self->_current_service->{description} :
                     $self->_current_service->{service_name};
 
     print $self->_current_service->{service_code} . ': ' . $category .  "\n" if $self->verbose >= 2;
     my $contacts = FixMyStreet::App->model( 'DB::Contact')->search(
         {
-            area_id => $self->_current_council->area_id,
+            body_id => $self->_current_body->id,
             -OR => [
                 email => $self->_current_service->{service_code},
                 category => $category,
@@ -125,7 +126,7 @@ sub _handle_existing_contact {
 
     my $service_name = $self->_normalize_service_name;
 
-    print $self->_current_council->area_id . " already has a contact for service code " . $self->_current_service->{service_code} . "\n" if $self->verbose >= 2;
+    print $self->_current_body->id . " already has a contact for service code " . $self->_current_service->{service_code} . "\n" if $self->verbose >= 2;
 
     if ( $contact->deleted || $service_name ne $contact->category || $self->_current_service->{service_code} ne $contact->email ) {
         eval {
@@ -143,7 +144,7 @@ sub _handle_existing_contact {
         };
 
         if ( $@ ) {
-            warn "Failed to update contact for service code " . $self->_current_service->{service_code} . " for council @{[$self->_current_council->area_id]}: $@\n"
+            warn "Failed to update contact for service code " . $self->_current_service->{service_code} . " for body @{[$self->_current_body->id]}: $@\n"
                 if $self->verbose >= 1;
             return;
         }
@@ -168,7 +169,7 @@ sub _create_contact {
         $contact = FixMyStreet::App->model( 'DB::Contact')->create(
             {
                 email => $self->_current_service->{service_code},
-                area_id => $self->_current_council->area_id,
+                body_id => $self->_current_body->id,
                 category => $service_name,
                 confirmed => 1,
                 deleted => 0,
@@ -180,7 +181,7 @@ sub _create_contact {
     };
 
     if ( $@ ) {
-        warn "Failed to create contact for service code " . $self->_current_service->{service_code} . " for council @{[$self->_current_council->area_id]}: $@\n"
+        warn "Failed to create contact for service code " . $self->_current_service->{service_code} . " for body @{[$self->_current_body->id]}: $@\n"
             if $self->verbose >= 1;
         return;
     }
@@ -191,7 +192,7 @@ sub _create_contact {
 
     if ( $contact ) {
         push @{ $self->found_contacts }, $self->_current_service->{service_code};
-        print "created contact for service code " . $self->_current_service->{service_code} . " for council @{[$self->_current_council->area_id]}\n" if $self->verbose >= 2;
+        print "created contact for service code " . $self->_current_service->{service_code} . " for body @{[$self->_current_body->id]}\n" if $self->verbose >= 2;
     }
 }
 
@@ -210,7 +211,7 @@ sub _add_meta_to_contact {
     if ( ! $meta_data->{attributes}->{attribute} ) {
         warn sprintf( "Empty meta data for %s at %s",
                       $self->_current_service->{service_code},
-                      $self->_current_council->endpoint )
+                      $self->_current_body->endpoint )
         if $self->verbose;
         return;
     }
@@ -225,7 +226,7 @@ sub _add_meta_to_contact {
 
     # we add these later on from bromley so don't list them here
     # as we don't want to display them
-    if ( $self->_current_council->area_id == 2482 ) {
+    if ( $self->_current_body->areas->{2482} ) {
         my %ignore = map { $_ => 1 } qw/
             service_request_id_ext
             requested_datetime
@@ -256,8 +257,8 @@ sub _normalize_service_name {
 
     # FIXME - at the moment it makes more sense to use the description
     # for cambridgeshire but need a more flexible way to set this
-    my $service_name = $self->_current_council->area_id == 2218 ? 
-                        $self->_current_service->{description} : 
+    my $service_name = $self->_current_body->areas->{2218} ?
+                        $self->_current_service->{description} :
                         $self->_current_service->{service_name};
     # remove trailing whitespace as it upsets db queries
     # to look up contact details when creating problem
@@ -272,7 +273,7 @@ sub _delete_contacts_not_in_service_list {
     my $found_contacts = FixMyStreet::App->model( 'DB::Contact')->search(
         {
             email => { -not_in => $self->found_contacts },
-            area_id => $self->_current_council->area_id,
+            body_id => $self->_current_body->id,
             deleted => 0,
         }
     );