Merge remote branch 'origin/master' into fmb-read-only

Conflicts:
	bin/send-reports
	perllib/FixMyStreet/Cobrand/Default.pm
	perllib/FixMyStreet/Cobrand/FixMyStreet.pm
	templates/web/fixmystreet/alert/index.html
	templates/web/fixmystreet/around/display_location.html
	web/cobrands/fixmystreet/_layout.scss
	web/js/map-OpenLayers.js

6 files changed, 313 insertions, 8 deletions

diff --git a/perllib/FixMyStreet/DB/Result/Comment.pm b/perllib/FixMyStreet/DB/Result/Comment.pm
index 5b45c63a8..91695d7d0 100644
--- a/perllib/FixMyStreet/DB/Result/Comment.pm
+++ b/perllib/FixMyStreet/DB/Result/Comment.pm
@@ -54,6 +54,18 @@ __PACKAGE__->add_columns(
   { data_type => "boolean", default_value => \"false", is_nullable => 0 },
   "problem_state",
   { data_type => "text", is_nullable => 1 },
+  "external_id",
+  { data_type => "text", is_nullable => 1 },
+  "extra",
+  { data_type => "text", is_nullable => 1 },
+  "send_fail_count",
+  { data_type => "integer", default_value => 0, is_nullable => 0 },
+  "send_fail_reason",
+  { data_type => "text", is_nullable => 1 },
+  "send_fail_timestamp",
+  { data_type => "timestamp", is_nullable => 1 },
+  "whensent",
+  { data_type => "timestamp", is_nullable => 1 },
 );
 __PACKAGE__->set_primary_key("id");
 __PACKAGE__->belongs_to(
@@ -70,13 +82,34 @@ __PACKAGE__->belongs_to(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07017 @ 2012-03-08 17:19:55
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:E+96vo/AB0zz1jAEPj/OKw
+# Created by DBIx::Class::Schema::Loader v0.07017 @ 2012-03-26 15:44:18
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:nvkElEgSU6XcLd9znSqhmQ
+
+__PACKAGE__->filter_column(
+    extra => {
+        filter_from_storage => sub {
+            my $self = shift;
+            my $ser  = shift;
+            return undef unless defined $ser;
+            my $h = new IO::String($ser);
+            return RABX::wire_rd($h);
+        },
+        filter_to_storage => sub {
+            my $self = shift;
+            my $data = shift;
+            my $ser  = '';
+            my $h    = new IO::String($ser);
+            RABX::wire_wr( $data, $h );
+            return $ser;
+        },
+    }
+);
 
 use DateTime::TimeZone;
 use Image::Size;
 use Moose;
 use namespace::clean -except => [ 'meta' ];
+use RABX;
 
 with 'FixMyStreet::Roles::Abuser';
 
diff --git a/perllib/FixMyStreet/DB/Result/Open311conf.pm b/perllib/FixMyStreet/DB/Result/Open311conf.pm
index 742a12ebd..c95b0c8f2 100644
--- a/perllib/FixMyStreet/DB/Result/Open311conf.pm
+++ b/perllib/FixMyStreet/DB/Result/Open311conf.pm
@@ -28,13 +28,30 @@ __PACKAGE__->add_columns(
   { data_type => "text", is_nullable => 1 },
   "send_method",
   { data_type => "text", is_nullable => 1 },
+  "send_comments",
+  { data_type => "boolean", default_value => \"false", is_nullable => 0 },
+  "comment_user_id",
+  { data_type => "integer", is_foreign_key => 1, is_nullable => 1 },
+  "suppress_alerts",
+  { data_type => "boolean", default_value => \"false", is_nullable => 0 },
 );
 __PACKAGE__->set_primary_key("id");
 __PACKAGE__->add_unique_constraint("open311conf_area_id_key", ["area_id"]);
+__PACKAGE__->belongs_to(
+  "comment_user",
+  "FixMyStreet::DB::Result::User",
+  { id => "comment_user_id" },
+  {
+    is_deferrable => 1,
+    join_type     => "LEFT",
+    on_delete     => "CASCADE",
+    on_update     => "CASCADE",
+  },
+);
 
 
-# Created by DBIx::Class::Schema::Loader v0.07017 @ 2012-03-08 17:19:55
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ClYnPB2gsKapnfHuco5d/w
+# Created by DBIx::Class::Schema::Loader v0.07017 @ 2012-05-11 13:30:31
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ByJbRe/Y/9Z1WHdG8kaIHg
 
 
 # You can replace this text with custom code or comments, and it will be preserved on regeneration
diff --git a/perllib/FixMyStreet/DB/Result/Problem.pm b/perllib/FixMyStreet/DB/Result/Problem.pm
index 192e539bd..9cbaef6c2 100644
--- a/perllib/FixMyStreet/DB/Result/Problem.pm
+++ b/perllib/FixMyStreet/DB/Result/Problem.pm
@@ -85,7 +85,7 @@ __PACKAGE__->add_columns(
   "geocode",
   { data_type => "bytea", is_nullable => 1 },
   "send_fail_count",
-  { data_type => "integer", default_value => 0, is_nullable => 0 },
+  { data_type => "integer", is_nullable => 1 },
   "send_fail_reason",
   { data_type => "text", is_nullable => 1 },
   "send_fail_timestamp",
@@ -578,7 +578,7 @@ sub body {
 #     Note:   this only makes sense when called on a problem that has been sent!
 sub can_display_external_id {
     my $self = shift;
-    if ($self->external_id && $self->send_method_used eq 'barnet') {
+    if ($self->external_id && $self->send_method_used && $self->send_method_used eq 'barnet') {
         return 1;
     }
     return 0;    
@@ -704,6 +704,17 @@ sub update_from_open311_service_request {
     return 1;
 }
 
+sub update_send_failed {
+    my $self = shift;
+    my $msg  = shift;
+
+    $self->update( {
+        send_fail_count => $self->send_fail_count + 1,
+        send_fail_timestamp => \'ms_current_timestamp()',
+        send_fail_reason => $msg
+    } );
+}
+
 # we need the inline_constructor bit as we don't inherit from Moose
 __PACKAGE__->meta->make_immutable( inline_constructor => 0 );
 
diff --git a/perllib/FixMyStreet/DB/Result/User.pm b/perllib/FixMyStreet/DB/Result/User.pm
index e13d88b88..7f43d1a52 100644
--- a/perllib/FixMyStreet/DB/Result/User.pm
+++ b/perllib/FixMyStreet/DB/Result/User.pm
@@ -30,6 +30,8 @@ __PACKAGE__->add_columns(
   { data_type => "integer", is_nullable => 1 },
   "flagged",
   { data_type => "boolean", default_value => \"false", is_nullable => 0 },
+  "title",
+  { data_type => "text", is_nullable => 1 },
 );
 __PACKAGE__->set_primary_key("id");
 __PACKAGE__->add_unique_constraint("users_email_key", ["email"]);
@@ -46,6 +48,12 @@ __PACKAGE__->has_many(
   { cascade_copy => 0, cascade_delete => 0 },
 );
 __PACKAGE__->has_many(
+  "open311confs",
+  "FixMyStreet::DB::Result::Open311conf",
+  { "foreign.comment_user_id" => "self.id" },
+  { cascade_copy => 0, cascade_delete => 0 },
+);
+__PACKAGE__->has_many(
   "problems",
   "FixMyStreet::DB::Result::Problem",
   { "foreign.user_id" => "self.id" },
@@ -53,8 +61,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07017 @ 2012-03-08 17:19:55
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:tM1LUGrqDeQnF4BDgnYXGQ
+# Created by DBIx::Class::Schema::Loader v0.07017 @ 2012-05-01 16:20:29
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:LKi8u5IYnHW1+Mez64nvGg
 
 __PACKAGE__->add_columns(
     "password" => {
@@ -172,4 +180,22 @@ sub belongs_to_council {
     return 0;
 }
 
+=head2 split_name
+
+    $name = $user->split_name;
+    printf( 'Welcome %s %s', $name->{first}, $name->{last} );
+
+Returns a hashref with first and last keys with first name(s) and last name.
+NB: the spliting algorithm is extremely basic.
+
+=cut
+
+sub split_name {
+    my $self = shift;
+
+    my ($first, $last) = $self->name =~ /^(\S*)(?: (.*))?$/;
+
+    return { first => $first || '', last => $last || '' };
+}
+
 1;
diff --git a/perllib/FixMyStreet/DB/ResultSet/Problem.pm b/perllib/FixMyStreet/DB/ResultSet/Problem.pm
index 3557e77c7..a7738becf 100644
--- a/perllib/FixMyStreet/DB/ResultSet/Problem.pm
+++ b/perllib/FixMyStreet/DB/ResultSet/Problem.pm
@@ -4,6 +4,16 @@ use base 'DBIx::Class::ResultSet';
 use strict;
 use warnings;
 
+use CronFns;
+
+use Utils;
+use mySociety::Config;
+use mySociety::EmailUtil;
+use mySociety::MaPit;
+
+use FixMyStreet::App;
+use FixMyStreet::SendReport;
+
 my $site_restriction;
 my $site_key;
 
@@ -209,4 +219,209 @@ sub categories_summary {
     return \%categories;
 }
 
+sub send_reports {
+    # Set up site, language etc.
+    my ($verbose, $nomail) = CronFns::options();
+    my $base_url = mySociety::Config::get('BASE_URL');
+    my $site = CronFns::site($base_url);
+
+    my $unsent = FixMyStreet::App->model("DB::Problem")->search( {
+        state => [ 'confirmed', 'fixed' ],
+        whensent => undef,
+        council => { '!=', undef },
+    } );
+    my (%notgot, %note);
+
+    my $send_report = FixMyStreet::SendReport->new();
+    my $senders = $send_report->get_senders;
+    my %sending_skipped_by_method;
+
+    while (my $row = $unsent->next) {
+
+        my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($row->cobrand)->new();
+
+        # Cobranded and non-cobranded messages can share a database. In this case, the conf file 
+        # should specify a vhost to send the reports for each cobrand, so that they don't get sent 
+        # more than once if there are multiple vhosts running off the same database. The email_host
+        # call checks if this is the host that sends mail for this cobrand.
+        next unless $cobrand->email_host();
+        $cobrand->set_lang_and_domain($row->lang, 1);
+        if ( $row->is_from_abuser ) {
+            $row->update( { state => 'hidden' } );
+            next;
+        }
+
+        # Template variables for the email
+        my $email_base_url = $cobrand->base_url_for_emails($row->cobrand_data);
+        my %h = map { $_ => $row->$_ } qw/id title detail name category latitude longitude used_map/;
+        map { $h{$_} = $row->user->$_ } qw/email phone/;
+        $h{confirmed} = DateTime::Format::Pg->format_datetime( $row->confirmed->truncate (to => 'second' ) );
+
+        $h{query} = $row->postcode;
+        $h{url} = $email_base_url . '/report/' . $row->id;
+        $h{phone_line} = $h{phone} ? _('Phone:') . " $h{phone}\n\n" : '';
+        if ($row->photo) {
+            $h{has_photo} = _("This web page also contains a photo of the problem, provided by the user.") . "\n\n";
+            $h{image_url} = $email_base_url . '/photo/' . $row->id . '.full.jpeg';
+        } else {
+            $h{has_photo} = '';
+            $h{image_url} = '';
+        }
+        $h{fuzzy} = $row->used_map ? _('To view a map of the precise location of this issue')
+            : _('The user could not locate the problem on a map, but to see the area around the location they entered');
+        $h{closest_address} = '';
+
+        # If we are in the UK include eastings and northings, and nearest stuff
+        $h{easting_northing} = '';
+        if ( $cobrand->country eq 'GB' ) {
+
+            ( $h{easting}, $h{northing} ) = Utils::convert_latlon_to_en( $h{latitude}, $h{longitude} );
+
+            # email templates don't have conditionals so we need to farmat this here
+            $h{easting_northing}                             #
+              = "Easting: $h{easting}\n\n"                   #
+              . "Northing: $h{northing}\n\n";
+
+        }
+
+        if ( $row->used_map ) {
+            $h{closest_address} = $cobrand->find_closest( $h{latitude}, $h{longitude}, $row );
+        }
+
+        my %reporters = ();
+        my (@to, @recips, $template, $areas_info, $sender_count );
+        if ($site eq 'emptyhomes') {
+
+            my $council = $row->council;
+            $areas_info = mySociety::MaPit::call('areas', $council);
+            my $name = $areas_info->{$council}->{name};
+            my $sender = "FixMyStreet::SendReport::EmptyHomes";
+            $reporters{ $sender } = $sender->new() unless $reporters{$sender};
+            $reporters{ $sender }->add_council( $council, $name );
+            $template = Utils::read_file("$FindBin::Bin/../templates/email/emptyhomes/" . $row->lang . "/submit.txt");
+
+        } else {
+
+            # XXX Needs locks!
+            my @all_councils = split /,|\|/, $row->council;
+            my ($councils, $missing) = $row->council =~ /^([\d,]+)(?:\|([\d,]+))?/;
+            my @councils = split(/,/, $councils);
+            $areas_info = mySociety::MaPit::call('areas', \@all_councils);
+            my @dear;
+
+            foreach my $council (@councils) {
+                my $name = $areas_info->{$council}->{name};
+
+                my $sender = $cobrand->get_council_sender( $council, $areas_info->{$council} );
+                $sender = "FixMyStreet::SendReport::$sender";
+
+                if ( ! exists $senders->{ $sender } ) {
+                    warn "No such sender [ $sender ] for council $name ( $council )";
+                    next;
+                }
+                $reporters{ $sender } ||= $sender->new();
+
+                if ( $reporters{ $sender }->should_skip( $row ) ) {
+                    $sending_skipped_by_method{ $sender }++ if 
+                        $reporters{ $sender }->skipped;
+                } else {
+                    push @dear, $name;
+                    $reporters{ $sender }->add_council( $council, $name );
+                }
+            }
+
+            $template = 'submit.txt';
+            $template = 'submit-brent.txt' if $row->council eq 2488 || $row->council eq 2237;
+            my $template_path = FixMyStreet->path_to( "templates", "email", $cobrand->moniker, $template )->stringify;
+            $template_path = FixMyStreet->path_to( "templates", "email", "default", $template )->stringify
+                unless -e $template_path;
+            $template = Utils::read_file( $template_path );
+
+            if ($h{category} eq _('Other')) {
+                $h{category_footer} = _('this type of local problem');
+                $h{category_line} = '';
+            } else {
+                $h{category_footer} = "'" . $h{category} . "'";
+                $h{category_line} = sprintf(_("Category: %s"), $h{category}) . "\n\n";
+            }
+
+            $h{councils_name} = join(_(' and '), @dear);
+            if ($h{category} eq _('Other')) {
+                $h{multiple} = @dear>1 ? "[ " . _("This email has been sent to both councils covering the location of the problem, as the user did not categorise it; please ignore it if you're not the correct council to deal with the issue, or let us know what category of problem this is so we can add it to our system.") . " ]\n\n"
+                    : '';
+            } else {
+                $h{multiple} = @dear>1 ? "[ " . _("This email has been sent to several councils covering the location of the problem, as the category selected is provided for all of them; please ignore it if you're not the correct council to deal with the issue.") . " ]\n\n"
+                    : '';
+            }
+            $h{missing} = ''; 
+            if ($missing) {
+                my $name = $areas_info->{$missing}->{name};
+                $h{missing} = '[ '
+                  . sprintf(_('We realise this problem might be the responsibility of %s; however, we don\'t currently have any contact details for them. If you know of an appropriate contact address, please do get in touch.'), $name)
+                  . " ]\n\n";
+            }
+
+            $sender_count = scalar @dear;
+        }
+
+        unless ( keys %reporters ) {
+            die 'Report not going anywhere for ID ' . $row->id . '!';
+        }
+
+        next unless $sender_count;
+
+        if (mySociety::Config::get('STAGING_SITE')) {
+            # on a staging server send emails to ourselves rather than the councils
+            my @testing_councils = split( '\|', mySociety::Config::get('TESTING_COUNCILS') );
+            unless ( grep { $row->council eq $_ } @testing_councils ) {
+                %reporters = (
+                    'FixMyStreet::SendReport::Email' => $reporters{ 'FixMyStreet::SendReport::Email' } || FixMyStreet::SendReport::Email->new()
+                );
+            }
+        }
+
+        # Multiply results together, so one success counts as a success.
+        my $result = -1;
+
+        for my $sender ( keys %reporters ) {
+            $result *= $reporters{ $sender }->send(
+                $row, \%h, \@to, $template, \@recips, $nomail, $areas_info
+            );
+        }
+
+        if ($result == mySociety::EmailUtil::EMAIL_SUCCESS) {
+            $row->update( {
+                whensent => \'ms_current_timestamp()',
+                lastupdate => \'ms_current_timestamp()',
+            } );
+        } else {
+            my @errors;
+            for my $sender ( keys %reporters ) {
+                unless ( $reporters{ $sender }->success ) {
+                    push @errors, $reporters{ $sender }->error;
+                }
+            }
+            $row->update_send_failed( join( '|', @errors ) );
+        }
+    }
+
+    if ($verbose) {
+        print "Council email addresses that need checking:\n" if keys %notgot;
+        foreach my $e (keys %notgot) {
+            foreach my $c (keys %{$notgot{$e}}) {
+                print $notgot{$e}{$c} . " problem, to $e category $c (" . $note{$e}{$c}. ")\n";
+            }
+        }
+        if (keys %sending_skipped_by_method) {
+            my $c = 0;
+            print "\nProblem reports that send-reports did not attempt to send the following:\n";
+            foreach my $send_method (sort keys %sending_skipped_by_method) {
+                printf "    %-24s %4d\n", "$send_method:", $sending_skipped_by_method{$send_method};
+                $c+=$sending_skipped_by_method{$send_method};
+            }
+            printf "    %-24s %4d\n", "Total:", $c;
+        }
+    }
+}
+
 1;
diff --git a/perllib/FixMyStreet/DB/ResultSet/Questionnaire.pm b/perllib/FixMyStreet/DB/ResultSet/Questionnaire.pm
index 0cf01b6d1..753eb2084 100644
--- a/perllib/FixMyStreet/DB/ResultSet/Questionnaire.pm
+++ b/perllib/FixMyStreet/DB/ResultSet/Questionnaire.pm
@@ -48,6 +48,9 @@ sub send_questionnaires_period {
         my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($row->cobrand)->new();
         $cobrand->set_lang_and_domain($row->lang, 1);
 
+        # Not all cobrands send questionnaires
+        next unless $cobrand->send_questionnaires;
+
         # Cobranded and non-cobranded messages can share a database. In this case, the conf file 
         # should specify a vhost to send the reports for each cobrand, so that they don't get sent 
         # more than once if there are multiple vhosts running off the same database. The email_host