More cron quietening, only show full Open311 first time, then summarise.

6 files changed, 33 insertions, 10 deletions

diff --git a/bin/fetch-comments b/bin/fetch-comments
index 4bbcc9d21..ef099fcc9 100755
--- a/bin/fetch-comments
+++ b/bin/fetch-comments
@@ -5,11 +5,13 @@ use warnings;
 require 5.8.0;
 
 use FixMyStreet::App;
+use CronFns;
+my ($verbose, $nomail) = CronFns::options();
 
 use Open311;
 use Open311::GetServiceRequestUpdates;
 
 
-my $updates = Open311::GetServiceRequestUpdates->new;
+my $updates = Open311::GetServiceRequestUpdates->new( verbose => $verbose );
 
 $updates->fetch;
diff --git a/conf/crontab.ugly b/conf/crontab.ugly
index ff0203cae..3e4767111 100644
--- a/conf/crontab.ugly
+++ b/conf/crontab.ugly
@@ -22,12 +22,15 @@ MAILTO=!!(*= $user *)!!@mysociety.org
 
 !!(* } elsif (($vhost eq 'www.fixmystreet.com') || ($vhost eq 'bromley.test.mysociety.org') || ($vhost eq 'integration-staging.fixmystreet.com') || ($vhost eq 'fixmybarangay.test.mysociety.org')) { *)!!
 5,10,15,20,25,30,35,40,45,50,55 * * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/send-reports.lock "/data/vhost/!!(*= $vhost *)!!/fixmystreet/bin/cron-wrapper send-reports" || echo "stalled?"
-0 0-8,10,11,13,14,16,17,19-23 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/send-reports.lock "/data/vhost/!!(*= $vhost *)!!/fixmystreet/bin/cron-wrapper send-reports" || echo "stalled?"
-0 9,12,15,18 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/send-reports.lock "/data/vhost/!!(*= $vhost *)!!/fixmystreet/bin/cron-wrapper send-reports --verbose" || echo "stalled?"
+0 0-7,9-11,13-15,17-23 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/send-reports.lock "/data/vhost/!!(*= $vhost *)!!/fixmystreet/bin/cron-wrapper send-reports" || echo "stalled?"
+0 8,12,16 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/send-reports.lock "/data/vhost/!!(*= $vhost *)!!/fixmystreet/bin/cron-wrapper send-reports --verbose" || echo "stalled?"
 2 * * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/send-alerts.lock "/data/vhost/!!(*= $vhost *)!!/fixmystreet/bin/cron-wrapper send-alerts" || echo "stalled?"
 0,30 * * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/send-questionnaires.lock "/data/vhost/!!(*= $vhost *)!!/fixmystreet/bin/cron-wrapper send-questionnaires" || echo "stalled?"
 5,10,15,20,25,30,35,40,45,50,55 * * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/send-comments.lock "/data/vhost/!!(*= $vhost *)!!/fixmystreet/bin/cron-wrapper send-comments" || echo "stalled?"
-5,10,15,20,25,30,35,40,45,50,55 * * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/fetch-comments.lock "/data/vhost/!!(*= $vhost *)!!/fixmystreet/bin/cron-wrapper fetch-comments" || echo "stalled?"
+
+5,10,15,20,25,30,35,40,45,50,55 0-7,9-23 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/fetch-comments.lock "/data/vhost/!!(*= $vhost *)!!/fixmystreet/bin/cron-wrapper fetch-comments" || echo "stalled?"
+10,15,20,25,30,35,40,45,50,55 8 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/fetch-comments.lock "/data/vhost/!!(*= $vhost *)!!/fixmystreet/bin/cron-wrapper fetch-comments" || echo "stalled?"
+5 8 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/fetch-comments.lock "/data/vhost/!!(*= $vhost *)!!/fixmystreet/bin/cron-wrapper fetch-comments --verbose" || echo "stalled?"
 
 30 * * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/open311-populate-service-list.lock "/data/vhost/!!(*= $vhost *)!!/fixmystreet/bin/cron-wrapper open311-populate-service-list" || echo "stalled?"
 0 0-7,9-23 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/open311-populate-service-list.lock "/data/vhost/!!(*= $vhost *)!!/fixmystreet/bin/cron-wrapper open311-populate-service-list" || echo "stalled?"
diff --git a/perllib/FixMyStreet/App/Controller/Admin.pm b/perllib/FixMyStreet/App/Controller/Admin.pm
index 2d1cf2c5a..f72478cff 100644
--- a/perllib/FixMyStreet/App/Controller/Admin.pm
+++ b/perllib/FixMyStreet/App/Controller/Admin.pm
@@ -661,7 +661,6 @@ sub report_edit : Path('report_edit') : Args(1) {
             || $flagged != $problem->flagged
             || $non_public != $problem->non_public )
         {
-            warn "edited";
             $edited = 1;
         }
 
diff --git a/perllib/FixMyStreet/DB/ResultSet/Problem.pm b/perllib/FixMyStreet/DB/ResultSet/Problem.pm
index f7f88edf0..fa9baf1d9 100644
--- a/perllib/FixMyStreet/DB/ResultSet/Problem.pm
+++ b/perllib/FixMyStreet/DB/ResultSet/Problem.pm
@@ -422,6 +422,19 @@ sub send_reports {
             }
             printf "    %-24s %4d\n", "Total:", $c;
         }
+        my $sending_errors = '';
+        my $unsent = FixMyStreet::App->model("DB::Problem")->search( {
+            state => [ 'confirmed', 'fixed' ],
+            whensent => undef,
+            council => { '!=', undef },
+            send_fail_count => { '>', 0 }
+        } );
+        while (my $row = $unsent->next) {
+            $sending_errors .= "* http://www.fixmystreet.com/report/$row->id, failed $row->send_fail_count times, last at $row->send_fail_timestamp, reason $row->send_fail_reason\n"
+        }
+        if ($sending_errors) {
+            print "The following reports had problems sending:\n$sending_errors";
+        }
     }
 }
 
diff --git a/perllib/Open311.pm b/perllib/Open311.pm
index 1027c756f..ef430d628 100644
--- a/perllib/Open311.pm
+++ b/perllib/Open311.pm
@@ -78,9 +78,11 @@ sub send_service_request {
             }
         }
 
-        warn sprintf( "Failed to submit problem %s over Open311, response\n: %s\n%s", $problem->id, $response, $self->debug_details );
+        warn sprintf( "Failed to submit problem %s over Open311, response\n: %s\n%s", $problem->id, $response, $self->debug_details )
+            unless $problem->send_fail_count;
     } else {
-        warn sprintf( "Failed to submit problem %s over Open311, details:\n%s", $problem->id, $self->error);
+        warn sprintf( "Failed to submit problem %s over Open311, details:\n%s", $problem->id, $self->error)
+            unless $problem->send_fail_count;
     }
     return 0;
 }
@@ -264,9 +266,11 @@ sub post_service_request_update {
             }
         }
 
-        warn sprintf( "Failed to submit comment %s over Open311, response - %s\n%s\n", $comment->id, $response, $self->debug_details );
+        warn sprintf( "Failed to submit comment %s over Open311, response - %s\n%s\n", $comment->id, $response, $self->debug_details )
+            unless $comment->send_fail_count;
     } else {
-        warn sprintf( "Failed to submit comment %s over Open311, details\n%s\n", $comment->id, $self->error);
+        warn sprintf( "Failed to submit comment %s over Open311, details\n%s\n", $comment->id, $self->error)
+            unless $comment->send_fail_count;
     }
     return 0;
 }
diff --git a/perllib/Open311/GetServiceRequestUpdates.pm b/perllib/Open311/GetServiceRequestUpdates.pm
index c30d05b22..6e6e3b905 100644
--- a/perllib/Open311/GetServiceRequestUpdates.pm
+++ b/perllib/Open311/GetServiceRequestUpdates.pm
@@ -10,6 +10,7 @@ has system_user => ( is => 'rw' );
 has start_date => ( is => 'ro', default => undef );
 has end_date => ( is => 'ro', default => undef );
 has suppress_alerts => ( is => 'rw', default => 0 );
+has verbose => ( is => 'ro', default => 0 );
 
 sub fetch {
     my $self = shift;
@@ -52,7 +53,8 @@ sub update_comments {
     my $requests = $open311->get_service_request_updates( @args );
 
     unless ( $open311->success ) {
-        warn "Failed to fetch ServiceRequest Updates: " . $open311->error;
+        warn "Failed to fetch ServiceRequest Updates: " . $open311->error
+            if $self->verbose;
         return 0;
     }