Generate /reports data on cron.

8 files changed, 96 insertions, 9 deletions

diff --git a/bin/update-all-reports b/bin/update-all-reports
new file mode 100755
index 000000000..2263a3d9d
--- /dev/null
+++ b/bin/update-all-reports
@@ -0,0 +1,69 @@
+#!/usr/bin/perl
+
+# update-all-reports:
+# Generate the data for the /reports page
+#
+# Copyright (c) 2011 UK Citizens Online Democracy. All rights reserved.
+# Email: matthew@mysociety.org. WWW: http://www.mysociety.org
+
+use strict;
+use warnings;
+require 5.8.0;
+
+use FixMyStreet::App;
+use File::Path ();
+use File::Slurp;
+use JSON;
+use List::MoreUtils qw(zip);
+
+my $fourweeks = 4*7*24*60*60;
+
+my $problems = FixMyStreet::App->model("DB::Problem")->search(
+    {
+        state => [ 'confirmed', 'fixed' ]
+    },
+    {
+        columns => [
+            'id', 'council', 'state', 'areas',
+            { duration => { extract => "epoch from current_timestamp-lastupdate" } },
+            { age      => { extract => "epoch from current_timestamp-confirmed"  } },
+        ]
+    }
+);
+$problems = $problems->cursor; # Raw DB cursor for speed
+
+my ( %fixed, %open );
+my @cols = ( 'id', 'council', 'state', 'areas', 'duration', 'age' );
+while ( my @problem = $problems->next ) {
+    my %problem = zip @cols, @problem;
+    my @areas;
+    if ( !$problem{council} ) {
+        # Problem was not sent to any council, add to all areas
+        @areas = grep { $_ } split( /,/, $problem{areas} );
+        $problem{councils} = 0;
+    } else {
+        # Add to councils it was sent to
+        (my $council = $problem{council}) =~ s/\|.*$//;
+        @areas = split( /,/, $council );
+        $problem{councils} = scalar @areas;
+    }
+    foreach my $council ( @areas ) {
+        my $duration_str = ( $problem{duration} > 2 * $fourweeks ) ? 'old' : 'new';
+        my $type = ( $problem{duration} > 2 * $fourweeks )
+            ? 'unknown'
+            : ($problem{age} > $fourweeks ? 'older' : 'new');
+        # Fixed problems are either old or new
+        $fixed{$council}{$duration_str}++ if $problem{state} eq 'fixed';
+        # Open problems are either unknown, older, or new
+        $open{$council}{$type}++ if $problem{state} eq 'confirmed';
+    }
+}
+
+my $body = JSON->new->utf8(1)->encode( {
+    fixed => \%fixed,
+    open  => \%open,
+} );
+
+File::Path::mkpath( FixMyStreet->path_to( '../data/' )->stringify );
+File::Slurp::write_file( FixMyStreet->path_to( '../data/all-reports.json' )->stringify, \$body );
+
diff --git a/conf/crontab.ugly b/conf/crontab.ugly
index 8fc1ecf40..d1779ff73 100644
--- a/conf/crontab.ugly
+++ b/conf/crontab.ugly
@@ -27,6 +27,9 @@ MAILTO=cron-!!(*= $site *)!!@mysociety.org
 
 !!(* } *)!!
 
+# Once an hour, update the all reports stats
+13 * * * * !!(*= $user *)!! /data/vhost/!!(*= $vhost *)!!/fixmystreet/bin/cron-wrapper update-all-reports
+
 # Once a day on all servers
 39 2 * * * !!(*= $user *)!! /data/vhost/!!(*= $vhost *)!!/fixmystreet/bin/problems-filed-graph
 43 2 * * * !!(*= $user *)!! /data/vhost/!!(*= $vhost *)!!/fixmystreet/bin/problem-creation-graph
diff --git a/perllib/FixMyStreet/App/Controller/Questionnaire.pm b/perllib/FixMyStreet/App/Controller/Questionnaire.pm
index 658d32e74..5506e2dbd 100755
--- a/perllib/FixMyStreet/App/Controller/Questionnaire.pm
+++ b/perllib/FixMyStreet/App/Controller/Questionnaire.pm
@@ -44,7 +44,7 @@ sub load_questionnaire : Private {
         my $problem_url = $c->uri_for( "/report/$problem_id" );
         my $contact_url = $c->uri_for( "/contact" );
         $c->stash->{message} = sprintf(_("You have already answered this questionnaire. If you have a question, please <a href='%s'>get in touch</a>, or <a href='%s'>view your problem</a>.\n"), $contact_url, $problem_url);
-        $c->stash->{template} = 'questionnaire/error.html';
+        $c->stash->{template} = 'errors/generic.html';
         $c->detach;
     }
 
@@ -95,7 +95,7 @@ sub missing_problem : Private {
     my ( $self, $c ) = @_;
 
     $c->stash->{message} = _("I'm afraid we couldn't locate your problem in the database.\n");
-    $c->stash->{template} = 'questionnaire/error.html';
+    $c->stash->{template} = 'errors/generic.html';
 }
 
 sub submit_creator_fixed : Private {
diff --git a/perllib/FixMyStreet/App/Controller/Reports.pm b/perllib/FixMyStreet/App/Controller/Reports.pm
index 850433d3c..e1983c855 100644
--- a/perllib/FixMyStreet/App/Controller/Reports.pm
+++ b/perllib/FixMyStreet/App/Controller/Reports.pm
@@ -2,6 +2,7 @@ package FixMyStreet::App::Controller::Reports;
 use Moose;
 use namespace::autoclean;
 
+use File::Slurp;
 use List::MoreUtils qw(zip);
 use POSIX qw(strcoll);
 use mySociety::MaPit;
@@ -51,7 +52,18 @@ sub index : Path : Args(0) {
     my @keys = sort { strcoll($areas_info->{$a}{name}, $areas_info->{$b}{name}) } keys %$areas_info;
     $c->stash->{areas_info_sorted} = [ map { $areas_info->{$_} } @keys ];
 
-    $c->forward( 'load_and_group_problems' );
+    eval {
+        my $data = File::Slurp::read_file(
+            FixMyStreet->path_to( '../data/all-reports.json' )->stringify
+        );
+        my $j = JSON->new->utf8->decode($data);
+        $c->stash->{fixed} = $j->{fixed};
+        $c->stash->{open} = $j->{open};
+    };
+    if ($@) {
+        $c->stash->{message} = _("There was a problem showing the All Reports page. Please try again later.");
+        $c->stash->{template} = 'errors/generic.html';
+    }
 }
 
 =head2 index
diff --git a/perllib/FixMyStreet/App/Controller/Tokens.pm b/perllib/FixMyStreet/App/Controller/Tokens.pm
index c75fcc9ee..111508e60 100644
--- a/perllib/FixMyStreet/App/Controller/Tokens.pm
+++ b/perllib/FixMyStreet/App/Controller/Tokens.pm
@@ -158,7 +158,7 @@ sub load_questionnaire_id : Private {
     my ( $self, $c, $token_code ) = @_;
 
     # Set up error handling
-    $c->stash->{error_template} = 'questionnaire/error.html';
+    $c->stash->{error_template} = 'errors/generic.html';
     $c->stash->{message} = _("I'm afraid we couldn't validate that token. If you've copied the URL from an email, please check that you copied it exactly.\n");
 
     my $auth_token = $c->forward( 'load_auth_token', [ $token_code, 'questionnaire' ] );
diff --git a/t/app/controller/reports.t b/t/app/controller/reports.t
index 6cb12e20f..58803d778 100644
--- a/t/app/controller/reports.t
+++ b/t/app/controller/reports.t
@@ -6,6 +6,9 @@ use mySociety::MaPit;
 
 ok( my $mech = Test::WWW::Mechanize::Catalyst->new, 'Created mech object' );
 
+# Run the cron script that makes the data for /reports so we don't get an error.
+system( "bin/cron-wrapper update-all-reports" );
+
 # check that we can get the page
 $mech->get_ok('/reports');
 $mech->title_like(qr{Summary reports});
diff --git a/templates/web/default/questionnaire/error.html b/templates/web/default/errors/generic.html
index 12aa8e170..12aa8e170 100755
--- a/templates/web/default/questionnaire/error.html
+++ b/templates/web/default/errors/generic.html
diff --git a/templates/web/default/reports/index.html b/templates/web/default/reports/index.html
index bb7824cad..f14fd079e 100755
--- a/templates/web/default/reports/index.html
+++ b/templates/web/default/reports/index.html
@@ -22,11 +22,11 @@
 [%- END -%]
 >
 <td align="left"><a href="[% area.url %]">[% area.name %]</a></td>
-<td>[% open.${area.id}.new.size or 0 %]</td>
-<td>[% open.${area.id}.older.size or 0 %]</td>
-<td>[% open.${area.id}.unknown.size or 0 %]</td>
-<td>[% fixed.${area.id}.new.size or 0 %]</td>
-<td>[% fixed.${area.id}.old.size or 0 %]</td>
+<td>[% open.${area.id}.new or 0 %]</td>
+<td>[% open.${area.id}.older or 0 %]</td>
+<td>[% open.${area.id}.unknown or 0 %]</td>
+<td>[% fixed.${area.id}.new or 0 %]</td>
+<td>[% fixed.${area.id}.old or 0 %]</td>
 </tr>
 [% END %]
 </table>