Offline process for CSV generation.

Include a status page, the option for access token requests to use this
system, and a script for manual generation.

8 files changed, 308 insertions, 7 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index de9b71d38..6a27ea704 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -34,6 +34,7 @@
         - Better sort admin user table.
         - Centralise update creation to include fields.
         - Add full text index to speed up admin search.
+        - Offline process for CSV generation.
     - Development improvements:
         - `#geolocate_link` is now easier to re-style. #3006
         - Links inside `#front-main` can be customised using `$primary_link_*` Sass variables. #3007
diff --git a/bin/csv-export b/bin/csv-export
new file mode 100755
index 000000000..29ca3388f
--- /dev/null
+++ b/bin/csv-export
@@ -0,0 +1,77 @@
+#!/usr/bin/env perl
+
+# csv-export
+# Offline creation of CSV export, first take
+
+use v5.14;
+use warnings;
+
+BEGIN {
+    use File::Basename qw(dirname);
+    use File::Spec;
+    my $d = dirname(File::Spec->rel2abs($0));
+    require "$d/../setenv.pl";
+}
+
+use open ':std', ':encoding(UTF-8)';
+use Getopt::Long::Descriptive;
+use Path::Tiny;
+use CronFns;
+use FixMyStreet::Cobrand;
+use FixMyStreet::DB;
+use FixMyStreet::Reporting;
+
+my $site = CronFns::site(FixMyStreet->config('BASE_URL'));
+CronFns::language($site);
+
+my ($opts, $usage) = describe_options(
+    '%c %o',
+    ['cobrand=s', 'which cobrand is asking for the data', { required => 1 }],
+    ['type=s', 'whether to export problems or updates', { required => 1 }],
+    ['out=s', 'where to output CSV data'],
+
+    ['body=i', 'Body ID to restrict export to'],
+    ['wards=s', 'Ward area IDs to restrict export to'],
+    ['category=s', 'Category to restrict export to'],
+    ['state=s', 'State to restrict export to'],
+    ['start_date=s', 'Start date for export (default 30 days ago)'],
+    ['end_date=s', 'End date for export'],
+
+    ['user=i', 'user ID which requested this export'],
+    ['verbose|v', 'more verbose output'],
+    ['help|h', "print usage message and exit" ],
+);
+$usage->die if $opts->help;
+
+my $use_stdout = !$opts->out || $opts->out eq '-';
+my ($file, $fh);
+if ($use_stdout) {
+    $fh = *STDOUT;
+} else {
+    $file = path($opts->out . '-part');
+    $fh = $file->openw_utf8;
+}
+
+my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($opts->cobrand);
+FixMyStreet::DB->schema->cobrand($cobrand);
+
+my $user = FixMyStreet::DB->resultset("User")->find($opts->user) if $opts->user;
+my $body = FixMyStreet::DB->resultset("Body")->find($opts->body) if $opts->body;
+my $wards = $opts->wards ? [split',', $opts->wards] : [];
+
+my $reporting = FixMyStreet::Reporting->new(
+    type => $opts->type,
+    user => $user,
+    category => $opts->category,
+    state => $opts->state,
+    wards => $wards,
+    body => $body,
+    $opts->start_date ? (start_date => $opts->start_date) : (),
+    end_date => $opts->end_date,
+);
+$reporting->construct_rs_filter;
+$reporting->csv_parameters;
+$reporting->generate_csv($fh);
+unless ($use_stdout) {
+    $file->move($opts->out);
+}
diff --git a/perllib/Catalyst/Authentication/Credential/AccessToken.pm b/perllib/Catalyst/Authentication/Credential/AccessToken.pm
index 24398823d..39364ad99 100644
--- a/perllib/Catalyst/Authentication/Credential/AccessToken.pm
+++ b/perllib/Catalyst/Authentication/Credential/AccessToken.pm
@@ -15,12 +15,18 @@ sub new {
     return $self;
 }
 
-sub authenticate {
-    my ( $self, $c, $realm, $authinfo_ignored ) = @_;
-
+sub get_token {
+    my ($self, $c) = @_;
     my $auth_header = $c->req->header('Authorization') || '';
     my ($token) = $auth_header =~ /^Bearer (.*)/i;
     $token ||= $c->get_param('access_token');
+    return $token;
+}
+
+sub authenticate {
+    my ( $self, $c, $realm, $authinfo_ignored ) = @_;
+
+    my $token = $self->get_token($c);
     return unless $token;
 
     my $id;
diff --git a/perllib/FixMyStreet/App/Controller/Dashboard.pm b/perllib/FixMyStreet/App/Controller/Dashboard.pm
index 0d0a704bb..5400a6209 100644
--- a/perllib/FixMyStreet/App/Controller/Dashboard.pm
+++ b/perllib/FixMyStreet/App/Controller/Dashboard.pm
@@ -137,9 +137,24 @@ sub index : Path : Args(0) {
 
     my $reporting = $c->forward('construct_rs_filter', [ $c->get_param('updates') ]);
 
-    if ( $c->get_param('export') ) {
+    if ( my $export = $c->get_param('export') ) {
         $reporting->csv_parameters;
-        $reporting->generate_csv_http($c);
+        if ($export == 1) {
+            # Existing method, generate and serve
+            $reporting->generate_csv_http($c);
+        } elsif ($export == 2) {
+            # New offline method
+            $reporting->kick_off_process;
+            my ($redirect, $code) = ('/dashboard/status', 303);
+            if (Catalyst::Authentication::Credential::AccessToken->get_token($c)) {
+                # Client knows to re-request until ready
+                $redirect = '/dashboard/csv/' . $reporting->filename . '.csv';
+                $c->res->body('');
+                $code = 202;
+            }
+            $c->res->redirect($redirect, $code);
+            $c->detach;
+        }
     } else {
         $c->forward('generate_grouped_data');
         $self->generate_summary_figures($c);
@@ -276,6 +291,62 @@ sub generate_summary_figures {
     }
 }
 
+sub status : Local : Args(0) {
+    my ($self, $c) = @_;
+
+    my $body = $c->stash->{body} = $c->forward('check_page_allowed');
+    $c->stash->{body_name} = $body->name if $body;
+
+    my $reporting = FixMyStreet::Reporting->new(
+        user => $c->user_exists ? $c->user->obj : undef,
+    );
+    my $dir = $reporting->cache_dir;
+    my @data;
+    foreach ($dir->children) {
+        my $stat = $_->stat;
+        my $name = $_->basename;
+        my $finished = $name =~ /part$/ ? 0 : 1;
+        $name =~ s/-part$//;
+        push @data, {
+            ctime => $stat->ctime,
+            size => $stat->size,
+            name => $name,
+            finished => $finished,
+        };
+    }
+    @data = sort { $b->{ctime} <=> $a->{ctime} } @data;
+    $c->stash->{rows} = \@data;
+}
+
+sub csv : Local : Args(1) {
+    my ($self, $c, $filename) = @_;
+
+    $c->authenticate(undef, "access_token");
+
+    my $body = $c->stash->{body} = $c->forward('check_page_allowed');
+
+    (my $basename = $filename) =~ s/\.csv$//;
+    my $reporting = FixMyStreet::Reporting->new(
+        user => $c->user_exists ? $c->user->obj : undef,
+        filename => $basename,
+    );
+    my $dir = $reporting->cache_dir;
+    my $csv = path($dir, $filename);
+
+    if (!$csv->exists) {
+        if (path($dir, "$filename-part")->exists && Catalyst::Authentication::Credential::AccessToken->get_token($c)) {
+            $c->res->body('');
+            $c->res->status(202);
+            $c->detach;
+        } else {
+            $c->detach( '/page_error_404_not_found', [] ) unless $csv->exists;
+        }
+    }
+
+    $reporting->http_setup($c);
+    $c->res->body($csv->openr_raw);
+}
+
 sub generate_body_response_time : Private {
     my ( $self, $c ) = @_;
 
diff --git a/perllib/FixMyStreet/Reporting.pm b/perllib/FixMyStreet/Reporting.pm
index 7ee9fc41a..08cdf1544 100644
--- a/perllib/FixMyStreet/Reporting.pm
+++ b/perllib/FixMyStreet/Reporting.pm
@@ -2,6 +2,7 @@ package FixMyStreet::Reporting;
 
 use DateTime;
 use Moo;
+use Path::Tiny;
 use Text::CSV;
 use Types::Standard qw(ArrayRef CodeRef Enum HashRef InstanceOf Int Maybe Str);
 use FixMyStreet::DB;
@@ -305,6 +306,46 @@ sub generate_csv {
 
 # Output code
 
+sub cache_dir {
+    my $self = shift;
+
+    my $cfg = FixMyStreet->config('PHOTO_STORAGE_OPTIONS');
+    my $dir = $cfg ? $cfg->{UPLOAD_DIR} : FixMyStreet->config('UPLOAD_DIR');
+    $dir = path($dir, "dashboard_csv")->absolute(FixMyStreet->path_to());
+    my $subdir = $self->user ? $self->user->id : 0;
+    $dir = $dir->child($subdir);
+    $dir->mkpath;
+    $dir;
+}
+
+sub kick_off_process {
+    my $self = shift;
+
+    return $self->_process if FixMyStreet->test_mode;
+
+    my $pid = fork;
+    unless ($pid) {
+        unless (fork) {
+            # eval so that it will definitely exit cleanly. Otherwise, an
+            # exception would turn this grandchild into a zombie app process
+            eval { $self->_process };
+            exit 0;
+        }
+        exit 0;
+    }
+    waitpid($pid, 0);
+}
+
+sub _process {
+    my $self = shift;
+    my $out = path($self->cache_dir, $self->filename . '.csv');
+    my $file = path($out . '-part');
+    if (!$file->exists) {
+        $self->generate_csv($file->openw_utf8);
+        $file->move($out);
+    }
+}
+
 # Outputs relevant CSV HTTP headers, and then streams the CSV
 sub generate_csv_http {
     my ($self, $c) = @_;
diff --git a/t/app/controller/dashboard.t b/t/app/controller/dashboard.t
index 526b1a4c3..fd491b540 100644
--- a/t/app/controller/dashboard.t
+++ b/t/app/controller/dashboard.t
@@ -20,6 +20,8 @@ use strict;
 use warnings;
 
 use FixMyStreet::TestMech;
+use File::Temp 'tempdir';
+use Path::Tiny;
 use Web::Scraper;
 
 set_absolute_time('2014-02-01T12:00:00');
@@ -81,10 +83,15 @@ my $categories = scraper {
     },
 };
 
+my $UPLOAD_DIR = tempdir( CLEANUP => 1 );
+
 FixMyStreet::override_config {
     ALLOWED_COBRANDS => 'no2fa',
     COBRAND_FEATURES => { category_groups => { no2fa => 1 } },
     MAPIT_URL => 'http://mapit.uk/',
+    PHOTO_STORAGE_OPTIONS => {
+        UPLOAD_DIR => $UPLOAD_DIR,
+    },
 }, sub {
 
     subtest 'not logged in, redirected to login' => sub {
@@ -252,7 +259,37 @@ FixMyStreet::override_config {
         like $mech->res->header('Content-type'), qr'text/csv';
         $mech->content_contains('Report ID');
         $mech->delete_header('Authorization');
+
+        my $token = 'access_token=' . $counciluser->id . '-1234567890abcdefgh';
+        $mech->get_ok("/dashboard?export=2&$token");
+        is $mech->res->code, 202;
+        my $loc = $mech->res->header('Location');
+        like $loc, qr{/dashboard/csv/.*\.csv$};
+        $mech->get_ok("$loc?$token");
+        like $mech->res->header('Content-type'), qr'text/csv';
+        $mech->content_contains('Report ID');
     };
+
+    subtest 'view status page' => sub {
+        # Simulate a partly done file
+        my $f = Path::Tiny->tempfile(SUFFIX => '.csv-part', DIR => path($UPLOAD_DIR, 'dashboard_csv', $counciluser->id));
+        (my $name = $f->basename) =~ s/-part$//;;
+
+        my $token = 'access_token=' . $counciluser->id . '-1234567890abcdefgh';
+        $mech->get_ok("/dashboard/csv/$name?$token");
+        is $mech->res->code, 202;
+
+        $mech->log_in_ok( $counciluser->email );
+        $mech->get_ok('/dashboard/status');
+        $mech->content_contains('/dashboard/csv/www.example.org-body-' . $body->id . '-start_date-2014-01-02.csv');
+        $mech->content_like(qr/$name\s*<br>0KB\s*<i>In progress/);
+
+        $f->remove;
+        $mech->get_ok('/dashboard/status');
+        $mech->content_contains('/dashboard/csv/www.example.org-body-' . $body->id . '-start_date-2014-01-02.csv');
+        $mech->content_lacks('In progress');
+        $mech->content_lacks('setTimeout');
+    }
 };
 
 FixMyStreet::override_config {
diff --git a/templates/web/base/dashboard/index.html b/templates/web/base/dashboard/index.html
index 5ac414bde..6a3075c61 100644
--- a/templates/web/base/dashboard/index.html
+++ b/templates/web/base/dashboard/index.html
@@ -105,8 +105,8 @@
     <li>[% INCLUDE gb new_gb='device+site' text=loc('Device and Site') %]</li>
     <li class="pull-right">
         <span>[% loc('Export as CSV') %]:</span>
-        <a href="[% c.uri_with({ export => 1 }) %]">[% loc('Reports') %]</a>
-        <a href="[% c.uri_with({ export => 1, updates => 1 }) %]">[% loc('Updates') %]</a>
+        <a href="[% c.uri_with({ export => 2 }) %]">[% loc('Reports') %]</a>
+        <a href="[% c.uri_with({ export => 2, updates => 1 }) %]">[% loc('Updates') %]</a>
     </li>
 </ul>
 
diff --git a/templates/web/base/dashboard/status.html b/templates/web/base/dashboard/status.html
new file mode 100644
index 000000000..734bb9ad3
--- /dev/null
+++ b/templates/web/base/dashboard/status.html
@@ -0,0 +1,68 @@
+[% USE date %]
+[% IF NOT c.get_param('ajax') %]
+[% INCLUDE 'header.html'
+    title = loc('Dashboard')
+    robots = 'noindex, nofollow'
+    bodyclass = 'fullwidthpage';
+%]
+
+[% IF body %]
+<hgroup>
+    [% tprintf(loc('<h2>Reports, Statistics and Actions for</h2> <h1>%s</h1>'), body_name) %]
+</hgroup>
+[% ELSE %]
+<h1>[% loc('Summary statistics') %]</h1>
+[% END %]
+
+<p><a href="[% c.uri_for_action('dashboard/index') %]">[% loc('Back') %]</a></p>
+
+[% END %]
+
+<table id="overview" cellpadding=8 cellspacing=0>
+    <tr>
+        <th scope="col">[% loc('Created') %]</th>
+        <th scope="col">[% loc('CSV File') %]</th>
+    </tr>
+  [% in_progress = 0 %]
+  [% FOR file IN rows %]
+    <tr>
+        <td>[% date.format(file.ctime, format = '%Y-%m-%d %H:%M') %]</td>
+        <td>
+            [% IF file.finished %]
+                <a href="/dashboard/csv/[% file.name %]">[% file.name %]</a>
+                <br>[% file.size div 1024 %]KB
+            [% ELSE %]
+                [% file.name %]
+                <br>[% file.size div 1024 %]KB
+                <i>[% loc('In progress') %]</i>
+                [% in_progress = 1 %]
+            [% END %]
+        </td>
+    </tr>
+  [% END %]
+</table>
+
+[% IF NOT c.get_param('ajax') %]
+
+[% IF in_progress %]
+<script nonce="[% csp_nonce %]">
+(function() {
+    var wait = 1;
+    setTimeout(function refresh() {
+        $('#overview').load('[% c.uri_for_action('dashboard/status') %]?ajax=1', function() {
+            if ($(this).html().indexOf('<i>[% loc('In progress', "JS") %]</i>') === -1) {
+                return;
+            }
+            wait += 1;
+            if (wait > 10) {
+                wait = 10;
+            }
+            setTimeout(refresh, wait * 1000);
+        });
+    }, wait * 1000);
+})();
+</script>
+[% END %]
+
+[% INCLUDE 'footer.html' %]
+[% END %]