Stream dashboard CSV output.

Rather than compile it all into on big string for output, write the
output as we go. Also output headers to encourage intermediaries to
stream as well.

1 files changed, 25 insertions, 11 deletions

diff --git a/perllib/FixMyStreet/App/Controller/Dashboard.pm b/perllib/FixMyStreet/App/Controller/Dashboard.pm
index 5bc82444d..fcd5bb748 100644
--- a/perllib/FixMyStreet/App/Controller/Dashboard.pm
+++ b/perllib/FixMyStreet/App/Controller/Dashboard.pm
@@ -3,6 +3,7 @@ use Moose;
 use namespace::autoclean;
 
 use DateTime;
+use Encode;
 use JSON::MaybeXS;
 use Path::Tiny;
 use Text::CSV;
@@ -408,12 +409,29 @@ hashref of extra data to include that can be used by 'columns'.
 sub generate_csv : Private {
     my ($self, $c) = @_;
 
+    my $filename = $c->stash->{csv}->{filename};
+    $c->res->content_type('text/csv; charset=utf-8');
+    $c->res->header('content-disposition' => "attachment; filename=\"${filename}.csv\"");
+
+    # Emit a header (copying Drupal's naming) telling an intermediary (e.g.
+    # Varnish) not to buffer the output. Varnish will need to know this, e.g.:
+    #   if (beresp.http.Surrogate-Control ~ "BigPipe/1.0") {
+    #     set beresp.do_stream = true;
+    #     set beresp.ttl = 0s;
+    #   }
+    $c->res->header('Surrogate-Control' => 'content="BigPipe/1.0"');
+
+    # Tell nginx not to buffer this response
+    $c->res->header('X-Accel-Buffering' => 'no');
+
+    # Define an empty body so the web view doesn't get added at the end
+    $c->res->body("");
+
     # Old parameter renaming
     $c->stash->{csv}->{objects} //= $c->stash->{csv}->{problems};
 
     my $csv = Text::CSV->new({ binary => 1, eol => "\n" });
-    $csv->combine(@{$c->stash->{csv}->{headers}});
-    my @body = ($csv->string);
+    $csv->print($c->response, $c->stash->{csv}->{headers});
 
     my $fixed_states = FixMyStreet::DB::Result::Problem->fixed_states;
     my $closed_states = FixMyStreet::DB::Result::Problem->closed_states;
@@ -467,19 +485,15 @@ sub generate_csv : Private {
             $hashref = { %$hashref, %$extra };
         }
 
-        $csv->combine(
+        $csv->print($c->response, [
+            map {
+                $_ = encode('UTF-8', $_) if $_;
+            }
             @{$hashref}{
                 @{$c->stash->{csv}->{columns}}
             },
-        );
-
-        push @body, $csv->string;
+        ] );
     }
-
-    my $filename = $c->stash->{csv}->{filename};
-    $c->res->content_type('text/csv; charset=utf-8');
-    $c->res->header('content-disposition' => "attachment; filename=\"${filename}.csv\"");
-    $c->res->body( join "", @body );
 }
 
 =head1 AUTHOR