Merge branch 'stream-csv-output'

1 files changed, 25 insertions, 11 deletions

diff --git a/perllib/FixMyStreet/App/Controller/Dashboard.pm b/perllib/FixMyStreet/App/Controller/Dashboard.pm
index a7de23ce3..bd60f8570 100644
--- a/perllib/FixMyStreet/App/Controller/Dashboard.pm
+++ b/perllib/FixMyStreet/App/Controller/Dashboard.pm
@@ -3,6 +3,7 @@ use Moose;
 use namespace::autoclean;
 
 use DateTime;
+use Encode;
 use JSON::MaybeXS;
 use Path::Tiny;
 use Text::CSV;
@@ -409,12 +410,29 @@ hashref of extra data to include that can be used by 'columns'.
 sub generate_csv : Private {
     my ($self, $c) = @_;
 
+    my $filename = $c->stash->{csv}->{filename};
+    $c->res->content_type('text/csv; charset=utf-8');
+    $c->res->header('content-disposition' => "attachment; filename=\"${filename}.csv\"");
+
+    # Emit a header (copying Drupal's naming) telling an intermediary (e.g.
+    # Varnish) not to buffer the output. Varnish will need to know this, e.g.:
+    #   if (beresp.http.Surrogate-Control ~ "BigPipe/1.0") {
+    #     set beresp.do_stream = true;
+    #     set beresp.ttl = 0s;
+    #   }
+    $c->res->header('Surrogate-Control' => 'content="BigPipe/1.0"');
+
+    # Tell nginx not to buffer this response
+    $c->res->header('X-Accel-Buffering' => 'no');
+
+    # Define an empty body so the web view doesn't get added at the end
+    $c->res->body("");
+
     # Old parameter renaming
     $c->stash->{csv}->{objects} //= $c->stash->{csv}->{problems};
 
     my $csv = Text::CSV->new({ binary => 1, eol => "\n" });
-    $csv->combine(@{$c->stash->{csv}->{headers}});
-    my @body = ($csv->string);
+    $csv->print($c->response, $c->stash->{csv}->{headers});
 
     my $fixed_states = FixMyStreet::DB::Result::Problem->fixed_states;
     my $closed_states = FixMyStreet::DB::Result::Problem->closed_states;
@@ -468,19 +486,15 @@ sub generate_csv : Private {
             $hashref = { %$hashref, %$extra };
         }
 
-        $csv->combine(
+        $csv->print($c->response, [
+            map {
+                $_ = encode('UTF-8', $_) if $_;
+            }
             @{$hashref}{
                 @{$c->stash->{csv}->{columns}}
             },
-        );
-
-        push @body, $csv->string;
+        ] );
     }
-
-    my $filename = $c->stash->{csv}->{filename};
-    $c->res->content_type('text/csv; charset=utf-8');
-    $c->res->header('content-disposition' => "attachment; filename=\"${filename}.csv\"");
-    $c->res->body( join "", @body );
 }
 
 =head1 AUTHOR