diff options
Diffstat (limited to 'perllib/FixMyStreet/App/Controller/Dashboard.pm')
| -rw-r--r-- | perllib/FixMyStreet/App/Controller/Dashboard.pm | 320 |
1 files changed, 90 insertions, 230 deletions
diff --git a/perllib/FixMyStreet/App/Controller/Dashboard.pm b/perllib/FixMyStreet/App/Controller/Dashboard.pm index ad6c9ba98..5400a6209 100644 --- a/perllib/FixMyStreet/App/Controller/Dashboard.pm +++ b/perllib/FixMyStreet/App/Controller/Dashboard.pm @@ -6,9 +6,9 @@ use DateTime; use Encode; use JSON::MaybeXS; use Path::Tiny; -use Text::CSV; use Time::Piece; use FixMyStreet::DateRange; +use FixMyStreet::Reporting; BEGIN { extends 'Catalyst::Controller'; } @@ -135,13 +135,25 @@ sub index : Path : Args(0) { $c->stash->{end_date} = $c->get_param('end_date'); $c->stash->{q_state} = $c->get_param('state') || ''; - $c->forward('construct_rs_filter', [ $c->get_param('updates') ]); - - if ( $c->get_param('export') ) { - if ($c->get_param('updates')) { - $c->forward('export_as_csv_updates'); - } else { - $c->forward('export_as_csv'); + my $reporting = $c->forward('construct_rs_filter', [ $c->get_param('updates') ]); + + if ( my $export = $c->get_param('export') ) { + $reporting->csv_parameters; + if ($export == 1) { + # Existing method, generate and serve + $reporting->generate_csv_http($c); + } elsif ($export == 2) { + # New offline method + $reporting->kick_off_process; + my ($redirect, $code) = ('/dashboard/status', 303); + if (Catalyst::Authentication::Credential::AccessToken->get_token($c)) { + # Client knows to re-request until ready + $redirect = '/dashboard/csv/' . $reporting->filename . '.csv'; + $c->res->body(''); + $code = 202; + } + $c->res->redirect($redirect, $code); + $c->detach; } } else { $c->forward('generate_grouped_data'); @@ -152,37 +164,19 @@ sub index : Path : Args(0) { sub construct_rs_filter : Private { my ($self, $c, $updates) = @_; - my %where; - $where{areas} = [ map { { 'like', "%,$_,%" } } @{$c->stash->{ward}} ] - if @{$c->stash->{ward}}; - $where{category} = $c->stash->{category} - if $c->stash->{category}; - - my $table_name = $updates ? 'problem' : 'me'; - - my $state = $c->stash->{q_state}; - if ( FixMyStreet::DB::Result::Problem->fixed_states->{$state} ) { # Probably fixed - council - $where{"$table_name.state"} = [ FixMyStreet::DB::Result::Problem->fixed_states() ]; - } elsif ( $state ) { - $where{"$table_name.state"} = $state; - } else { - $where{"$table_name.state"} = [ FixMyStreet::DB::Result::Problem->visible_states() ]; - } - - my $days30 = DateTime->now(time_zone => FixMyStreet->time_zone || FixMyStreet->local_time_zone)->subtract(days => 30); - $days30->truncate( to => 'day' ); - - my $range = FixMyStreet::DateRange->new( + my $reporting = FixMyStreet::Reporting->new( + type => $updates ? 'updates' : 'problems', + category => $c->stash->{category}, + state => $c->stash->{q_state}, + wards => $c->stash->{ward}, + body => $c->stash->{body} || undef, start_date => $c->stash->{start_date}, - start_default => $days30, end_date => $c->stash->{end_date}, - formatter => $c->model('DB')->storage->datetime_parser, + user => $c->user_exists ? $c->user->obj : undef, ); - $where{"$table_name.confirmed"} = $range->sql; - $c->stash->{params} = \%where; - my $rs = $updates ? $c->cobrand->updates : $c->cobrand->problems; - $c->stash->{objects_rs} = $rs->to_body($c->stash->{body})->search( \%where ); + $c->stash($reporting->construct_rs_filter); + return $reporting; } sub generate_grouped_data : Private { @@ -297,210 +291,67 @@ sub generate_summary_figures { } } -sub generate_body_response_time : Private { - my ( $self, $c ) = @_; - - my $avg = $c->stash->{body}->calculate_average($c->cobrand->call_hook("body_responsiveness_threshold")); - $c->stash->{body_average} = $avg ? int($avg / 60 / 60 / 24 + 0.5) : 0; -} - -sub csv_filename { - my ($self, $c, $updates) = @_; - my %where = ( - category => $c->stash->{category}, - state => $c->stash->{q_state}, - ward => join(',', @{$c->stash->{ward}}), - ); - $where{body} = $c->stash->{body}->id if $c->stash->{body}; - join '-', - $c->req->uri->host, - $updates ? ('updates') : (), - map { - my $value = $where{$_}; - (defined $value and length $value) ? ($_, $value) : () - } sort keys %where -}; - -sub export_as_csv_updates : Private { +sub status : Local : Args(0) { my ($self, $c) = @_; - my $csv = $c->stash->{csv} = { - objects => $c->stash->{objects_rs}->search_rs({}, { - order_by => ['me.confirmed', 'me.id'], - '+columns' => ['problem.bodies_str'], - cursor_page_size => 1000, - }), - headers => [ - 'Report ID', 'Update ID', 'Date', 'Status', 'Problem state', - 'Text', 'User Name', 'Reported As', - ], - columns => [ - 'problem_id', 'id', 'confirmed', 'state', 'problem_state', - 'text', 'user_name_display', 'reported_as', - ], - filename => $self->csv_filename($c, 1), - }; - $c->cobrand->call_hook("dashboard_export_updates_add_columns"); - $c->forward('generate_csv'); -} - -sub export_as_csv : Private { - my ($self, $c) = @_; + my $body = $c->stash->{body} = $c->forward('check_page_allowed'); + $c->stash->{body_name} = $body->name if $body; - my $csv = $c->stash->{csv} = { - objects => $c->stash->{objects_rs}->search_rs({}, { - join => 'comments', - '+columns' => ['comments.problem_state', 'comments.state', 'comments.confirmed', 'comments.mark_fixed'], - order_by => ['me.confirmed', 'me.id'], - cursor_page_size => 1000, - }), - headers => [ - 'Report ID', - 'Title', - 'Detail', - 'User Name', - 'Category', - 'Created', - 'Confirmed', - 'Acknowledged', - 'Fixed', - 'Closed', - 'Status', - 'Latitude', 'Longitude', - 'Query', - 'Ward', - 'Easting', - 'Northing', - 'Report URL', - 'Site Used', - 'Reported As', - ], - columns => [ - 'id', - 'title', - 'detail', - 'user_name_display', - 'category', - 'created', - 'confirmed', - 'acknowledged', - 'fixed', - 'closed', - 'state', - 'latitude', 'longitude', - 'postcode', - 'wards', - 'local_coords_x', - 'local_coords_y', - 'url', - 'site_used', - 'reported_as', - ], - filename => $self->csv_filename($c, 0), - }; - $c->cobrand->call_hook("dashboard_export_problems_add_columns"); - $c->forward('generate_csv'); + my $reporting = FixMyStreet::Reporting->new( + user => $c->user_exists ? $c->user->obj : undef, + ); + my $dir = $reporting->cache_dir; + my @data; + foreach ($dir->children) { + my $stat = $_->stat; + my $name = $_->basename; + my $finished = $name =~ /part$/ ? 0 : 1; + $name =~ s/-part$//; + push @data, { + ctime => $stat->ctime, + size => $stat->size, + name => $name, + finished => $finished, + }; + } + @data = sort { $b->{ctime} <=> $a->{ctime} } @data; + $c->stash->{rows} = \@data; } -=head2 generate_csv - -Generates a CSV output, given a 'csv' stash hashref containing: -* filename: filename to be used in output -* problems: a resultset of the rows to output -* headers: an arrayref of the header row strings -* columns: an arrayref of the columns (looked up in the row's as_hashref, plus -the following: user_name_display, acknowledged, fixed, closed, wards, -local_coords_x, local_coords_y, url). -* extra_data: If present, a function that is passed the report and returns a -hashref of extra data to include that can be used by 'columns'. - -=cut +sub csv : Local : Args(1) { + my ($self, $c, $filename) = @_; -sub generate_csv : Private { - my ($self, $c) = @_; + $c->authenticate(undef, "access_token"); - my $filename = $c->stash->{csv}->{filename}; - $c->res->content_type('text/csv; charset=utf-8'); - $c->res->header('content-disposition' => "attachment; filename=\"${filename}.csv\""); - - # Emit a header (copying Drupal's naming) telling an intermediary (e.g. - # Varnish) not to buffer the output. Varnish will need to know this, e.g.: - # if (beresp.http.Surrogate-Control ~ "BigPipe/1.0") { - # set beresp.do_stream = true; - # set beresp.ttl = 0s; - # } - $c->res->header('Surrogate-Control' => 'content="BigPipe/1.0"'); - - # Tell nginx not to buffer this response - $c->res->header('X-Accel-Buffering' => 'no'); - - # Define an empty body so the web view doesn't get added at the end - $c->res->body(""); - - # Old parameter renaming - $c->stash->{csv}->{objects} //= $c->stash->{csv}->{problems}; - - my $csv = Text::CSV->new({ binary => 1, eol => "\n" }); - $csv->print($c->response, $c->stash->{csv}->{headers}); - - my $fixed_states = FixMyStreet::DB::Result::Problem->fixed_states; - my $closed_states = FixMyStreet::DB::Result::Problem->closed_states; - - my %asked_for = map { $_ => 1 } @{$c->stash->{csv}->{columns}}; - - my $objects = $c->stash->{csv}->{objects}; - while ( my $obj = $objects->next ) { - my $hashref = $obj->as_hashref($c, \%asked_for); - - $hashref->{user_name_display} = $obj->anonymous - ? '(anonymous)' : $obj->name; - - if ($asked_for{acknowledged}) { - for my $comment ($obj->comments) { - my $problem_state = $comment->problem_state or next; - next unless $comment->state eq 'confirmed'; - next if $problem_state eq 'confirmed'; - $hashref->{acknowledged} //= $comment->confirmed; - $hashref->{fixed} //= $fixed_states->{ $problem_state } || $comment->mark_fixed ? - $comment->confirmed : undef; - if ($closed_states->{ $problem_state }) { - $hashref->{closed} = $comment->confirmed; - last; - } - } - } - - if ($asked_for{wards}) { - $hashref->{wards} = join ', ', - map { $c->stash->{children}->{$_}->{name} } - grep {$c->stash->{children}->{$_} } - split ',', $hashref->{areas}; - } + my $body = $c->stash->{body} = $c->forward('check_page_allowed'); - if ($obj->can('local_coords') && $asked_for{local_coords_x}) { - ($hashref->{local_coords_x}, $hashref->{local_coords_y}) = - $obj->local_coords; - } - if ($obj->can('url')) { - my $base = $c->cobrand->base_url_for_report($obj->can('problem') ? $obj->problem : $obj); - $hashref->{url} = join '', $base, $obj->url; + (my $basename = $filename) =~ s/\.csv$//; + my $reporting = FixMyStreet::Reporting->new( + user => $c->user_exists ? $c->user->obj : undef, + filename => $basename, + ); + my $dir = $reporting->cache_dir; + my $csv = path($dir, $filename); + + if (!$csv->exists) { + if (path($dir, "$filename-part")->exists && Catalyst::Authentication::Credential::AccessToken->get_token($c)) { + $c->res->body(''); + $c->res->status(202); + $c->detach; + } else { + $c->detach( '/page_error_404_not_found', [] ) unless $csv->exists; } + } - $hashref->{site_used} = $obj->can('service') ? ($obj->service || $obj->cobrand) : $obj->cobrand; - - $hashref->{reported_as} = $obj->get_extra_metadata('contributed_as') || ''; + $reporting->http_setup($c); + $c->res->body($csv->openr_raw); +} - if (my $fn = $c->stash->{csv}->{extra_data}) { - my $extra = $fn->($obj); - $hashref = { %$hashref, %$extra }; - } +sub generate_body_response_time : Private { + my ( $self, $c ) = @_; - $csv->print($c->response, [ - @{$hashref}{ - @{$c->stash->{csv}->{columns}} - }, - ] ); - } + my $avg = $c->stash->{body}->calculate_average($c->cobrand->call_hook("body_responsiveness_threshold")); + $c->stash->{body_average} = $avg ? int($avg / 60 / 60 / 24 + 0.5) : 0; } sub heatmap : Local : Args(0) { @@ -534,7 +385,7 @@ sub heatmap : Local : Args(0) { if ($c->get_param('ajax')) { my @pins; while ( my $problem = $problems->next ) { - push @pins, $problem->pin_data($c, 'reports'); + push @pins, $problem->pin_data('reports'); } $c->stash->{pins} = \@pins; $c->detach('/reports/ajax', [ 'dashboard/heatmap-list.html' ]); @@ -544,7 +395,8 @@ sub heatmap : Local : Args(0) { $c->stash->{children} = $children; $c->stash->{ward_hash} = { map { $_->{id} => 1 } @{$c->stash->{wards}} } if $c->stash->{wards}; - $c->forward('/reports/setup_categories_and_map'); + $c->forward('/reports/setup_categories'); + $c->forward('/reports/setup_map'); } sub heatmap_filters :Private { @@ -591,7 +443,15 @@ sub heatmap_sidebar :Private { order_by => 'lastupdate', })->all ]; - my $params = { map { my $n = $_; s/me\./problem\./; $_ => $where->{$n} } keys %$where }; + my $params = { map { + my $v = $where->{$_}; + if (ref $v eq 'HASH') { + $v = { map { my $vv = $v->{$_}; s/me\./problem\./; $_ => $vv } keys %$v }; + } else { + s/me\./problem\./; + } + $_ => $v; + } keys %$where }; my $body = $c->stash->{body}; my @user; |