path: root/perllib/FixMyStreet/Reporting.pm

package FixMyStreet::Reporting;

use DateTime;
use Moo;
use Path::Tiny;
use Text::CSV;
use Types::Standard qw(ArrayRef CodeRef Enum HashRef InstanceOf Int Maybe Str);
use FixMyStreet::DB;

# What are we reporting on

has type => ( is => 'ro', isa => Enum['problems','updates'] );
has on_problems => ( is => 'lazy', default => sub { $_[0]->type eq 'problems' } );
has on_updates => ( is => 'lazy', default => sub { $_[0]->type eq 'updates' } );

# Filters to restrict the reporting to

has body => ( is => 'ro', isa => Maybe[InstanceOf['FixMyStreet::DB::Result::Body']] );
has wards => ( is => 'ro', isa => ArrayRef[Int], default => sub { [] } );
has category => ( is => 'ro', isa => Maybe[Str] );
has state => ( is => 'ro', isa => Maybe[Str] );
has start_date => ( is => 'ro',
    isa => Str,
    default => sub {
        my $days30 = DateTime->now(time_zone => FixMyStreet->time_zone || FixMyStreet->local_time_zone)->subtract(days => 30);
        $days30->truncate( to => 'day' );
        $days30->strftime('%Y-%m-%d');
    }
);
has end_date => ( is => 'ro', isa => Maybe[Str] );

# Things needed for cobrand specific extra data or checks

has cobrand => ( is => 'ro', default => sub { FixMyStreet::DB->schema->cobrand } ); # Which cobrand is asking, to get the right data / hooks / base URL
has user => ( is => 'ro', isa => Maybe[InstanceOf['FixMyStreet::DB::Result::User']] );

# Things created in the process, that can be manually overridden

has objects_rs => ( is => 'rwp' ); # ResultSet of rows

sub objects_attrs {
    my ($self, $attrs) = @_;
    my $rs = $self->objects_rs->search(undef, $attrs);
    $self->_set_objects_rs($rs);
    return $rs;
}

# CSV header strings and column keys (looked up in the row's as_hashref, plus
# the following: user_name_display, acknowledged, fixed, closed, wards,
# local_coords_x, local_coords_y, url, subcategory, site_used, reported_as)
has csv_headers => ( is => 'rwp', isa => ArrayRef[Str], default => sub { [] } );
has csv_columns => ( is => 'rwp', isa => ArrayRef[Str], default => sub { [] } );

sub modify_csv_header {
    my ($self, %mapping) = @_;
    $self->_set_csv_headers([
        map { $mapping{$_} || $_ } @{ $self->csv_headers },
    ]);
}

sub splice_csv_column {
    my ($self, $before, $column, $header) = @_;

    for (my $i = 0; $i < @{$self->csv_columns}; $i++) {
        my $col = $self->csv_columns->[$i];
        if ($col eq $before) {
            splice @{$self->csv_columns}, $i, 0, $column;
            splice @{$self->csv_headers}, $i, 0, $header;
            last;
        }
    }
}

sub add_csv_columns {
    my $self = shift;
    for (my $i = 0; $i < @_; $i += 2) {
        my $column = $_[$i];
        my $header = $_[$i+1];
        push @{$self->csv_columns}, $column;
        push @{$self->csv_headers}, $header;
    }
}

# A function that is passed the report and returns a hashref of extra data to
# include that can be used by 'columns'
has csv_extra_data => ( is => 'rw', isa => CodeRef );

has filename => ( is => 'rw', isa => Str, lazy => 1, default => sub {
    my $self = shift;
    my %where = (
        category => $self->category,
        state => $self->state,
        ward => join(',', @{$self->wards}),
        start_date => $self->start_date,
        end_date => $self->end_date,
    );
    $where{body} = $self->body->id if $self->body;
    my $host = URI->new($self->cobrand->base_url)->host;
    join '-',
        $host,
        $self->on_updates ? ('updates') : (),
        map {
            my $value = $where{$_};
            (my $nosp = $value || '') =~ s/ /-/g;
            (defined $value and length $value) ? ($_, $nosp) : ()
        } sort keys %where
});

# Generation code

sub construct_rs_filter {
    my $self = shift;

    my $table_name = $self->on_updates ? 'problem' : 'me';

    my %where;
    $where{areas} = [ map { { 'like', "%,$_,%" } } @{$self->wards} ]
        if @{$self->wards};
    $where{"$table_name.category"} = $self->category
        if $self->category;

    if ( $self->state && FixMyStreet::DB::Result::Problem->fixed_states->{$self->state} ) { # Probably fixed - council
        $where{"$table_name.state"} = [ FixMyStreet::DB::Result::Problem->fixed_states() ];
    } elsif ( $self->state ) {
        $where{"$table_name.state"} = $self->state;
    } else {
        $where{"$table_name.state"} = [ FixMyStreet::DB::Result::Problem->visible_states() ];
    }

    my $range = FixMyStreet::DateRange->new(
        start_date => $self->start_date,
        end_date => $self->end_date,
        formatter => FixMyStreet::DB->schema->storage->datetime_parser,
    );
    $where{"$table_name.confirmed"} = $range->sql;

    my $rs = $self->on_updates ? $self->cobrand->updates : $self->cobrand->problems;
    my $objects_rs = $rs->to_body($self->body)->search( \%where );
    $self->_set_objects_rs($objects_rs);
    return {
        params => \%where,
        objects_rs => $objects_rs,
    }
}

sub csv_parameters {
    my $self = shift;
    if ($self->on_updates) {
        $self->_csv_parameters_updates;
    } else {
        $self->_csv_parameters_problems;
    }
}

sub _csv_parameters_updates {
    my $self = shift;

    $self->objects_attrs({
        join => 'problem',
        order_by => ['me.confirmed', 'me.id'],
        '+columns' => ['problem.bodies_str'],
        cursor_page_size => 1000,
    });
    $self->_set_csv_headers([
        'Report ID', 'Update ID', 'Date', 'Status', 'Problem state',
        'Text', 'User Name', 'Reported As',
    ]);
    $self->_set_csv_columns([
        'problem_id', 'id', 'confirmed', 'state', 'problem_state',
        'text', 'user_name_display', 'reported_as',
    ]);
    $self->cobrand->call_hook(dashboard_export_updates_add_columns => $self);
}

sub _csv_parameters_problems {
    my $self = shift;

    my $groups = $self->cobrand->enable_category_groups ? 1 : 0;
    my $join = ['comments'];
    my $columns = ['comments.id', 'comments.problem_state', 'comments.state', 'comments.confirmed', 'comments.mark_fixed'];
    if ($groups) {
        push @$join, 'contact';
        push @$columns, 'contact.id', 'contact.extra';
    }
    $self->objects_attrs({
        join => $join,
        collapse => 1,
        '+columns' => $columns,
        order_by => ['me.confirmed', 'me.id'],
        cursor_page_size => 1000,
    });
    $self->_set_csv_headers([
        'Report ID',
        'Title',
        'Detail',
        'User Name',
        'Category',
        $groups ? ('Subcategory') : (),
        'Created',
        'Confirmed',
        'Acknowledged',
        'Fixed',
        'Closed',
        'Status',
        'Latitude', 'Longitude',
        'Query',
        'Ward',
        'Easting',
        'Northing',
        'Report URL',
        'Site Used',
        'Reported As',
    ]);
    $self->_set_csv_columns([
        'id',
        'title',
        'detail',
        'user_name_display',
        'category',
        $groups ? ('subcategory') : (),
        'created',
        'confirmed',
        'acknowledged',
        'fixed',
        'closed',
        'state',
        'latitude', 'longitude',
        'postcode',
        'wards',
        'local_coords_x',
        'local_coords_y',
        'url',
        'site_used',
        'reported_as',
    ]);
    $self->cobrand->call_hook(dashboard_export_problems_add_columns => $self);
}

=head2 generate_csv

Generates a CSV output to a file handler provided

=cut

sub generate_csv {
    my ($self, $handle) = @_;

    my $csv = Text::CSV->new({ binary => 1, eol => "\n" });
    $csv->print($handle, $self->csv_headers);

    my $fixed_states = FixMyStreet::DB::Result::Problem->fixed_states;
    my $closed_states = FixMyStreet::DB::Result::Problem->closed_states;

    my %asked_for = map { $_ => 1 } @{$self->csv_columns};

    my $children = $self->body ? $self->body->first_area_children : {};

    my $objects = $self->objects_rs;
    while ( my $obj = $objects->next ) {
        my $hashref = $obj->as_hashref(\%asked_for);

        $hashref->{user_name_display} = $obj->anonymous
            ? '(anonymous)' : $obj->name;

        if ($asked_for{acknowledged}) {
            for my $comment ($obj->comments) {
                my $problem_state = $comment->problem_state or next;
                next unless $comment->state eq 'confirmed';
                next if $problem_state eq 'confirmed';
                $hashref->{acknowledged} //= $comment->confirmed;
                $hashref->{action_scheduled} //= $problem_state eq 'action scheduled' ? $comment->confirmed : undef;
                $hashref->{fixed} //= $fixed_states->{ $problem_state } || $comment->mark_fixed ?
                    $comment->confirmed : undef;
                if ($closed_states->{ $problem_state }) {
                    $hashref->{closed} = $comment->confirmed;
                    last;
                }
            }
        }

        if ($asked_for{wards}) {
            $hashref->{wards} = join ', ',
              map { $children->{$_}->{name} }
              grep { $children->{$_} }
              split ',', $hashref->{areas};
        }

        if ($obj->can('local_coords') && $asked_for{local_coords_x}) {
            ($hashref->{local_coords_x}, $hashref->{local_coords_y}) =
                $obj->local_coords;
        }

        if ($asked_for{subcategory}) {
            my $group = $obj->contact ? $obj->contact->groups : [];
            $group = join(',', @$group);
            if ($group) {
                $hashref->{subcategory} = $obj->category;
                $hashref->{category} = $group;
            }
        }

        my $base = $self->cobrand->base_url_for_report($obj->can('problem') ? $obj->problem : $obj);
        $hashref->{url} = join '', $base, $obj->url;

        $hashref->{site_used} = $obj->can('service') ? ($obj->service || $obj->cobrand) : $obj->cobrand;

        $hashref->{reported_as} = $obj->get_extra_metadata('contributed_as') || '';

        if (my $fn = $self->csv_extra_data) {
            my $extra = $fn->($obj);
            $hashref = { %$hashref, %$extra };
        }

        $csv->print($handle, [
            @{$hashref}{
                @{$self->csv_columns}
            },
        ] );
    }
}

# Output code

sub cache_dir {
    my $self = shift;

    my $cfg = FixMyStreet->config('PHOTO_STORAGE_OPTIONS');
    my $dir = $cfg ? $cfg->{UPLOAD_DIR} : FixMyStreet->config('UPLOAD_DIR');
    $dir = path($dir, "dashboard_csv")->absolute(FixMyStreet->path_to());
    my $subdir = $self->user ? $self->user->id : 0;
    $dir = $dir->child($subdir);
    $dir->mkpath;
    $dir;
}

sub kick_off_process {
    my $self = shift;

    my $out = path($self->cache_dir, $self->filename . '.csv');
    my $file = path($out . '-part');
    return if $file->exists;
    $file->touch; # So status page shows it even if process takes short while to spin up

    my $cmd = FixMyStreet->path_to('bin/csv-export');
    $cmd .= ' --cobrand ' . $self->cobrand->moniker;
    $cmd .= " --out \Q$out\E";
    foreach (qw(type category state start_date end_date)) {
        $cmd .= " --$_ " . quotemeta($self->$_) if $self->$_;
    }
    foreach (qw(body user)) {
        $cmd .= " --$_ " . $self->$_->id if $self->$_;
    }
    $cmd .= " --wards " . join(',', map { quotemeta } @{$self->wards}) if @{$self->wards};
    $cmd .= ' &' unless FixMyStreet->test_mode;

    system($cmd);
}

# Outputs relevant CSV HTTP headers, and then streams the CSV
sub generate_csv_http {
    my ($self, $c) = @_;
    $self->http_setup($c);
    $self->generate_csv($c->response);
}

sub http_setup {
    my ($self, $c) = @_;
    my $filename = $self->filename;

    $c->res->content_type('text/csv; charset=utf-8');
    $c->res->header('content-disposition' => "attachment; filename=\"${filename}.csv\"");

    # Emit a header (copying Drupal's naming) telling an intermediary (e.g.
    # Varnish) not to buffer the output. Varnish will need to know this, e.g.:
    #   if (beresp.http.Surrogate-Control ~ "BigPipe/1.0") {
    #     set beresp.do_stream = true;
    #     set beresp.ttl = 0s;
    #   }
    $c->res->header('Surrogate-Control' => 'content="BigPipe/1.0"');

    # Tell nginx not to buffer this response
    $c->res->header('X-Accel-Buffering' => 'no');

    # Define an empty body so the web view doesn't get added at the end
    $c->res->body("");
}

1;