diff options
Diffstat (limited to 'perllib/FixMyStreet')
| -rwxr-xr-x | perllib/FixMyStreet/App/Controller/About.pm | 1 | ||||
| -rw-r--r-- | perllib/FixMyStreet/App/Controller/Reports.pm | 110 | ||||
| -rw-r--r-- | perllib/FixMyStreet/Cobrand/FixMyStreet.pm | 98 | ||||
| -rw-r--r-- | perllib/FixMyStreet/DB/Result/Body.pm | 5 | ||||
| -rwxr-xr-x | perllib/FixMyStreet/Script/UpdateAllReports.pm | 157 |
5 files changed, 271 insertions, 100 deletions
diff --git a/perllib/FixMyStreet/App/Controller/About.pm b/perllib/FixMyStreet/App/Controller/About.pm index 233da25d3..48a5dfffd 100755 --- a/perllib/FixMyStreet/App/Controller/About.pm +++ b/perllib/FixMyStreet/App/Controller/About.pm @@ -23,6 +23,7 @@ sub page : Path("/about") : Args(1) { my $template = $c->forward('find_template'); $c->detach('/page_error_404_not_found', []) unless $template; $c->stash->{template} = $template; + $c->cobrand->call_hook('about_hook'); } sub index : Path("/about") : Args(0) { diff --git a/perllib/FixMyStreet/App/Controller/Reports.pm b/perllib/FixMyStreet/App/Controller/Reports.pm index 8beb2c091..b6281f0ca 100644 --- a/perllib/FixMyStreet/App/Controller/Reports.pm +++ b/perllib/FixMyStreet/App/Controller/Reports.pm @@ -2,9 +2,9 @@ package FixMyStreet::App::Controller::Reports; use Moose; use namespace::autoclean; -use File::Slurp; use JSON::MaybeXS; use List::MoreUtils qw(any); +use Path::Tiny; use POSIX qw(strcoll); use RABX; use mySociety::MaPit; @@ -69,34 +69,27 @@ sub index : Path : Args(0) { } } - # Fetch all bodies - my @bodies = $c->model('DB::Body')->search({ - deleted => 0, - }, { - '+select' => [ { count => 'area_id' } ], - '+as' => [ 'area_count' ], - join => 'body_areas', - distinct => 1, - })->all; - @bodies = sort { strcoll($a->name, $b->name) } @bodies; - $c->stash->{bodies} = \@bodies; - $c->stash->{any_empty_bodies} = any { $_->get_column('area_count') == 0 } @bodies; - my $dashboard = eval { - my $data = File::Slurp::read_file( - FixMyStreet->path_to( '../data/all-reports-dashboard.json' )->stringify - ); - $c->stash(decode_json($data)); + my $data = FixMyStreet->config('TEST_DASHBOARD_DATA'); + # uncoverable branch true + unless ($data) { + my $fn = '../data/all-reports-dashboard'; + if ($c->stash->{body}) { + $fn .= '-' . $c->stash->{body}->id; + } + $data = decode_json(path(FixMyStreet->path_to($fn . '.json'))->slurp_utf8); + } + $c->stash($data); return 1; }; - my $table = eval { - my $data = File::Slurp::read_file( - FixMyStreet->path_to( '../data/all-reports.json' )->stringify - ); + my $table = !$c->stash->{body} && eval { + my $data = path(FixMyStreet->path_to('../data/all-reports.json'))->slurp_utf8; $c->stash(decode_json($data)); return 1; }; if (!$dashboard && !$table) { + $c->detach('/page_error_404_not_found') if $c->stash->{body}; + my $message = _("There was a problem showing the All Reports page. Please try again later."); if ($c->config->{STAGING_SITE}) { $message .= '</p><p>Perhaps the bin/update-all-reports script needs running. Use: bin/update-all-reports</p><p>' @@ -105,6 +98,26 @@ sub index : Path : Args(0) { $c->detach('/page_error_500_internal_error', [ $message ]); } + if ($c->stash->{body}) { + my $children = $c->stash->{body}->first_area_children; + unless ($children->{error}) { + $c->stash->{children} = $children; + } + } else { + # Fetch all bodies + my @bodies = $c->model('DB::Body')->search({ + deleted => 0, + }, { + '+select' => [ { count => 'area_id' } ], + '+as' => [ 'area_count' ], + join => 'body_areas', + distinct => 1, + })->all; + @bodies = sort { strcoll($a->name, $b->name) } @bodies; + $c->stash->{bodies} = \@bodies; + $c->stash->{any_empty_bodies} = any { $_->get_column('area_count') == 0 } @bodies; + } + # Down here so that error pages aren't cached. $c->response->header('Cache-Control' => 'max-age=3600'); } @@ -133,6 +146,20 @@ sub ward : Path : Args(2) { my @wards = split /\|/, $ward || ""; $c->forward( 'body_check', [ $body ] ); + + my $body_short = $c->cobrand->short_name( $c->stash->{body} ); + $c->stash->{body_url} = '/reports/' . $body_short; + + if ($ward && $ward eq 'summary') { + if (my $actual_ward = $c->get_param('ward')) { + $ward = $c->cobrand->short_name({ name => $actual_ward }); + $c->res->redirect($ward); + $c->detach; + } + $c->cobrand->call_hook('council_dashboard_hook'); + $c->go('index'); + } + $c->forward( 'ward_check', [ @wards ] ) if @wards; $c->forward( 'check_canonical_url', [ $body ] ); @@ -143,13 +170,10 @@ sub ward : Path : Args(2) { $c->detach('ajax', [ 'reports/_problem-list.html' ]); } - my $body_short = $c->cobrand->short_name( $c->stash->{body} ); $c->stash->{rss_url} = '/rss/reports/' . $body_short; $c->stash->{rss_url} .= '/' . $c->cobrand->short_name( $c->stash->{ward} ) if $c->stash->{ward}; - $c->stash->{body_url} = '/reports/' . $body_short; - $c->stash->{stats} = $c->cobrand->get_report_stats(); my @categories = $c->stash->{body}->contacts->not_deleted->search( undef, { @@ -178,9 +202,7 @@ sub ward : Path : Args(2) { # List of wards if ( !$c->stash->{wards} && $c->stash->{body}->id && $c->stash->{body}->body_areas->first ) { - my $children = mySociety::MaPit::call('area/children', [ $c->stash->{body}->body_areas->first->area_id ], - type => $c->cobrand->area_types_children, - ); + my $children = $c->stash->{body}->first_area_children; unless ($children->{error}) { foreach (values %$children) { $_->{url} = $c->uri_for( $c->stash->{body_url} @@ -310,17 +332,35 @@ sub body_check : Private { # Oslo/ kommunes sharing a name in Norway return if $c->cobrand->reports_body_check( $c, $q_body ); + my $body = $c->forward('body_find', [ $q_body ]); + if ($body) { + $c->stash->{body} = $body; + return; + } + + # No result, bad body name. + $c->detach( 'redirect_index' ); +} + +=head2 + +Given a string, try and find a body starting with/matching that string. +Returns the matching body object if found. + +=cut + +sub body_find : Private { + my ($self, $c, $q_body) = @_; + # We must now have a string to check my @bodies = $c->model('DB::Body')->search( { name => { -like => "$q_body%" } } )->all; if (@bodies == 1) { - $c->stash->{body} = $bodies[0]; - return; + return $bodies[0]; } else { foreach (@bodies) { if (lc($_->name) eq lc($q_body) || $_->name =~ /^\Q$q_body\E (Borough|City|District|County) Council$/i) { - $c->stash->{body} = $_; - return; + return $_; } } } @@ -333,13 +373,9 @@ sub body_check : Private { if (@translations == 1) { if ( my $body = $c->model('DB::Body')->find( { id => $translations[0]->object_id } ) ) { - $c->stash->{body} = $body; - return; + return $body; } } - - # No result, bad body name. - $c->detach( 'redirect_index' ); } =head2 ward_check diff --git a/perllib/FixMyStreet/Cobrand/FixMyStreet.pm b/perllib/FixMyStreet/Cobrand/FixMyStreet.pm index c50721334..a50a22ff9 100644 --- a/perllib/FixMyStreet/Cobrand/FixMyStreet.pm +++ b/perllib/FixMyStreet/Cobrand/FixMyStreet.pm @@ -1,6 +1,9 @@ package FixMyStreet::Cobrand::FixMyStreet; use base 'FixMyStreet::Cobrand::UK'; +use strict; +use warnings; + use mySociety::Random; use constant COUNCIL_ID_BROMLEY => 2482; @@ -62,5 +65,98 @@ sub extra_contact_validation { return %errors; } -1; +=head2 council_dashboard_hook + +This is for council-specific dashboard pages, which can only be seen by +superusers and logged-in users with an email domain matching a body name. + +=cut + +sub council_dashboard_hook { + my $self = shift; + my $c = $self->{c}; + + unless ( $c->user_exists ) { + $c->res->redirect('/about/council-dashboard'); + $c->detach; + } + + return if $c->user->is_superuser; + + my $body = $c->user->from_body || _user_to_body($c); + if ($body) { + # Matching URL and user's email body + return if $body->id eq $c->stash->{body}->id; + + # Matched /a/ body, redirect to its summary page + $c->stash->{body} = $body; + $c->stash->{wards} = [ { name => 'summary' } ]; + $c->detach('/reports/redirect_body'); + } + + $c->res->redirect('/about/council-dashboard'); +} +sub _user_to_body { + my $c = shift; + my $email = lc $c->user->email; + return _email_to_body($c, $email); +} + +sub _email_to_body { + my ($c, $email) = @_; + my ($domain) = $email =~ m{ @ (.*) \z }x; + + my @data = eval { FixMyStreet->path_to('../data/fixmystreet-councils.csv')->slurp }; + my $body; + foreach (@data) { + chomp; + my ($d, $b) = split /\|/; + if ($d eq $domain) { + $body = $b; + last; + } + } + # If we didn't find a lookup entry, default to the first part of the domain + unless ($body) { + $domain =~ s/\.gov\.uk$//; + $body = ucfirst $domain; + } + + $body = $c->forward('/reports/body_find', [ $body ]); + return $body; +} + +sub about_hook { + my $self = shift; + my $c = $self->{c}; + + if ($c->stash->{template} eq 'about/council-dashboard.html') { + $c->stash->{form_name} = $c->get_param('name') || ''; + $c->stash->{email} = $c->get_param('username') || ''; + if ($c->user_exists) { + my $body = _user_to_body($c); + if ($body) { + $c->stash->{body} = $body; + $c->stash->{wards} = [ { name => 'summary' } ]; + $c->detach('/reports/redirect_body'); + } + } + if (my $email = $c->get_param('username')) { + $email = lc $email; + $email =~ s/\s+//g; + my $body = _email_to_body($c, $email); + if ($body) { + # Send confirmation email (hopefully) + $c->stash->{template} = 'auth/general.html'; + $c->detach('/auth/general'); + } else { + $c->stash->{no_body_found} = 1; + $c->set_param('em', $email); # What the contact form wants + $c->detach('/contact/submit'); + } + } + } +} + +1; diff --git a/perllib/FixMyStreet/DB/Result/Body.pm b/perllib/FixMyStreet/DB/Result/Body.pm index 6481d5cfc..da5c38168 100644 --- a/perllib/FixMyStreet/DB/Result/Body.pm +++ b/perllib/FixMyStreet/DB/Result/Body.pm @@ -156,12 +156,13 @@ sub areas { } sub first_area_children { - my ( $self, $c ) = @_; + my ( $self ) = @_; my $area_id = $self->body_areas->first->area_id; + my $cobrand = $self->result_source->schema->cobrand; my $children = mySociety::MaPit::call('area/children', $area_id, - type => $c->cobrand->area_types_children, + type => $cobrand->area_types_children, ); return $children; diff --git a/perllib/FixMyStreet/Script/UpdateAllReports.pm b/perllib/FixMyStreet/Script/UpdateAllReports.pm index 1bd069ee8..f4f444d5b 100755 --- a/perllib/FixMyStreet/Script/UpdateAllReports.pm +++ b/perllib/FixMyStreet/Script/UpdateAllReports.pm @@ -4,11 +4,9 @@ use strict; use warnings; use FixMyStreet; +use FixMyStreet::Cobrand; use FixMyStreet::DB; -use File::Path (); -use File::Slurp; -use JSON::MaybeXS; use List::MoreUtils qw(zip); use List::Util qw(sum); @@ -21,6 +19,11 @@ if ( FixMyStreet->config('BASE_URL') =~ /zurich|zueri/ ) { $age_column = 'created'; } +my $dtf = FixMyStreet::DB->schema->storage->datetime_parser; + +my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker('default')->new; +FixMyStreet::DB->schema->cobrand($cobrand); + sub generate { my $include_areas = shift; @@ -81,13 +84,10 @@ sub generate { } } - my $body = encode_json( { + return { fixed => \%fixed, open => \%open, - } ); - - File::Path::mkpath( FixMyStreet->path_to( '../data/' )->stringify ); - File::Slurp::write_file( FixMyStreet->path_to( '../data/all-reports.json' )->stringify, \$body ); + }; } sub end_period { @@ -107,10 +107,18 @@ sub loop_period { } sub generate_dashboard { + my $body = shift; + my %data; + my $rs = FixMyStreet::DB->resultset('Problem'); + $rs = $rs->to_body($body) if $body; + + my $rs_c = FixMyStreet::DB->resultset('Comment'); + $rs_c = $rs_c->to_body($body) if $body; + my $end_today = end_period('day'); - my $min_confirmed = FixMyStreet::DB->resultset('Problem')->search({ + my $min_confirmed = $rs->search({ state => [ FixMyStreet::DB::Result::Problem->visible_states() ], }, { select => [ { min => 'confirmed' } ], @@ -134,11 +142,11 @@ sub generate_dashboard { my @problem_periods = loop_period($min_confirmed, $group_by, $extra); my %problems_reported_by_period = stuff_by_day_or_year( - $group_by, 'Problem', + $group_by, $rs, state => [ FixMyStreet::DB::Result::Problem->visible_states() ], ); my %problems_fixed_by_period = stuff_by_day_or_year( - $group_by, 'Problem', + $group_by, $rs, state => [ FixMyStreet::DB::Result::Problem->fixed_states() ], ); @@ -158,24 +166,23 @@ sub generate_dashboard { ); $data{last_seven_days} = \%last_seven_days; - my $dtf = FixMyStreet::DB->schema->storage->datetime_parser; my $eight_ago = $dtf->format_datetime(DateTime->now->subtract(days => 8)); %problems_reported_by_period = stuff_by_day_or_year('day', - 'Problem', + $rs, state => [ FixMyStreet::DB::Result::Problem->visible_states() ], - confirmed => { '>=', $eight_ago }, + 'me.confirmed' => { '>=', $eight_ago }, ); %problems_fixed_by_period = stuff_by_day_or_year('day', - 'Comment', - confirmed => { '>=', $eight_ago }, + $rs_c, + 'me.confirmed' => { '>=', $eight_ago }, -or => [ problem_state => [ FixMyStreet::DB::Result::Problem->fixed_states() ], mark_fixed => 1, ], ); my %problems_updated_by_period = stuff_by_day_or_year('day', - 'Comment', - confirmed => { '>=', $eight_ago }, + $rs_c, + 'me.confirmed' => { '>=', $eight_ago }, ); my $date = DateTime->today->subtract(days => 7); @@ -189,47 +196,17 @@ sub generate_dashboard { $last_seven_days{fixed_total} = sum @{$last_seven_days{fixed}}; $last_seven_days{updated_total} = sum @{$last_seven_days{updated}}; - my(@top_five_bodies); - $data{top_five_bodies} = \@top_five_bodies; - - my $bodies = FixMyStreet::DB->resultset('Body')->search; - my $substmt = "select min(id) from comment where me.problem_id=comment.problem_id and (problem_state in ('fixed', 'fixed - council', 'fixed - user') or mark_fixed)"; - while (my $body = $bodies->next) { - my $subquery = FixMyStreet::DB->resultset('Comment')->to_body($body)->search({ - -or => [ - problem_state => [ FixMyStreet::DB::Result::Problem->fixed_states() ], - mark_fixed => 1, - ], - 'me.id' => \"= ($substmt)", - 'me.state' => 'confirmed', - }, { - select => [ - { extract => "epoch from me.confirmed-problem.confirmed", -as => 'time' }, - ], - as => [ qw/time/ ], - rows => 100, - order_by => { -desc => 'me.confirmed' }, - join => 'problem' - })->as_subselect_rs; - my $avg = $subquery->search({ - }, { - select => [ { avg => "time" } ], - as => [ qw/avg/ ], - })->first->get_column('avg'); - push @top_five_bodies, { name => $body->name, days => int($avg / 60 / 60 / 24 + 0.5) } - if defined $avg; + if ($body) { + calculate_top_five_wards(\%data, $rs, $body); + } else { + calculate_top_five_bodies(\%data, $rs_c); } - @top_five_bodies = sort { $a->{days} <=> $b->{days} } @top_five_bodies; - $data{average} = @top_five_bodies - ? int((sum map { $_->{days} } @top_five_bodies) / @top_five_bodies + 0.5) : undef; - - @top_five_bodies = @top_five_bodies[0..4] if @top_five_bodies > 5; my $week_ago = $dtf->format_datetime(DateTime->now->subtract(days => 7)); - my $last_seven_days = FixMyStreet::DB->resultset("Problem")->search({ + my $last_seven_days = $rs->search({ confirmed => { '>=', $week_ago }, })->count; - my @top_five_categories = FixMyStreet::DB->resultset("Problem")->search({ + my @top_five_categories = $rs->search({ confirmed => { '>=', $week_ago }, category => { '!=', 'Other' }, }, { @@ -247,19 +224,17 @@ sub generate_dashboard { } $data{other_categories} = $last_seven_days; - my $body = encode_json( \%data ); - File::Path::mkpath( FixMyStreet->path_to( '../data/' )->stringify ); - File::Slurp::write_file( FixMyStreet->path_to( '../data/all-reports-dashboard.json' )->stringify, \$body ); + return \%data; } sub stuff_by_day_or_year { my $period = shift; - my $table = shift; + my $rs = shift; my %params = @_; - my $results = FixMyStreet::DB->resultset($table)->search({ + my $results = $rs->search({ %params }, { - select => [ { extract => \"$period from confirmed", -as => $period }, { count => 'id' } ], + select => [ { extract => \"$period from me.confirmed", -as => $period }, { count => 'me.id' } ], as => [ $period, 'count' ], group_by => [ $period ], }); @@ -271,4 +246,66 @@ sub stuff_by_day_or_year { return %out; } +sub calculate_top_five_bodies { + my ($data, $rs_c) = @_; + + my(@top_five_bodies); + + my $bodies = FixMyStreet::DB->resultset('Body')->search; + my $substmt = "select min(id) from comment where me.problem_id=comment.problem_id and (problem_state in ('fixed', 'fixed - council', 'fixed - user') or mark_fixed)"; + while (my $body = $bodies->next) { + my $subquery = $rs_c->to_body($body)->search({ + -or => [ + problem_state => [ FixMyStreet::DB::Result::Problem->fixed_states() ], + mark_fixed => 1, + ], + 'me.id' => \"= ($substmt)", + 'me.state' => 'confirmed', + }, { + select => [ + { extract => "epoch from me.confirmed-problem.confirmed", -as => 'time' }, + ], + as => [ qw/time/ ], + rows => 100, + order_by => { -desc => 'me.confirmed' }, + join => 'problem' + })->as_subselect_rs; + my $avg = $subquery->search({ + }, { + select => [ { avg => "time" } ], + as => [ qw/avg/ ], + })->first->get_column('avg'); + push @top_five_bodies, { name => $body->name, days => int($avg / 60 / 60 / 24 + 0.5) } + if defined $avg; + } + @top_five_bodies = sort { $a->{days} <=> $b->{days} } @top_five_bodies; + $data->{average} = @top_five_bodies + ? int((sum map { $_->{days} } @top_five_bodies) / @top_five_bodies + 0.5) : undef; + + @top_five_bodies = @top_five_bodies[0..4] if @top_five_bodies > 5; + $data->{top_five_bodies} = \@top_five_bodies; +} + +sub calculate_top_five_wards { + my ($data, $rs, $body) = @_; + + my $children = $body->first_area_children; + die $children->{error} if $children->{error}; + + my $week_ago = $dtf->format_datetime(DateTime->now->subtract(days => 7)); + my $last_seven_days = $rs->search({ confirmed => { '>=', $week_ago } }); + my $last_seven_days_count = $last_seven_days->count; + $last_seven_days = $last_seven_days->search(undef, { select => 'areas' }); + + while (my $row = $last_seven_days->next) { + $children->{$_}{reports}++ foreach grep { $children->{$_} } split /,/, $row->areas; + } + my @wards = sort { $b->{reports} <=> $a->{reports} } grep { $_->{reports} } values %$children; + @wards = @wards[0..4] if @wards > 5; + + my $sum_five = (sum map { $_->{reports} } @wards) || 0; + $data->{other_wards} = $last_seven_days_count - $sum_five; + $data->{wards} = \@wards; +} + 1; |