diff options
Diffstat (limited to 'perllib/FixMyStreet')
107 files changed, 4458 insertions, 1412 deletions
diff --git a/perllib/FixMyStreet/App.pm b/perllib/FixMyStreet/App.pm index 6a41d93a9..e367f0332 100644 --- a/perllib/FixMyStreet/App.pm +++ b/perllib/FixMyStreet/App.pm @@ -101,9 +101,6 @@ __PACKAGE__->config( use_session => 0, credential => { class => 'AccessToken', - token_field => 'extra', - # This means the token has to be 18 characters long (as generated by AuthToken) - token_lookup => { like => "%access_token,T18:TOKEN,%" }, }, store => $store, }, @@ -517,6 +514,7 @@ Sets the query parameter to the passed variable. sub set_param { my ($c, $param, $value) = @_; $c->req->params->{$param} = $value; + $c->req->body_params->{$param} = $value; } =head2 check_2fa @@ -536,6 +534,11 @@ sub check_2fa { return 0; } +sub user_country { + my $c = shift; + return FixMyStreet::Gaze::get_country_from_ip($c->req->address); +} + =head1 SEE ALSO L<FixMyStreet::App::Controller::Root>, L<Catalyst> diff --git a/perllib/FixMyStreet/App/Controller/Admin.pm b/perllib/FixMyStreet/App/Controller/Admin.pm index 038cba9e5..8d6a41b9f 100644 --- a/perllib/FixMyStreet/App/Controller/Admin.pm +++ b/perllib/FixMyStreet/App/Controller/Admin.pm @@ -71,13 +71,16 @@ sub index : Path : Args(0) { } my @unsent = $c->cobrand->problems->search( { - state => [ FixMyStreet::DB::Result::Problem::open_states() ], + 'me.state' => [ FixMyStreet::DB::Result::Problem::open_states() ], whensent => undef, bodies_str => { '!=', undef }, # Ignore very recent ones that probably just haven't been sent yet confirmed => { '<', \"current_timestamp - '5 minutes'::interval" }, }, { + '+columns' => ['user.email'], + prefetch => 'contact', + join => 'user', order_by => 'confirmed', } )->all; $c->stash->{unsent_reports} = \@unsent; @@ -301,7 +304,14 @@ sub add_flags : Private { sub flagged : Path('flagged') : Args(0) { my ( $self, $c ) = @_; - my $problems = $c->cobrand->problems->search( { flagged => 1 } ); + my $problems = $c->cobrand->problems->search( + { 'me.flagged' => 1 }, + { + '+columns' => ['user.email'], + join => 'user', + prefetch => 'contact', + } + ); # pass in as array ref as using same template as search_reports # which has to use an array ref for sql quoting reasons diff --git a/perllib/FixMyStreet/App/Controller/Admin/Bodies.pm b/perllib/FixMyStreet/App/Controller/Admin/Bodies.pm index 6ae068cd9..31a717a9c 100644 --- a/perllib/FixMyStreet/App/Controller/Admin/Bodies.pm +++ b/perllib/FixMyStreet/App/Controller/Admin/Bodies.pm @@ -151,8 +151,7 @@ sub category : Chained('body') : PathPart('') { my $history = $c->model('DB::ContactsHistory')->search( { - body_id => $c->stash->{body_id}, - category => $c->stash->{contact}->category + contact_id => $c->stash->{contact}->id, }, { order_by => ['contacts_history_id'] @@ -241,17 +240,27 @@ sub update_contact : Private { if ($current_contact && $contact->id && $contact->id != $current_contact->id) { $errors{category} = _('You cannot rename a category to an existing category'); } elsif ($current_contact && !$contact->id) { - # Changed name $contact = $current_contact; - $c->model('DB::Problem')->to_body($c->stash->{body_id})->search({ category => $current_category })->update({ category => $category }); - $contact->category($category); + # Set the flag here so we can run the editable test on it + $contact->set_extra_metadata(open311_protect => $c->get_param('open311_protect')); + if (!$contact->category_uneditable) { + # Changed name + $c->model('DB::Problem')->to_body($c->stash->{body_id})->search({ category => $current_category })->update({ category => $category }); + $contact->category($category); + } } my $email = $c->get_param('email'); - $email =~ s/\s+//g; my $send_method = $c->get_param('send_method') || $contact->body->send_method || ""; + if ($send_method eq 'Open311') { + $email =~ s/^\s+|\s+$//g; + } else { + $email =~ s/\s+//g; + } my $email_unchanged = $contact->email && $email && $contact->email eq $email; - unless ( $send_method eq 'Open311' || $email_unchanged ) { + my $cobrand = $contact->body->get_cobrand_handler; + my $cobrand_valid = $cobrand && $cobrand->call_hook(validate_contact_email => $email); + unless ( $send_method eq 'Open311' || $email_unchanged || $cobrand_valid ) { $errors{email} = _('Please enter a valid email') unless is_valid_email_list($email) || $email eq 'REFUSED'; } @@ -267,15 +276,19 @@ sub update_contact : Private { $contact->send_method( $c->get_param('send_method') ); # Set flags in extra to the appropriate values - if ( $c->get_param('photo_required') ) { - $contact->set_extra_metadata_if_undefined( photo_required => 1 ); - } else { - $contact->unset_extra_metadata( 'photo_required' ); + foreach (qw(photo_required open311_protect updates_disallowed reopening_disallowed assigned_users_only anonymous_allowed)) { + if ( $c->get_param($_) ) { + $contact->set_extra_metadata( $_ => 1 ); + } else { + $contact->unset_extra_metadata($_); + } } - if ( $c->get_param('open311_protect') ) { - $contact->set_extra_metadata( open311_protect => 1 ); - } else { - $contact->unset_extra_metadata( 'open311_protect' ); + if ( $c->user->is_superuser ) { + if ( $c->get_param('hardcoded') ) { + $contact->set_extra_metadata( hardcoded => 1 ); + } else { + $contact->unset_extra_metadata('hardcoded'); + } } if ( my @group = $c->get_param_list('group') ) { @group = grep { $_ } @group; diff --git a/perllib/FixMyStreet/App/Controller/Admin/Reports.pm b/perllib/FixMyStreet/App/Controller/Admin/Reports.pm index 7300fe676..20801e0cf 100644 --- a/perllib/FixMyStreet/App/Controller/Admin/Reports.pm +++ b/perllib/FixMyStreet/App/Controller/Admin/Reports.pm @@ -4,6 +4,7 @@ use namespace::autoclean; BEGIN { extends 'Catalyst::Controller'; } +use utf8; use List::MoreUtils 'uniq'; use FixMyStreet::SMS; use Utils; @@ -50,6 +51,8 @@ sub index : Path { return if $c->cobrand->call_hook(report_search_query => $query, $p_page, $u_page, $order); + my $problems = $c->cobrand->problems; + if (my $search = $c->get_param('search')) { $search = $self->trim($search); @@ -62,9 +65,6 @@ sub index : Path { $c->stash->{searched} = $search; - my $search_n = 0; - $search_n = int($search) if $search =~ /^\d+$/; - my $like_search = "%$search%"; my $parsed = FixMyStreet::SMS->parse_username($search); @@ -92,24 +92,18 @@ sub index : Path { 'me.external_id' => { like => "%$1%" } ]; } else { - $query->{'-or'} = [ - 'me.id' => $search_n, - 'user.email' => { ilike => $like_search }, - 'user.phone' => { ilike => $like_search }, - 'me.external_id' => { ilike => $like_search }, - 'me.name' => { ilike => $like_search }, - 'me.title' => { ilike => $like_search }, - detail => { ilike => $like_search }, - bodies_str => { like => $like_search }, - cobrand_data => { like => $like_search }, - ]; + $problems = $problems->search_text($search); + # The below is added so that PostgreSQL does not try and use other indexes + # besides the full text search. It should have no impact on results shown. + $order = [ $order, { -desc => "me.id" }, { -desc => "me.created" } ]; } - my $problems = $c->cobrand->problems->search( + $problems = $problems->search( $query, { join => 'user', '+columns' => 'user.email', + prefetch => 'contact', rows => 50, order_by => $order, } @@ -118,6 +112,8 @@ sub index : Path { $c->stash->{problems} = [ $problems->all ]; $c->stash->{problems_pager} = $problems->pager; + my $updates = $c->cobrand->updates; + $order = { -desc => 'me.id' }; if ($valid_email) { $query = [ 'user.email' => { ilike => $like_search }, @@ -132,30 +128,25 @@ sub index : Path { 'me.problem_id' => int($1), ]; } elsif ($search =~ /^area:(\d+)$/) { - $query = []; + $query = 0; } else { - $query = [ - 'me.id' => $search_n, - 'problem.id' => $search_n, - 'user.email' => { ilike => $like_search }, - 'user.phone' => { ilike => $like_search }, - 'me.name' => { ilike => $like_search }, - text => { ilike => $like_search }, - 'me.cobrand_data' => { ilike => $like_search }, - ]; + $updates = $updates->search_text($search); + $order = [ $order, { -desc => "me.created" } ]; + $query = 1; } - if (@$query) { - my $updates = $c->cobrand->updates->search( - { - -or => $query, - }, + $query = { -or => $query } if ref $query; + + if ($query) { + $query = undef unless ref $query; + $updates = $updates->search( + $query, { '+columns' => ['user.email'], join => 'user', prefetch => [qw/problem/], rows => 50, - order_by => { -desc => 'me.id' } + order_by => $order, } )->page( $u_page ); $c->stash->{updates} = [ $updates->all ]; @@ -164,9 +155,15 @@ sub index : Path { } else { - my $problems = $c->cobrand->problems->search( + $problems = $problems->search( $query, - { order_by => $order, rows => 50 } + { + '+columns' => ['user.email'], + join => 'user', + prefetch => 'contact', + order_by => $order, + rows => 50 + } )->page( $p_page ); $c->stash->{problems} = [ $problems->all ]; $c->stash->{problems_pager} = $problems->pager; @@ -227,6 +224,26 @@ sub edit : Path('/admin/report_edit') : Args(1) { push @fields, { name => 'Duplicates', val => join( ',', @{ $problem->get_extra_metadata('duplicates') } ) }; delete $extra->{duplicates}; } + + if ( $extra->{contributed_by} ) { + my $u = $c->cobrand->users->find({id => $extra->{contributed_by}}); + if ( $u ) { + my $uri = $c->uri_for_action('admin/users/index', { search => $u->email } ); + push @fields, { + name => _('Created By'), + val => FixMyStreet::Template::SafeString->new( "<a href=\"$uri\">@{[$u->name]} (@{[$u->email]})</a>" ) + }; + if ( $u->from_body ) { + push @fields, { name => _('Created Body'), val => $u->from_body->name }; + } elsif ( $u->is_superuser ) { + push @fields, { name => _('Created Body'), val => _('Superuser') }; + } + } else { + push @fields, { name => 'contributed_by', val => $extra->{contributed_by} }; + } + delete $extra->{contributed_by}; + } + for my $key ( keys %$extra ) { push @fields, { name => $key, val => $extra->{$key} }; } @@ -341,24 +358,10 @@ sub edit : Path('/admin/report_edit') : Args(1) { if ( $problem->state ne $old_state ) { $c->forward( '/admin/log_edit', [ $id, 'problem', 'state_change' ] ); - my $name = $c->user->moderating_user_name; - my $extra = { is_superuser => 1 }; - if ($c->user->from_body) { - delete $extra->{is_superuser}; - $extra->{is_body_user} = $c->user->from_body->id; - } - my $timestamp = \'current_timestamp'; $problem->add_to_comments( { text => $c->stash->{update_text} || '', - created => $timestamp, - confirmed => $timestamp, - user_id => $c->user->id, - name => $name, - mark_fixed => 0, - anonymous => 0, - state => 'confirmed', + user => $c->user->obj, problem_state => $problem->state, - extra => $extra } ); } $c->forward( '/admin/log_edit', [ $id, 'problem', 'edit' ] ); @@ -417,13 +420,7 @@ sub edit_category : Private { } else { $problem->add_to_comments({ text => $update_text, - created => \'current_timestamp', - confirmed => \'current_timestamp', - user_id => $c->user->id, - name => $c->user->from_body ? $c->user->from_body->name : $c->user->name, - state => 'confirmed', - mark_fixed => 0, - anonymous => 0, + user => $c->user->obj, }); } $c->forward( '/admin/log_edit', [ $problem->id, 'problem', 'category_change' ] ); diff --git a/perllib/FixMyStreet/App/Controller/Admin/Templates.pm b/perllib/FixMyStreet/App/Controller/Admin/Templates.pm index efff1b488..9fb401e2b 100644 --- a/perllib/FixMyStreet/App/Controller/Admin/Templates.pm +++ b/perllib/FixMyStreet/App/Controller/Admin/Templates.pm @@ -69,7 +69,7 @@ sub edit : Path : Args(2) { category => $_->category_display, active => $active_contacts{$_->id}, email => $_->email, - group => $_->get_extra_metadata('group') // '', + group => $_->groups, } } @live_contacts; $c->stash->{contacts} = \@all_contacts; $c->forward('/report/stash_category_groups', [ \@all_contacts, 1 ]) if $c->cobrand->enable_category_groups; @@ -100,7 +100,10 @@ sub edit : Path : Args(2) { $template->text( $c->get_param('text') ); $template->state( $c->get_param('state') ); - $template->external_status_code( $c->get_param('external_status_code') ); + + my $ext_code = $c->cobrand->call_hook('admin_templates_external_status_code_hook'); + $ext_code ||= $c->get_param('external_status_code'); + $template->external_status_code($ext_code); if ( $template->state && $template->external_status_code ) { $c->stash->{errors} ||= {}; diff --git a/perllib/FixMyStreet/App/Controller/Admin/Triage.pm b/perllib/FixMyStreet/App/Controller/Admin/Triage.pm index 428c35073..c5bb6628d 100644 --- a/perllib/FixMyStreet/App/Controller/Admin/Triage.pm +++ b/perllib/FixMyStreet/App/Controller/Admin/Triage.pm @@ -90,10 +90,8 @@ sub setup_categories : Private { delete $c->stash->{categories_hash}; my %category_groups = (); for my $category (@{$c->stash->{end_options}}) { - my $group = $category->{group} // $category->get_extra_metadata('group') // ['']; - # this could be an array ref or a string - my @groups = ref $group eq 'ARRAY' ? @$group : ($group); - push( @{$category_groups{$_}}, $category ) for @groups; + my $groups = $category->groups; + push( @{$category_groups{$_}}, $category ) for @$groups; } my @category_groups = (); for my $group ( grep { $_ ne _('Other') } sort keys %category_groups ) { @@ -119,27 +117,14 @@ sub update : Private { $c->stash->{problem}->update( { state => 'confirmed' } ); $c->forward( '/admin/log_edit', [ $problem->id, 'problem', 'triage' ] ); - my $name = $c->user->moderating_user_name; - my $extra = { is_superuser => 1 }; - if ($c->user->from_body) { - delete $extra->{is_superuser}; - $extra->{is_body_user} = $c->user->from_body->id; - } - + my $extra; $extra->{triage_report} = 1; $extra->{holding_category} = $current_category; $extra->{new_category} = $new_category; - my $timestamp = \'current_timestamp'; my $comment = $problem->add_to_comments( { text => "Report triaged from $current_category to $new_category", - created => $timestamp, - confirmed => $timestamp, - user_id => $c->user->id, - name => $name, - mark_fixed => 0, - anonymous => 0, - state => 'confirmed', + user => $c->user->obj, problem_state => $problem->state, extra => $extra, whensent => \'current_timestamp', diff --git a/perllib/FixMyStreet/App/Controller/Admin/Users.pm b/perllib/FixMyStreet/App/Controller/Admin/Users.pm index 046e19126..7ebfb9bbd 100644 --- a/perllib/FixMyStreet/App/Controller/Admin/Users.pm +++ b/perllib/FixMyStreet/App/Controller/Admin/Users.pm @@ -29,17 +29,29 @@ sub index :Path : Args(0) { if ($c->req->method eq 'POST') { my @uids = $c->get_param_list('uid'); - my @role_ids = $c->get_param_list('roles'); my $user_rs = FixMyStreet::DB->resultset("User")->search({ id => \@uids }); - foreach my $user ($user_rs->all) { - $user->admin_user_body_permissions->delete; - $user->user_roles->search({ - role_id => { -not_in => \@role_ids }, - })->delete; - foreach my $role (@role_ids) { - $user->user_roles->find_or_create({ - role_id => $role, + if ( $c->get_param('remove-staff') ) { + foreach my $user ($user_rs->all) { + $user->update({ + from_body => undef, + email_verified => 0, + phone_verified => 0, }); + $user->user_roles->delete; + $user->admin_user_body_permissions->delete; + } + } else { + my @role_ids = $c->get_param_list('roles'); + foreach my $user ($user_rs->all) { + $user->admin_user_body_permissions->delete; + $user->user_roles->search({ + role_id => { -not_in => \@role_ids }, + })->delete; + foreach my $role (@role_ids) { + $user->user_roles->find_or_create({ + role_id => $role, + }); + } } } $c->stash->{status_message} = _('Updated!'); @@ -47,28 +59,13 @@ sub index :Path : Args(0) { my $search = $c->get_param('search'); my $role = $c->get_param('role'); + my $users = $c->cobrand->users; if ($search || $role) { - my $users = $c->cobrand->users; - my $isearch; if ($search) { $search = $self->trim($search); $search =~ s/^<(.*)>$/$1/; # In case email wrapped in <...> $c->stash->{searched} = $search; - - $isearch = '%' . $search . '%'; - my $search_n = 0; - $search_n = int($search) if $search =~ /^\d+$/; - - $users = $users->search( - { - -or => [ - email => { ilike => $isearch }, - phone => { ilike => $isearch }, - name => { ilike => $isearch }, - from_body => $search_n, - ] - } - ); + $users = $users->search_text($search); } if ($role) { $c->stash->{role_selected} = $role; @@ -78,25 +75,23 @@ sub index :Path : Args(0) { join => 'user_roles', }); } - - my @users = $users->all; - $c->stash->{users} = [ @users ]; - if ($search) { - $c->forward('/admin/add_flags', [ { email => { ilike => $isearch } } ]); - } - } else { $c->forward('/auth/get_csrf_token'); $c->forward('/admin/fetch_all_bodies'); $c->cobrand->call_hook('admin_user_edit_extra_data'); # Admin users by default - my $users = $c->cobrand->users->search( - { from_body => { '!=', undef } }, - { order_by => 'name' } - ); - my @users = $users->all; - $c->stash->{users} = \@users; + $users = $users->search({ from_body => { '!=', undef } }); + } + + $users = $users->search(undef, { + prefetch => 'from_body', + order_by => [ \"me.name = ''", 'me.name' ], + }); + my @users = $users->all; + $c->stash->{users} = \@users; + if ($search) { + $c->forward('/admin/add_flags', [ { email => { ilike => "%$search%" } } ]); } my $rs; @@ -373,6 +368,11 @@ sub edit : Chained('user') : PathPart('') : Args(0) { my @live_contact_ids = map { $_->id } @live_contacts; my @new_contact_ids = grep { $c->get_param("contacts[$_]") } @live_contact_ids; $user->set_extra_metadata('categories', \@new_contact_ids); + if ($c->get_param('assigned_categories_only')) { + $user->set_extra_metadata(assigned_categories_only => 1); + } else { + $user->unset_extra_metadata('assigned_categories_only'); + } } else { $user->unset_extra_metadata('categories'); } @@ -396,7 +396,7 @@ sub edit : Chained('user') : PathPart('') : Args(0) { id => $_->id, category => $_->category, active => $active_contacts{$_->id}, - group => $_->get_extra_metadata('group') // '', + group => $_->groups, } } @live_contacts; $c->stash->{contacts} = \@all_contacts; $c->forward('/report/stash_category_groups', [ \@all_contacts, 1 ]) if $c->cobrand->enable_category_groups; diff --git a/perllib/FixMyStreet/App/Controller/Around.pm b/perllib/FixMyStreet/App/Controller/Around.pm index af50f1883..803bfad58 100644 --- a/perllib/FixMyStreet/App/Controller/Around.pm +++ b/perllib/FixMyStreet/App/Controller/Around.pm @@ -240,12 +240,8 @@ sub check_and_stash_category : Private { $c->stash->{bodies_ids} = [ map { $_->id } @bodies]; $c->{stash}->{list_of_names_as_string} = $csv->string; - my $where = { body_id => [ keys %bodies ], }; - - my $cobrand_where = $c->cobrand->call_hook('munge_around_category_where', $where ); - if ( $cobrand_where ) { - $where = $cobrand_where; - } + my $where = { body_id => [ keys %bodies ], }; + $c->cobrand->call_hook('munge_around_category_where', $where); my @categories = $c->model('DB::Contact')->not_deleted->search( $where, @@ -254,6 +250,9 @@ sub check_and_stash_category : Private { distinct => 1 } )->all_sorted; + # Ensure only uniquely named categories are shown + my %seen; + @categories = grep { !$seen{$_->category_display}++ } @categories; $c->stash->{filter_categories} = \@categories; my %categories_mapped = map { $_->category => 1 } @categories; $c->forward('/report/stash_category_groups', [ \@categories ]) if $c->cobrand->enable_category_groups; @@ -262,6 +261,8 @@ sub check_and_stash_category : Private { my %valid_categories = map { $_ => 1 } grep { $_ && $categories_mapped{$_} } @$categories; $c->stash->{filter_category} = \%valid_categories; $c->cobrand->call_hook('munge_around_filter_category_list'); + + $c->forward('/report/assigned_users_only', [ \@categories ]); } sub map_features : Private { @@ -293,7 +294,7 @@ sub map_features : Private { @pins = map { # Here we might have a DB::Problem or a DB::Result::Nearby, we always want the problem. my $p = (ref $_ eq 'FixMyStreet::DB::Result::Nearby') ? $_->problem : $_; - $p->pin_data($c, 'around'); + $p->pin_data('around'); } @$on_map, @$nearby; } diff --git a/perllib/FixMyStreet/App/Controller/Auth.pm b/perllib/FixMyStreet/App/Controller/Auth.pm index cecfa318c..0d0c2240a 100644 --- a/perllib/FixMyStreet/App/Controller/Auth.pm +++ b/perllib/FixMyStreet/App/Controller/Auth.pm @@ -448,6 +448,9 @@ sub check_csrf_token : Private { unless $time && $time > time() - 3600 && $token eq $gen_token; + + # Also check recaptcha if needed + $c->cobrand->call_hook('check_recaptcha'); } sub no_csrf_token : Private { @@ -457,7 +460,7 @@ sub no_csrf_token : Private { =item common_password -Returns 1/0 depending on if password is common or not. +Returns 1/0 depending on if password is common/breached or not. =cut @@ -466,10 +469,8 @@ sub common_password : Local : Args(0) { my $password = $c->get_param('password_register'); - my $return = JSON->true; - if (!$c->cobrand->call_hook('bypass_password_checks') && found($password)) { - $return = _('Please choose a less commonly-used password'); - } + my $pass = $c->forward('test_password', [ $password ]); + my $return = $pass ? JSON->true : $c->stash->{field_errors}->{password_register}; my $body = JSON->new->utf8->allow_nonref->encode($return); $c->res->content_type('application/json; charset=utf-8'); @@ -488,22 +489,50 @@ sub test_password : Private { return 1 if $c->cobrand->call_hook('bypass_password_checks'); - my @errors; - + my $error; my $min_length = $c->cobrand->password_minimum_length; - push @errors, sprintf(_('Please make sure your password is at least %d characters long'), $min_length) - if length($password) < $min_length; - - push @errors, _('Please choose a less commonly-used password') - if found($password); + if (length($password) < $min_length) { + $error = sprintf(_('Please make sure your password is at least %d characters long'), $min_length); + } elsif (found($password)) { + $error = _('Please choose a less commonly-used password'); + } elsif (hibp($password)) { + $error = _('That password has appeared in a known third-party data breach (<a href="https://haveibeenpwned.com/Passwords" target="_blank">more information</a>); please choose another'); + } - if (@errors) { - $c->stash->{field_errors}->{password_register} = join('<br>', @errors); + if ($error) { + $c->stash->{field_errors}->{password_register} = $error; return 0; } return 1; } +=item hibp + +Returns true if we should check Have I Been Pwned and the check +comes back positive for a password that has been breached. + +=cut + +use Encode qw(encode); +use Digest::SHA qw(sha1_hex); +use LWP::Simple; +use Unicode::Normalize; + +sub hibp : Private { + my $password = shift; + + return 0 unless FixMyStreet->config('CHECK_HAVEIBEENPWNED'); + my $sha1 = uc sha1_hex(encode('UTF-8', NFD($password))); + my $url = 'https://api.pwnedpasswords.com/range/' . substr($sha1, 0, 5); + my $response = LWP::Simple::get($url); + my $remainder = substr($sha1, 5); + foreach my $line (split /\r\n/, $response) { + my ($part, $count) = split /:/, $line; + return $count if $part eq $remainder; + } + return 0; +} + =head2 sign_out Log the user out. Tell them we've done so. diff --git a/perllib/FixMyStreet/App/Controller/Auth/Profile.pm b/perllib/FixMyStreet/App/Controller/Auth/Profile.pm index a89c6f539..a5dc5d3e7 100644 --- a/perllib/FixMyStreet/App/Controller/Auth/Profile.pm +++ b/perllib/FixMyStreet/App/Controller/Auth/Profile.pm @@ -188,9 +188,10 @@ sub generate_token : Path('/auth/generate_token') { if ($c->get_param('generate_token')) { my $token = mySociety::AuthToken::random_token(); - $c->user->set_extra_metadata('access_token', $token); + my $u = FixMyStreet::DB->resultset("User")->new({ password => $token }); + $c->user->set_extra_metadata('access_token', $u->password); $c->user->update; - $c->stash->{token_generated} = 1; + $c->stash->{token_generated} = $c->user->id . '-' . $token; } my $action = $c->get_param('2fa_action') || ''; @@ -224,7 +225,7 @@ sub generate_token : Path('/auth/generate_token') { } $c->stash->{has_2fa} = $has_2fa ? 1 : 0; - $c->stash->{existing_token} = $c->user->get_extra_metadata('access_token'); + $c->stash->{existing_token} = $c->user->get_extra_metadata('access_token') ? 1 : 0; } __PACKAGE__->meta->make_immutable; diff --git a/perllib/FixMyStreet/App/Controller/Auth/Social.pm b/perllib/FixMyStreet/App/Controller/Auth/Social.pm index 06e67573f..ce94fe256 100644 --- a/perllib/FixMyStreet/App/Controller/Auth/Social.pm +++ b/perllib/FixMyStreet/App/Controller/Auth/Social.pm @@ -6,7 +6,7 @@ BEGIN { extends 'Catalyst::Controller'; } use Net::Facebook::Oauth2; use Net::Twitter::Lite::WithAPIv1_1; -use OIDC::Lite::Client::WebServer::Azure; +use OIDC::Lite::Client::WebServer::AuthCodeFlow; use URI::Escape; use mySociety::AuthToken; @@ -167,7 +167,7 @@ sub oidc : Private { my $config = $c->cobrand->feature('oidc_login'); - OIDC::Lite::Client::WebServer::Azure->new( + OIDC::Lite::Client::WebServer::AuthCodeFlow->new( id => $config->{client_id}, secret => $config->{secret}, authorize_uri => $config->{auth_uri}, @@ -179,7 +179,9 @@ sub oidc_sign_in : Private { my ( $self, $c ) = @_; $c->detach( '/page_error_403_access_denied', [] ) if FixMyStreet->config('SIGNUPS_DISABLED'); - $c->detach( '/page_error_400_bad_request', [] ) unless $c->cobrand->feature('oidc_login'); + + my $cfg = $c->cobrand->feature('oidc_login'); + $c->detach( '/page_error_400_bad_request', [] ) unless $cfg; my $oidc = $c->forward('oidc'); my $nonce = $self->generate_nonce(); @@ -190,6 +192,15 @@ sub oidc_sign_in : Private { extra => { response_mode => 'form_post', nonce => $nonce, + # auth_extra_params provides a way to pass custom parameters + # to the OIDC endpoint for the intial authentication request. + # This allows, for example, a custom scope to be used, + # or the `hd` parameter which customises the appearance of + # the login form. + # This is primarily useful for Google G Suite authentication - see + # available parameters here: + # https://developers.google.com/identity/protocols/oauth2/openid-connect#authenticationuriparameters + %{ $cfg->{auth_extra_params} || {} } , }, ); @@ -201,14 +212,14 @@ sub oidc_sign_in : Private { # The OIDC endpoint may require a specific URI to be called to log the user # out when they log out of FMS. - if ( my $redirect_uri = $c->cobrand->feature('oidc_login')->{logout_uri} ) { + if ( my $redirect_uri = $cfg->{logout_uri} ) { $redirect_uri .= "?post_logout_redirect_uri="; $redirect_uri .= URI::Escape::uri_escape( $c->uri_for('/auth/sign_out') ); $oauth{logout_redirect_uri} = $redirect_uri; } # The OIDC endpoint may provide a specific URI for changing the user's password. - if ( my $password_change_uri = $c->cobrand->feature('oidc_login')->{password_change_uri} ) { + if ( my $password_change_uri = $cfg->{password_change_uri} ) { $oauth{change_password_uri} = $oidc->uri_to_redirect( uri => $password_change_uri, redirect_uri => $c->uri_for('/auth/OIDC'), @@ -279,6 +290,7 @@ sub oidc_callback: Path('/auth/OIDC') : Args(0) { eval { $id_token = $oidc->get_access_token( code => $c->get_param('code'), + redirect_uri => $c->uri_for('/auth/OIDC') ); }; if ($@) { @@ -294,10 +306,18 @@ sub oidc_callback: Path('/auth/OIDC') : Args(0) { # check that the nonce matches what we set in the user session $c->detach('/page_error_500_internal_error', ['invalid id_token']) unless $id_token->payload->{nonce} eq $c->session->{oauth}{nonce}; + if (my $domains = $c->cobrand->feature('oidc_login')->{allowed_domains}) { + # Check that the hd payload is present in the token and matches the + # list of allowed domains from the config + my $hd = $id_token->payload->{hd}; + my %allowed_domains = map { $_ => 1} @$domains; + $c->detach('oauth_failure') unless $allowed_domains{$hd}; + } + # Some claims need parsing into a friendlier format - # XXX check how much of this is Westminster/Azure-specific - my $name = join(" ", $id_token->payload->{given_name}, $id_token->payload->{family_name}); + my $name = $id_token->payload->{name} || join(" ", $id_token->payload->{given_name}, $id_token->payload->{family_name}); my $email = $id_token->payload->{email}; + # WCC Azure provides a single email address as an array for some reason my $emails = $id_token->payload->{emails}; if ($emails && @$emails) { diff --git a/perllib/FixMyStreet/App/Controller/Contact.pm b/perllib/FixMyStreet/App/Controller/Contact.pm index 9ce89a9e2..f919cbeff 100644 --- a/perllib/FixMyStreet/App/Controller/Contact.pm +++ b/perllib/FixMyStreet/App/Controller/Contact.pm @@ -199,7 +199,7 @@ sub prepare_params_for_email : Private { my $base_url = $c->cobrand->base_url(); my $admin_url = $c->cobrand->admin_base_url; - my $user = $c->cobrand->users->find( { email => $c->stash->{em} } ); + my $user = $c->cobrand->users->find( { email => lc $c->stash->{em} } ); if ( $user ) { $c->stash->{user_admin_url} = $admin_url . '/users/' . $user->id; $c->stash->{user_reports_admin_url} = $admin_url . '/reports?search=' . $user->email; diff --git a/perllib/FixMyStreet/App/Controller/Contact/Enquiry.pm b/perllib/FixMyStreet/App/Controller/Contact/Enquiry.pm index 5b1c4980f..da5b9906e 100644 --- a/perllib/FixMyStreet/App/Controller/Contact/Enquiry.pm +++ b/perllib/FixMyStreet/App/Controller/Contact/Enquiry.pm @@ -49,15 +49,9 @@ sub submit : Path('submit') : Args(0) { $c->set_param('pc', ''); $c->set_param('skipped', 1); - $c->forward('/report/new/initialize_report'); - $c->forward('/report/new/check_for_category'); - $c->forward('/auth/check_csrf_token'); - $c->forward('/report/new/process_report'); - $c->forward('/report/new/process_user'); - $c->forward('handle_uploads'); - $c->forward('/photo/process_photo'); - $c->go('index', [ 1 ]) unless $c->forward('/report/new/check_for_errors'); - $c->forward('/report/new/save_user_and_report'); + $c->forward('/report/new/non_map_creation', [ [ '/contact/enquiry/handle_uploads' ] ]) + or $c->go('index', [ 1 ]); + $c->forward('confirm_report'); $c->stash->{success} = 1; diff --git a/perllib/FixMyStreet/App/Controller/Dashboard.pm b/perllib/FixMyStreet/App/Controller/Dashboard.pm index ad6c9ba98..5400a6209 100644 --- a/perllib/FixMyStreet/App/Controller/Dashboard.pm +++ b/perllib/FixMyStreet/App/Controller/Dashboard.pm @@ -6,9 +6,9 @@ use DateTime; use Encode; use JSON::MaybeXS; use Path::Tiny; -use Text::CSV; use Time::Piece; use FixMyStreet::DateRange; +use FixMyStreet::Reporting; BEGIN { extends 'Catalyst::Controller'; } @@ -135,13 +135,25 @@ sub index : Path : Args(0) { $c->stash->{end_date} = $c->get_param('end_date'); $c->stash->{q_state} = $c->get_param('state') || ''; - $c->forward('construct_rs_filter', [ $c->get_param('updates') ]); - - if ( $c->get_param('export') ) { - if ($c->get_param('updates')) { - $c->forward('export_as_csv_updates'); - } else { - $c->forward('export_as_csv'); + my $reporting = $c->forward('construct_rs_filter', [ $c->get_param('updates') ]); + + if ( my $export = $c->get_param('export') ) { + $reporting->csv_parameters; + if ($export == 1) { + # Existing method, generate and serve + $reporting->generate_csv_http($c); + } elsif ($export == 2) { + # New offline method + $reporting->kick_off_process; + my ($redirect, $code) = ('/dashboard/status', 303); + if (Catalyst::Authentication::Credential::AccessToken->get_token($c)) { + # Client knows to re-request until ready + $redirect = '/dashboard/csv/' . $reporting->filename . '.csv'; + $c->res->body(''); + $code = 202; + } + $c->res->redirect($redirect, $code); + $c->detach; } } else { $c->forward('generate_grouped_data'); @@ -152,37 +164,19 @@ sub index : Path : Args(0) { sub construct_rs_filter : Private { my ($self, $c, $updates) = @_; - my %where; - $where{areas} = [ map { { 'like', "%,$_,%" } } @{$c->stash->{ward}} ] - if @{$c->stash->{ward}}; - $where{category} = $c->stash->{category} - if $c->stash->{category}; - - my $table_name = $updates ? 'problem' : 'me'; - - my $state = $c->stash->{q_state}; - if ( FixMyStreet::DB::Result::Problem->fixed_states->{$state} ) { # Probably fixed - council - $where{"$table_name.state"} = [ FixMyStreet::DB::Result::Problem->fixed_states() ]; - } elsif ( $state ) { - $where{"$table_name.state"} = $state; - } else { - $where{"$table_name.state"} = [ FixMyStreet::DB::Result::Problem->visible_states() ]; - } - - my $days30 = DateTime->now(time_zone => FixMyStreet->time_zone || FixMyStreet->local_time_zone)->subtract(days => 30); - $days30->truncate( to => 'day' ); - - my $range = FixMyStreet::DateRange->new( + my $reporting = FixMyStreet::Reporting->new( + type => $updates ? 'updates' : 'problems', + category => $c->stash->{category}, + state => $c->stash->{q_state}, + wards => $c->stash->{ward}, + body => $c->stash->{body} || undef, start_date => $c->stash->{start_date}, - start_default => $days30, end_date => $c->stash->{end_date}, - formatter => $c->model('DB')->storage->datetime_parser, + user => $c->user_exists ? $c->user->obj : undef, ); - $where{"$table_name.confirmed"} = $range->sql; - $c->stash->{params} = \%where; - my $rs = $updates ? $c->cobrand->updates : $c->cobrand->problems; - $c->stash->{objects_rs} = $rs->to_body($c->stash->{body})->search( \%where ); + $c->stash($reporting->construct_rs_filter); + return $reporting; } sub generate_grouped_data : Private { @@ -297,210 +291,67 @@ sub generate_summary_figures { } } -sub generate_body_response_time : Private { - my ( $self, $c ) = @_; - - my $avg = $c->stash->{body}->calculate_average($c->cobrand->call_hook("body_responsiveness_threshold")); - $c->stash->{body_average} = $avg ? int($avg / 60 / 60 / 24 + 0.5) : 0; -} - -sub csv_filename { - my ($self, $c, $updates) = @_; - my %where = ( - category => $c->stash->{category}, - state => $c->stash->{q_state}, - ward => join(',', @{$c->stash->{ward}}), - ); - $where{body} = $c->stash->{body}->id if $c->stash->{body}; - join '-', - $c->req->uri->host, - $updates ? ('updates') : (), - map { - my $value = $where{$_}; - (defined $value and length $value) ? ($_, $value) : () - } sort keys %where -}; - -sub export_as_csv_updates : Private { +sub status : Local : Args(0) { my ($self, $c) = @_; - my $csv = $c->stash->{csv} = { - objects => $c->stash->{objects_rs}->search_rs({}, { - order_by => ['me.confirmed', 'me.id'], - '+columns' => ['problem.bodies_str'], - cursor_page_size => 1000, - }), - headers => [ - 'Report ID', 'Update ID', 'Date', 'Status', 'Problem state', - 'Text', 'User Name', 'Reported As', - ], - columns => [ - 'problem_id', 'id', 'confirmed', 'state', 'problem_state', - 'text', 'user_name_display', 'reported_as', - ], - filename => $self->csv_filename($c, 1), - }; - $c->cobrand->call_hook("dashboard_export_updates_add_columns"); - $c->forward('generate_csv'); -} - -sub export_as_csv : Private { - my ($self, $c) = @_; + my $body = $c->stash->{body} = $c->forward('check_page_allowed'); + $c->stash->{body_name} = $body->name if $body; - my $csv = $c->stash->{csv} = { - objects => $c->stash->{objects_rs}->search_rs({}, { - join => 'comments', - '+columns' => ['comments.problem_state', 'comments.state', 'comments.confirmed', 'comments.mark_fixed'], - order_by => ['me.confirmed', 'me.id'], - cursor_page_size => 1000, - }), - headers => [ - 'Report ID', - 'Title', - 'Detail', - 'User Name', - 'Category', - 'Created', - 'Confirmed', - 'Acknowledged', - 'Fixed', - 'Closed', - 'Status', - 'Latitude', 'Longitude', - 'Query', - 'Ward', - 'Easting', - 'Northing', - 'Report URL', - 'Site Used', - 'Reported As', - ], - columns => [ - 'id', - 'title', - 'detail', - 'user_name_display', - 'category', - 'created', - 'confirmed', - 'acknowledged', - 'fixed', - 'closed', - 'state', - 'latitude', 'longitude', - 'postcode', - 'wards', - 'local_coords_x', - 'local_coords_y', - 'url', - 'site_used', - 'reported_as', - ], - filename => $self->csv_filename($c, 0), - }; - $c->cobrand->call_hook("dashboard_export_problems_add_columns"); - $c->forward('generate_csv'); + my $reporting = FixMyStreet::Reporting->new( + user => $c->user_exists ? $c->user->obj : undef, + ); + my $dir = $reporting->cache_dir; + my @data; + foreach ($dir->children) { + my $stat = $_->stat; + my $name = $_->basename; + my $finished = $name =~ /part$/ ? 0 : 1; + $name =~ s/-part$//; + push @data, { + ctime => $stat->ctime, + size => $stat->size, + name => $name, + finished => $finished, + }; + } + @data = sort { $b->{ctime} <=> $a->{ctime} } @data; + $c->stash->{rows} = \@data; } -=head2 generate_csv - -Generates a CSV output, given a 'csv' stash hashref containing: -* filename: filename to be used in output -* problems: a resultset of the rows to output -* headers: an arrayref of the header row strings -* columns: an arrayref of the columns (looked up in the row's as_hashref, plus -the following: user_name_display, acknowledged, fixed, closed, wards, -local_coords_x, local_coords_y, url). -* extra_data: If present, a function that is passed the report and returns a -hashref of extra data to include that can be used by 'columns'. - -=cut +sub csv : Local : Args(1) { + my ($self, $c, $filename) = @_; -sub generate_csv : Private { - my ($self, $c) = @_; + $c->authenticate(undef, "access_token"); - my $filename = $c->stash->{csv}->{filename}; - $c->res->content_type('text/csv; charset=utf-8'); - $c->res->header('content-disposition' => "attachment; filename=\"${filename}.csv\""); - - # Emit a header (copying Drupal's naming) telling an intermediary (e.g. - # Varnish) not to buffer the output. Varnish will need to know this, e.g.: - # if (beresp.http.Surrogate-Control ~ "BigPipe/1.0") { - # set beresp.do_stream = true; - # set beresp.ttl = 0s; - # } - $c->res->header('Surrogate-Control' => 'content="BigPipe/1.0"'); - - # Tell nginx not to buffer this response - $c->res->header('X-Accel-Buffering' => 'no'); - - # Define an empty body so the web view doesn't get added at the end - $c->res->body(""); - - # Old parameter renaming - $c->stash->{csv}->{objects} //= $c->stash->{csv}->{problems}; - - my $csv = Text::CSV->new({ binary => 1, eol => "\n" }); - $csv->print($c->response, $c->stash->{csv}->{headers}); - - my $fixed_states = FixMyStreet::DB::Result::Problem->fixed_states; - my $closed_states = FixMyStreet::DB::Result::Problem->closed_states; - - my %asked_for = map { $_ => 1 } @{$c->stash->{csv}->{columns}}; - - my $objects = $c->stash->{csv}->{objects}; - while ( my $obj = $objects->next ) { - my $hashref = $obj->as_hashref($c, \%asked_for); - - $hashref->{user_name_display} = $obj->anonymous - ? '(anonymous)' : $obj->name; - - if ($asked_for{acknowledged}) { - for my $comment ($obj->comments) { - my $problem_state = $comment->problem_state or next; - next unless $comment->state eq 'confirmed'; - next if $problem_state eq 'confirmed'; - $hashref->{acknowledged} //= $comment->confirmed; - $hashref->{fixed} //= $fixed_states->{ $problem_state } || $comment->mark_fixed ? - $comment->confirmed : undef; - if ($closed_states->{ $problem_state }) { - $hashref->{closed} = $comment->confirmed; - last; - } - } - } - - if ($asked_for{wards}) { - $hashref->{wards} = join ', ', - map { $c->stash->{children}->{$_}->{name} } - grep {$c->stash->{children}->{$_} } - split ',', $hashref->{areas}; - } + my $body = $c->stash->{body} = $c->forward('check_page_allowed'); - if ($obj->can('local_coords') && $asked_for{local_coords_x}) { - ($hashref->{local_coords_x}, $hashref->{local_coords_y}) = - $obj->local_coords; - } - if ($obj->can('url')) { - my $base = $c->cobrand->base_url_for_report($obj->can('problem') ? $obj->problem : $obj); - $hashref->{url} = join '', $base, $obj->url; + (my $basename = $filename) =~ s/\.csv$//; + my $reporting = FixMyStreet::Reporting->new( + user => $c->user_exists ? $c->user->obj : undef, + filename => $basename, + ); + my $dir = $reporting->cache_dir; + my $csv = path($dir, $filename); + + if (!$csv->exists) { + if (path($dir, "$filename-part")->exists && Catalyst::Authentication::Credential::AccessToken->get_token($c)) { + $c->res->body(''); + $c->res->status(202); + $c->detach; + } else { + $c->detach( '/page_error_404_not_found', [] ) unless $csv->exists; } + } - $hashref->{site_used} = $obj->can('service') ? ($obj->service || $obj->cobrand) : $obj->cobrand; - - $hashref->{reported_as} = $obj->get_extra_metadata('contributed_as') || ''; + $reporting->http_setup($c); + $c->res->body($csv->openr_raw); +} - if (my $fn = $c->stash->{csv}->{extra_data}) { - my $extra = $fn->($obj); - $hashref = { %$hashref, %$extra }; - } +sub generate_body_response_time : Private { + my ( $self, $c ) = @_; - $csv->print($c->response, [ - @{$hashref}{ - @{$c->stash->{csv}->{columns}} - }, - ] ); - } + my $avg = $c->stash->{body}->calculate_average($c->cobrand->call_hook("body_responsiveness_threshold")); + $c->stash->{body_average} = $avg ? int($avg / 60 / 60 / 24 + 0.5) : 0; } sub heatmap : Local : Args(0) { @@ -534,7 +385,7 @@ sub heatmap : Local : Args(0) { if ($c->get_param('ajax')) { my @pins; while ( my $problem = $problems->next ) { - push @pins, $problem->pin_data($c, 'reports'); + push @pins, $problem->pin_data('reports'); } $c->stash->{pins} = \@pins; $c->detach('/reports/ajax', [ 'dashboard/heatmap-list.html' ]); @@ -544,7 +395,8 @@ sub heatmap : Local : Args(0) { $c->stash->{children} = $children; $c->stash->{ward_hash} = { map { $_->{id} => 1 } @{$c->stash->{wards}} } if $c->stash->{wards}; - $c->forward('/reports/setup_categories_and_map'); + $c->forward('/reports/setup_categories'); + $c->forward('/reports/setup_map'); } sub heatmap_filters :Private { @@ -591,7 +443,15 @@ sub heatmap_sidebar :Private { order_by => 'lastupdate', })->all ]; - my $params = { map { my $n = $_; s/me\./problem\./; $_ => $where->{$n} } keys %$where }; + my $params = { map { + my $v = $where->{$_}; + if (ref $v eq 'HASH') { + $v = { map { my $vv = $v->{$_}; s/me\./problem\./; $_ => $vv } keys %$v }; + } else { + s/me\./problem\./; + } + $_ => $v; + } keys %$where }; my $body = $c->stash->{body}; my @user; diff --git a/perllib/FixMyStreet/App/Controller/JSON.pm b/perllib/FixMyStreet/App/Controller/JSON.pm index ccc5b31dc..4657fcf2c 100644 --- a/perllib/FixMyStreet/App/Controller/JSON.pm +++ b/perllib/FixMyStreet/App/Controller/JSON.pm @@ -7,7 +7,6 @@ BEGIN { extends 'Catalyst::Controller'; } use JSON::MaybeXS; use DateTime; use DateTime::Format::ISO8601; -use List::MoreUtils 'uniq'; use FixMyStreet::DateRange; =head1 NAME @@ -102,6 +101,7 @@ sub problems : Local { } ); foreach my $problem (@problems) { + $c->cobrand->call_hook(munge_problem_list => $problem); $problem->name( '' ) if $problem->anonymous == 1; $problem->service( 'Web interface' ) if $problem->service eq ''; my $body_names = $problem->body_names; diff --git a/perllib/FixMyStreet/App/Controller/Location.pm b/perllib/FixMyStreet/App/Controller/Location.pm index 416fb942a..3869ef8d3 100644 --- a/perllib/FixMyStreet/App/Controller/Location.pm +++ b/perllib/FixMyStreet/App/Controller/Location.pm @@ -6,6 +6,7 @@ BEGIN {extends 'Catalyst::Controller'; } use Encode; use FixMyStreet::Geocode; +use Geo::OLC; use Try::Tiny; use Utils; @@ -74,6 +75,30 @@ sub determine_location_from_pc : Private { map { Utils::truncate_coordinate($_) } ($1, $2); return $c->forward( 'check_location' ); } + + if (Geo::OLC::is_full($pc)) { + my $ref = Geo::OLC::decode($pc); + ($c->stash->{latitude}, $c->stash->{longitude}) = + map { Utils::truncate_coordinate($_) } @{$ref->{center}}; + return $c->forward( 'check_location' ); + } + + if ($pc =~ /^\s*([2-9CFGHJMPQRVWX]{4,6}\+[2-9CFGHJMPQRVWX]{2,3})\s+(.*)$/i) { + my ($code, $rest) = ($1, $2); + my ($lat, $lon, $error) = FixMyStreet::Geocode::lookup($rest, $c); + if (ref($error) eq 'ARRAY') { # Just take the first result + $lat = $error->[0]{latitude}; + $lon = $error->[0]{longitude}; + } + if (defined $lat && defined $lon) { + $code = Geo::OLC::recover_nearest($code, $lat, $lon); + my $ref = Geo::OLC::decode($code); + ($c->stash->{latitude}, $c->stash->{longitude}) = + map { Utils::truncate_coordinate($_) } @{$ref->{center}}; + return $c->forward( 'check_location' ); + } + } + if ( $c->cobrand->country eq 'GB' && $pc =~ /^([A-Z])([A-Z])([\d\s]{4,})$/i) { if (my $convert = gridref_to_latlon( $1, $2, $3 )) { ($c->stash->{latitude}, $c->stash->{longitude}) = diff --git a/perllib/FixMyStreet/App/Controller/Moderate.pm b/perllib/FixMyStreet/App/Controller/Moderate.pm index f4143f0b4..c936f13c0 100644 --- a/perllib/FixMyStreet/App/Controller/Moderate.pm +++ b/perllib/FixMyStreet/App/Controller/Moderate.pm @@ -340,13 +340,8 @@ sub moderate_state : Private { $problem->state($new_state); $problem->add_to_comments( { text => $c->stash->{moderation_reason}, - created => \'current_timestamp', - confirmed => \'current_timestamp', - user_id => $c->user->id, - name => $c->user->from_body ? $c->user->from_body->name : $c->user->name, - state => 'confirmed', - mark_fixed => 0, - anonymous => $c->user->from_body ? 0 : 1, + user => $c->user->obj, + anonymous => $c->user->is_superuser || $c->user->from_body ? 0 : 1, problem_state => $new_state, } ); return 'state'; diff --git a/perllib/FixMyStreet/App/Controller/My.pm b/perllib/FixMyStreet/App/Controller/My.pm index 3328caac0..52a3a8cef 100644 --- a/perllib/FixMyStreet/App/Controller/My.pm +++ b/perllib/FixMyStreet/App/Controller/My.pm @@ -135,13 +135,14 @@ sub get_problems : Private { my $problems = []; my $states = $c->stash->{filter_problem_states}; + my $table = $c->action eq 'my/planned' ? 'report' : 'me'; my $params = { - state => [ keys %$states ], + "$table.state" => [ keys %$states ], }; my $categories = [ $c->get_param_list('filter_category', 1) ]; if ( @$categories ) { - $params->{category} = $categories; + $params->{"$table.category"} = $categories; $c->stash->{filter_category} = { map { $_ => 1 } @$categories }; } @@ -149,13 +150,14 @@ sub get_problems : Private { $rows = 5000 if $c->stash->{sort_key} eq 'shortlist'; # Want all reports my $rs = $c->stash->{problems_rs}->search( $params, { + prefetch => 'contact', order_by => $c->stash->{sort_order}, rows => $rows, } )->include_comment_counts->page( $p_page ); while ( my $problem = $rs->next ) { $c->stash->{has_content}++; - push @$pins, $problem->pin_data($c, 'my', private => 1); + push @$pins, $problem->pin_data('my', private => 1); push @$problems, $problem; } @@ -186,15 +188,28 @@ sub get_updates : Private { sub setup_page_data : Private { my ($self, $c) = @_; + my $table = $c->action eq 'my/planned' ? 'report' : 'me'; my @categories = $c->stash->{problems_rs}->search({ - state => [ FixMyStreet::DB::Result::Problem->visible_states() ], + "$table.state" => [ FixMyStreet::DB::Result::Problem->visible_states() ], }, { - columns => [ 'category', 'bodies_str', 'extra' ], + join => 'contact', + columns => [ "$table.category", 'contact.extra', 'contact.category' ], distinct => 1, - order_by => [ 'category' ], + order_by => [ "$table.category" ], } )->all; + # Ensure only uniquely named categories are shown + my %seen; + @categories = grep { !$seen{$_->category_display}++ } @categories; $c->stash->{filter_categories} = \@categories; - $c->forward('/report/stash_category_groups', [ \@categories ]) if $c->cobrand->enable_category_groups; + + if ($c->cobrand->enable_category_groups) { + my @contacts = map { { + category => $_->category, + category_display => $_->category_display, + group => [''], + } } @categories; + $c->forward('/report/stash_category_groups', [ \@contacts ]); + } my $pins = $c->stash->{pins}; FixMyStreet::Map::display_map( diff --git a/perllib/FixMyStreet/App/Controller/Offline.pm b/perllib/FixMyStreet/App/Controller/Offline.pm index adb3de14d..af05c585f 100644 --- a/perllib/FixMyStreet/App/Controller/Offline.pm +++ b/perllib/FixMyStreet/App/Controller/Offline.pm @@ -22,6 +22,7 @@ Offline pages Catalyst Controller - service worker handling sub service_worker : Path("/service-worker.js") { my ($self, $c) = @_; + $c->res->headers->header('Cache-Control' => 'max-age=0'); $c->res->content_type('application/javascript'); } diff --git a/perllib/FixMyStreet/App/Controller/Open311.pm b/perllib/FixMyStreet/App/Controller/Open311.pm index b4b5d5e3a..73a91a62a 100644 --- a/perllib/FixMyStreet/App/Controller/Open311.pm +++ b/perllib/FixMyStreet/App/Controller/Open311.pm @@ -210,7 +210,7 @@ sub output_requests : Private { }; # Look up categories for this council or councils - my $problems = $c->cobrand->problems->search( $criteria, $attr ); + my $problems = $c->stash->{rs}->search( $criteria, $attr ); my %statusmap = ( map( { $_ => 'open' } FixMyStreet::DB::Result::Problem->open_states() ), @@ -220,6 +220,8 @@ sub output_requests : Private { my @problemlist; while ( my $problem = $problems->next ) { + $c->cobrand->call_hook(munge_problem_list => $problem); + my $id = $problem->id; $problem->service( 'Web interface' ) unless $problem->service; @@ -323,24 +325,8 @@ sub get_requests : Private { if ( 'status' eq $param ) { $value = { 'open' => [ FixMyStreet::DB::Result::Problem->open_states() ], - 'closed' => [ FixMyStreet::DB::Result::Problem->fixed_states(), 'closed' ], + 'closed' => [ FixMyStreet::DB::Result::Problem->fixed_states(), FixMyStreet::DB::Result::Problem->closed_states() ], }->{$value}; - } elsif ( 'agency_responsible' eq $param ) { - my @valuelist; - for my $agency (split(/\|/, $value)) { - unless ($agency =~ m/^(\d+)$/) { - $c->detach( 'error', [ - sprintf(_('Invalid agency_responsible value %s'), - $value) - ] ); - } - my $agencyid = $1; - # FIXME This seem to match the wrong entries - # some times. Not sure when or why - my $re = "(\\y$agencyid\\y|^$agencyid\\y|\\y$agencyid\$)"; - push(@valuelist, $re); - } - $value = \@valuelist; } elsif ( 'has_photo' eq $param ) { $value = undef; $op = '!=' if 'true' eq $value; @@ -363,6 +349,11 @@ sub get_requests : Private { $criteria->{confirmed} = { '<', $c->get_param('end_date') }; } + $c->stash->{rs} = $c->cobrand->problems; + if (my $bodies = $c->get_param('agency_responsible')) { + $c->stash->{rs} = $c->stash->{rs}->to_body([ split(/\|/, $bodies) ]); + } + if ('rss' eq $c->stash->{format}) { $c->stash->{type} = 'new_problems'; $c->forward( '/rss/lookup_type' ); @@ -384,7 +375,7 @@ sub rss_query : Private { rows => $limit }; - my $problems = $c->cobrand->problems->search( $criteria, $attr ); + my $problems = $c->stash->{rs}->search( $criteria, $attr ); $c->stash->{problems} = $problems; } @@ -411,6 +402,7 @@ sub get_request : Private { id => $id, non_public => 0, }; + $c->stash->{rs} = $c->cobrand->problems; $c->forward( 'output_requests', [ $criteria ] ); } diff --git a/perllib/FixMyStreet/App/Controller/Open311/Updates.pm b/perllib/FixMyStreet/App/Controller/Open311/Updates.pm index 105400a8a..8881a1b87 100644 --- a/perllib/FixMyStreet/App/Controller/Open311/Updates.pm +++ b/perllib/FixMyStreet/App/Controller/Open311/Updates.pm @@ -3,7 +3,6 @@ package FixMyStreet::App::Controller::Open311::Updates; use utf8; use Moose; use namespace::autoclean; -use Open311; use Open311::GetServiceRequestUpdates; BEGIN { extends 'Catalyst::Controller'; } @@ -31,7 +30,6 @@ sub receive : Regex('^open311/v2/servicerequestupdates.(xml|json)$') : Args(0) { $body = $c->model('DB::Body')->find({ id => $c->get_param('jurisdiction_id') }); } $c->detach('bad_request', ['jurisdiction_id']) unless $body; - my $user = $body->comment_user; my $key = $c->get_param('api_key') || ''; my $token = $c->cobrand->feature('open311_token') || ''; @@ -45,37 +43,36 @@ sub receive : Regex('^open311/v2/servicerequestupdates.(xml|json)$') : Args(0) { $request->{$_} = $c->get_param($_) || $c->detach('bad_request', [ $_ ]); } - my %open311_conf = ( - endpoint => $body->endpoint, - api_key => $body->api_key, - jurisdiction => $body->jurisdiction, - extended_statuses => $body->send_extended_statuses, - ); + $c->forward('process_update', [ $body, $request ]); +} - my $cobrand = $body->get_cobrand_handler; - $cobrand->call_hook(open311_config_updates => \%open311_conf) - if $cobrand; +sub process_update : Private { + my ($self, $c, $body, $request) = @_; - my $open311 = Open311->new(%open311_conf); my $updates = Open311::GetServiceRequestUpdates->new( - system_user => $user, - current_open311 => $open311, + system_user => $body->comment_user, current_body => $body, ); my $p = $updates->find_problem($request); $c->detach('bad_request', [ 'not found' ]) unless $p; - my $comment = $p->comments->search( { external_id => $request->{update_id} } )->first; - $c->detach('bad_request', [ 'already exists' ]) if $comment; + $c->forward('check_existing', [ $p, $request, $updates ]); - $comment = $updates->process_update($request, $p); + my $comment = $updates->process_update($request, $p); my $data = { service_request_updates => { update_id => $comment->id } }; $c->forward('/open311/format_output', [ $data ]); } +sub check_existing : Private { + my ($self, $c, $p, $request, $updates) = @_; + + my $comment = $p->comments->search( { external_id => $request->{update_id} } )->first; + $c->detach('bad_request', [ 'already exists' ]) if $comment; +} + sub bad_request : Private { my ($self, $c, $comment) = @_; $c->response->status(400); diff --git a/perllib/FixMyStreet/App/Controller/Questionnaire.pm b/perllib/FixMyStreet/App/Controller/Questionnaire.pm index ab6117ae4..ef6152c30 100755 --- a/perllib/FixMyStreet/App/Controller/Questionnaire.pm +++ b/perllib/FixMyStreet/App/Controller/Questionnaire.pm @@ -206,16 +206,12 @@ sub submit_standard : Private { $update = $c->model('DB::Comment')->new( { problem => $problem, - name => $problem->name, user => $problem->user, text => $update, - state => 'confirmed', mark_fixed => $c->stash->{new_state} eq 'fixed - user' ? 1 : 0, mark_open => $c->stash->{new_state} eq 'confirmed' ? 1 : 0, lang => $c->stash->{lang_code}, cobrand => $c->cobrand->moniker, - cobrand_data => '', - confirmed => \'current_timestamp', anonymous => $problem->anonymous, } ); diff --git a/perllib/FixMyStreet/App/Controller/Report.pm b/perllib/FixMyStreet/App/Controller/Report.pm index 72f96013a..1e5751588 100644 --- a/perllib/FixMyStreet/App/Controller/Report.pm +++ b/perllib/FixMyStreet/App/Controller/Report.pm @@ -85,14 +85,32 @@ sub display :PathPart('') :Chained('id') :Args(0) { $c->forward( 'load_updates' ); $c->forward( 'format_problem_for_display' ); - my $permissions = $c->stash->{_permissions} ||= $c->forward( 'check_has_permission_to', - [ qw/report_inspect report_edit_category report_edit_priority report_mark_private triage/ ] ); - if (any { $_ } values %$permissions) { + my $permissions = $c->stash->{permissions} ||= $c->forward('fetch_permissions'); + + my $staff_user = $c->user_exists && ($c->user->is_superuser || $c->user->belongs_to_body($c->stash->{problem}->bodies_str)); + + if ($staff_user) { + # Check assigned categories feature + my $okay = 1; + my $contact = $c->stash->{problem}->contact; + if ($contact && ($c->user->get_extra_metadata('assigned_categories_only') || $contact->get_extra_metadata('assigned_users_only'))) { + my $user_cats = $c->user->get_extra_metadata('categories') || []; + $okay = any { $contact->id eq $_ } @$user_cats; + } + if ($okay) { + $c->stash->{relevant_staff_user} = 1; + } else { + # Remove all staff permissions + $permissions = $c->stash->{permissions} = {}; + } + } + + if (grep { $permissions->{$_} } qw/report_inspect report_edit_category report_edit_priority report_mark_private triage/) { $c->stash->{template} = 'report/inspect.html'; $c->forward('inspect'); } - if ($c->user_exists && $c->user->has_permission_to(contribute_as_another_user => $c->stash->{problem}->bodies_str_ids)) { + if ($permissions->{contribute_as_another_user}) { $c->stash->{email} = $c->user->email; } } @@ -133,11 +151,13 @@ sub support :Chained('id') :Args(0) { sub load_problem_or_display_error : Private { my ( $self, $c, $id ) = @_; + my $attrs = { prefetch => 'contact' }; + # try to load a report if the id is a number my $problem = ( !$id || $id =~ m{\D} ) # is id non-numeric? ? undef # ...don't even search - : $c->cobrand->problems->find( { id => $id } ) + : $c->cobrand->problems->find( { id => $id }, $attrs ) or $c->detach( '/page_error_404_not_found', [ _('Unknown problem ID') ] ); # check that the problem is suitable to show. @@ -158,8 +178,7 @@ sub load_problem_or_display_error : Private { } elsif ( $problem->non_public ) { # Creator, and inspection users can see non_public reports $c->stash->{problem} = $problem; - my $permissions = $c->stash->{_permissions} = $c->forward( 'check_has_permission_to', - [ qw/report_inspect report_edit_category report_edit_priority report_mark_private / ] ); + my $permissions = $c->stash->{permissions} = $c->forward('fetch_permissions'); # If someone has clicked a unique token link in an email to them my $from_email = $c->sessionid && $c->flash->{alert_to_reporter} && $c->flash->{alert_to_reporter} == $problem->id; @@ -178,6 +197,7 @@ sub load_problem_or_display_error : Private { } } + $c->cobrand->call_hook(munge_problem_list => $problem); $c->stash->{problem} = $problem; if ( $c->user_exists && $c->user->can_moderate($problem) ) { $c->stash->{problem_original} = $problem->find_or_new_related( @@ -232,6 +252,7 @@ sub load_updates : Private { my @combined; my %questionnaires_with_updates; while (my $update = $updates->next) { + $c->cobrand->call_hook(munge_update_list => $update); push @combined, [ $update->confirmed, $update ]; if (my $qid = $update->get_extra_metadata('questionnaire_id')) { $questionnaires_with_updates{$qid} = $update; @@ -304,7 +325,7 @@ sub format_problem_for_display : Private { $c->res->content_type('application/json; charset=utf-8'); # encode_json doesn't like DateTime objects, so strip them out - my $report_hashref = $c->cobrand->problem_as_hashref( $problem, $c ); + my $report_hashref = $c->cobrand->problem_as_hashref( $problem ); delete $report_hashref->{created}; delete $report_hashref->{confirmed}; @@ -312,7 +333,7 @@ sub format_problem_for_display : Private { my $content = $json->encode( { report => $report_hashref, - updates => $c->cobrand->updates_as_hashref( $problem, $c ), + updates => $c->cobrand->updates_as_hashref( $problem ), } ); $c->res->body( $content ); @@ -333,7 +354,7 @@ sub generate_map_tags : Private { latitude => $problem->latitude, longitude => $problem->longitude, pins => $problem->used_map - ? [ $problem->pin_data($c, 'report', type => 'big', draggable => 1) ] + ? [ $problem->pin_data('report', type => 'big', draggable => 1) ] : [], ); @@ -384,7 +405,7 @@ sub delete :Chained('id') :Args(0) { sub inspect : Private { my ( $self, $c ) = @_; my $problem = $c->stash->{problem}; - my $permissions = $c->stash->{_permissions}; + my $permissions = $c->stash->{permissions}; $c->forward('/admin/reports/categories_for_point'); $c->stash->{report_meta} = { map { 'x' . $_->{name} => $_ } @{ $c->stash->{problem}->get_extra_fields() } }; @@ -392,7 +413,8 @@ sub inspect : Private { if ($c->cobrand->can('body')) { my $priorities_by_category = FixMyStreet::App->model('DB::ResponsePriority')->by_categories( $c->stash->{contacts}, - body_id => $c->cobrand->body->id + body_id => $c->cobrand->body->id, + problem => $problem, ); $c->stash->{priorities_by_category} = $priorities_by_category; my $templates_by_category = FixMyStreet::App->model('DB::ResponseTemplate')->by_categories( @@ -445,7 +467,7 @@ sub inspect : Private { } } - if ( $c->get_param('include_update') ) { + if ( $c->get_param('include_update') or $c->get_param('raise_defect') ) { $update_text = Utils::cleanup_text( $c->get_param('public_update'), { allow_multiline => 1 } ); if (!$update_text) { $valid = 0; @@ -490,6 +512,14 @@ sub inspect : Private { }; $c->user->create_alert($problem->id, $options); } + + # If the state has been changed to action scheduled and they've said + # they want to raise a defect, consider the report to be inspected. + if ($problem->state eq 'action scheduled' && $c->get_param('raise_defect') && !$problem->get_extra_metadata('inspected')) { + $update_params{extra} = { 'defect_raised' => 1 }; + $problem->set_extra_metadata( inspected => 1 ); + $c->forward( '/admin/log_edit', [ $problem->id, 'problem', 'inspected' ] ); + } } $problem->non_public($c->get_param('non_public') ? 1 : 0); @@ -533,6 +563,12 @@ sub inspect : Private { $c->cobrand->call_hook(report_inspect_update_extra => $problem); + $c->forward('/photo/process_photo'); + if ( my $photo_error = delete $c->stash->{photo_error} ) { + $valid = 0; + push @{ $c->stash->{errors} }, $photo_error; + } + if ($valid) { $problem->lastupdate( \'current_timestamp' ); $problem->update; @@ -548,16 +584,12 @@ sub inspect : Private { epoch => $saved_at ); } - my $name = $c->user->from_body ? $c->user->from_body->name : $c->user->name; $problem->add_to_comments( { text => $update_text, created => $timestamp, confirmed => $timestamp, - user_id => $c->user->id, - name => $name, - state => 'confirmed', - mark_fixed => 0, - anonymous => 0, + user => $c->user->obj, + photo => $c->stash->{upload_fileid} || undef, %update_params, } ); } @@ -647,7 +679,7 @@ sub _nearby_json :Private { # Want to treat these as if they were on map $nearby = [ map { $_->problem } @$nearby ]; my @pins = map { - my $p = $_->pin_data($c, 'around'); + my $p = $_->pin_data('around'); [ $p->{latitude}, $p->{longitude}, $p->{colour}, $p->{id}, $p->{title}, $pin_size, JSON->false ] @@ -666,30 +698,26 @@ sub _nearby_json :Private { } -=head2 check_has_permission_to +=head2 fetch_permissions -Ensure the currently logged-in user has any of the provided permissions applied -to the current Problem in $c->stash->{problem}. Shows the 403 page if not. +Returns a hash of the user's permissions, applied to the problem +in $c->stash->{problem}. =cut -sub check_has_permission_to : Private { - my ( $self, $c, @permissions ) = @_; +sub fetch_permissions : Private { + my ( $self, $c ) = @_; return {} unless $c->user_exists; - my $bodies = $c->stash->{problem}->bodies_str_ids; - my %permissions = map { $_ => $c->user->has_permission_to($_, $bodies) } @permissions; - return \%permissions; + return $c->user->permissions($c->stash->{problem}); }; - sub stash_category_groups : Private { my ( $self, $c, $contacts, $combine_multiple ) = @_; my %category_groups = (); for my $category (@$contacts) { - my $group = $category->{group} // $category->get_extra_metadata('group') // ['']; - # this could be an array ref or a string - my @groups = ref $group eq 'ARRAY' ? @$group : ($group); + my $group = $category->{group} // $category->groups; + my @groups = @$group; if (scalar @groups > 1 && $combine_multiple) { @groups = sort @groups; $category->{group} = \@groups; @@ -708,6 +736,19 @@ sub stash_category_groups : Private { $c->stash->{category_groups} = \@category_groups; } +sub assigned_users_only : Private { + my ($self, $c, $categories) = @_; + + # Assigned only category checking + if ($c->user_exists && $c->user->from_body) { + my @assigned_users_only = grep { $_->get_extra_metadata('assigned_users_only') } @$categories; + $c->stash->{assigned_users_only} = { map { $_->category => 1 } @assigned_users_only }; + $c->stash->{assigned_categories_only} = $c->user->get_extra_metadata('assigned_categories_only'); + + $c->stash->{user_categories} = { map { $_ => 1 } @{$c->user->categories} }; + } +} + __PACKAGE__->meta->make_immutable; 1; diff --git a/perllib/FixMyStreet/App/Controller/Report/New.pm b/perllib/FixMyStreet/App/Controller/Report/New.pm index fc1a78cd5..f64a109e8 100644 --- a/perllib/FixMyStreet/App/Controller/Report/New.pm +++ b/perllib/FixMyStreet/App/Controller/Report/New.pm @@ -6,8 +6,7 @@ BEGIN { extends 'Catalyst::Controller'; } use utf8; use Encode; -use List::MoreUtils qw(uniq); -use List::Util 'first'; +use List::Util qw(uniq); use HTML::Entities; use Path::Class; use Utils; @@ -30,8 +29,8 @@ Create a new report, or complete a partial one. submit_problem: true if a problem has been submitted, at all. submit_sign_in: true if the sign in button has been clicked by logged out user. -submit_register: true if the register/confirm by email button has been clicked -by logged out user. +submit_register(_mobile): true if the register/confirm by email button has been clicked +by logged out user, or submit button clicked by logged in user. =head2 location (required) @@ -127,8 +126,10 @@ sub report_new_ajax : Path('mobile') : Args(0) { # Apps are sending email as username # Prepare for when they upgrade - if (!$c->get_param('username')) { - $c->set_param('username', $c->get_param('email')); + my $username_field = ( $c->get_param('submit_sign_in') || $c->get_param('password_sign_in') ) + ? 'username': 'username_register'; + if (!$c->get_param($username_field)) { + $c->set_param($username_field, $c->get_param('email')); } # create the report - loading a partial if available @@ -333,12 +334,14 @@ sub disable_form_message : Private { my %category; foreach my $opt (@{$_->{values}}) { if ($opt->{disable}) { - $category{message} = $opt->{disable_message} || $_->{datatype_description}; - $category{code} = $_->{code}; - push @{$category{answers}}, $opt->{key}; + my $message = $opt->{disable_message} || $_->{datatype_description}; + $category{$message} ||= {}; + $category{$message}->{message} = $message; + $category{$message}->{code} = $_->{code}; + push @{$category{$message}->{answers}}, $opt->{key}; } } - push @{$out{questions}}, \%category if %category; + push @{$out{questions}}, $_ for values %category; } } @@ -759,14 +762,20 @@ sub setup_categories_and_bodies : Private { if !$c->stash->{unresponsive}{ALL} && ($contact->email =~ /^REFUSED$/i || $body_send_method eq 'Refused'); - push @category_options, $contact unless $seen{$contact->category}; - $seen{$contact->category} = $contact; + if (my $cat = $seen{$contact->category}) { + # Make sure the category is listed in all its groups, not just the first set + my @groups = uniq @{$cat->groups}, @{$contact->groups}; + $cat->set_extra_metadata(group => \@groups); + } else { + push @category_options, $contact; + $seen{$contact->category} = $contact; + } } if (@category_options) { # If there's an Other category present, put it at the bottom @category_options = ( - { category => _('-- Pick a category --'), category_display => _('-- Pick a category --'), group => '' }, + { category => _('-- Pick a category --'), category_display => _('-- Pick a category --'), group => [''] }, grep { $_->category ne _('Other') } @category_options ); push @category_options, $seen{_('Other')} if $seen{_('Other')}; } @@ -837,10 +846,16 @@ sub process_user : Private { # Extract all the params to a hash to make them easier to work with my %params = map { $_ => $c->get_param($_) } - ( 'email', 'name', 'phone', 'password_register', 'fms_extra_title' ); + qw( email name phone password_register fms_extra_title update_method ); - # Report form includes two username fields: #form_username_register and #form_username_sign_in - $params{username} = (first { $_ } $c->get_param_list('username')) || ''; + if ($c->user_exists) { + $params{username} = $c->get_param('username'); + } elsif ($c->get_param('submit_sign_in') || $c->get_param('password_sign_in')) { + $params{username} = $c->get_param('username'); + } else { + $params{username} = $c->get_param('username_register'); + } + $params{username} ||= ''; my $anon_button = $c->cobrand->allow_anonymous_reports eq 'button' && $c->get_param('report_anonymously'); my $anon_fallback = $c->cobrand->allow_anonymous_reports eq '1' && !$c->user_exists && !$params{username}; @@ -891,11 +906,24 @@ sub process_user : Private { $params{username} = $params{phone}; } + # Code to deal with SMS being switched on and so the user being asked to + # pick a method and no username field + if (!$params{username} && !$params{update_method}) { + $c->stash->{field_errors}->{update_method} = _('Please pick your update preference'); + } + if (!$params{username} && $params{update_method}) { + if ($params{update_method} eq 'phone') { + $params{username} = $params{phone}; + } else { + $params{username} = $params{email}; + } + $c->stash->{update_method} = $params{update_method}; + } + my $parsed = FixMyStreet::SMS->parse_username($params{username}); my $type = $parsed->{type} || 'email'; $type = 'email' unless FixMyStreet->config('SMS_AUTHENTICATION') || $c->stash->{contributing_as_another_user}; - $report->user( $c->model('DB::User')->find_or_new( { $type => $parsed->{username} } ) ) - unless $report->user; + $report->user( $c->model('DB::User')->find_or_new( { $type => $parsed->{username} } ) ); $c->stash->{phone_may_be_mobile} = $type eq 'phone' && $parsed->{may_be_mobile}; @@ -1018,7 +1046,13 @@ sub process_report : Private { $report->detail( $detail ); # mobile device type - $report->service( $params{service} ) if $params{service}; + if ($params{service}) { + $report->service($params{service}); + } elsif ($c->get_param('submit_register_mobile')) { + $report->service('mobile'); + } elsif ($c->get_param('submit_register')) { + $report->service('desktop'); + } # set these straight from the params $report->category( _ $params{category} ) if $params{category}; @@ -1106,7 +1140,7 @@ sub process_report : Private { # save the cobrand and language related information $report->cobrand( $c->cobrand->moniker ); - $report->cobrand_data( '' ); + $report->cobrand_data( $c->stash->{cobrand_data} || '' ); $report->lang( $c->stash->{lang_code} ); return 1; @@ -1228,6 +1262,7 @@ sub check_for_errors : Private { # if using social login then we don't care about other errors $c->stash->{is_social_user} = $c->get_param('social_sign_in') ? 1 : 0; if ( $c->stash->{is_social_user} ) { + delete $field_errors{update_method}; delete $field_errors{name}; delete $field_errors{username}; } @@ -1243,6 +1278,16 @@ sub check_for_errors : Private { $field_errors{photo} = $photo_error; } + # Now assign the username error according to where it came from + if ($field_errors{username}) { + if ($c->get_param('submit_sign_in') || $c->get_param('password_sign_in')) { + $field_errors{username_sign_in} = $field_errors{username}; + } else { + $field_errors{username_register} = $field_errors{username}; + } + delete $field_errors{username}; + } + # all good if no errors return 1 unless scalar keys %field_errors || $c->stash->{login_success}; @@ -1301,6 +1346,11 @@ sub send_problem_confirm_text : Private { $data->{id} = $report->id; $c->forward('/auth/phone/send_token', [ $data, 'problem', $report->user->phone ]); + my $error = $c->render_fragment( 'auth/_username_error.html', { default => 'phone' }); + if ($error) { + $c->stash->{field_errors}{phone} = $error; + $c->forward('generate_map'); + } $c->stash->{submit_url} = '/report/new/text'; } @@ -1626,7 +1676,7 @@ sub redirect_or_confirm_creation : Private { $c->forward( 'create_related_things' ); if ($c->stash->{contributing_as_another_user} && $report->user->email && $report->user->id != $c->user->id - && !$c->cobrand->report_sent_confirmation_email) { + && !$c->cobrand->report_sent_confirmation_email($report)) { $c->send_email( 'other-reported.txt', { to => [ [ $report->user->email, $report->name ] ], } ); @@ -1674,21 +1724,8 @@ sub create_related_things : Private { foreach my $body (values %{$problem->bodies}) { my $user = $body->comment_user or next; - my %open311_conf = ( - endpoint => $body->endpoint || '', - api_key => $body->api_key || '', - jurisdiction => $body->jurisdiction || '', - extended_statuses => $body->send_extended_statuses, - ); - - my $cobrand = $body->get_cobrand_handler; - $cobrand->call_hook(open311_config_updates => \%open311_conf) - if $cobrand; - - my $open311 = Open311->new(%open311_conf); my $updates = Open311::GetServiceRequestUpdates->new( system_user => $user, - current_open311 => $open311, current_body => $body, blank_updates_permitted => 1, ); @@ -1699,7 +1736,10 @@ sub create_related_things : Private { my $request = { service_request_id => $problem->id, update_id => 'auto-internal', - comment_time => DateTime->now, + # Add a second so it is definitely later than problem confirmed timestamp, + # which uses current_timestamp (and thus microseconds) whilst this update + # is rounded down to the nearest second + comment_time => DateTime->now->add( seconds => 1 ), status => 'open', description => $description, }; @@ -1793,6 +1833,24 @@ sub generate_category_extra_json : Private { return \@fields; } +sub non_map_creation : Private { + my ($self, $c, $extras) = @_; + + $c->forward('initialize_report'); + $c->forward('check_for_category'); + $c->forward('/auth/check_csrf_token'); + $c->forward('process_report'); + $c->forward('process_user'); + if ($extras) { + $c->forward($_) for @$extras; + } + $c->forward('/photo/process_photo'); + return 0 unless $c->forward('check_for_errors'); + $c->forward('save_user_and_report'); + return 1; + +} + __PACKAGE__->meta->make_immutable; 1; diff --git a/perllib/FixMyStreet/App/Controller/Report/Update.pm b/perllib/FixMyStreet/App/Controller/Report/Update.pm index 41c42b8a1..2acafc654 100644 --- a/perllib/FixMyStreet/App/Controller/Report/Update.pm +++ b/perllib/FixMyStreet/App/Controller/Report/Update.pm @@ -6,7 +6,6 @@ BEGIN { extends 'Catalyst::Controller'; } use utf8; use Path::Class; -use List::Util 'first'; use Utils; =head1 NAME @@ -103,8 +102,14 @@ sub process_user : Private { my %params = map { $_ => $c->get_param($_) } ( 'name', 'password_register', 'fms_extra_title' ); - # Update form includes two username fields: #form_username_register and #form_username_sign_in - $params{username} = (first { $_ } $c->get_param_list('username')) || ''; + if ($c->user_exists) { + $params{username} = $c->get_param('username'); + } elsif ($c->get_param('submit_sign_in') || $c->get_param('password_sign_in')) { + $params{username} = $c->get_param('username'); + } else { + $params{username} = $c->get_param('username_register'); + } + $params{username} ||= ''; my $anon_button = $c->cobrand->allow_anonymous_reports eq 'button' && $c->get_param('report_anonymously'); if ($anon_button) { @@ -121,7 +126,7 @@ sub process_user : Private { if ( $c->user_exists ) { { my $user = $c->user->obj; - if ($c->stash->{contributing_as_another_user} = $user->contributing_as('another_user', $c, $update->problem->bodies_str_ids)) { + if ($c->stash->{contributing_as_another_user} = $user->contributing_as('another_user', $c, $c->stash->{problem}->bodies_str_ids)) { # Act as if not logged in (and it will be auto-confirmed later on) last; } @@ -145,8 +150,7 @@ sub process_user : Private { my $parsed = FixMyStreet::SMS->parse_username($params{username}); my $type = $parsed->{type} || 'email'; $type = 'email' unless FixMyStreet->config('SMS_AUTHENTICATION') || $c->stash->{contributing_as_another_user}; - $update->user( $c->model('DB::User')->find_or_new( { $type => $parsed->{username} } ) ) - unless $update->user; + $update->user( $c->model('DB::User')->find_or_new( { $type => $parsed->{username} } ) ); $c->stash->{phone_may_be_mobile} = $type eq 'phone' && $parsed->{may_be_mobile}; @@ -248,7 +252,7 @@ sub load_problem : Private { # Problem ID could come from existing update in token, or from query parameter my $problem_id = $update->problem_id || $c->get_param('id'); $c->forward( '/report/load_problem_or_display_error', [ $problem_id ] ); - $update->problem($c->stash->{problem}); + $update->problem_id($c->stash->{problem}->id); } =head2 check_form_submitted @@ -282,7 +286,8 @@ sub process_update : Private { my $name = Utils::trim_text( $params{name} ); - $params{reopen} = 0 unless $c->user && $c->user->id == $c->stash->{problem}->user->id; + my $problem = $c->stash->{problem}; + $params{reopen} = 0 unless $c->user && $c->user->id == $problem->user->id; my $update = $c->stash->{update}; $update->text($params{update}); @@ -290,11 +295,11 @@ sub process_update : Private { $update->mark_fixed($params{fixed} ? 1 : 0); $update->mark_open($params{reopen} ? 1 : 0); - $c->stash->{contributing_as_body} = $c->user_exists && $c->user->contributing_as('body', $c, $update->problem->bodies_str_ids); - $c->stash->{contributing_as_anonymous_user} = $c->user_exists && $c->user->contributing_as('anonymous_user', $c, $update->problem->bodies_str_ids); + $c->stash->{contributing_as_body} = $c->user_exists && $c->user->contributing_as('body', $c, $problem->bodies_str_ids); + $c->stash->{contributing_as_anonymous_user} = $c->user_exists && $c->user->contributing_as('anonymous_user', $c, $problem->bodies_str_ids); # This is also done in process_user, but is needed here for anonymous() just below - my $anon_button = $c->cobrand->allow_anonymous_reports($update->problem->category) eq 'button' && $c->get_param('report_anonymously'); + my $anon_button = $c->cobrand->allow_anonymous_reports($problem->category) eq 'button' && $c->get_param('report_anonymously'); if ($anon_button) { $c->stash->{contributing_as_anonymous_user} = 1; $c->stash->{contributing_as_body} = undef; @@ -323,7 +328,6 @@ sub process_update : Private { # then we are not changing the state of the problem so can use the current # problem state } else { - my $problem = $c->stash->{problem} || $update->problem; $update->problem_state( $problem->state ); } } @@ -332,7 +336,7 @@ sub process_update : Private { my @extra; # Next function fills this, but we don't need it here. # This is just so that the error checking for these extra fields runs. # TODO Use extra here as it is used on reports. - my $body = (values %{$update->problem->bodies})[0]; + my $body = (values %{$problem->bodies})[0]; $c->cobrand->process_open311_extras( $c, $body, \@extra ); if ( $c->get_param('fms_extra_title') ) { @@ -404,6 +408,16 @@ sub check_for_errors : Private { $field_errors{photo} = $photo_error; } + # Now assign the username error according to where it came from + if ($field_errors{username}) { + if ($c->get_param('submit_sign_in') || $c->get_param('password_sign_in')) { + $field_errors{username_sign_in} = $field_errors{username}; + } else { + $field_errors{username_register} = $field_errors{username}; + } + delete $field_errors{username}; + } + # all good if no errors return 1 unless ( scalar keys %field_errors @@ -484,6 +498,13 @@ sub save_update : Private { $update->confirm(); } elsif ($c->stash->{contributing_as_anonymous_user}) { $update->set_extra_metadata( contributed_as => 'anonymous_user' ); + if ( $c->user_exists && $c->user->from_body ) { + # If a staff user has clicked the 'report anonymously' button then + # there would be no record of who that staff member was as we've + # used the cobrand's anonymous_account for the report. In this case + # record the staff user ID in the report metadata. + $update->set_extra_metadata( contributed_by => $c->user->id ); + } $update->confirm(); } elsif ( !$update->user->in_storage ) { # User does not exist. @@ -571,6 +592,11 @@ sub send_confirmation_text : Private { my ( $self, $c ) = @_; my $update = $c->stash->{update}; $c->forward('/auth/phone/send_token', [ $c->stash->{token_data}, 'comment', $update->user->phone ]); + my $error = $c->render_fragment( 'auth/_username_error.html', { default => 'phone' }); + if ($error) { + $c->stash->{field_errors}{username_register} = $error; + $c->go( '/report/display', [ $c->stash->{problem}->id ], [] ); + } $c->stash->{submit_url} = '/report/update/text'; } diff --git a/perllib/FixMyStreet/App/Controller/Reports.pm b/perllib/FixMyStreet/App/Controller/Reports.pm index 97976ebe3..e65810b91 100644 --- a/perllib/FixMyStreet/App/Controller/Reports.pm +++ b/perllib/FixMyStreet/App/Controller/Reports.pm @@ -154,6 +154,8 @@ sub ward : Path : Args(2) { $c->forward('stash_report_sort', [ $c->cobrand->reports_ordering ]); $c->forward( 'load_and_group_problems' ); + $c->forward('setup_categories'); + if ($c->get_param('ajax')) { my $ajax_template = $c->stash->{ajax_template} || 'reports/_problem-list.html'; $c->detach('ajax', [ $ajax_template ]); @@ -165,7 +167,7 @@ sub ward : Path : Args(2) { $c->stash->{stats} = $c->cobrand->get_report_stats(); - $c->forward('setup_categories_and_map'); + $c->forward('setup_map'); # List of wards if ( !$c->stash->{wards} && $c->stash->{body}->id && $c->stash->{body}->body_areas->first ) { @@ -181,7 +183,7 @@ sub ward : Path : Args(2) { } } -sub setup_categories_and_map :Private { +sub setup_categories :Private { my ($self, $c) = @_; my @categories = $c->stash->{body}->contacts->not_deleted->search( undef, { @@ -191,9 +193,15 @@ sub setup_categories_and_map :Private { $c->cobrand->call_hook('munge_reports_category_list', \@categories); + $c->forward('/report/assigned_users_only', [ \@categories ]); + $c->stash->{filter_categories} = \@categories; $c->stash->{filter_category} = { map { $_ => 1 } $c->get_param_list('filter_category', 1) }; $c->forward('/report/stash_category_groups', [ \@categories ]) if $c->cobrand->enable_category_groups; +} + +sub setup_map :Private { + my ($self, $c) = @_; my $pins = $c->stash->{pins} || []; @@ -473,10 +481,10 @@ sub summary : Private { $c->forward('/admin/fetch_contacts'); $c->stash->{contacts} = [ $c->stash->{contacts}->all ]; - $c->forward('/dashboard/construct_rs_filter', []); + my $reporting = $c->forward('/dashboard/construct_rs_filter', []); if ( $c->get_param('csv') ) { - $c->detach('export_summary_csv'); + $c->detach('export_summary_csv', [ $reporting ]); } $c->forward('/dashboard/generate_grouped_data'); @@ -486,38 +494,26 @@ sub summary : Private { } sub export_summary_csv : Private { - my ( $self, $c ) = @_; + my ( $self, $c, $reporting ) = @_; - $c->stash->{csv} = { - objects => $c->stash->{objects_rs}->search_rs({}, { - rows => 100, - order_by => { '-desc' => 'me.confirmed' }, - }), - headers => [ - 'Report ID', - 'Title', - 'Category', - 'Created', - 'Confirmed', - 'Status', - 'Latitude', 'Longitude', - 'Query', - 'Report URL', - ], - columns => [ - 'id', - 'title', - 'category', - 'created', - 'confirmed', - 'state', - 'latitude', 'longitude', - 'postcode', - 'url', - ], - filename => 'fixmystreet-data', - }; - $c->forward('/dashboard/generate_csv'); + $reporting->objects_attrs({ + rows => 100, + order_by => { '-desc' => 'me.confirmed' }, + }); + $reporting->add_csv_columns( + id => 'Report ID', + title => 'Title', + category => 'Category', + created => 'Created', + confirmed => 'Confirmed', + state => 'Status', + latitude => 'Latitude', + longitude => 'Longitude', + postcode => 'Query', + url => 'Report URL', + ); + $reporting->filename('fixmystreet-data'); + $reporting->generate_csv_http($c); } =head2 check_canonical_url @@ -620,6 +616,9 @@ sub load_problems_parameters : Private { }; if ($c->user_exists && $body) { my $prefetch = []; + if ($c->user->from_body || $c->user->is_superuser) { + push @$prefetch, 'contact'; + } if ($c->user->has_permission_to('planned_reports', $body->id)) { push @$prefetch, 'user_planned_reports'; } @@ -646,7 +645,7 @@ sub load_problems_parameters : Private { } if (@$category) { - $where->{category} = $category; + $where->{'me.category'} = $category; } if ($c->stash->{wards}) { @@ -687,12 +686,12 @@ sub check_non_public_reports_permission : Private { } if ( $user_has_permission ) { - $where->{non_public} = 1 if $c->stash->{only_non_public}; + $where->{'me.non_public'} = 1 if $c->stash->{only_non_public}; } else { - $where->{non_public} = 0; + $where->{'me.non_public'} = 0; } } else { - $where->{non_public} = 0; + $where->{'me.non_public'} = 0; } } @@ -815,7 +814,7 @@ sub stash_report_sort : Private { sub add_row { my ( $c, $problem, $body, $problems, $pins ) = @_; push @{$problems->{$body}}, $problem; - push @$pins, $problem->pin_data($c, 'reports'); + push @$pins, $problem->pin_data('reports'); } sub ajax : Private { diff --git a/perllib/FixMyStreet/App/Controller/Test.pm b/perllib/FixMyStreet/App/Controller/Test.pm index 5ec4bebf3..ce54f004f 100644 --- a/perllib/FixMyStreet/App/Controller/Test.pm +++ b/perllib/FixMyStreet/App/Controller/Test.pm @@ -42,6 +42,28 @@ sub setup : Path('/_test/setup') : Args(1) { my $problem = FixMyStreet::DB->resultset("Problem")->find(1); $problem->update({ category => 'Skips' }); $c->response->body("OK"); + } elsif ( $test eq 'regression-duplicate-stopper') { + my $problem = FixMyStreet::DB->resultset("Problem")->find(1); + $problem->update({ category => 'Flytipping' }); + my $category = FixMyStreet::DB->resultset('Contact')->search({ + category => 'Flytipping', + })->first; + $category->push_extra_fields({ + code => 'hazardous', + datatype => 'singlevaluelist', + description => 'Hazardous material', + order => 0, + variable => 'true', + values => [ + { key => 'yes', name => 'Yes', disable => 1, disable_message => 'Please phone' }, + { key => 'no', name => 'No' }, + ], + }); + $category->update; + $c->response->body("OK"); + } elsif ($test eq 'simple-service-check') { + my $problem = FixMyStreet::DB->resultset("Problem")->search(undef, { order_by => { -desc => 'id' } })->first; + $c->response->body($problem->service); } } @@ -51,6 +73,15 @@ sub teardown : Path('/_test/teardown') : Args(1) { my $problem = FixMyStreet::DB->resultset("Problem")->find(1); $problem->update({ category => 'Potholes' }); $c->response->body("OK"); + } elsif ( $test eq 'regression-duplicate-stopper') { + my $problem = FixMyStreet::DB->resultset("Problem")->find(1); + $problem->update({ category => 'Potholes' }); + my $category = FixMyStreet::DB->resultset('Contact')->search({ + category => 'Flytipping', + })->first; + $category->remove_extra_field('hazardous'); + $category->update; + $c->response->body("OK"); } } diff --git a/perllib/FixMyStreet/App/Controller/Waste.pm b/perllib/FixMyStreet/App/Controller/Waste.pm new file mode 100644 index 000000000..fe177e9fe --- /dev/null +++ b/perllib/FixMyStreet/App/Controller/Waste.pm @@ -0,0 +1,569 @@ +package FixMyStreet::App::Controller::Waste; +use Moose; +use namespace::autoclean; + +BEGIN { extends 'Catalyst::Controller' } + +use utf8; +use Lingua::EN::Inflect qw( NUMWORDS ); +use FixMyStreet::App::Form::Waste::UPRN; +use FixMyStreet::App::Form::Waste::AboutYou; +use FixMyStreet::App::Form::Waste::Request; +use FixMyStreet::App::Form::Waste::Report; +use FixMyStreet::App::Form::Waste::Enquiry; +use Open311::GetServiceRequestUpdates; + +sub auto : Private { + my ( $self, $c ) = @_; + my $cobrand_check = $c->cobrand->feature('waste'); + $c->detach( '/page_error_404_not_found' ) if !$cobrand_check; + return 1; +} + +sub index : Path : Args(0) { + my ( $self, $c ) = @_; + + if (my $id = $c->get_param('address')) { + $c->detach('redirect_to_id', [ $id ]); + } + + $c->stash->{title} = 'What is your address?'; + my $form = FixMyStreet::App::Form::Waste::UPRN->new( cobrand => $c->cobrand ); + $form->process( params => $c->req->body_params ); + if ($form->validated) { + my $addresses = $form->value->{postcode}; + $form = address_list_form($addresses); + } + $c->stash->{form} = $form; +} + +sub address_list_form { + my $addresses = shift; + HTML::FormHandler->new( + field_list => [ + address => { + required => 1, + type => 'Select', + widget => 'RadioGroup', + label => 'Select an address', + tags => { last_differs => 1, small => 1 }, + options => $addresses, + }, + go => { + type => 'Submit', + value => 'Continue', + element_attr => { class => 'govuk-button' }, + }, + ], + ); +} + +sub redirect_to_id : Private { + my ($self, $c, $id) = @_; + my $uri = '/waste/' . $id; + my $type = $c->get_param('type') || ''; + $uri .= '/request' if $type eq 'request'; + $uri .= '/report' if $type eq 'report'; + $c->res->redirect($uri); + $c->detach; +} + +sub property : Chained('/') : PathPart('waste') : CaptureArgs(1) { + my ($self, $c, $id) = @_; + + if ($id eq 'missing') { + $c->stash->{template} = 'waste/missing.html'; + $c->detach; + } + + $c->forward('/auth/get_csrf_token'); + + my $property = $c->stash->{property} = $c->cobrand->call_hook(look_up_property => $id); + $c->detach( '/page_error_404_not_found', [] ) unless $property; + + $c->stash->{latitude} = $property->{latitude}; + $c->stash->{longitude} = $property->{longitude}; + + $c->stash->{service_data} = $c->cobrand->call_hook(bin_services_for_address => $property) || []; + $c->stash->{services} = { map { $_->{service_id} => $_ } @{$c->stash->{service_data}} }; +} + +sub bin_days : Chained('property') : PathPart('') : Args(0) { + my ($self, $c) = @_; +} + +sub calendar : Chained('property') : PathPart('calendar.ics') : Args(0) { + my ($self, $c) = @_; + $c->res->header(Content_Type => 'text/calendar'); + require Data::ICal::RFC7986; + require Data::ICal::Entry::Event; + my $calendar = Data::ICal::RFC7986->new( + calname => 'Bin calendar', + rfc_strict => 1, + auto_uid => 1, + ); + $calendar->add_properties( + prodid => '//FixMyStreet//Bin Collection Calendars//EN', + method => 'PUBLISH', + 'refresh-interval' => [ 'P1D', { value => 'DURATION' } ], + 'x-published-ttl' => 'P1D', + calscale => 'GREGORIAN', + 'x-wr-timezone' => 'Europe/London', + source => [ $c->uri_for_action($c->action, [ $c->stash->{property}{id} ]), { value => 'URI' } ], + url => $c->uri_for_action('waste/bin_days', [ $c->stash->{property}{id} ]), + ); + + my $events = $c->cobrand->bin_future_collections; + my $stamp = DateTime->now->strftime('%Y%m%dT%H%M%SZ'); + foreach (@$events) { + my $event = Data::ICal::Entry::Event->new; + $event->add_properties( + summary => $_->{summary}, + description => $_->{desc}, + dtstamp => $stamp, + dtstart => [ $_->{date}->ymd(''), { value => 'DATE' } ], + dtend => [ $_->{date}->add(days=>1)->ymd(''), { value => 'DATE' } ], + ); + $calendar->add_entry($event); + } + + $c->res->body($calendar->as_string); +} + +sub construct_bin_request_form { + my $c = shift; + + my $field_list = []; + + foreach (@{$c->stash->{service_data}}) { + next unless $_->{next} && !$_->{request_open}; + my $name = $_->{service_name}; + my $containers = $_->{request_containers}; + my $max = $_->{request_max}; + foreach my $id (@$containers) { + push @$field_list, "container-$id" => { + type => 'Checkbox', + apply => [ + { + when => { "quantity-$id" => sub { $_[0] > 0 } }, + check => qr/^1$/, + message => 'Please tick the box', + }, + ], + label => $name, + option_label => $c->stash->{containers}->{$id}, + tags => { toggle => "form-quantity-$id-row" }, + }; + $name = ''; # Only on first container + push @$field_list, "quantity-$id" => { + type => 'Select', + label => 'Quantity', + tags => { + hint => "You can request a maximum of " . NUMWORDS($max) . " containers", + initial_hidden => 1, + }, + options => [ + { value => "", label => '-' }, + map { { value => $_, label => $_ } } (1..$max), + ], + required_when => { "container-$id" => 1 }, + }; + } + } + + return $field_list; +} + +sub request : Chained('property') : Args(0) { + my ($self, $c) = @_; + + my $field_list = construct_bin_request_form($c); + + $c->stash->{first_page} = 'request'; + $c->stash->{form_class} = 'FixMyStreet::App::Form::Waste::Request'; + $c->stash->{page_list} = [ + request => { + fields => [ grep { ! ref $_ } @$field_list, 'submit' ], + title => 'Which containers do you need?', + next => 'about_you', + }, + ]; + $c->stash->{field_list} = $field_list; + $c->forward('form'); +} + +sub process_request_data : Private { + my ($self, $c, $form) = @_; + my $data = $form->saved_data; + my $address = $c->stash->{property}->{address}; + my @services = grep { /^container-/ && $data->{$_} } keys %$data; + foreach (@services) { + my ($id) = /container-(.*)/; + my $container = $c->stash->{containers}{$id}; + my $quantity = $data->{"quantity-$id"}; + $data->{title} = "Request new $container"; + $data->{detail} = "Quantity: $quantity\n\n$address"; + $c->set_param('Container_Type', $id); + $c->set_param('Quantity', $quantity); + $c->forward('add_report', [ $data ]) or return; + push @{$c->stash->{report_ids}}, $c->stash->{report}->id; + } + return 1; +} + +sub construct_bin_report_form { + my $c = shift; + + my $field_list = []; + + foreach (@{$c->stash->{service_data}}) { + next unless $_->{last} && $_->{report_allowed} && !$_->{report_open}; + my $id = $_->{service_id}; + my $name = $_->{service_name}; + push @$field_list, "service-$id" => { + type => 'Checkbox', + label => $name, + option_label => $name, + }; + } + + return $field_list; +} + +sub report : Chained('property') : Args(0) { + my ($self, $c) = @_; + + my $field_list = construct_bin_report_form($c); + + $c->stash->{first_page} = 'report'; + $c->stash->{form_class} = 'FixMyStreet::App::Form::Waste::Report'; + $c->stash->{page_list} = [ + report => { + fields => [ grep { ! ref $_ } @$field_list, 'submit' ], + title => 'Select your missed collection', + next => 'about_you', + }, + ]; + $c->stash->{field_list} = $field_list; + $c->forward('form'); +} + +sub process_report_data : Private { + my ($self, $c, $form) = @_; + my $data = $form->saved_data; + my $address = $c->stash->{property}->{address}; + my @services = grep { /^service-/ && $data->{$_} } keys %$data; + foreach (@services) { + my ($id) = /service-(.*)/; + my $service = $c->stash->{services}{$id}{service_name}; + $data->{title} = "Report missed $service"; + $data->{detail} = "$data->{title}\n\n$address"; + $c->set_param('service_id', $id); + $c->forward('add_report', [ $data ]) or return; + push @{$c->stash->{report_ids}}, $c->stash->{report}->id; + } + return 1; +} + +sub enquiry : Chained('property') : Args(0) { + my ($self, $c) = @_; + + if (my $template = $c->get_param('template')) { + $c->stash->{template} = "waste/enquiry-$template.html"; + $c->detach; + } + + $c->forward('setup_categories_and_bodies'); + + my $category = $c->get_param('category'); + my $service = $c->get_param('service_id'); + if (!$category || !$service || !$c->stash->{services}{$service}) { + $c->res->redirect('/waste/' . $c->stash->{property}{id}); + $c->detach; + } + my ($contact) = grep { $_->category eq $category } @{$c->stash->{contacts}}; + if (!$contact) { + $c->res->redirect('/waste/' . $c->stash->{property}{id}); + $c->detach; + } + + my $field_list = []; + foreach (@{$contact->get_metadata_for_input}) { + next if $_->{code} eq 'service_id' || $_->{code} eq 'uprn' || $_->{code} eq 'property_id'; + my $type = 'Text'; + $type = 'TextArea' if 'text' eq ($_->{datatype} || ''); + my $required = $_->{required} eq 'true' ? 1 : 0; + push @$field_list, "extra_$_->{code}" => { + type => $type, label => $_->{description}, required => $required + }; + } + + $c->stash->{first_page} = 'enquiry'; + $c->stash->{form_class} = 'FixMyStreet::App::Form::Waste::Enquiry'; + $c->stash->{page_list} = [ + enquiry => { + fields => [ 'category', 'service_id', grep { ! ref $_ } @$field_list, 'continue' ], + title => $category, + next => 'about_you', + update_field_list => sub { + my $form = shift; + my $c = $form->c; + return { + category => { default => $c->get_param('category') }, + service_id => { default => $c->get_param('service_id') }, + } + } + }, + ]; + $c->stash->{field_list} = $field_list; + $c->forward('form'); +} + +sub process_enquiry_data : Private { + my ($self, $c, $form) = @_; + my $data = $form->saved_data; + my $address = $c->stash->{property}->{address}; + $data->{title} = $data->{category}; + $data->{detail} = "$data->{category}\n\n$address"; + # Read extra details in loop + foreach (grep { /^extra_/ } keys %$data) { + my ($id) = /^extra_(.*)/; + $c->set_param($id, $data->{$_}); + } + $c->set_param('service_id', $data->{service_id}); + $c->forward('add_report', [ $data ]) or return; + push @{$c->stash->{report_ids}}, $c->stash->{report}->id; + return 1; +} + +sub load_form { + my ($c, $previous_form) = @_; + + my $page; + if ($previous_form) { + $page = $previous_form->next; + } else { + $page = $c->forward('get_page'); + } + + my $form = $c->stash->{form_class}->new( + page_list => $c->stash->{page_list}, + $c->stash->{field_list} ? (field_list => $c->stash->{field_list}) : (), + page_name => $page, + csrf_token => $c->stash->{csrf_token}, + c => $c, + previous_form => $previous_form, + saved_data_encoded => $c->get_param('saved_data'), + no_preload => 1, + ); + + if (!$form->has_current_page) { + $c->detach('/page_error_400_bad_request', [ 'Bad request' ]); + } + + return $form; +} + +sub form : Private { + my ($self, $c) = @_; + + my $form = load_form($c); + if ($c->get_param('process')) { + $c->forward('/auth/check_csrf_token'); + $form->process(params => $c->req->body_params); + if ($form->validated) { + $form = load_form($c, $form); + } + } + + $form->process unless $form->processed; + + $c->stash->{template} = $form->template || 'waste/index.html'; + $c->stash->{form} = $form; +} + +sub get_page : Private { + my ($self, $c) = @_; + + my $goto = $c->get_param('goto') || ''; + my $process = $c->get_param('process') || ''; + $goto = $c->stash->{first_page} unless $goto || $process; + if ($goto && $process) { + $c->detach('/page_error_400_bad_request', [ 'Bad request' ]); + } + + return $goto || $process; +} + +sub add_report : Private { + my ( $self, $c, $data ) = @_; + + $c->stash->{cobrand_data} = 'waste'; + + # XXX Is this best way to do this? + if ($c->user_exists && $c->user->from_body && $c->user->email ne $data->{email}) { + $c->set_param('form_as', 'another_user'); + $c->set_param('username', $data->{email} || $data->{phone}); + } else { + $c->set_param('username_register', $data->{email} || $data->{phone}); + } + + # Set the data as if a new report form has been submitted + + $c->set_param('submit_problem', 1); + $c->set_param('pc', ''); + $c->set_param('non_public', 1); + + $c->set_param('name', $data->{name}); + $c->set_param('phone', $data->{phone}); + + $c->set_param('category', $data->{category}); + $c->set_param('title', $data->{title}); + $c->set_param('detail', $data->{detail}); + $c->set_param('uprn', $c->stash->{property}{uprn}); + $c->set_param('property_id', $c->stash->{property}{id}); + + $c->forward('setup_categories_and_bodies') unless $c->stash->{contacts}; + $c->forward('/report/new/non_map_creation', [['/waste/remove_name_errors']]) or return; + my $report = $c->stash->{report}; + $report->confirm; + $report->update; + + $c->model('DB::Alert')->find_or_create({ + user => $report->user, + alert_type => 'new_updates', + parameter => $report->id, + cobrand => $report->cobrand, + lang => $report->lang, + })->confirm; + + return 1; +} + +sub remove_name_errors : Private { + my ($self, $c) = @_; + # We do not mind about missing title/split name here + my $field_errors = $c->stash->{field_errors}; + delete $field_errors->{fms_extra_title}; + delete $field_errors->{first_name}; + delete $field_errors->{last_name}; +} + +sub setup_categories_and_bodies : Private { + my ($self, $c) = @_; + + $c->stash->{all_areas} = $c->stash->{all_areas_mapit} = { $c->cobrand->council_area_id => { id => $c->cobrand->council_area_id } }; + $c->forward('/report/new/setup_categories_and_bodies'); + my $contacts = $c->stash->{contacts}; + @$contacts = grep { grep { $_ eq 'Waste' } @{$_->groups} } @$contacts; +} + +sub receive_echo_event_notification : Path('/waste/echo') : Args(0) { + my ($self, $c) = @_; + $c->stash->{format} = 'xml'; + $c->response->header(Content_Type => 'application/soap+xml'); + + require SOAP::Lite; + + $c->detach('soap_error', [ 'Invalid method', 405 ]) unless $c->req->method eq 'POST'; + + my $echo = $c->cobrand->feature('echo'); + $c->detach('soap_error', [ 'Missing config', 500 ]) unless $echo; + + # Make sure we log entire request for debugging + $c->detach('soap_error', [ 'Missing body' ]) unless $c->req->body; + my $soap = join('', $c->req->body->getlines); + $c->log->info($soap); + + my $body = $c->cobrand->body; + $c->detach('soap_error', [ 'Bad jurisdiction' ]) unless $body; + + my $env = SOAP::Deserializer->deserialize($soap); + + my $header = $env->header; + $c->detach('soap_error', [ 'Missing SOAP header' ]) unless $header; + my $action = $header->{Action}; + $c->detach('soap_error', [ 'Incorrect Action' ]) unless $action && $action eq $echo->{receive_action}; + $header = $header->{Security}; + $c->detach('soap_error', [ 'Missing Security header' ]) unless $header; + my $token = $header->{UsernameToken}; + $c->detach('soap_error', [ 'Authentication failed' ]) + unless $token && $token->{Username} eq $echo->{receive_username} && $token->{Password} eq $echo->{receive_password}; + + my $event = $env->result; + + my $cfg = { echo => Integrations::Echo->new(%$echo) }; + my $request = $c->cobrand->construct_waste_open311_update($cfg, $event); + $request->{updated_datetime} = DateTime::Format::W3CDTF->format_datetime(DateTime->now); + $request->{service_request_id} = $event->{Guid}; + + my $updates = Open311::GetServiceRequestUpdates->new( + system_user => $body->comment_user, + current_body => $body, + ); + + my $p = $updates->find_problem($request); + if ($p) { + $c->forward('check_existing_update', [ $p, $request, $updates ]); + my $comment = $updates->process_update($request, $p); + } + # Still want to say it is okay, even if we did nothing with it + $c->forward('soap_ok'); +} + +sub soap_error : Private { + my ($self, $c, $comment, $code) = @_; + $code ||= 400; + $c->response->status($code); + my $type = $code == 500 ? 'Server' : 'Client'; + $c->response->body(SOAP::Serializer->fault($type, "Bad request: $comment", soap_header())); +} + +sub soap_ok : Private { + my ($self, $c) = @_; + $c->response->status(200); + my $method = SOAP::Data->name("NotifyEventUpdatedResponse")->attr({ + xmlns => "http://www.twistedfish.com/xmlns/echo/api/v1" + }); + $c->response->body(SOAP::Serializer->envelope(method => $method, soap_header())); +} + +sub soap_header { + my $attr = "http://www.twistedfish.com/xmlns/echo/api/v1"; + my $action = "NotifyEventUpdatedResponse"; + my $header = SOAP::Header->name("Action")->attr({ + xmlns => 'http://www.w3.org/2005/08/addressing', + 'soap:mustUnderstand' => 1, + })->value("$attr/ReceiverService/$action"); + + my $dt = DateTime->now(); + my $dt2 = $dt->clone->add(minutes => 5); + my $w3c = DateTime::Format::W3CDTF->new; + my $header2 = SOAP::Header->name("Security")->attr({ + 'soap:mustUnderstand' => 'true', + 'xmlns' => 'http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd' + })->value( + \SOAP::Header->name( + "Timestamp" => \SOAP::Header->value( + SOAP::Header->name('Created', $w3c->format_datetime($dt)), + SOAP::Header->name('Expires', $w3c->format_datetime($dt2)), + ) + )->attr({ + xmlns => "http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd", + }) + ); + return ($header, $header2); +} + +sub check_existing_update : Private { + my ($self, $c, $p, $request, $updates) = @_; + + my $cfg = { updates => $updates }; + $c->detach('soap_ok') + unless $c->cobrand->waste_check_last_update( + $cfg, $p, $request->{status}, $request->{external_status_code}); +} + +__PACKAGE__->meta->make_immutable; + +1; diff --git a/perllib/FixMyStreet/App/Form/Field/JSON.pm b/perllib/FixMyStreet/App/Form/Field/JSON.pm new file mode 100644 index 000000000..4da4ef2b0 --- /dev/null +++ b/perllib/FixMyStreet/App/Form/Field/JSON.pm @@ -0,0 +1,42 @@ +package FixMyStreet::App::Form::Field::JSON; + +use HTML::FormHandler::Moose; +extends 'HTML::FormHandler::Field::Hidden'; + +use JSON::MaybeXS; +use MIME::Base64; + +has '+inflate_method' => ( default => sub { \&inflate_json } ); +has '+deflate_method' => ( default => sub { \&deflate_json } ); +has '+fif_from_value' => ( default => 1 ); + +sub inflate_json { + my ($self, $value) = @_; + return $value unless $value; + $value = decode_json(decode_base64($value)); + return $value; +} + +sub deflate_json { + my ($self, $value) = @_; + return $value unless $value; + $value = encode_base64(encode_json($value), ""); + return $value; +} + +__PACKAGE__->meta->make_immutable; +use namespace::autoclean; + +1; + +__END__ + +=pod + +=encoding UTF-8 + +=head1 NAME + +FixMyStreet::App::Form::Field::JSON - used to store some data in a hidden field + +=cut diff --git a/perllib/FixMyStreet/App/Form/Field/Postcode.pm b/perllib/FixMyStreet/App/Form/Field/Postcode.pm new file mode 100644 index 000000000..093ae66a3 --- /dev/null +++ b/perllib/FixMyStreet/App/Form/Field/Postcode.pm @@ -0,0 +1,50 @@ +package FixMyStreet::App::Form::Field::Postcode; + +use HTML::FormHandler::Moose; +extends 'HTML::FormHandler::Field::Text'; + +use mySociety::PostcodeUtil; + +apply( + [ + { + transform => sub { + my ( $value, $field ) = @_; + $value =~ s/[^A-Z0-9]//i; + return mySociety::PostcodeUtil::canonicalise_postcode($value); + } + }, + { + check => sub { mySociety::PostcodeUtil::is_valid_postcode(shift) }, + message => 'Sorry, we did not recognise that postcode.', + } + ] +); + + +__PACKAGE__->meta->make_immutable; +use namespace::autoclean; + +1; + +__END__ + +=pod + +=encoding UTF-8 + +=head1 NAME + +FixMyStreet::App::Form::Field::Postcode - validates postcode using mySociety::PostcodeUtil + +=head1 DESCRIPTION + +Validates that the input looks like a postcode using L<mySociety::PostcodeUtil>. +Widget type is 'text'. + +=head1 DEPENDENCIES + +L<mySociety::PostcodeUtil> + +=cut + diff --git a/perllib/FixMyStreet/App/Form/Page/Simple.pm b/perllib/FixMyStreet/App/Form/Page/Simple.pm new file mode 100644 index 000000000..89a871e2e --- /dev/null +++ b/perllib/FixMyStreet/App/Form/Page/Simple.pm @@ -0,0 +1,25 @@ +package FixMyStreet::App::Form::Page::Simple; +use Moose; +extends 'HTML::FormHandler::Page'; + +# What page to go to after successful submission of this page +has next => ( is => 'ro', isa => 'Str|CodeRef' ); + +# A function that will be called to generate an update_field_list parameter +has update_field_list => ( + is => 'ro', + isa => 'CodeRef', + predicate => 'has_update_field_list', +); + +# A function called after all form processing, just before template display +# (to e.g. set up the map) +has post_process => ( + is => 'ro', + isa => 'CodeRef', +); + +# Catalyst action to forward to once this page has been reached +has finished => ( is => 'ro', isa => 'CodeRef' ); + +1; diff --git a/perllib/FixMyStreet/App/Form/Page/Waste.pm b/perllib/FixMyStreet/App/Form/Page/Waste.pm new file mode 100644 index 000000000..5275cae7f --- /dev/null +++ b/perllib/FixMyStreet/App/Form/Page/Waste.pm @@ -0,0 +1,11 @@ +package FixMyStreet::App::Form::Page::Waste; +use Moose; +extends 'FixMyStreet::App::Form::Page::Simple'; + +# Title to use for this page +has title => ( is => 'ro', isa => 'Str' ); + +# Special template to use in preference to the default +has template => ( is => 'ro', isa => 'Str' ); + +1; diff --git a/perllib/FixMyStreet/App/Form/Waste.pm b/perllib/FixMyStreet/App/Form/Waste.pm new file mode 100644 index 000000000..c430506c7 --- /dev/null +++ b/perllib/FixMyStreet/App/Form/Waste.pm @@ -0,0 +1,52 @@ +package FixMyStreet::App::Form::Waste; + +use HTML::FormHandler::Moose; +extends 'FixMyStreet::App::Form::Wizard'; + +has c => ( is => 'ro' ); + +has default_page_type => ( is => 'ro', isa => 'Str', default => 'Waste' ); + +has finished_action => ( is => 'ro' ); + +before _process_page_array => sub { + my ($self, $pages) = @_; + foreach my $page (@$pages) { + $page->{type} = $self->default_page_type + unless $page->{type}; + } +}; + +# Add some functions to the form to pass through to the current page +has '+current_page' => ( + handles => { + title => 'title', + template => 'template', + } +); + +sub wizard_finished { + my ($form, $action) = @_; + my $c = $form->c; + my $success = $c->forward($action, [ $form ]); + if (!$success) { + $form->add_form_error('Something went wrong, please try again'); + foreach (keys %{$c->stash->{field_errors}}) { + $form->add_form_error("$_: " . $c->stash->{field_errors}{$_}); + } + } + return $success; +} + +# Make sure we can have pre-ticked things on the first page +before after_build => sub { + my $self = shift; + + my $saved_data = $self->previous_form ? $self->previous_form->saved_data : $self->saved_data; + + my $c = $self->c; + + map { $saved_data->{$_} = 1 } grep { /^(service|container)-/ && $c->req->params->{$_} } keys %{$c->req->params}; +}; + +1; diff --git a/perllib/FixMyStreet/App/Form/Waste/AboutYou.pm b/perllib/FixMyStreet/App/Form/Waste/AboutYou.pm new file mode 100644 index 000000000..d5bb3df2b --- /dev/null +++ b/perllib/FixMyStreet/App/Form/Waste/AboutYou.pm @@ -0,0 +1,38 @@ +package FixMyStreet::App::Form::Waste::AboutYou; + +use utf8; +use HTML::FormHandler::Moose::Role; +use FixMyStreet::SMS; + +has_field name => ( + type => 'Text', + label => 'Your name', + required => 1, + validate_method => sub { + my $self = shift; + $self->add_error('Please enter your full name.') + if length($self->value) < 5 + || $self->value !~ m/\s/ + || $self->value =~ m/\ba\s*n+on+((y|o)mo?u?s)?(ly)?\b/i; + }, +); + +has_field phone => ( + type => 'Text', + label => 'Telephone number', + validate_method => sub { + my $self = shift; + my $parsed = FixMyStreet::SMS->parse_username($self->value); + $self->add_error('Please provide a valid phone number') + unless $parsed->{phone}; + } +); + +has_field email => ( + type => 'Email', + tags => { + hint => 'If you provide an email address, we can send you order status updates' + }, +); + +1; diff --git a/perllib/FixMyStreet/App/Form/Waste/Enquiry.pm b/perllib/FixMyStreet/App/Form/Waste/Enquiry.pm new file mode 100644 index 000000000..fa85d5d4c --- /dev/null +++ b/perllib/FixMyStreet/App/Form/Waste/Enquiry.pm @@ -0,0 +1,48 @@ +package FixMyStreet::App::Form::Waste::Enquiry; + +use utf8; +use HTML::FormHandler::Moose; +extends 'FixMyStreet::App::Form::Waste'; + +# First page has dynamic fields, so is set in code + +has_field category => ( type => 'Hidden' ); +has_field service_id => ( type => 'Hidden' ); + +has_page about_you => ( + fields => ['name', 'phone', 'email', 'continue'], + title => 'About you', + next => 'summary', +); + +with 'FixMyStreet::App::Form::Waste::AboutYou'; + +has_page summary => ( + fields => ['submit'], + title => 'Submit missed collection', + template => 'waste/summary_enquiry.html', + finished => sub { + return $_[0]->wizard_finished('process_enquiry_data'); + }, + next => 'done', +); + +has_page done => ( + title => 'Enquiry sent', + template => 'waste/confirmation.html', +); + +has_field continue => ( + type => 'Submit', + value => 'Continue', + element_attr => { class => 'govuk-button' }, + order => 999 +); + +has_field submit => ( + type => 'Submit', + value => 'Submit', + element_attr => { class => 'govuk-button' } +); + +1; diff --git a/perllib/FixMyStreet/App/Form/Waste/Report.pm b/perllib/FixMyStreet/App/Form/Waste/Report.pm new file mode 100644 index 000000000..589e75d48 --- /dev/null +++ b/perllib/FixMyStreet/App/Form/Waste/Report.pm @@ -0,0 +1,65 @@ +package FixMyStreet::App::Form::Waste::Report; + +use utf8; +use HTML::FormHandler::Moose; +extends 'FixMyStreet::App::Form::Waste'; + +# First page has dynamic fields, so is set in code + +has_page about_you => ( + fields => ['name', 'email', 'phone', 'continue'], + title => 'About you', + next => 'summary', +); + +with 'FixMyStreet::App::Form::Waste::AboutYou'; + +has_page summary => ( + fields => ['submit'], + title => 'Submit missed collection', + template => 'waste/summary_report.html', + finished => sub { + return $_[0]->wizard_finished('process_report_data'); + }, + next => 'done', +); + +has_page done => ( + title => 'Missed collection sent', + template => 'waste/confirmation.html', +); + +has_field category => ( + type => 'Hidden', + default => 'Report missed collection' +); + +has_field continue => ( + type => 'Submit', + value => 'Continue', + element_attr => { class => 'govuk-button' }, +); + +has_field submit => ( + type => 'Submit', + value => 'Report collection as missed', + element_attr => { class => 'govuk-button' }, + order => 999, +); + +sub validate { + my $self = shift; + my $any = 0; + + foreach ($self->all_fields) { + $any = 1 if $_->name =~ /^service-/ && ($_->value || $self->saved_data->{$_->name}); + } + $self->add_form_error('Please specify what was missed') + unless $any; + + $self->add_form_error('Please specify at least one of phone or email') + unless $self->field('phone')->is_inactive || $self->field('phone')->value || $self->field('email')->value; +} + +1; + diff --git a/perllib/FixMyStreet/App/Form/Waste/Request.pm b/perllib/FixMyStreet/App/Form/Waste/Request.pm new file mode 100644 index 000000000..e7caaa206 --- /dev/null +++ b/perllib/FixMyStreet/App/Form/Waste/Request.pm @@ -0,0 +1,64 @@ +package FixMyStreet::App::Form::Waste::Request; + +use utf8; +use HTML::FormHandler::Moose; +extends 'FixMyStreet::App::Form::Waste'; + +# First page has dynamic fields, so is set in code + +has_page about_you => ( + fields => ['name', 'email', 'phone', 'continue'], + title => 'About you', + next => 'summary', +); + +with 'FixMyStreet::App::Form::Waste::AboutYou'; + +has_page summary => ( + fields => ['submit'], + title => 'Submit container request', + template => 'waste/summary_request.html', + finished => sub { + return $_[0]->wizard_finished('process_request_data'); + }, + next => 'done', +); + +has_page done => ( + title => 'Container request sent', + template => 'waste/confirmation.html', +); + +has_field category => ( + type => 'Hidden', + default => 'Request new container', +); + +has_field continue => ( + type => 'Submit', + value => 'Continue', + element_attr => { class => 'govuk-button' }, +); + +has_field submit => ( + type => 'Submit', + value => 'Request new containers', + element_attr => { class => 'govuk-button' }, + order => 999, +); + +sub validate { + my $self = shift; + my $any = 0; + + foreach ($self->all_fields) { + $any = 1 if $_->name =~ /^container-/ && ($_->value || $self->saved_data->{$_->name}); + } + $self->add_form_error('Please specify what you need') + unless $any; + + $self->add_form_error('Please specify at least one of phone or email') + unless $self->field('phone')->is_inactive || $self->field('phone')->value || $self->field('email')->value; +} + +1; diff --git a/perllib/FixMyStreet/App/Form/Waste/UPRN.pm b/perllib/FixMyStreet/App/Form/Waste/UPRN.pm new file mode 100644 index 000000000..d0ac7b3cb --- /dev/null +++ b/perllib/FixMyStreet/App/Form/Waste/UPRN.pm @@ -0,0 +1,37 @@ +package FixMyStreet::App::Form::Waste::UPRN; + +use utf8; +use HTML::FormHandler::Moose; +extends 'HTML::FormHandler'; + +use mySociety::PostcodeUtil qw(is_valid_postcode); + +has '+field_name_space' => ( default => 'FixMyStreet::App::Form::Field' ); + +has cobrand => ( is => 'ro' ); + +has_field postcode => ( + required => 1, + type => 'Postcode', + validate_method => sub { + my $self = shift; + return if $self->has_errors; # Called even if already failed + my $data = $self->form->cobrand->bin_addresses_for_postcode($self->value); + if (!@$data) { + $self->add_error('Sorry, we did not find any results for that postcode'); + } + push @$data, { value => 'missing', label => 'I can’t find my address' }; + $self->value($data); + }, + tags => { autofocus => 1 }, +); + +has_field go => ( + type => 'Submit', + value => 'Go', + element_attr => { class => 'govuk-button' }, +); + +__PACKAGE__->meta->make_immutable; + +1; diff --git a/perllib/FixMyStreet/App/Form/Wizard.pm b/perllib/FixMyStreet/App/Form/Wizard.pm new file mode 100644 index 000000000..edb7b0c5c --- /dev/null +++ b/perllib/FixMyStreet/App/Form/Wizard.pm @@ -0,0 +1,115 @@ +package FixMyStreet::App::Form::Wizard; +# ABSTRACT: create a multi-page form, based on HTML::FormHandler::Wizard, but not numbered + +use HTML::FormHandler::Moose; +extends 'HTML::FormHandler'; +with ('HTML::FormHandler::BuildPages', 'HTML::FormHandler::Pages' ); + +sub is_wizard { 1 } # So build_active is called + +sub build_page_name_space { 'FixMyStreet::App::Form::Page' } +has '+field_name_space' => ( default => 'FixMyStreet::App::Form::Field' ); + +# Internal attributes and fields to handle multi-page forms +has page_name => ( is => 'ro', isa => 'Str' ); +has current_page => ( is => 'ro', lazy => 1, + default => sub { $_[0]->page($_[0]->page_name) }, + predicate => 'has_current_page', +); + +has saved_data_encoded => ( is => 'ro', isa => 'Maybe[Str]' ); +has saved_data => ( is => 'rw', lazy => 1, isa => 'HashRef', default => sub { + $_[0]->field('saved_data')->inflate_json($_[0]->saved_data_encoded) || {}; +}); +has previous_form => ( is => 'ro', isa => 'Maybe[HTML::FormHandler]' ); +has csrf_token => ( is => 'ro', isa => 'Str' ); + +has_field saved_data => ( type => 'JSON' ); +has_field token => ( type => 'Hidden', required => 1 ); +has_field process => ( type => 'Hidden', required => 1 ); + +sub next { + my $self = shift; + my $next = $self->current_page->next; + if (ref $next eq 'CODE') { + $next = $next->($self->saved_data); + } + return $next; +} + +# Override HFH default and set current page only to active +sub build_active { + my $self = shift; + + my %active; + foreach my $fname ($self->current_page->all_fields) { + $active{$fname} = 1; + } + + foreach my $page ( $self->all_pages ) { + foreach my $fname ( $page->all_fields ) { + my $field = $self->field($fname); + $field->inactive(1) unless $active{$fname}; + } + } +} + +# Stuff to set up as soon as we have a form +sub after_build { + my $self = shift; + my $page = $self->current_page; + + my $saved_data = $self->previous_form ? $self->previous_form->saved_data : $self->saved_data; + + $self->init_object($saved_data); # For filling in existing values + $self->saved_data($saved_data); + + # Fill in internal fields + $self->update_field(saved_data => { default => $saved_data }); + $self->update_field(token => { default => $self->csrf_token }); + $self->update_field(process => { default => $page->name }); + + # Update field list with any dynamic things (eg user-based, address lookup, geocoding) + if ($page->has_update_field_list) { + my $updates = $page->update_field_list->($self) || {}; + foreach my $field_name (keys %$updates) { + $self->update_field($field_name, $updates->{$field_name}); + } + } +} + +# After a form has been processed, run any post process functions +after 'process' => sub { + my $self = shift; + my $page = $self->current_page; + $page->post_process->($self) if $page->post_process; +}; + +after 'validate_form' => sub { + my $self = shift; + + if ($self->validated) { + # Update saved_data for the next page + my $saved_data = { %{$self->saved_data}, %{$self->value} }; + delete $saved_data->{process}; + delete $saved_data->{token}; + delete $saved_data->{saved_data}; + $self->saved_data($saved_data); + $self->field('saved_data')->_set_value($saved_data); + + # And check to see if there is a function to call on the page + my $page = $self->current_page; + if ($page->finished) { + my $success = $page->finished->($self); + if (!$success) { + $self->add_form_error('Something went wrong, please try again') + unless $self->has_form_errors; + $self->validated(0); + } + } + } +}; + +__PACKAGE__->meta->make_immutable; +use namespace::autoclean; +1; diff --git a/perllib/FixMyStreet/App/Model/PhotoSet.pm b/perllib/FixMyStreet/App/Model/PhotoSet.pm index 76a287e71..85e457856 100644 --- a/perllib/FixMyStreet/App/Model/PhotoSet.pm +++ b/perllib/FixMyStreet/App/Model/PhotoSet.pm @@ -8,6 +8,7 @@ use Scalar::Util 'openhandle', 'blessed'; use Image::Size; use IPC::Cmd qw(can_run); use IPC::Open3; +use Try::Tiny; use FixMyStreet; use FixMyStreet::ImageMagick; @@ -149,7 +150,9 @@ has ids => ( # Arrayref of $fileid tuples (always, so post upload/raw data proc } # we have an image we can use - save it to storage - $photo_blob = FixMyStreet::ImageMagick->new(blob => $photo_blob)->shrink('2048x2048')->as_blob; + $photo_blob = try { + FixMyStreet::ImageMagick->new(blob => $photo_blob)->shrink('2048x2048')->as_blob; + } catch { $photo_blob }; return $self->storage->store_photo($photo_blob); } @@ -201,18 +204,20 @@ sub get_image_data { } my $im = FixMyStreet::ImageMagick->new(blob => $image->{data}); - my $photo; - if ( $size eq 'tn' ) { - $photo = $im->shrink('x100'); - } elsif ( $size eq 'fp' ) { - $photo = $im->crop; - } elsif ( $size eq 'og' ) { - $photo = $im->crop('1200x630'); - } elsif ( $size eq 'full' ) { - $photo = $im - } else { - $photo = $im->shrink($args{default} || '250x250'); - } + my $photo = try { + if ( $size eq 'tn' ) { + $im->shrink('x100'); + } elsif ( $size eq 'fp' ) { + $im->crop; + } elsif ( $size eq 'og' ) { + $im->crop('1200x630'); + } elsif ( $size eq 'full' ) { + $im + } else { + $im->shrink($args{default} || '250x250'); + } + }; + return unless $photo; return { data => $photo->as_blob, diff --git a/perllib/FixMyStreet/App/View/Web.pm b/perllib/FixMyStreet/App/View/Web.pm index 1e1b50094..8d3d53d0d 100644 --- a/perllib/FixMyStreet/App/View/Web.pm +++ b/perllib/FixMyStreet/App/View/Web.pm @@ -25,7 +25,7 @@ __PACKAGE__->config( FILTERS => { add_links => \&add_links, escape_js => \&escape_js, - markup => [ \&markup_factory, 1 ], + staff_html_markup => [ \&staff_html_markup_factory, 1 ], }, COMPILE_EXT => '.ttc', STAT_TTL => FixMyStreet->config('STAGING_SITE') ? 1 : 86400, @@ -98,32 +98,26 @@ Add some links to some text (and thus HTML-escapes the other text). sub add_links { my $text = shift; - $text = FixMyStreet::Template::conditional_escape($text); - $text =~ s/\r//g; - $text =~ s{(https?://)([^\s]+)}{"<a href=\"$1$2\">$1" . _space_slash($2) . '</a>'}ge; - return FixMyStreet::Template::SafeString->new($text); + return FixMyStreet::Template::add_links($text); } -sub _space_slash { - my $t = shift; - $t =~ s{/(?!$)}{/ }g; - return $t; -} +=head2 staff_html_markup_factory -=head2 markup_factory +This returns a function that processes the text body of an update, applying +HTML sanitization and markdown-style italics if it was made by a staff user. -This returns a function that will allow updates to have markdown-style italics. -Pass in the user that wrote the text, so we know whether it can be privileged. +Pass in the update extra, so we can determine if it was made by a staff user. =cut -sub markup_factory { - my ($c, $user) = @_; +sub staff_html_markup_factory { + my ($c, $extra) = @_; + + my $staff = $extra->{is_superuser} || $extra->{is_body_user}; + return sub { my $text = shift; - return $text unless $user && ($user->from_body || $user->is_superuser); - $text =~ s{\*(\S.*?\S)\*}{<i>$1</i>}; - FixMyStreet::Template::SafeString->new($text); + return FixMyStreet::Template::_staff_html_markup($text, $staff); } } diff --git a/perllib/FixMyStreet/Cobrand/BathNES.pm b/perllib/FixMyStreet/Cobrand/BathNES.pm index 06095734b..e8e2c5427 100644 --- a/perllib/FixMyStreet/Cobrand/BathNES.pm +++ b/perllib/FixMyStreet/Cobrand/BathNES.pm @@ -90,14 +90,6 @@ sub send_questionnaires { 0 } sub default_map_zoom { 3 } -sub category_extra_hidden { - my ($self, $meta) = @_; - my $code = $meta->{code}; - # These two are used in the non-Open311 'Street light fault' category. - return 1 if $code eq 'unitid' || $code eq 'asset_details'; - return $self->SUPER::category_extra_hidden($meta); -} - sub available_permissions { my $self = shift; @@ -171,9 +163,8 @@ sub categories_restriction { # Do a manual prefetch of all staff users for contributed_by lookup sub _dashboard_user_lookup { my $self = shift; - my $c = $self->{c}; - my @user_ids = $c->model('DB::User')->search( + my @user_ids = FixMyStreet::DB->resultset('User')->search( { from_body => { '!=' => undef } }, { columns => [ 'id', 'email' ] })->all; @@ -182,23 +173,22 @@ sub _dashboard_user_lookup { } sub dashboard_export_updates_add_columns { - my $self = shift; - my $c = $self->{c}; + my ($self, $csv) = @_; - return unless $c->user->has_body_permission_to('export_extra_columns'); + return unless $csv->user->has_body_permission_to('export_extra_columns'); - push @{$c->stash->{csv}->{headers}}, "Staff User"; - push @{$c->stash->{csv}->{headers}}, "User Email"; - push @{$c->stash->{csv}->{columns}}, "staff_user"; - push @{$c->stash->{csv}->{columns}}, "user_email"; + $csv->add_csv_columns( + staff_user => 'Staff User', + user_email => 'User Email', + ); - $c->stash->{csv}->{objects} = $c->stash->{csv}->{objects}->search(undef, { + $csv->objects_attrs({ '+columns' => ['user.email'], join => 'user', }); my $user_lookup = $self->_dashboard_user_lookup; - $c->stash->{csv}->{extra_data} = sub { + $csv->csv_extra_data(sub { my $report = shift; my $staff_user = ''; @@ -210,38 +200,28 @@ sub dashboard_export_updates_add_columns { user_email => $report->user->email || '', staff_user => $staff_user, }; - }; + }); } sub dashboard_export_problems_add_columns { - my $self = shift; - my $c = $self->{c}; - - return unless $c->user->has_body_permission_to('export_extra_columns'); - - $c->stash->{csv}->{headers} = [ - @{ $c->stash->{csv}->{headers} }, - "User Email", - "User Phone", - "Staff User", - "Attribute Data", - ]; - - $c->stash->{csv}->{columns} = [ - @{ $c->stash->{csv}->{columns} }, - "user_email", - "user_phone", - "staff_user", - "attribute_data", - ]; - - $c->stash->{csv}->{objects} = $c->stash->{csv}->{objects}->search(undef, { + my ($self, $csv) = @_; + + return unless $csv->user->has_body_permission_to('export_extra_columns'); + + $csv->add_csv_columns( + user_email => 'User Email', + user_phone => 'User Phone', + staff_user => 'Staff User', + attribute_data => "Attribute Data", + ); + + $csv->objects_attrs({ '+columns' => ['user.email', 'user.phone'], join => 'user', }); my $user_lookup = $self->_dashboard_user_lookup; - $c->stash->{csv}->{extra_data} = sub { + $csv->csv_extra_data(sub { my $report = shift; my $staff_user = ''; @@ -255,7 +235,7 @@ sub dashboard_export_problems_add_columns { staff_user => $staff_user, attribute_data => $attribute_data, }; - }; + }); } 1; diff --git a/perllib/FixMyStreet/Cobrand/Bexley.pm b/perllib/FixMyStreet/Cobrand/Bexley.pm index 481926e72..063a225b7 100644 --- a/perllib/FixMyStreet/Cobrand/Bexley.pm +++ b/perllib/FixMyStreet/Cobrand/Bexley.pm @@ -3,10 +3,6 @@ use parent 'FixMyStreet::Cobrand::Whitelabel'; use strict; use warnings; -use Encode; -use JSON::MaybeXS; -use LWP::Simple qw($ua); -use Path::Tiny; use Time::Piece; sub council_area_id { 2494 } @@ -54,7 +50,7 @@ sub open311_munge_update_params { $params->{service_request_id_ext} = $comment->problem->id; - my $contact = $comment->problem->category_row; + my $contact = $comment->problem->contact; $params->{service_code} = $contact->email; } @@ -88,8 +84,8 @@ sub open311_config { $params->{multi_photos} = 1; } -sub open311_extra_data { - my ($self, $row, $h, $extra, $contact) = @_; +sub open311_extra_data_include { + my ($self, $row, $h, $contact) = @_; my $open311_only; if ($contact->email =~ /^Confirm/) { @@ -103,7 +99,7 @@ sub open311_extra_data { if (!$row->get_extra_field_value('site_code')) { if (my $ref = $self->lookup_site_code($row, 'NSG_REF')) { - push @$extra, { name => 'site_code', value => $ref, description => 'Site code' }; + $row->update_extra_field({ name => 'site_code', value => $ref, description => 'Site code' }); } } } elsif ($contact->email =~ /^Uniform/) { @@ -112,7 +108,7 @@ sub open311_extra_data { # WFS service at the point we're sending the report over Open311. if (!$row->get_extra_field_value('uprn')) { if (my $ref = $self->lookup_site_code($row, 'UPRN')) { - push @$extra, { name => 'uprn', description => 'UPRN', value => $ref }; + $row->update_extra_field({ name => 'uprn', description => 'UPRN', value => $ref }); } } } else { # Symology @@ -121,7 +117,7 @@ sub open311_extra_data { # WFS service at the point we're sending the report over Open311. if (!$row->get_extra_field_value('NSGRef')) { if (my $ref = $self->lookup_site_code($row, 'NSG_REF')) { - push @$extra, { name => 'NSGRef', description => 'NSG Ref', value => $ref }; + $row->update_extra_field({ name => 'NSGRef', description => 'NSG Ref', value => $ref }); } } } @@ -202,9 +198,6 @@ sub open311_post_send { $self->open311_config($row, $h, {}, $contact); # Populate NSGRef again if needed - my $extra_data = join "; ", map { "$_->{description}: $_->{value}" } @{$row->get_extra_fields}; - $h->{additional_information} = $extra_data; - $sender->send($row, $h); } @@ -216,71 +209,21 @@ sub email_list { return @to; } -sub dashboard_export_problems_add_columns { - my $self = shift; - my $c = $self->{c}; - - my %groups; - if ($c->stash->{body}) { - %groups = FixMyStreet::DB->resultset('Contact')->search({ - body_id => $c->stash->{body}->id, - })->group_lookup; - } - - splice @{$c->stash->{csv}->{headers}}, 5, 0, 'Subcategory'; - splice @{$c->stash->{csv}->{columns}}, 5, 0, 'subcategory'; - - $c->stash->{csv}->{extra_data} = sub { - my $report = shift; - - if ($groups{$report->category}) { - return { - category => $groups{$report->category}, - subcategory => $report->category, - }; - } - return {}; - }; -} - sub _is_out_of_hours { my $time = localtime; return 1 if $time->hour > 16 || ($time->hour == 16 && $time->min >= 45); return 1 if $time->hour < 8; return 1 if $time->wday == 1 || $time->wday == 7; - return 1 if _is_bank_holiday(); + return 1 if FixMyStreet::Cobrand::UK::is_public_holiday(); return 0; } -sub _is_bank_holiday { - my $json = _get_bank_holiday_json(); - my $today = localtime->date; - for my $event (@{$json->{'england-and-wales'}{events}}) { - if ($event->{date} eq $today) { - return 1; - } - } -} +sub update_anonymous_message { + my ($self, $update) = @_; + my $t = Utils::prettify_dt( $update->confirmed ); -sub _get_bank_holiday_json { - my $file = 'bank-holidays.json'; - my $cache_file = path(FixMyStreet->path_to("../data/$file")); - my $js; - if (-s $cache_file && -M $cache_file <= 7 && !FixMyStreet->config('STAGING_SITE')) { - # uncoverable statement - $js = $cache_file->slurp_utf8; - } else { - $ua->timeout(5); - $js = LWP::Simple::get("https://www.gov.uk/$file"); - # uncoverable branch false - $js = decode_utf8($js) if !utf8::is_utf8($js); - if ($js && !FixMyStreet->config('STAGING_SITE')) { - # uncoverable statement - $cache_file->spew_utf8($js); - } - } - $js = JSON->new->decode($js) if $js; - return $js; + my $staff = $update->user->from_body || $update->get_extra_metadata('is_body_user') || $update->get_extra_metadata('is_superuser'); + return sprintf('Posted anonymously by a non-staff user at %s', $t) if !$staff; } 1; diff --git a/perllib/FixMyStreet/Cobrand/Bristol.pm b/perllib/FixMyStreet/Cobrand/Bristol.pm index 6e3160c89..5e70c9456 100644 --- a/perllib/FixMyStreet/Cobrand/Bristol.pm +++ b/perllib/FixMyStreet/Cobrand/Bristol.pm @@ -52,7 +52,10 @@ sub categories_restriction { # Email categories with a devolved send_method, so can identify Open311 # categories as those which have a blank send_method. # Also Highways England categories have a blank send_method. - return $rs->search( { 'me.send_method' => undef } ); + return $rs->search( { -or => [ + 'me.send_method' => undef, # Open311 categories + 'me.send_method' => '', # Open311 categories that have been edited in the admin + ] } ); } sub open311_config { @@ -68,8 +71,10 @@ sub open311_contact_meta_override { $service->{group} = []; my %server_set = (easting => 1, northing => 1); + my %hidden_field = (usrn => 1, asset_id => 1); foreach (@$meta) { $_->{automated} = 'server_set' if $server_set{$_->{code}}; + $_->{automated} = 'hidden_field' if $hidden_field{$_->{code}}; } } diff --git a/perllib/FixMyStreet/Cobrand/Bromley.pm b/perllib/FixMyStreet/Cobrand/Bromley.pm index 8f82817a8..cd923c19d 100644 --- a/perllib/FixMyStreet/Cobrand/Bromley.pm +++ b/perllib/FixMyStreet/Cobrand/Bromley.pm @@ -6,8 +6,17 @@ use warnings; use utf8; use DateTime::Format::W3CDTF; use DateTime::Format::Flexible; +use File::Temp; +use Integrations::Echo; +use JSON::MaybeXS; +use Parallel::ForkManager; +use Sort::Key::Natural qw(natkeysort_inplace); +use Storable; use Try::Tiny; use FixMyStreet::DateRange; +use FixMyStreet::WorkingDays; +use Open311::GetServiceRequestUpdates; +use Memcached; sub council_area_id { return 2482; } sub council_area { return 'Bromley'; } @@ -171,11 +180,12 @@ sub open311_config { $params->{extended_description} = 0; } -sub open311_extra_data { - my ($self, $row, $h, $extra) = @_; +sub open311_extra_data_include { + my ($self, $row, $h) = @_; my $title = $row->title; + my $extra = $row->get_extra_fields; foreach (@$extra) { next unless $_->{value}; $title .= ' | ID: ' . $_->{value} if $_->{name} eq 'feature_id'; @@ -207,7 +217,11 @@ sub open311_extra_data { push @$open311_only, { name => 'fms_extra_title', value => $row->user->title }; } - return ($open311_only, [ 'feature_id', 'prow_reference' ]); + return $open311_only; +} + +sub open311_extra_data_exclude { + [ 'feature_id', 'prow_reference' ] } sub open311_config_updates { @@ -215,7 +229,7 @@ sub open311_config_updates { $params->{endpoints} = { service_request_updates => 'update.xml', update => 'update.xml' - }; + } if $params->{endpoint} =~ /bromley.gov.uk/; } sub open311_pre_send { @@ -228,6 +242,11 @@ sub open311_pre_send { } } +sub open311_pre_send_updates { + my ($self, $row) = @_; + return $self->open311_pre_send($row); +} + sub open311_munge_update_params { my ($self, $params, $comment, $body) = @_; delete $params->{update_id}; @@ -317,6 +336,8 @@ sub add_admin_subcategories { my $c = $self->{c}; my $user = $c->stash->{user}; + return $c->stash->{contacts} unless $user; # e.g. admin templates, not user + my @subcategories = @{$user->get_extra_metadata('subcategories') || []}; my %active_contacts = map { $_ => 1 } @subcategories; @@ -328,7 +349,7 @@ sub add_admin_subcategories { foreach (@{$subcats{$_->{id}}}) { push @new_contacts, { id => $_->{key}, - category => (" " x 4) . $_->{name}, + category => (" " x 4) . $_->{name}, # nbsp active => $active_contacts{$_->{key}}, }; } @@ -344,25 +365,637 @@ sub munge_load_and_group_problems { return unless $c->action eq 'dashboard/heatmap'; # Bromley subcategory stuff - if (!$where->{category}) { + if (!$where->{'me.category'}) { my $cats = $c->user->categories; my $subcats = $c->user->get_extra_metadata('subcategories') || []; - $where->{category} = [ @$cats, @$subcats ] if @$cats || @$subcats; + $where->{'me.category'} = [ @$cats, @$subcats ] if @$cats || @$subcats; } my %subcats = $self->subcategories; my $subcat; - my %chosen = map { $_ => 1 } @{$where->{category} || []}; + my %chosen = map { $_ => 1 } @{$where->{'me.category'} || []}; my @subcat = grep { $chosen{$_} } map { $_->{key} } map { @$_ } values %subcats; if (@subcat) { my %chosen = map { $_ => 1 } @subcat; $where->{'-or'} = { - category => [ grep { !$chosen{$_} } @{$where->{category}} ], - subcategory => \@subcat, + 'me.category' => [ grep { !$chosen{$_} } @{$where->{'me.category'}} ], + 'me.subcategory' => \@subcat, }; - delete $where->{category}; + delete $where->{'me.category'}; } } -1; +# We want to send confirmation emails only for Waste reports +sub report_sent_confirmation_email { + my ($self, $report) = @_; + my $contact = $report->contact or return; + return 'id' if grep { $_ eq 'Waste' } @{$report->contact->groups}; + return ''; +} + +sub munge_around_category_where { + my ($self, $where) = @_; + $where->{extra} = [ undef, { -not_like => '%Waste%' } ]; +} + +sub munge_reports_category_list { + my ($self, $categories) = @_; + @$categories = grep { grep { $_ ne 'Waste' } @{$_->groups} } @$categories; +} + +sub munge_report_new_contacts { + my ($self, $categories) = @_; + + return if $self->{c}->action =~ /^waste/; + + @$categories = grep { grep { $_ ne 'Waste' } @{$_->groups} } @$categories; + $self->SUPER::munge_report_new_contacts($categories); +} + +sub updates_disallowed { + my $self = shift; + my ($problem) = @_; + + # No updates on waste reports + return 'waste' if $problem->cobrand_data eq 'waste'; + + return $self->next::method(@_); +} + +sub bin_addresses_for_postcode { + my $self = shift; + my $pc = shift; + + my $echo = $self->feature('echo'); + $echo = Integrations::Echo->new(%$echo); + my $points = $echo->FindPoints($pc); + my $data = [ map { { + value => $_->{Id}, + label => FixMyStreet::Template::title($_->{Description}), + } } @$points ]; + natkeysort_inplace { $_->{label} } @$data; + return $data; +} + +sub look_up_property { + my $self = shift; + my $id = shift; + + my $cfg = $self->feature('echo'); + if ($cfg->{max_per_day}) { + my $today = DateTime->today->set_time_zone(FixMyStreet->local_time_zone)->ymd; + my $ip = $self->{c}->req->address; + my $key = FixMyStreet->test_mode ? "bromley-test" : "bromley-$ip-$today"; + my $count = Memcached::increment($key, 86400) || 0; + $self->{c}->detach('/page_error_403_access_denied', []) if $count > $cfg->{max_per_day}; + } + + my $calls = $self->call_api( + GetPointAddress => [ $id ], + GetServiceUnitsForObject => [ $id ], + GetEventsForObject => [ 'PointAddress', $id ], + ); + + $self->{api_serviceunits} = $calls->{"GetServiceUnitsForObject $id"}; + $self->{api_events} = $calls->{"GetEventsForObject PointAddress $id"}; + my $result = $calls->{"GetPointAddress $id"}; + return { + id => $result->{Id}, + uprn => $result->{SharedRef}{Value}{anyType}, + address => FixMyStreet::Template::title($result->{Description}), + latitude => $result->{Coordinates}{GeoPoint}{Latitude}, + longitude => $result->{Coordinates}{GeoPoint}{Longitude}, + }; +} + +my %irregulars = ( 1 => 'st', 2 => 'nd', 3 => 'rd', 11 => 'th', 12 => 'th', 13 => 'th'); +sub ordinal { + my $n = shift; + $irregulars{$n % 100} || $irregulars{$n % 10} || 'th'; +} + +sub construct_bin_date { + my $str = shift; + return unless $str; + my $offset = ($str->{OffsetMinutes} || 0) * 60; + my $zone = DateTime::TimeZone->offset_as_string($offset); + my $date = DateTime::Format::W3CDTF->parse_datetime($str->{DateTime}); + $date->set_time_zone($zone); + return $date; +} + +sub image_for_service { + my ($self, $service_id) = @_; + my $base = '/cobrands/bromley/images/container-images'; + my $images = { + 531 => "$base/refuse-black-sack", + 532 => "$base/refuse-black-sack", + 533 => "$base/large-communal-black", + 535 => "$base/kerbside-green-box-mix", + 536 => "$base/small-communal-mix", + 537 => "$base/kerbside-black-box-paper", + 541 => "$base/small-communal-paper", + 542 => "$base/food-green-caddy", + 544 => "$base/food-communal", + 545 => "$base/garden-waste-bin", + }; + return $images->{$service_id}; +} + +sub bin_services_for_address { + my $self = shift; + my $property = shift; + + my %service_name_override = ( + 531 => 'Non-Recyclable Refuse', + 532 => 'Non-Recyclable Refuse', + 533 => 'Non-Recyclable Refuse', + 535 => 'Mixed Recycling (Cans, Plastics & Glass)', + 536 => 'Mixed Recycling (Cans, Plastics & Glass)', + 537 => 'Paper & Cardboard', + 541 => 'Paper & Cardboard', + 542 => 'Food Waste', + 544 => 'Food Waste', + 545 => 'Garden Waste', + ); + + $self->{c}->stash->{containers} = { + 1 => 'Green Box (Plastic)', + 2 => 'Wheeled Bin (Plastic)', + 12 => 'Black Box (Paper)', + 13 => 'Wheeled Bin (Paper)', + 9 => 'Kitchen Caddy', + 10 => 'Outside Food Waste Container', + 45 => 'Wheeled Bin (Food)', + }; + my %service_to_containers = ( + 535 => [ 1 ], + 536 => [ 2 ], + 537 => [ 12 ], + 541 => [ 13 ], + 542 => [ 9, 10 ], + 544 => [ 45 ], + ); + my %request_allowed = map { $_ => 1 } keys %service_to_containers; + my %quantity_max = ( + 535 => 6, + 536 => 4, + 537 => 6, + 541 => 4, + 542 => 6, + 544 => 4, + ); + + my $result = $self->{api_serviceunits}; + return [] unless @$result; + + my $events = $self->{api_events}; + my $open = $self->_parse_open_events($events); + + my @to_fetch; + my %schedules; + my @task_refs; + foreach (@$result) { + next unless $_->{ServiceTasks}; + + my $servicetask = $_->{ServiceTasks}{ServiceTask}; + my $schedules = _parse_schedules($servicetask); + + next unless $schedules->{next} or $schedules->{last}; + $schedules{$_->{Id}} = $schedules; + push @to_fetch, GetEventsForObject => [ ServiceUnit => $_->{Id} ]; + push @task_refs, $schedules->{last}{ref} if $schedules->{last}; + } + push @to_fetch, GetTasks => \@task_refs if @task_refs; + + my $calls = $self->call_api(@to_fetch); + + my @out; + my %task_ref_to_row; + foreach (@$result) { + next unless $schedules{$_->{Id}}; + my $schedules = $schedules{$_->{Id}}; + my $servicetask = $_->{ServiceTasks}{ServiceTask}; + + my $events = $calls->{"GetEventsForObject ServiceUnit $_->{Id}"}; + my $open_unit = $self->_parse_open_events($events); + + my $containers = $service_to_containers{$_->{ServiceId}}; + my ($open_request) = grep { $_ } map { $open->{request}->{$_} } @$containers; + my $row = { + id => $_->{Id}, + service_id => $_->{ServiceId}, + service_name => $service_name_override{$_->{ServiceId}} || $_->{ServiceName}, + report_open => $open->{missed}->{$_->{ServiceId}} || $open_unit->{missed}->{$_->{ServiceId}}, + request_allowed => $request_allowed{$_->{ServiceId}}, + request_open => $open_request, + request_containers => $containers, + request_max => $quantity_max{$_->{ServiceId}}, + enquiry_open_events => $open->{enquiry}, + service_task_id => $servicetask->{Id}, + service_task_name => $servicetask->{TaskTypeName}, + service_task_type_id => $servicetask->{TaskTypeId}, + schedule => $servicetask->{ScheduleDescription}, + last => $schedules->{last}, + next => $schedules->{next}, + }; + if ($row->{last}) { + my $ref = join(',', @{$row->{last}{ref}}); + $task_ref_to_row{$ref} = $row; + } + push @out, $row; + } + if (%task_ref_to_row) { + my $tasks = $calls->{GetTasks}; + my $now = DateTime->now->set_time_zone(FixMyStreet->local_time_zone); + foreach (@$tasks) { + my $ref = join(',', @{$_->{Ref}{Value}{anyType}}); + my $completed = construct_bin_date($_->{CompletedDate}); + my $state = $_->{State}{Name} || ''; + my $task_type_id = $_->{TaskTypeId} || ''; + + my $orig_resolution = $_->{Resolution}{Name} || ''; + my $resolution = $orig_resolution; + my $resolution_id = $_->{Resolution}{Ref}{Value}{anyType}; + if ($resolution_id) { + my $template = FixMyStreet::DB->resultset('ResponseTemplate')->search({ + 'me.body_id' => $self->body->id, + 'me.external_status_code' => [ + "$resolution_id,$task_type_id,$state", + "$resolution_id,$task_type_id,", + "$resolution_id,,$state", + "$resolution_id,,", + $resolution_id, + ], + })->first; + $resolution = $template->text if $template; + } + + my $row = $task_ref_to_row{$ref}; + $row->{last}{state} = $state; + $row->{last}{completed} = $completed; + $row->{last}{resolution} = $resolution; + $row->{report_allowed} = within_working_days($row->{last}{date}, 2); + + # Special handling if last instance is today + if ($row->{last}{date}->ymd eq $now->ymd) { + # If it's before 5pm and outstanding, show it as in progress + if ($state eq 'Outstanding' && $now->hour < 17) { + $row->{next} = $row->{last}; + $row->{next}{state} = 'In progress'; + delete $row->{last}; + } + if (!$completed && $now->hour < 17) { + $row->{report_allowed} = 0; + } + } + + # If the task is ended and could not be done, do not allow reporting + if ($state eq 'Not Completed' || ($state eq 'Completed' && $orig_resolution eq 'Excess Waste')) { + $row->{report_allowed} = 0; + $row->{report_locked_out} = 1; + } + } + } + + return \@out; +} + +sub _parse_open_events { + my $self = shift; + my $events = shift; + my $open; + foreach (@$events) { + next if $_->{ResolvedDate}; + next if $_->{ResolutionCodeId} && $_->{ResolutionCodeId} != 584; # Out of Stock + my $event_type = $_->{EventTypeId}; + my $service_id = $_->{ServiceId}; + if ($event_type == 2104) { # Request + my $data = $_->{Data}{ExtensibleDatum}; + my $container; + DATA: foreach (@$data) { + if ($_->{ChildData}) { + foreach (@{$_->{ChildData}{ExtensibleDatum}}) { + if ($_->{DatatypeName} eq 'Container Type') { + $container = $_->{Value}; + last DATA; + } + } + } + } + my $report = $self->problems->search({ external_id => $_->{Guid} })->first; + $open->{request}->{$container} = $report ? { report => $report } : 1; + } elsif (2095 <= $event_type && $event_type <= 2103) { # Missed collection + my $report = $self->problems->search({ external_id => $_->{Guid} })->first; + $open->{missed}->{$service_id} = $report ? { report => $report } : 1; + } else { # General enquiry of some sort + $open->{enquiry}->{$event_type} = 1; + } + } + return $open; +} + +sub _parse_schedules { + my $servicetask = shift; + return unless $servicetask->{ServiceTaskSchedules}; + my $schedules = $servicetask->{ServiceTaskSchedules}{ServiceTaskSchedule}; + $schedules = [ $schedules ] unless ref $schedules eq 'ARRAY'; + + my $today = DateTime->now->set_time_zone(FixMyStreet->local_time_zone)->strftime("%F"); + my ($min_next, $max_last, $next_changed); + foreach my $schedule (@$schedules) { + my $end_date = construct_bin_date($schedule->{EndDate})->strftime("%F"); + next if $end_date lt $today; + + my $next = $schedule->{NextInstance}; + my $d = construct_bin_date($next->{CurrentScheduledDate}); + if ($d && (!$min_next || $d < $min_next->{date})) { + $next_changed = $next->{CurrentScheduledDate}{DateTime} ne $next->{OriginalScheduledDate}{DateTime}; + $min_next = { + date => $d, + ordinal => ordinal($d->day), + changed => $next_changed, + }; + } + + my $last = $schedule->{LastInstance}; + $d = construct_bin_date($last->{CurrentScheduledDate}); + # It is possible the last instance for this schedule has been rescheduled to + # be in the future. If so, we should treat it like it is a next instance. + if ($d && $d->strftime("%F") gt $today && (!$min_next || $d < $min_next->{date})) { + my $last_changed = $last->{CurrentScheduledDate}{DateTime} ne $last->{OriginalScheduledDate}{DateTime}; + $min_next = { + date => $d, + ordinal => ordinal($d->day), + changed => $last_changed, + }; + } elsif ($d && (!$max_last || $d > $max_last->{date})) { + my $last_changed = $last->{CurrentScheduledDate}{DateTime} ne $last->{OriginalScheduledDate}{DateTime}; + $max_last = { + date => $d, + ordinal => ordinal($d->day), + changed => $last_changed, + ref => $last->{Ref}{Value}{anyType}, + }; + } + } + + return { + next => $min_next, + last => $max_last, + }; +} + +sub bin_future_collections { + my $self = shift; + + my $services = $self->{c}->stash->{service_data}; + my @tasks; + my %names; + foreach (@$services) { + push @tasks, $_->{service_task_id}; + $names{$_->{service_task_id}} = $_->{service_name}; + } + + my $echo = $self->feature('echo'); + $echo = Integrations::Echo->new(%$echo); + my $result = $echo->GetServiceTaskInstances(@tasks); + + my $events = []; + foreach (@$result) { + my $task_id = $_->{ServiceTaskRef}{Value}{anyType}; + my $tasks = Integrations::Echo::force_arrayref($_->{Instances}, 'ScheduledTaskInfo'); + foreach (@$tasks) { + my $dt = construct_bin_date($_->{CurrentScheduledDate}); + my $summary = $names{$task_id} . ' collection'; + my $desc = ''; + push @$events, { date => $dt, summary => $summary, desc => $desc }; + } + } + return $events; +} + +=over + +=item within_working_days + +Given a DateTime object and a number, return true if today is less than or +equal to that number of working days (excluding weekends and bank holidays) +after the date. + +=cut + +sub within_working_days { + my ($dt, $days) = @_; + my $wd = FixMyStreet::WorkingDays->new(public_holidays => FixMyStreet::Cobrand::UK::public_holidays()); + $dt = $wd->add_days($dt, $days)->ymd; + my $today = DateTime->now->set_time_zone(FixMyStreet->local_time_zone)->ymd; + return $today le $dt; +} + +=item waste_fetch_events + +Loop through all open waste events to see if there have been any updates + +=back + +=cut + +sub waste_fetch_events { + my ($self, $verbose) = @_; + + my $body = $self->body; + my @contacts = $body->contacts->search({ + send_method => 'Open311', + endpoint => { '!=', '' }, + })->all; + die "Could not find any devolved contacts\n" unless @contacts; + + my %open311_conf = ( + endpoint => $contacts[0]->endpoint || '', + api_key => $contacts[0]->api_key || '', + jurisdiction => $contacts[0]->jurisdiction || '', + extended_statuses => $body->send_extended_statuses, + ); + my $cobrand = $body->get_cobrand_handler; + $cobrand->call_hook(open311_config_updates => \%open311_conf) + if $cobrand; + my $open311 = Open311->new(%open311_conf); + + my $updates = Open311::GetServiceRequestUpdates->new( + current_open311 => $open311, + current_body => $body, + system_user => $body->comment_user, + suppress_alerts => 0, + blank_updates_permitted => $body->blank_updates_permitted, + ); + + my $echo = $self->feature('echo'); + $echo = Integrations::Echo->new(%$echo); + + my $cfg = { + verbose => $verbose, + updates => $updates, + echo => $echo, + event_types => {}, + }; + + my $reports = $self->problems->search({ + external_id => { '!=', '' }, + state => [ FixMyStreet::DB::Result::Problem->open_states() ], + category => [ map { $_->category } @contacts ], + }); + + while (my $report = $reports->next) { + print 'Fetching data for report ' . $report->id . "\n" if $verbose; + + my $event = $cfg->{echo}->GetEvent($report->external_id); + my $request = $self->construct_waste_open311_update($cfg, $event) or next; + + next if !$request->{status} || $request->{status} eq 'confirmed'; # Still in initial state + next unless $self->waste_check_last_update( + $cfg, $report, $request->{status}, $request->{external_status_code}); + my $last_updated = construct_bin_date($event->{LastUpdatedDate}); + $request->{comment_time} = $last_updated; + + print " Updating report to state $request->{status}, $request->{description} ($request->{external_status_code})\n" if $cfg->{verbose}; + $cfg->{updates}->process_update($request, $report); + } +} + +sub construct_waste_open311_update { + my ($self, $cfg, $event) = @_; + + my $event_type = $cfg->{event_types}{$event->{EventTypeId}} ||= $self->waste_get_event_type($cfg, $event->{EventTypeId}); + my $state_id = $event->{EventStateId}; + my $resolution_id = $event->{ResolutionCodeId} || ''; + my $status = $event_type->{states}{$state_id}{state}; + my $description = $event_type->{resolution}{$resolution_id} || $event_type->{states}{$state_id}{name}; + return { + description => $description, + status => $status, + update_id => 'waste', + external_status_code => "$resolution_id,,", + }; +} + +sub waste_get_event_type { + my ($self, $cfg, $id) = @_; + + my $event_type = $cfg->{echo}->GetEventType($id); + + my $state_map = { + New => { New => 'confirmed' }, + Pending => { + Unallocated => 'investigating', + 'Allocated to Crew' => 'action scheduled', + }, + Closed => { + Completed => 'fixed - council', + 'Not Completed' => 'unable to fix', + Rejected => 'closed', + }, + }; + + my $states = $event_type->{Workflow}->{States}->{State}; + my $data; + foreach (@$states) { + my $core = $_->{CoreState}; # New/Pending/Closed + my $name = $_->{Name}; # New : Unallocated/Allocated to Crew : Completed/Not Completed/Rejected + $data->{states}{$_->{Id}} = { + core => $core, + name => $name, + state => $state_map->{$core}{$name}, + }; + my $codes = Integrations::Echo::force_arrayref($_->{ResolutionCodes}, 'StateResolutionCode'); + foreach (@$codes) { + my $name = $_->{Name}; + my $id = $_->{ResolutionCodeId}; + $data->{resolution}{$id} = $name; + } + } + return $data; +} + +# We only have the report's current state, no history, so must check current +# against latest received update to see if state the same, and skip if so +sub waste_check_last_update { + my ($self, $cfg, $report, $status, $resolution_id) = @_; + + my $latest = $report->comments->search( + { external_id => 'waste', }, + { order_by => { -desc => 'id' } } + )->first; + + if ($latest) { + my $state = $cfg->{updates}->current_open311->map_state($status); + my $code = $latest->get_extra_metadata('external_status_code') || ''; + if ($latest->problem_state eq $state && $code eq $resolution_id) { + print " Latest update matches fetched state, skipping\n" if $cfg->{verbose}; + return; + } + } + return 1; +} + +sub admin_templates_external_status_code_hook { + my ($self) = @_; + my $c = $self->{c}; + + my $res_code = $c->get_param('resolution_code') || ''; + my $task_type = $c->get_param('task_type') || ''; + my $task_state = $c->get_param('task_state') || ''; + + return "$res_code,$task_type,$task_state"; +} + +sub call_api { + my $self = shift; + + my $tmp = File::Temp->new; + my @cmd = ( + FixMyStreet->path_to('bin/fixmystreet.com/bromley-echo'), + '--out', $tmp, + '--calls', encode_json(\@_), + ); + + # We cannot fork directly under mod_fcgid, so + # call an external script that calls back in. + my $data; + if (FixMyStreet->test_mode) { + $data = $self->_parallel_api_calls(@_); + } else { + system(@cmd); + $data = Storable::fd_retrieve($tmp); + } + return $data; +} + +sub _parallel_api_calls { + my $self = shift; + my $echo = $self->feature('echo'); + $echo = Integrations::Echo->new(%$echo); + + my %calls; + my $pm = Parallel::ForkManager->new(FixMyStreet->test_mode ? 0 : 10); + $pm->run_on_finish(sub { + my ($pid, $exit_code, $ident, $exit_signal, $core_dump, $data) = @_; + %calls = ( %calls, %$data ); + }); + + while (@_) { + my $call = shift; + my $args = shift; + $pm->start and next; + my $result = $echo->$call(@$args); + my $key = "$call @$args"; + $key = $call if $call eq 'GetTasks'; + $pm->finish(0, { $key => $result }); + } + $pm->wait_all_children; + + return \%calls; +} + +1; diff --git a/perllib/FixMyStreet/Cobrand/Buckinghamshire.pm b/perllib/FixMyStreet/Cobrand/Buckinghamshire.pm index 117725273..f901c4e2f 100644 --- a/perllib/FixMyStreet/Cobrand/Buckinghamshire.pm +++ b/perllib/FixMyStreet/Cobrand/Buckinghamshire.pm @@ -45,16 +45,7 @@ sub send_questionnaires { return 0; } -sub open311_pre_send { - my ($self, $row, $open311) = @_; - - return unless $row->extra; - my $extra = $row->get_extra_fields; - if (@$extra) { - @$extra = grep { $_->{name} ne 'road-placement' } @$extra; - $row->set_extra_fields(@$extra); - } -} +sub open311_extra_data_exclude { [ 'road-placement' ] } sub open311_post_send { my ($self, $row, $h) = @_; @@ -103,6 +94,7 @@ sub report_new_munge_before_insert { my ($self, $report) = @_; return unless $report->category eq 'Flytipping'; + return unless $self->{c}->stash->{report}->to_body_named('Buckinghamshire'); my $placement = $self->{c}->get_param('road-placement'); return unless $placement && $placement eq 'off-road'; @@ -132,19 +124,17 @@ sub map_type { 'Buckinghamshire' } sub default_map_zoom { 3 } sub _dashboard_export_add_columns { - my $self = shift; - my $c = $self->{c}; + my ($self, $csv) = @_; - push @{$c->stash->{csv}->{headers}}, "Staff User"; - push @{$c->stash->{csv}->{columns}}, "staff_user"; + $csv->add_csv_columns( staff_user => 'Staff User' ); # All staff users, for contributed_by lookup - my @user_ids = $c->model('DB::User')->search( + my @user_ids = FixMyStreet::DB->resultset('User')->search( { from_body => $self->body->id }, { columns => [ 'id', 'email', ] })->all; my %user_lookup = map { $_->id => $_->email } @user_ids; - $c->stash->{csv}->{extra_data} = sub { + $csv->csv_extra_data(sub { my $report = shift; my $staff_user = ''; if (my $contributed_by = $report->get_extra_metadata('contributed_by')) { @@ -153,15 +143,15 @@ sub _dashboard_export_add_columns { return { staff_user => $staff_user, }; - }; + }); } sub dashboard_export_updates_add_columns { - shift->_dashboard_export_add_columns; + shift->_dashboard_export_add_columns(@_); } sub dashboard_export_problems_add_columns { - shift->_dashboard_export_add_columns; + shift->_dashboard_export_add_columns(@_); } # Enable adding/editing of parish councils in the admin diff --git a/perllib/FixMyStreet/Cobrand/CheshireEast.pm b/perllib/FixMyStreet/Cobrand/CheshireEast.pm index c5e5107f3..2a0423b7c 100644 --- a/perllib/FixMyStreet/Cobrand/CheshireEast.pm +++ b/perllib/FixMyStreet/Cobrand/CheshireEast.pm @@ -5,6 +5,7 @@ use strict; use warnings; use Moo; +with 'FixMyStreet::Roles::ConfirmOpen311'; with 'FixMyStreet::Roles::ConfirmValidation'; sub council_area_id { 21069 } @@ -56,39 +57,6 @@ sub abuse_reports_only { 1 } sub send_questionnaires { 0 } -sub open311_config { - my ($self, $row, $h, $params) = @_; - - $params->{multi_photos} = 1; -} - -sub open311_extra_data { - my ($self, $row, $h, $extra) = @_; - - my $open311_only = [ - { name => 'report_url', - value => $h->{url} }, - { name => 'title', - value => $row->title }, - { name => 'description', - value => $row->detail }, - ]; - - # Reports made via FMS.com or the app probably won't have a site code - # value because we don't display the adopted highways layer on those - # frontends. Instead we'll look up the closest asset from the WFS - # service at the point we're sending the report over Open311. - if (!$row->get_extra_field_value('site_code')) { - if (my $site_code = $self->lookup_site_code($row)) { - push @$extra, - { name => 'site_code', - value => $site_code }; - } - } - - return $open311_only; -} - # TODO These values may not be accurate sub lookup_site_code_config { { buffer => 200, # metres @@ -142,4 +110,7 @@ sub council_rss_alert_options { return ( \@options, undef ); } +# Make sure fetched report description isn't shown. +sub filter_report_description { "" } + 1; diff --git a/perllib/FixMyStreet/Cobrand/Default.pm b/perllib/FixMyStreet/Cobrand/Default.pm index 695487268..e58bceb2a 100644 --- a/perllib/FixMyStreet/Cobrand/Default.pm +++ b/perllib/FixMyStreet/Cobrand/Default.pm @@ -523,13 +523,29 @@ sub allow_update_reporting { return 0; } =item updates_disallowed Returns a boolean indicating whether updates on a particular report are allowed -or not. Default behaviour is disallowed if "closed_updates" metadata is set. +or not. Default behaviour is disallowed if "closed_updates" metadata is set, or +if the report's category has its "updates_disallowed" flag set. =cut sub updates_disallowed { my ($self, $problem) = @_; return 1 if $problem->get_extra_metadata('closed_updates'); + return 1 if $problem->contact && $problem->contact->get_extra_metadata('updates_disallowed'); + return 0; +} + +=item reopening_disallowed + +Returns a boolean indicating whether reopening of a particular report is +allowed or not. Default behaviour is allowed unless the report's category +has its reopening_disallowed flag set. + +=cut + +sub reopening_disallowed { + my ($self, $problem) = @_; + return 1 if $problem->contact && $problem->contact->get_extra_metadata('reopening_disallowed'); return 0; } @@ -941,11 +957,12 @@ Get stats to display on the council reports page sub get_report_stats { return 0; } sub get_body_sender { - my ( $self, $body, $category ) = @_; + my ( $self, $body, $problem ) = @_; # look up via category + my $category = $problem->category; my $contact = $body->contacts->search( { category => $category } )->first; - if ( $body->can_be_devolved && $contact->send_method ) { + if ( $body->can_be_devolved && $contact && $contact->send_method ) { return { method => $contact->send_method, config => $contact, contact => $contact }; } @@ -1055,7 +1072,7 @@ sub can_support_problems { return 0; } =item default_map_zoom default_map_zoom is used when displaying a map overriding the -default of max-4 or max-3 depending on population density. +default that depends on population density. =cut @@ -1107,7 +1124,22 @@ pressed in the front end, rather than whenever a username is not provided. =cut -sub allow_anonymous_reports { 0; } +sub allow_anonymous_reports { + my ($self, $category_name) = @_; + + $category_name ||= $self->{c}->stash->{category}; + if ( $category_name && $self->can('body') and $self->body ) { + my $category_rs = FixMyStreet::DB->resultset("Contact")->search({ + body_id => $self->body->id, + category => $category_name + }); + if ( my $category = $category_rs->first ) { + return 'button' if $category->get_extra_metadata('anonymous_allowed'); + } + } + + return 0; +} =item anonymous_account @@ -1216,15 +1248,13 @@ sub get_geocoder { sub problem_as_hashref { my $self = shift; my $problem = shift; - my $ctx = shift; - return $problem->as_hashref( $ctx ); + return $problem->as_hashref; } sub updates_as_hashref { my $self = shift; my $problem = shift; - my $ctx = shift; return {}; } @@ -1256,14 +1286,6 @@ sub category_extra_hidden { return 0; } -sub traffic_management_options { - return [ - _("Yes"), - _("No"), - ]; -} - - =item display_days_ago_threshold Used to control whether a relative 'n days ago' or absolute date is shown diff --git a/perllib/FixMyStreet/Cobrand/EastSussex.pm b/perllib/FixMyStreet/Cobrand/EastSussex.pm index e6c2da6c5..b2fd58dc1 100644 --- a/perllib/FixMyStreet/Cobrand/EastSussex.pm +++ b/perllib/FixMyStreet/Cobrand/EastSussex.pm @@ -7,11 +7,10 @@ use warnings; sub council_area_id { return 2224; } sub open311_extra_data { - my ($self, $row, $h, $extra, $contact) = @_; + my ($self, $row, $h, $contact) = @_; $h->{es_original_detail} = $row->detail; - $contact = $row->category_row; my $fields = $contact->get_extra_fields; my $text = ''; for my $field ( @$fields ) { @@ -21,7 +20,7 @@ sub open311_extra_data { } } $row->detail($row->detail . $text); - return (); + return (undef, ['sect_label', 'road_name', 'area_name']); } sub open311_post_send { diff --git a/perllib/FixMyStreet/Cobrand/FixMyStreet.pm b/perllib/FixMyStreet/Cobrand/FixMyStreet.pm index dfb511f39..ae96924d8 100644 --- a/perllib/FixMyStreet/Cobrand/FixMyStreet.pm +++ b/perllib/FixMyStreet/Cobrand/FixMyStreet.pm @@ -35,15 +35,19 @@ sub restriction { return {}; } -# FixMyStreet needs to not show TfL reports... +# FixMyStreet needs to not show TfL reports or Bromley waste reports sub problems_restriction { my ($self, $rs) = @_; my $table = ref $rs eq 'FixMyStreet::DB::ResultSet::Nearby' ? 'problem' : 'me'; - return $rs->search({ "$table.cobrand" => { '!=' => 'tfl' } }); + return $rs->search({ + "$table.cobrand" => { '!=' => 'tfl' }, + "$table.cobrand_data" => { '!=' => 'waste' }, + }); } sub problems_sql_restriction { my $self = shift; return "AND cobrand != 'tfl'"; + # Doesn't need Bromley one as all waste reports non-public } sub relative_url_for_report { @@ -54,32 +58,40 @@ sub relative_url_for_report { sub munge_around_category_where { my ($self, $where) = @_; + my $iow = grep { $_->name eq 'Isle of Wight Council' } @{ $self->{c}->stash->{around_bodies} }; + if ($iow) { + # display all the categories on Isle of Wight at the moment as there's no way to + # do the expand bit later as we fetch it using ajax which uses a bounding box so + # can't determine the body + $where->{send_method} = [ { '!=' => 'Triage' }, undef ]; + } + my $bromley = grep { $_->name eq 'Bromley Council' } @{ $self->{c}->stash->{around_bodies} }; + if ($bromley) { + $where->{extra} = [ undef, { -not_like => '%Waste%' } ]; + } +} + +sub _iow_category_munge { + my ($self, $body, $categories) = @_; my $user = $self->{c}->user; - my @iow = grep { $_->name eq 'Isle of Wight Council' } @{ $self->{c}->stash->{around_bodies} }; - return unless @iow; - - # display all the categories on Isle of Wight at the moment as there's no way to - # do the expand bit later as we fetch it using ajax which uses a bounding box so - # can't determine the body - $where->{send_method} = [ { '!=' => 'Triage' }, undef ]; - return $where; + + if ( $user && ( $user->is_superuser || $user->belongs_to_body( $body->id ) ) ) { + @$categories = grep { !$_->send_method || $_->send_method ne 'Triage' } @$categories; + return; + } + + @$categories = grep { $_->send_method && $_->send_method eq 'Triage' } @$categories; } -sub munge_reports_categories_list { +sub munge_reports_category_list { my ($self, $categories) = @_; my %bodies = map { $_->body->name => $_->body } @$categories; - if ( $bodies{'Isle of Wight Council'} ) { - my $user = $self->{c}->user; - my $b = $bodies{'Isle of Wight Council'}; - - if ( $user && ( $user->is_superuser || $user->belongs_to_body( $b->id ) ) ) { - @$categories = grep { !$_->send_method || $_->send_method ne 'Triage' } @$categories; - return @$categories; - } - - @$categories = grep { $_->send_method && $_->send_method eq 'Triage' } @$categories; - return @$categories; + if ( my $body = $bodies{'Isle of Wight Council'} ) { + return $self->_iow_category_munge($body, $categories); + } + if ( $bodies{'Bromley Council'} ) { + @$categories = grep { grep { $_ ne 'Waste' } @{$_->groups} } @$categories; } } @@ -118,16 +130,12 @@ sub munge_report_new_contacts { my %bodies = map { $_->body->name => $_->body } @$contacts; - if ( $bodies{'Isle of Wight Council'} ) { - my $user = $self->{c}->user; - if ( $user && ( $user->is_superuser || $user->belongs_to_body( $bodies{'Isle of Wight Council'}->id ) ) ) { - @$contacts = grep { !$_->send_method || $_->send_method ne 'Triage' } @$contacts; - return; - } - - @$contacts = grep { $_->send_method && $_->send_method eq 'Triage' } @$contacts; + if ( my $body = $bodies{'Isle of Wight Council'} ) { + return $self->_iow_category_munge($body, $contacts); + } + if ( $bodies{'Bromley Council'} ) { + @$contacts = grep { grep { $_ ne 'Waste' } @{$_->groups} } @$contacts; } - if ( $bodies{'TfL'} ) { # Presented categories vary if we're on/off a red route my $tfl = FixMyStreet::Cobrand->get_class_for_moniker( 'tfl' )->new({ c => $self->{c} }); @@ -139,10 +147,10 @@ sub munge_report_new_contacts { sub munge_load_and_group_problems { my ($self, $where, $filter) = @_; - return unless $where->{category} && $self->{c}->stash->{body}->name eq 'Isle of Wight Council'; + return unless $where->{'me.category'} && $self->{c}->stash->{body}->name eq 'Isle of Wight Council'; my $iow = FixMyStreet::Cobrand->get_class_for_moniker( 'isleofwight' )->new({ c => $self->{c} }); - $where->{category} = $iow->expand_triage_cat_list($where->{category}, $self->{c}->stash->{body}); + $where->{'me.category'} = $iow->expand_triage_cat_list($where->{'me.category'}, $self->{c}->stash->{body}); } sub title_list { @@ -310,6 +318,19 @@ sub updates_disallowed { return $self->next::method(@_); } +sub problem_state_processed { + my ($self, $comment) = @_; + + my $state = $comment->problem_state || ''; + my $code = $comment->get_extra_metadata('external_status_code') || ''; + + my ($cfg) = $self->per_body_config('extra_state_mapping', $comment->problem); + + $state = ( $cfg->{$state}->{$code} || $state ) if $cfg->{$state}; + + return $state; +} + sub suppress_reporter_alerts { my $self = shift; my $c = $self->{c}; @@ -347,4 +368,13 @@ sub manifest { }; } +sub report_new_munge_before_insert { + my ($self, $report) = @_; + + # Make sure TfL reports are marked safety critical + $self->SUPER::report_new_munge_before_insert($report); + + FixMyStreet::Cobrand::Buckinghamshire::report_new_munge_before_insert($self, $report); +} + 1; diff --git a/perllib/FixMyStreet/Cobrand/Greenwich.pm b/perllib/FixMyStreet/Cobrand/Greenwich.pm index be260d0c0..4cc4e4163 100644 --- a/perllib/FixMyStreet/Cobrand/Greenwich.pm +++ b/perllib/FixMyStreet/Cobrand/Greenwich.pm @@ -44,8 +44,8 @@ sub reports_per_page { return 20; } sub admin_user_domain { 'royalgreenwich.gov.uk' } -sub open311_extra_data { - my ($self, $row, $h, $extra) = @_; +sub open311_extra_data_include { + my ($self, $row, $h) = @_; # Greenwich doesn't have category metadata to fill this return [ diff --git a/perllib/FixMyStreet/Cobrand/Hackney.pm b/perllib/FixMyStreet/Cobrand/Hackney.pm new file mode 100644 index 000000000..b8f92f1ea --- /dev/null +++ b/perllib/FixMyStreet/Cobrand/Hackney.pm @@ -0,0 +1,207 @@ +package FixMyStreet::Cobrand::Hackney; +use parent 'FixMyStreet::Cobrand::Whitelabel'; + +use strict; +use warnings; +use mySociety::EmailUtil qw(is_valid_email is_valid_email_list); + +sub council_area_id { return 2508; } +sub council_area { return 'Hackney'; } +sub council_name { return 'Hackney Council'; } +sub council_url { return 'hackney'; } +sub send_questionnaires { 0 } + +sub disambiguate_location { + my $self = shift; + my $string = shift; + + my $town = 'Hackney'; + + # Teale Street is on the boundary with Tower Hamlets and + # shows the 'please use fixmystreet.com' message, but Hackney + # do provide services on that road. + ($string, $town) = ('E2 9AA', '') if $string =~ /^teale\s+st/i; + + return { + %{ $self->SUPER::disambiguate_location() }, + string => $string, + town => $town, + centre => '51.552267,-0.063316', + bounds => [ 51.519814, -0.104511, 51.577784, -0.016527 ], + }; +} + +sub do_not_reply_email { shift->feature('do_not_reply_email') } + +sub verp_email_domain { shift->feature('verp_email_domain') } + +sub get_geocoder { + return 'OSM'; # default of Bing gives poor results, let's try overriding. +} + +sub geocoder_munge_query_params { + my ($self, $params) = @_; + + $params->{addressdetails} = 1; +} + +sub geocoder_munge_results { + my ($self, $result) = @_; + if (my $a = $result->{address}) { + if ($a->{road} && $a->{suburb} && $a->{postcode}) { + $result->{display_name} = "$a->{road}, $a->{suburb}, $a->{postcode}"; + return; + } + } + $result->{display_name} = '' unless $result->{display_name} =~ /Hackney/; + $result->{display_name} =~ s/, United Kingdom$//; + $result->{display_name} =~ s/, London, Greater London, England//; + $result->{display_name} =~ s/, London Borough of Hackney//; +} + + +sub open311_config { + my ($self, $row, $h, $params) = @_; + + $params->{multi_photos} = 1; +} + +sub open311_extra_data { + my ($self, $row, $h, $contact) = @_; + + my $open311_only = [ + { name => 'report_url', + value => $h->{url} }, + { name => 'title', + value => $row->title }, + { name => 'description', + value => $row->detail }, + { name => 'category', + value => $row->category }, + ]; + + # Make sure contact 'email' set correctly for Open311 + if (my $sent_to = $row->get_extra_metadata('sent_to')) { + $row->unset_extra_metadata('sent_to'); + my $code = $sent_to->{$contact->email}; + $contact->email($code) if $code; + } + + return $open311_only; +} + +sub map_type { 'OSM' } + +sub default_map_zoom { 6 } + +sub admin_user_domain { 'hackney.gov.uk' } + +sub social_auth_enabled { + my $self = shift; + + return $self->feature('oidc_login') ? 1 : 0; +} + +sub anonymous_account { + my $self = shift; + return { + email => $self->feature('anonymous_account') . '@' . $self->admin_user_domain, + name => 'Anonymous user', + }; +} + +sub open311_skip_existing_contact { + my ($self, $contact) = @_; + + # For Hackney we want the 'protected' flag to prevent any changes to this + # contact at all. + return $contact->get_extra_metadata("open311_protect") ? 1 : 0; +} + +sub open311_filter_contacts_for_deletion { + my ($self, $contacts) = @_; + + # Don't delete open311 protected contacts when importing + return $contacts->search({ + extra => { -not_like => '%T15:open311_protect,I1:1%' }, + }); +} + +sub problem_is_within_area_type { + my ($self, $problem, $type) = @_; + my $layer_map = { + park => "greenspaces:hackney_park", + estate => "housing:lbh_estate", + }; + my $layer = $layer_map->{$type}; + return unless $layer; + + my ($x, $y) = $problem->local_coords; + + my $cfg = { + url => "https://map.hackney.gov.uk/geoserver/wfs", + srsname => "urn:ogc:def:crs:EPSG::27700", + typename => $layer, + outputformat => "json", + filter => "<Filter xmlns:gml=\"http://www.opengis.net/gml\"><Intersects><PropertyName>geom</PropertyName><gml:Point srsName=\"27700\"><gml:coordinates>$x,$y</gml:coordinates></gml:Point></Intersects></Filter>", + }; + + my $features = $self->_fetch_features($cfg, $x, $y) || []; + return scalar @$features ? 1 : 0; +} + +sub get_body_sender { + my ( $self, $body, $problem ) = @_; + + my $contact = $body->contacts->search( { category => $problem->category } )->first; + + if (my ($park, $estate, $other) = $self->_split_emails($contact->email)) { + my $to = $other; + if ($self->problem_is_within_area_type($problem, 'park')) { + $to = $park; + } elsif ($self->problem_is_within_area_type($problem, 'estate')) { + $to = $estate; + } + $problem->set_extra_metadata(sent_to => { $contact->email => $to }); + if (is_valid_email($to)) { + return { method => 'Email', contact => $contact }; + } + } + return $self->SUPER::get_body_sender($body, $problem); +} + +# Translate email address to actual delivery address +sub munge_sendreport_params { + my ($self, $row, $h, $params) = @_; + + my $sent_to = $row->get_extra_metadata('sent_to') or return; + $row->unset_extra_metadata('sent_to'); + for my $recip (@{$params->{To}}) { + my ($email, $name) = @$recip; + $recip->[0] = $sent_to->{$email} if $sent_to->{$email}; + } +} + +sub _split_emails { + my ($self, $email) = @_; + + my $parts = join '\s*', qw(^ park : (.*?) ; estate : (.*?) ; other : (.*?) $); + my $regex = qr/$parts/i; + + if (my ($park, $estate, $other) = $email =~ $regex) { + return ($park, $estate, $other); + } + return (); +} + +sub validate_contact_email { + my ( $self, $email ) = @_; + + return 1 if is_valid_email_list($email); + + my @emails = grep { $_ } $self->_split_emails($email); + return unless @emails; + return 1 if is_valid_email_list(join(",", @emails)); +} + +1; diff --git a/perllib/FixMyStreet/Cobrand/HighwaysEngland.pm b/perllib/FixMyStreet/Cobrand/HighwaysEngland.pm index ed58eb4f7..c282ac5ea 100644 --- a/perllib/FixMyStreet/Cobrand/HighwaysEngland.pm +++ b/perllib/FixMyStreet/Cobrand/HighwaysEngland.pm @@ -29,6 +29,15 @@ sub users_restriction { FixMyStreet::Cobrand::UKCouncils::users_restriction($_[0 sub updates_restriction { FixMyStreet::Cobrand::UKCouncils::updates_restriction($_[0], $_[1]) } sub base_url { FixMyStreet::Cobrand::UKCouncils::base_url($_[0]) } +sub munge_problem_list { + my ($self, $problem) = @_; + $problem->anonymous(1); +} +sub munge_update_list { + my ($self, $update) = @_; + $update->anonymous(1); +} + sub admin_allow_user { my ( $self, $user ) = @_; return 1 if $user->is_superuser; diff --git a/perllib/FixMyStreet/Cobrand/Hounslow.pm b/perllib/FixMyStreet/Cobrand/Hounslow.pm index 2fc949546..90d3b17dc 100644 --- a/perllib/FixMyStreet/Cobrand/Hounslow.pm +++ b/perllib/FixMyStreet/Cobrand/Hounslow.pm @@ -65,8 +65,14 @@ sub categories_restriction { # Email categories with a devolved send_method, so can identify Open311 # categories as those which have a blank send_method. return $rs->search({ - 'me.send_method' => undef, 'body.name' => [ 'Hounslow Borough Council', 'Highways England' ], + -or => [ + 'me.send_method' => undef, + 'me.category' => { -in => [ + 'Pavement Overcrowding', + 'Streetspace Suggestions and Feedback', + ] }, + ], }); } @@ -120,40 +126,25 @@ sub open311_skip_report_fetch { sub filter_report_description { "" } sub setup_general_enquiries_stash { - my $self = shift; - - my @bodies = $self->{c}->model('DB::Body')->active->for_areas(( $self->council_area_id ))->all; - my %bodies = map { $_->id => $_ } @bodies; - my @contacts # - = $self->{c} # - ->model('DB::Contact') # - ->active - ->search( - { - 'me.body_id' => [ keys %bodies ] - }, - { - prefetch => 'body', - order_by => 'me.category', - } - )->all; - @contacts = grep { - my $group = $_->get_extra_metadata('group') || ''; - $group eq 'Other' || $group eq 'General Enquiries'; - } @contacts; - $self->{c}->stash->{bodies} = \%bodies; - $self->{c}->stash->{bodies_to_list} = \%bodies; - $self->{c}->stash->{contacts} = \@contacts; - $self->{c}->stash->{missing_details_bodies} = []; - $self->{c}->stash->{missing_details_body_names} = []; - - $self->{c}->set_param('title', "General Enquiry"); - # Can't use (0, 0) for lat lon so default to the rough location - # of Hounslow Highways HQ. - $self->{c}->stash->{latitude} = 51.469; - $self->{c}->stash->{longitude} = -0.35; - - return 1; + my $self = shift; + my $c = $self->{c}; + + $c->set_param('title', "General Enquiry"); + # Can't use (0, 0) for lat lon so default to the rough location + # of Hounslow Highways HQ. + $c->stash->{latitude} = 51.469; + $c->stash->{longitude} = -0.35; + + $c->stash->{all_areas} = { $self->council_area_id => { id => $self->council_area_id } }; + $c->forward('/report/new/setup_categories_and_bodies'); + + my $contacts = $c->stash->{contacts}; + @$contacts = grep { + my $groups = $_->groups; + grep { $_ eq 'Other' || $_ eq 'General Enquiries' } @$groups; + } @$contacts; + + return 1; } sub abuse_reports_only { 1 } @@ -171,4 +162,29 @@ sub lookup_site_code_config { { # their cobrand at all. sub cut_off_date { '2019-05-06' } +sub front_stats_data { + my ( $self ) = @_; + + my $recency = '1 week'; + my $shorter_recency = '3 days'; + + my $completed = $self->problems->recent_completed(); + my $updates = $self->problems->number_comments(); + my $new = $self->problems->recent_new( $recency ); + + if ( $new > $completed ) { + $recency = $shorter_recency; + $new = $self->problems->recent_new( $recency ); + } + + my $stats = { + completed => $completed, + updates => $updates, + new => $new, + recency => $recency, + }; + + return $stats; +} + 1; diff --git a/perllib/FixMyStreet/Cobrand/IsleOfWight.pm b/perllib/FixMyStreet/Cobrand/IsleOfWight.pm index db0a20b9c..72555b9e6 100644 --- a/perllib/FixMyStreet/Cobrand/IsleOfWight.pm +++ b/perllib/FixMyStreet/Cobrand/IsleOfWight.pm @@ -6,6 +6,7 @@ use warnings; use Moo; with 'FixMyStreet::Roles::ConfirmOpen311'; +with 'FixMyStreet::Roles::ConfirmValidation'; sub council_area_id { 2636 } sub council_area { 'Isle of Wight' } @@ -63,20 +64,18 @@ sub lookup_site_code_config { { accept_feature => sub { 1 } } } -sub open311_pre_send { - my ($self, $row, $open311) = @_; - - return unless $row->extra; - my $extra = $row->get_extra_fields; - if (@$extra) { - @$extra = grep { $_->{name} ne 'urgent' } @$extra; - $row->set_extra_fields(@$extra); - } -} +sub open311_extra_data_exclude { [ '^urgent$' ] } # Make sure fetched report description isn't shown. sub filter_report_description { "" } +around 'open311_config' => sub { + my ($orig, $self, $row, $h, $params) = @_; + + $params->{upload_files} = 1; + $self->$orig($row, $h, $params); +}; + sub open311_munge_update_params { my ($self, $params, $comment, $body) = @_; @@ -130,19 +129,18 @@ sub munge_around_category_where { my $b = $self->{c}->model('DB::Body')->for_areas( $self->council_area_id )->first; if ( $user && ( $user->is_superuser || $user->belongs_to_body( $b->id ) ) ) { $where->{send_method} = [ { '!=' => 'Triage' }, undef ]; - return $where; + return; } $where->{'send_method'} = 'Triage'; - return $where; } sub munge_load_and_group_problems { my ($self, $where, $filter) = @_; - return unless $where->{category}; + return unless $where->{'me.category'}; - $where->{category} = $self->_expand_triage_cat_list($where->{category}); + $where->{'me.category'} = $self->_expand_triage_cat_list($where->{'me.category'}); } sub munge_around_filter_category_list { @@ -176,10 +174,7 @@ sub expand_triage_cat_list { my %group_to_category; while ( my $cat = $all_cats->next ) { - next unless $cat->get_extra_metadata('group'); - my $groups = $cat->get_extra_metadata('group'); - $groups = ref $groups eq 'ARRAY' ? $groups : [ $groups ]; - for my $group ( @$groups ) { + for my $group ( @{$cat->groups} ) { $group_to_category{$group} //= []; push @{ $group_to_category{$group} }, $cat->category; } diff --git a/perllib/FixMyStreet/Cobrand/Lincolnshire.pm b/perllib/FixMyStreet/Cobrand/Lincolnshire.pm index ee40bb173..d1fe319e1 100644 --- a/perllib/FixMyStreet/Cobrand/Lincolnshire.pm +++ b/perllib/FixMyStreet/Cobrand/Lincolnshire.pm @@ -77,4 +77,11 @@ sub pin_colour { return 'yellow'; } +around 'open311_config' => sub { + my ($orig, $self, $row, $h, $params) = @_; + + $params->{upload_files} = 1; + $self->$orig($row, $h, $params); +}; + 1; diff --git a/perllib/FixMyStreet/Cobrand/Northamptonshire.pm b/perllib/FixMyStreet/Cobrand/Northamptonshire.pm index 3e32b0856..2543f701d 100644 --- a/perllib/FixMyStreet/Cobrand/Northamptonshire.pm +++ b/perllib/FixMyStreet/Cobrand/Northamptonshire.pm @@ -91,10 +91,10 @@ sub open311_config { $params->{multi_photos} = 1; } -sub open311_extra_data { - my ($self, $row, $h, $extra) = @_; +sub open311_extra_data_include { + my ($self, $row, $h) = @_; - return ([ + return [ { name => 'report_url', value => $h->{url} }, { name => 'title', @@ -103,10 +103,9 @@ sub open311_extra_data { value => $row->detail }, { name => 'category', value => $row->category }, - ], [ - 'emergency' - ]); + ]; } +sub open311_extra_data_exclude { [ 'emergency' ] } sub open311_get_update_munging { my ($self, $comment) = @_; diff --git a/perllib/FixMyStreet/Cobrand/Oxfordshire.pm b/perllib/FixMyStreet/Cobrand/Oxfordshire.pm index 8ce12a81b..97174e1ce 100644 --- a/perllib/FixMyStreet/Cobrand/Oxfordshire.pm +++ b/perllib/FixMyStreet/Cobrand/Oxfordshire.pm @@ -122,8 +122,8 @@ sub open311_config { $params->{extended_description} = 'oxfordshire'; } -sub open311_extra_data { - my ($self, $row, $h, $extra) = @_; +sub open311_extra_data_include { + my ($self, $row, $h) = @_; return [ { name => 'external_id', value => $row->id }, @@ -138,6 +138,65 @@ sub open311_config_updates { $params->{use_customer_reference} = 1; } +sub open311_pre_send { + my ($self, $row, $open311) = @_; + + $self->{ox_original_detail} = $row->detail; + + if (my $fid = $row->get_extra_field_value('feature_id')) { + my $text = "Asset Id: $fid\n\n" . $row->detail; + $row->detail($text); + } +} + +sub open311_post_send { + my ($self, $row, $h, $contact) = @_; + + $row->detail($self->{ox_original_detail}); +} + +sub open311_munge_update_params { + my ($self, $params, $comment, $body) = @_; + + if ($comment->get_extra_metadata('defect_raised')) { + my $p = $comment->problem; + my ($e, $n) = $p->local_coords; + my $usrn = $p->get_extra_field_value('usrn'); + if (!$usrn) { + my $cfg = { + url => 'https://tilma.mysociety.org/mapserver/oxfordshire', + typename => "OCCRoads", + srsname => 'urn:ogc:def:crs:EPSG::27700', + accept_feature => sub { 1 }, + filter => "<Filter xmlns:gml=\"http://www.opengis.net/gml\"><DWithin><PropertyName>SHAPE_GEOMETRY</PropertyName><gml:Point><gml:coordinates>$e,$n</gml:coordinates></gml:Point><Distance units='m'>20</Distance></DWithin></Filter>", + }; + my $features = $self->_fetch_features($cfg); + my $feature = $self->_nearest_feature($cfg, $e, $n, $features); + if ($feature) { + my $props = $feature->{properties}; + $usrn = Utils::trim_text($props->{TYPE1_2_USRN}); + } + } + $params->{'attribute[usrn]'} = $usrn; + $params->{'attribute[raise_defect]'} = 1; + $params->{'attribute[easting]'} = $e; + $params->{'attribute[northing]'} = $n; + my $details = $comment->user->email . ' '; + if (my $traffic = $p->get_extra_metadata('traffic_information')) { + $details .= 'TM1 ' if $traffic eq 'Signs and Cones'; + $details .= 'TM2 ' if $traffic eq 'Stop and Go Boards'; + } + (my $type = $p->get_extra_metadata('defect_item_type')) =~ s/ .*//; + $details .= $type eq 'Sweep' ? 'S&F' : $type; + $details .= ' ' . ($p->get_extra_metadata('detailed_information') || ''); + $params->{'attribute[extra_details]'} = $details; + + foreach (qw(defect_item_category defect_item_type defect_item_detail defect_location_description)) { + $params->{"attribute[$_]"} = $p->get_extra_metadata($_); + } + } +} + sub should_skip_sending_update { my ($self, $update ) = @_; @@ -151,18 +210,20 @@ sub should_skip_sending_update { return 0; } -sub on_map_default_status { return 'open'; } -sub admin_user_domain { 'oxfordshire.gov.uk' } +sub report_inspect_update_extra { + my ( $self, $problem ) = @_; -sub traffic_management_options { - return [ - "Signs and Cones", - "Stop and Go Boards", - "High Speed Roads", - ]; + foreach (qw(defect_item_category defect_item_type defect_item_detail defect_location_description)) { + my $value = $self->{c}->get_param($_); + $problem->set_extra_metadata($_ => $value) if $value; + } } +sub on_map_default_status { return 'open'; } + +sub admin_user_domain { 'oxfordshire.gov.uk' } + sub admin_pages { my $self = shift; @@ -203,13 +264,11 @@ sub available_permissions { } sub dashboard_export_problems_add_columns { - my $self = shift; - my $c = $self->{c}; + my ($self, $csv) = @_; - push @{$c->stash->{csv}->{headers}}, "HIAMS/Exor Ref"; - push @{$c->stash->{csv}->{columns}}, "external_ref"; + $csv->add_csv_columns( external_ref => 'HIAMS/Exor Ref' ); - $c->stash->{csv}->{extra_data} = sub { + $csv->csv_extra_data(sub { my $report = shift; # Try and get a HIAMS reference first of all my $ref = $report->get_extra_metadata('customer_reference'); @@ -222,7 +281,7 @@ sub dashboard_export_problems_add_columns { return { external_ref => ( $ref || '' ), }; - }; + }); } 1; diff --git a/perllib/FixMyStreet/Cobrand/Peterborough.pm b/perllib/FixMyStreet/Cobrand/Peterborough.pm index 0ddaeacb6..b10367cfd 100644 --- a/perllib/FixMyStreet/Cobrand/Peterborough.pm +++ b/perllib/FixMyStreet/Cobrand/Peterborough.pm @@ -13,6 +13,7 @@ sub council_area { 'Peterborough' } sub council_name { 'Peterborough City Council' } sub council_url { 'peterborough' } sub map_type { 'MasterMap' } +sub default_map_zoom { 5 } sub send_questionnaires { 0 } @@ -31,6 +32,8 @@ sub disambiguate_location { sub get_geocoder { 'OSM' } +sub contact_extra_fields { [ 'display_name' ] } + sub geocoder_munge_results { my ($self, $result) = @_; $result->{display_name} = '' unless $result->{display_name} =~ /City of Peterborough/; @@ -40,30 +43,29 @@ sub geocoder_munge_results { sub admin_user_domain { "peterborough.gov.uk" } -around open311_extra_data => sub { - my ($orig, $self, $row, $h, $extra) = @_; +around open311_extra_data_include => sub { + my ($orig, $self, $row, $h) = @_; - my $open311_only = $self->$orig($row, $h, $extra); + my $open311_only = $self->$orig($row, $h); foreach (@$open311_only) { if ($_->{name} eq 'description') { my ($ref) = grep { $_->{name} =~ /pcc-Skanska-csc-ref/i } @{$row->get_extra_fields}; $_->{value} .= "\n\nSkanska CSC ref: $ref->{value}" if $ref; } } + if ( $row->geocode && $row->contact->email =~ /Bartec/ ) { + my $address = $row->geocode->{resourceSets}->[0]->{resources}->[0]->{address}; + my ($number, $street) = $address->{addressLine} =~ /\s*(\d*)\s*(.*)/; + push @$open311_only, ( + { name => 'postcode', value => $address->{postalCode} }, + { name => 'house_no', value => $number }, + { name => 'street', value => $street } + ); + } return $open311_only; }; - # remove categories which are informational only -sub open311_pre_send { - my ($self, $row, $open311) = @_; - - return unless $row->extra; - my $extra = $row->get_extra_fields; - if (@$extra) { - @$extra = grep { $_->{name} !~ /^(PCC-|emergency$|private_land$)/i } @$extra; - $row->set_extra_fields(@$extra); - } -} +sub open311_extra_data_exclude { [ '^PCC-', '^emergency$', '^private_land$' ] } sub lookup_site_code_config { { buffer => 50, # metres @@ -85,8 +87,37 @@ sub open311_munge_update_params { # Send the FMS problem ID with the update. $params->{service_request_id_ext} = $comment->problem->id; - my $contact = $comment->problem->category_row; + my $contact = $comment->problem->contact; $params->{service_code} = $contact->email; } +around 'open311_config' => sub { + my ($orig, $self, $row, $h, $params) = @_; + + $params->{upload_files} = 1; + $self->$orig($row, $h, $params); +}; + +sub dashboard_export_problems_add_columns { + my ($self, $csv) = @_; + + $csv->add_csv_columns( + usrn => 'USRN', + nearest_address => 'Nearest address', + ); + + $csv->csv_extra_data(sub { + my $report = shift; + + my $address = ''; + $address = $report->geocode->{resourceSets}->[0]->{resources}->[0]->{name} + if $report->geocode; + + return { + usrn => $report->get_extra_field_value('site_code'), + nearest_address => $address, + }; + }); +} + 1; diff --git a/perllib/FixMyStreet/Cobrand/Rutland.pm b/perllib/FixMyStreet/Cobrand/Rutland.pm index 63a20d893..bc8eff6d2 100644 --- a/perllib/FixMyStreet/Cobrand/Rutland.pm +++ b/perllib/FixMyStreet/Cobrand/Rutland.pm @@ -29,8 +29,8 @@ sub open311_config { $params->{multi_photos} = 1; } -sub open311_extra_data { - my ($self, $row, $h, $extra) = @_; +sub open311_extra_data_include { + my ($self, $row, $h) = @_; return [ { name => 'external_id', value => $row->id }, diff --git a/perllib/FixMyStreet/Cobrand/TfL.pm b/perllib/FixMyStreet/Cobrand/TfL.pm index b98ad1d8b..b04841c39 100644 --- a/perllib/FixMyStreet/Cobrand/TfL.pm +++ b/perllib/FixMyStreet/Cobrand/TfL.pm @@ -209,7 +209,7 @@ sub around_nearby_filter { sub state_groups_inspect { my $rs = FixMyStreet::DB->resultset("State"); - my @open = grep { $_ !~ /^(planned|action scheduled|for triage)$/ } FixMyStreet::DB::Result::Problem->open_states; + my @open = grep { $_ !~ /^(planned|investigating|for triage)$/ } FixMyStreet::DB::Result::Problem->open_states; my @closed = grep { $_ ne 'closed' } FixMyStreet::DB::Result::Problem->closed_states; [ [ $rs->display('confirmed'), \@open ], @@ -242,51 +242,32 @@ sub available_permissions { } sub dashboard_export_problems_add_columns { - my $self = shift; - my $c = $self->{c}; + my ($self, $csv) = @_; - my %groups; - if ($c->stash->{body}) { - %groups = FixMyStreet::DB->resultset('Contact')->search({ - body_id => $c->stash->{body}->id, - })->group_lookup; - } + $csv->modify_csv_header( Ward => 'Borough' ); + + $csv->add_csv_columns( + agent_responsible => "Agent responsible", + safety_critical => "Safety critical", + delivered_to => "Delivered to", + closure_email_at => "Closure email at", + reassigned_at => "Reassigned at", + reassigned_by => "Reassigned by", + ); + $csv->splice_csv_column('fixed', action_scheduled => 'Action scheduled'); - splice @{$c->stash->{csv}->{headers}}, 5, 0, 'Subcategory'; - splice @{$c->stash->{csv}->{columns}}, 5, 0, 'subcategory'; - - $c->stash->{csv}->{headers} = [ - map { $_ eq 'Ward' ? 'Borough' : $_ } @{ $c->stash->{csv}->{headers} }, - "Agent responsible", - "Safety critical", - "Delivered to", - "Closure email at", - "Reassigned at", - "Reassigned by", - ]; - - $c->stash->{csv}->{columns} = [ - @{ $c->stash->{csv}->{columns} }, - "agent_responsible", - "safety_critical", - "delivered_to", - "closure_email_at", - "reassigned_at", - "reassigned_by", - ]; - - if ($c->stash->{category}) { - my ($contact) = grep { $_->category eq $c->stash->{category} } @{$c->stash->{contacts}}; + if ($csv->category) { + my @contacts = $csv->body->contacts->search(undef, { order_by => [ 'category' ] } )->all; + my ($contact) = grep { $_->category eq $csv->category } @contacts; if ($contact) { foreach (@{$contact->get_metadata_for_storage}) { next if $_->{code} eq 'safety_critical'; - push @{$c->stash->{csv}->{columns}}, "extra.$_->{code}"; - push @{$c->stash->{csv}->{headers}}, $_->{description}; + $csv->add_csv_columns( "extra.$_->{code}" => $_->{description} ); } } } - $c->stash->{csv}->{extra_data} = sub { + $csv->csv_extra_data(sub { my $report = shift; my $agent = $report->shortlisted_user; @@ -315,8 +296,6 @@ sub dashboard_export_problems_add_columns { my $fields = { acknowledged => $report->whensent, agent_responsible => $agent ? $agent->name : '', - category => $groups{$report->category}, - subcategory => $report->category, user_name_display => $user_name_display, safety_critical => $safety_critical, delivered_to => join(',', @$delivered_to), @@ -329,7 +308,7 @@ sub dashboard_export_problems_add_columns { $fields->{"extra.$_->{name}"} = $_->{value}; } return $fields; - }; + }); } sub must_have_2fa { @@ -449,6 +428,13 @@ sub munge_surrounding_london { # Don't send any TfL categories %$bodies = map { $_->id => $_ } grep { $_->name ne 'TfL' } values %$bodies; } + + # Hackney doesn't have any of the council TfL categories so don't show + # any Hackney categories on red routes + my %bodies = map { $_->name => $_->id } values %$bodies; + if ( $bodies{'Hackney Council'} && $self->report_new_is_on_tlrn ) { + delete $bodies->{ $bodies{'Hackney Council'} }; + } } sub munge_red_route_categories { @@ -498,6 +484,7 @@ sub _tlrn_categories { [ "Mobile Crane Operation", "Other (TfL)", "Pavement Defect (uneven surface / cracked paving slab)", + "Pavement Overcrowding", "Pothole", "Pothole (minor)", "Roadworks", @@ -505,6 +492,7 @@ sub _tlrn_categories { [ "Single Light out (street light)", "Standing water", "Street Light - Equipment damaged, pole leaning", + "Streetspace Feedback", "Unstable hoardings", "Unstable scaffolding", "Worn out road markings", diff --git a/perllib/FixMyStreet/Cobrand/UK.pm b/perllib/FixMyStreet/Cobrand/UK.pm index a42ff58a6..988458e0f 100644 --- a/perllib/FixMyStreet/Cobrand/UK.pm +++ b/perllib/FixMyStreet/Cobrand/UK.pm @@ -2,7 +2,11 @@ package FixMyStreet::Cobrand::UK; use base 'FixMyStreet::Cobrand::Default'; use strict; +use Encode; use JSON::MaybeXS; +use LWP::UserAgent; +use Path::Tiny; +use Time::Piece; use mySociety::MaPit; use mySociety::VotingArea; use Utils; @@ -397,9 +401,9 @@ sub link_to_council_cobrand { $handler->moniker ne $self->{c}->cobrand->moniker ) { my $url = sprintf("%s%s", $handler->base_url, $problem->url); - return sprintf("<a href='%s'>%s</a>", $url, $problem->body( $self->{c} )); + return sprintf("<a href='%s'>%s</a>", $url, $problem->body); } else { - return $problem->body( $self->{c} ); + return $problem->body; } } @@ -407,12 +411,6 @@ sub lookup_by_ref_regex { return qr/^\s*(\d+)\s*$/; } -sub category_extra_hidden { - my ($self, $meta) = @_; - return 1 if $meta->{code} eq 'usrn' || $meta->{code} eq 'asset_id'; - return $self->SUPER::category_extra_hidden($meta); -} - sub report_new_munge_before_insert { my ($self, $report) = @_; @@ -422,4 +420,82 @@ sub report_new_munge_before_insert { } } +# To use recaptcha, add a RECAPTCHA key to your config, with subkeys secret and +# site_key, taken from the recaptcha site. This shows it to non-UK IP addresses +# on alert and report pages. + +sub requires_recaptcha { + my $self = shift; + my $c = $self->{c}; + + return 0 if $c->user_exists; + return 0 if !FixMyStreet->config('RECAPTCHA'); + return 0 unless $c->action =~ /^(alert|report|around)/; + return 0 if $c->user_country eq 'GB'; + return 1; +} + +sub check_recaptcha { + my $self = shift; + my $c = $self->{c}; + + return unless $self->requires_recaptcha; + + my $url = 'https://www.google.com/recaptcha/api/siteverify'; + my $res = LWP::UserAgent->new->post($url, { + secret => FixMyStreet->config('RECAPTCHA')->{secret}, + response => $c->get_param('g-recaptcha-response'), + remoteip => $c->req->address, + }); + $res = decode_json($res->content); + $c->detach('/page_error_400_bad_request', ['Bad recaptcha']) + unless $res->{success}; +} + +sub public_holidays { + my $nation = shift || 'england-and-wales'; + my $json = _get_bank_holiday_json(); + return [ map { $_->{date} } @{$json->{$nation}{events}} ]; +} + +sub is_public_holiday { + my %args = @_; + $args{date} ||= localtime; + $args{date} = $args{date}->date; + $args{nation} ||= 'england-and-wales'; + my $json = _get_bank_holiday_json(); + for my $event (@{$json->{$args{nation}}{events}}) { + if ($event->{date} eq $args{date}) { + return 1; + } + } +} + +sub _get_bank_holiday_json { + my $file = 'bank-holidays.json'; + my $cache_file = path(FixMyStreet->path_to("../data/$file")); + my $js; + if (-s $cache_file && -M $cache_file <= 7 && !FixMyStreet->config('STAGING_SITE')) { + # uncoverable statement + $js = $cache_file->slurp_utf8; + } else { + $js = _fetch_url("https://www.gov.uk/$file"); + # uncoverable branch false + $js = decode_utf8($js) if !utf8::is_utf8($js); + if ($js && !FixMyStreet->config('STAGING_SITE')) { + # uncoverable statement + $cache_file->spew_utf8($js); + } + } + $js = JSON->new->decode($js) if $js; + return $js; +} + +sub _fetch_url { + my $url = shift; + my $ua = LWP::UserAgent->new; + $ua->timeout(5); + $ua->get($url)->content; +} + 1; diff --git a/perllib/FixMyStreet/Cobrand/UKCouncils.pm b/perllib/FixMyStreet/Cobrand/UKCouncils.pm index 21dd2d455..0e8341d57 100644 --- a/perllib/FixMyStreet/Cobrand/UKCouncils.pm +++ b/perllib/FixMyStreet/Cobrand/UKCouncils.pm @@ -270,6 +270,19 @@ sub relative_url_for_report { return FixMyStreet->config('BASE_URL'); } +sub problem_state_processed { + my ($self, $comment) = @_; + + my $state = $comment->problem_state || ''; + my $code = $comment->get_extra_metadata('external_status_code') || ''; + + my $cfg = $self->feature('extra_state_mapping'); + + $state = ( $cfg->{$state}->{$code} || $state ) if $cfg->{$state}; + + return $state; +} + sub admin_allow_user { my ( $self, $user ) = @_; return 1 if $user->is_superuser; @@ -329,6 +342,13 @@ sub munge_report_new_contacts { } } +sub open311_extra_data { + my $self = shift; + my $include = $self->call_hook(open311_extra_data_include => @_); + my $exclude = $self->call_hook(open311_extra_data_exclude => @_); + push @$exclude, 'sect_label', 'road_name', 'area_name'; + return ($include, $exclude); +}; =head2 lookup_site_code @@ -392,7 +412,7 @@ sub _fetch_features_url { SRSNAME => $cfg->{srsname}, TYPENAME => $cfg->{typename}, VERSION => "1.1.0", - outputformat => "geojson", + outputformat => $cfg->{outputformat} || "geojson", $cfg->{filter} ? ( Filter => $cfg->{filter} ) : ( BBOX => $cfg->{bbox} ), ); @@ -405,7 +425,7 @@ sub _nearest_feature { # We have a list of features, and we want to find the one closest to the # report location. - my $site_code = ''; + my $chosen = ''; my $nearest; # We shouldn't receive anything aside from these geometry types, but belt and braces. @@ -432,14 +452,14 @@ sub _nearest_feature { for (my $i=0; $i<@$coordinates-1; $i++) { my $distance = $self->_distanceToLine($x, $y, $coordinates->[$i], $coordinates->[$i+1]); if ( !defined $nearest || $distance < $nearest ) { - $site_code = $feature->{properties}->{$cfg->{property}}; + $chosen = $feature; $nearest = $distance; } } } } - return $site_code; + return $cfg->{property} && $chosen ? $chosen->{properties}->{$cfg->{property}} : $chosen; } sub contact_name { diff --git a/perllib/FixMyStreet/Cobrand/Westminster.pm b/perllib/FixMyStreet/Cobrand/Westminster.pm index c9f31f7f9..e00a7c092 100644 --- a/perllib/FixMyStreet/Cobrand/Westminster.pm +++ b/perllib/FixMyStreet/Cobrand/Westminster.pm @@ -78,15 +78,15 @@ sub open311_config { $h->{account_id} = $id || '0'; } -sub open311_extra_data { - my ($self, $row, $h, $extra) = @_; +sub open311_extra_data_include { + my ($self, $row, $h) = @_; # Reports made via the app probably won't have a USRN because we don't # display the road layer. Instead we'll look up the closest asset from the # asset service at the point we're sending the report over Open311. if (!$row->get_extra_field_value('USRN')) { if (my $ref = $self->lookup_site_code($row, 'USRN')) { - push @$extra, { name => 'USRN', value => $ref }; + $row->update_extra_field({ name => 'USRN', value => $ref }); } } @@ -96,7 +96,7 @@ sub open311_extra_data { my ($uprn_field) = grep { $_->{name} eq 'UPRN' } @$fields; if ( $uprn_field && !$uprn_field->{value} ) { if (my $ref = $self->lookup_site_code($row, 'UPRN')) { - push @$extra, { name => 'UPRN', value => $ref }; + $row->update_extra_field({ name => 'UPRN', value => $ref }); } } diff --git a/perllib/FixMyStreet/Cobrand/Zurich.pm b/perllib/FixMyStreet/Cobrand/Zurich.pm index 3cf678f9c..c7b9f70ee 100644 --- a/perllib/FixMyStreet/Cobrand/Zurich.pm +++ b/perllib/FixMyStreet/Cobrand/Zurich.pm @@ -10,6 +10,8 @@ use DateTime::Format::Pg; use Try::Tiny; use FixMyStreet::Geocode::Zurich; +use FixMyStreet::Template; +use FixMyStreet::WorkingDays; use strict; use warnings; @@ -131,9 +133,8 @@ sub problem_has_user_response { sub problem_as_hashref { my $self = shift; my $problem = shift; - my $ctx = shift; - my $hashref = $problem->as_hashref( $ctx ); + my $hashref = $problem->as_hashref; if ( $problem->state eq 'submitted' ) { for my $var ( qw( photo is_fixed meta ) ) { @@ -171,7 +172,6 @@ sub problem_as_hashref { sub updates_as_hashref { my $self = shift; my $problem = shift; - my $ctx = shift; my $hashref = {}; @@ -179,10 +179,10 @@ sub updates_as_hashref { $hashref->{update_pp} = $self->prettify_dt( $problem->lastupdate ); if ( $problem->state ne 'external' ) { - $hashref->{details} = FixMyStreet::App::View::Web::add_links( + $hashref->{details} = FixMyStreet::Template::add_links( $problem->get_extra_metadata('public_response') || '' ); } else { - $hashref->{details} = sprintf( _('Assigned to %s'), $problem->body($ctx)->name ); + $hashref->{details} = sprintf( _('Assigned to %s'), $problem->body->name ); } } @@ -217,13 +217,13 @@ sub allow_photo_display { } sub get_body_sender { - my ( $self, $body, $category ) = @_; + my ( $self, $body, $problem ) = @_; return { method => 'Zurich' }; } # Report overdue functions -my %public_holidays = map { $_ => 1 } ( +my @public_holidays = ( # New Year's Day, Saint Berchtold, Good Friday, Easter Monday, # Sechseläuten, Labour Day, Ascension Day, Whit Monday, # Swiss National Holiday, Knabenschiessen, Christmas, St Stephen's Day @@ -249,53 +249,23 @@ my %public_holidays = map { $_ => 1 } ( '2021-09-13', ); -sub is_public_holiday { - my $dt = shift; - return $public_holidays{$dt->ymd}; -} - -sub is_weekend { - my $dt = shift; - return $dt->dow > 5; -} - -sub add_days { - my ( $dt, $days ) = @_; - $dt = $dt->clone; - while ( $days > 0 ) { - $dt->add ( days => 1 ); - next if is_public_holiday($dt) or is_weekend($dt); - $days--; - } - return $dt; -} - -sub sub_days { - my ( $dt, $days ) = @_; - $dt = $dt->clone; - while ( $days > 0 ) { - $dt->subtract ( days => 1 ); - next if is_public_holiday($dt) or is_weekend($dt); - $days--; - } - return $dt; -} - sub overdue { my ( $self, $problem ) = @_; my $w = $problem->created; return 0 unless $w; + my $wd = FixMyStreet::WorkingDays->new( public_holidays => \@public_holidays ); + # call with previous state if ( $problem->state eq 'submitted' ) { # One working day - $w = add_days( $w, 1 ); + $w = $wd->add_days( $w, 1 ); return $w < DateTime->now() ? 1 : 0; } elsif ( $problem->state eq 'confirmed' || $problem->state eq 'in progress' || $problem->state eq 'feedback pending' ) { # States which affect the subdiv_overdue statistic. TODO: this may no longer be required # Six working days from creation - $w = add_days( $w, 6 ); + $w = $wd->add_days( $w, 6 ); return $w < DateTime->now() ? 1 : 0; # call with new state @@ -303,7 +273,7 @@ sub overdue { # States which affect the closed_overdue statistic # Five working days from moderation (so 6 from creation) - $w = add_days( $w, 6 ); + $w = $wd->add_days( $w, 6 ); return $w < DateTime->now() ? 1 : 0; } } @@ -454,10 +424,21 @@ sub admin_type { return $type; } +sub _admin_index_order { + my $self = shift; + my $c = $self->{c}; + my $order = $c->get_param('o') || 'created'; + my $dir = defined $c->get_param('d') ? $c->get_param('d') : 1; + $c->stash->{order} = $order; + $c->stash->{dir} = $dir; + return $dir ? { -desc => $order } : $order; +} + sub admin { my $self = shift; my $c = $self->{c}; my $type = $c->stash->{admin_type}; + my $internal = $c->get_param('internal'); if ($type eq 'dm') { $c->stash->{template} = 'admin/index-dm.html'; @@ -466,22 +447,20 @@ sub admin { my @children = map { $_->id } $body->bodies->all; my @all = (@children, $body->id); - my $order = $c->get_param('o') || 'created'; - my $dir = defined $c->get_param('d') ? $c->get_param('d') : 1; - $c->stash->{order} = $order; - $c->stash->{dir} = $dir; - $order = { -desc => $order } if $dir; + my $order = $self->_admin_index_order; - # XXX No multiples or missing bodies + # No multiples or missing bodies $c->stash->{submitted} = $c->cobrand->problems->search({ state => [ 'submitted', 'confirmed' ], bodies_str => $c->stash->{body}->id, + non_public => $internal ? 1 : 0, }, { order_by => $order, }); $c->stash->{approval} = $c->cobrand->problems->search({ state => 'feedback pending', bodies_str => $c->stash->{body}->id, + non_public => $internal ? 1 : 0, }, { order_by => $order, }); @@ -490,6 +469,7 @@ sub admin { $c->stash->{other} = $c->cobrand->problems->search({ state => { -not_in => [ 'submitted', 'confirmed', 'feedback pending' ] }, bodies_str => \@all, + non_public => $internal ? 1 : 0, }, { order_by => $order, })->page( $page ); @@ -499,23 +479,20 @@ sub admin { $c->stash->{template} = 'admin/index-sdm.html'; my $body = $c->stash->{body}; + my $order = $self->_admin_index_order; - my $order = $c->get_param('o') || 'created'; - my $dir = defined $c->get_param('d') ? $c->get_param('d') : 1; - $c->stash->{order} = $order; - $c->stash->{dir} = $dir; - $order = { -desc => $order } if $dir; - - # XXX No multiples or missing bodies + # No multiples or missing bodies $c->stash->{reports_new} = $c->cobrand->problems->search( { state => 'in progress', bodies_str => $body->id, + non_public => $internal ? 1 : 0, }, { order_by => $order } ); $c->stash->{reports_unpublished} = $c->cobrand->problems->search( { state => 'feedback pending', bodies_str => $body->parent->id, + non_public => $internal ? 1 : 0, }, { order_by => $order } ); @@ -524,6 +501,7 @@ sub admin { $c->stash->{reports_published} = $c->cobrand->problems->search( { state => 'fixed - council', bodies_str => $body->parent->id, + non_public => $internal ? 1 : 0, }, { order_by => $order } )->page( $page ); @@ -544,6 +522,18 @@ sub category_options { $c->stash->{category_options} = \@categories; } +sub report_remove_internal_flag { + my $self = shift; + my $c = $self->{c}; + my $problem = $c->stash->{problem}; + $c->forward('/auth/check_csrf_token'); + $problem->non_public(0); + $problem->update; + $c->forward('/admin/log_edit', [ $problem->id, 'problem', 'Intern Flag entfernt' ]); + # Make sure the problem's time_spent is updated + $self->update_admin_log($c, $problem); +} + sub admin_report_edit { my $self = shift; my $c = $self->{c}; @@ -623,6 +613,10 @@ sub admin_report_edit { } } + if ( ($type eq 'super' || $type eq 'dm') && $c->get_param('stop_internal') ) { + $self->report_remove_internal_flag; + return $self->admin_report_edit_done; + } # Problem updates upon submission if ( ($type eq 'super' || $type eq 'dm') && $c->get_param('submit') ) { @@ -863,18 +857,12 @@ sub admin_report_edit { $c->go('index'); } - $c->stash->{updates} = [ $c->model('DB::Comment') - ->search( { problem_id => $problem->id }, { order_by => 'created' } ) - ->all ]; - - $self->stash_states($problem); - return 1; + return $self->admin_report_edit_done; } if ($type eq 'sdm') { - my $editable = $type eq 'sdm' && $body->id eq $problem->bodies_str; - $c->stash->{sdm_disabled} = $editable ? '' : 'disabled'; + my $editable = $body->id eq $problem->bodies_str; # Has cut-down edit template for adding update and sending back up only $c->stash->{template} = 'admin/report_edit-sdm.html'; @@ -905,6 +893,8 @@ sub admin_report_edit { # Make sure the problem's time_spent is updated $self->update_admin_log($c, $problem); $c->res->redirect( '/admin/summary' ); + } elsif ($editable && $c->get_param('stop_internal')) { + $self->report_remove_internal_flag; } elsif ($editable && $c->get_param('submit')) { $c->forward('/auth/check_csrf_token'); @@ -936,22 +926,25 @@ sub admin_report_edit { # If they clicked the no more updates button, we're done. if ($c->get_param('no_more_updates')) { - $problem->set_extra_metadata( subdiv_overdue => $self->overdue( $problem ) ); - $problem->bodies_str( $body->parent->id ); - $problem->whensent( undef ); - $self->set_problem_state($c, $problem, 'feedback pending'); + if ($problem->non_public) { + $problem->bodies_str( $body->parent->id ); + $self->set_problem_state($c, $problem, 'fixed - council'); + } else { + $problem->set_extra_metadata( subdiv_overdue => $self->overdue( $problem ) ); + $problem->bodies_str( $body->parent->id ); + $problem->whensent( undef ); + $self->set_problem_state($c, $problem, 'feedback pending'); + } $problem->update; $c->res->redirect( '/admin/summary' ); } } - $c->stash->{updates} = [ $c->model('DB::Comment') - ->search( { problem_id => $problem->id }, { order_by => 'created' } ) - ->all ]; - - $self->stash_states($problem); - return 1; + $c->stash->{sdm_disabled} = $editable ? '' : 'disabled'; + $c->stash->{sdm_disabled_internal} = $problem->non_public ? 'disabled' : ''; + $c->stash->{sdm_disabled_fixed} = $problem->is_fixed ? 'disabled' : ''; + return $self->admin_report_edit_done; } $self->stash_states($problem); @@ -959,6 +952,19 @@ sub admin_report_edit { } +sub admin_report_edit_done { + my $self = shift; + my $c = $self->{c}; + my $problem = $c->stash->{problem}; + $c->stash->{updates} = [ $c->model('DB::Comment') + ->search( { problem_id => $problem->id }, { order_by => 'created' } ) + ->all ]; + + $self->stash_states($problem); + return 1; +} + + sub admin_district_lookup { my ($self, $row) = @_; FixMyStreet::Geocode::Zurich::admin_district($row->local_coords); @@ -1053,6 +1059,7 @@ sub _admin_send_email { my ( $c, $template, $problem ) = @_; return unless $problem->get_extra_metadata('email_confirmed'); + return if $problem->non_public; my $to = $problem->name ? [ $problem->user->email, $problem->name ] @@ -1240,8 +1247,8 @@ sub admin_stats { sub export_as_csv { my ($self, $c, $params) = @_; - my $csv = $c->stash->{csv} = { - objects => $c->model('DB::Problem')->search_rs( + my $reporting = FixMyStreet::Reporting->new( + objects_rs => $c->model('DB::Problem')->search_rs( $params, { join => ['admin_log_entries', 'user'], @@ -1262,7 +1269,7 @@ sub export_as_csv { ] } ), - headers => [ + csv_headers => [ 'Report ID', 'Created', 'Sent to Agency', 'Last Updated', 'E', 'N', 'Category', 'Status', 'Closure Status', 'UserID', 'User email', 'User phone', 'User name', @@ -1270,7 +1277,7 @@ sub export_as_csv { 'Media URL', 'Interface Used', 'Council Response', 'Strasse', 'Mast-Nr.', 'Haus-Nr.', 'Hydranten-Nr.', ], - columns => [ + csv_columns => [ 'id', 'created', 'whensent',' lastupdate', 'local_coords_x', 'local_coords_y', 'category', 'state', 'closure_status', 'user_id', 'user_email', 'user_phone', 'user_name', @@ -1278,11 +1285,11 @@ sub export_as_csv { 'media_url', 'service', 'public_response', 'strasse', 'mast_nr',' haus_nr', 'hydranten_nr', ], - extra_data => sub { + csv_extra_data => sub { my $report = shift; my $body_name = ""; - if ( my $external_body = $report->body($c) ) { + if ( my $external_body = $report->body ) { $body_name = $external_body->name || '[Unknown body]'; } @@ -1325,8 +1332,8 @@ sub export_as_csv { }; }, filename => 'stats', - }; - $c->forward('/dashboard/generate_csv'); + ); + $reporting->generate_csv_http($c); } sub problem_confirm_email_extras { @@ -1389,4 +1396,13 @@ sub hook_report_filter_status { } @$status; } +# If report is made by a flagged user, mark as non-public +sub report_new_munge_before_insert { + my ($self, $report) = @_; + + if ($report->user->flagged) { + $report->non_public(1); + } +} + 1; diff --git a/perllib/FixMyStreet/DB/Result/Comment.pm b/perllib/FixMyStreet/DB/Result/Comment.pm index b217bf96c..82476ba10 100644 --- a/perllib/FixMyStreet/DB/Result/Comment.pm +++ b/perllib/FixMyStreet/DB/Result/Comment.pm @@ -110,6 +110,61 @@ with 'FixMyStreet::Roles::Abuser', 'FixMyStreet::Roles::Moderation', 'FixMyStreet::Roles::PhotoSet'; +=head2 FOREIGNBUILDARGS + +Make sure that when creating a new Comment object, certain +other fields are set based upon the supplied data. + +=cut + +sub FOREIGNBUILDARGS { + my ($class, $opts) = @_; + + if (my $user = $opts->{user}) { + my $name; + if ($user->is_superuser) { + $opts->{extra}->{is_superuser} = 1; + $name = _('an administrator'); + } elsif (my $body = $user->from_body) { + $opts->{extra}->{is_body_user} = $body->id; + $name = $body->name; + $name = 'Island Roads' if $name eq 'Isle of Wight Council'; + } else { + $name = $user->name; + } + $opts->{name} //= $name; + } + + $opts->{anonymous} //= 0; + $opts->{mark_fixed} //= 0; + $opts->{state} //= 'confirmed'; # it's only public updates that need to be unconfirmed + if ($opts->{state} eq 'confirmed') { + $opts->{confirmed} //= \'current_timestamp'; + } + + return $opts; +}; + +=head2 around user + +Also make sure we catch the setting of a user on an object at a time other than +object creation, to set the extra field needed. + +=cut + +around user => sub { + my ( $orig, $self ) = ( shift, shift ); + my $res = $self->$orig(@_); + if (@_) { + if ($_[0]->is_superuser) { + $self->set_extra_metadata( is_superuser => 1 ); + } elsif (my $body = $_[0]->from_body) { + $self->set_extra_metadata( is_body_user => $body->id ); + } + } + return $res; +}; + =head2 get_cobrand_logged Get a cobrand object for the cobrand the update was made on. @@ -207,13 +262,19 @@ about an update. Can include HTML. =cut sub meta_line { - my ( $self, $c ) = @_; + my ( $self, $user ) = @_; + my $cobrand = $self->result_source->schema->cobrand; my $meta = ''; - if ($self->anonymous or !$self->name) { - $meta = sprintf( _( 'Posted anonymously at %s' ), Utils::prettify_dt( $self->confirmed ) ) - } elsif ($self->user->from_body || $self->get_extra_metadata('is_body_user') || $self->get_extra_metadata('is_superuser') ) { + my $contributed_as = $self->get_extra_metadata('contributed_as') || ''; + my $staff = $self->user->from_body || $self->get_extra_metadata('is_body_user') || $self->get_extra_metadata('is_superuser'); + my $anon = $self->anonymous || !$self->name; + + if ($anon && (!$staff || $contributed_as eq 'anonymous_user' || $contributed_as eq 'another_user')) { + $meta = $cobrand->call_hook(update_anonymous_message => $self); + $meta ||= sprintf( _( 'Posted anonymously at %s' ), Utils::prettify_dt( $self->confirmed ) ) + } elsif ($staff) { my $user_name = FixMyStreet::Template::html_filter($self->user->name); my $body; if ($self->get_extra_metadata('is_superuser')) { @@ -237,9 +298,9 @@ sub meta_line { $body = 'Island Roads'; } } - my $cobrand_always_view_body_user = $c->cobrand->call_hook("always_view_body_contribute_details"); + my $cobrand_always_view_body_user = $cobrand->call_hook("always_view_body_contribute_details"); my $can_view_contribute = $cobrand_always_view_body_user || - ($c->user_exists && $c->user->has_permission_to('view_body_contribute_details', $self->problem->bodies_str_ids)); + ($user && $user->has_permission_to('view_body_contribute_details', $self->problem->bodies_str_ids)); if ($self->text) { if ($can_view_contribute) { $meta = sprintf( _( 'Posted by <strong>%s</strong> (%s) at %s' ), $body, $user_name, Utils::prettify_dt( $self->confirmed ) ); @@ -268,16 +329,20 @@ sub problem_state_processed { my $self = shift; return 'fixed - user' if $self->mark_fixed; return 'confirmed' if $self->mark_open; - return $self->problem_state; + my $cobrand = $self->result_source->schema->cobrand; + my $cobrand_state = $cobrand->call_hook(problem_state_processed => $self); + + return $cobrand_state || $self->problem_state; } sub problem_state_display { - my ( $self, $c ) = @_; + my $self = shift; my $state = $self->problem_state_processed; return '' unless $state; - my $cobrand_name = $c->cobrand->moniker; + my $cobrand = $self->result_source->schema->cobrand; + my $cobrand_name = $cobrand->moniker; my $names = join(',,', @{$self->problem->body_names}); if ($names =~ /(Bromley|Isle of Wight|TfL)/) { ($cobrand_name = lc $1) =~ s/ //g; @@ -313,7 +378,8 @@ sub hide { } sub as_hashref { - my ($self, $c, $cols) = @_; + my ($self, $cols) = @_; + my $cobrand = $self->result_source->schema->cobrand; my $out = { id => $self->id, @@ -329,7 +395,7 @@ sub as_hashref { if ($self->confirmed) { $out->{confirmed} = $self->confirmed if !$cols || $cols->{confirmed}; - $out->{confirmed_pp} = $c->cobrand->prettify_dt( $self->confirmed ) if !$cols || $cols->{confirmed_pp}; + $out->{confirmed_pp} = $cobrand->prettify_dt( $self->confirmed ) if !$cols || $cols->{confirmed_pp}; } return $out; diff --git a/perllib/FixMyStreet/DB/Result/Contact.pm b/perllib/FixMyStreet/DB/Result/Contact.pm index affc6d480..69f8886eb 100644 --- a/perllib/FixMyStreet/DB/Result/Contact.pm +++ b/perllib/FixMyStreet/DB/Result/Contact.pm @@ -113,9 +113,11 @@ sub category_display { $self->get_extra_metadata('display_name') || $self->translate_column('category'); } +# Returns an arrayref of groups this Contact is in; if it is +# not in any group, returns an arrayref of the empty string. sub groups { my $self = shift; - my $groups = $self->get_extra_metadata('group') || []; + my $groups = $self->get_extra_metadata('group') || ['']; $groups = [ $groups ] unless ref $groups eq 'ARRAY'; return $groups; } @@ -175,4 +177,31 @@ sub disable_form_field { return $field; } +sub sent_by_open311 { + my $self = shift; + my $body = $self->body; + my $method = $self->send_method || ''; + my $body_method = $body->send_method || ''; + return 1 if + (!$body->can_be_devolved && $body_method eq 'Open311') + || ($body->can_be_devolved && $body_method eq 'Open311' && !$method) + || ($body->can_be_devolved && $method eq 'Open311'); + return 0; +} + +# We do not want to allow editing of a category's name +# if it's Open311, unless it's marked as protected +# Also prevent editing of hardcoded categories +sub category_uneditable { + my $self = shift; + return 1 if + $self->in_storage + && !$self->get_extra_metadata('open311_protect') + && $self->sent_by_open311; + return 1 if + $self->in_storage + && $self->get_extra_metadata('hardcoded'); + return 0; +} + 1; diff --git a/perllib/FixMyStreet/DB/Result/ModerationOriginalData.pm b/perllib/FixMyStreet/DB/Result/ModerationOriginalData.pm index 1805e1fd2..dd76a52c0 100644 --- a/perllib/FixMyStreet/DB/Result/ModerationOriginalData.pm +++ b/perllib/FixMyStreet/DB/Result/ModerationOriginalData.pm @@ -156,6 +156,20 @@ sub compare_photo { return FixMyStreet::Template::SafeString->new($s); } +# This is a list of extra keys that could be set on a report after a moderation +# has occurred. This can confuse the display of the last moderation entry, as +# the comparison with the problem's extra will be wrong. +my @keys_to_ignore = ( + 'sent_to', # SendReport::Email adds this arrayref when sent + 'closed_updates', # Marked to close a report to updates + 'closure_alert_sent_at', # Set by alert sending if update closes a report + # Can be set/changed by an Open311 update + 'external_status_code', 'customer_reference', + # Can be set by inspectors + 'traffic_information', 'detailed_information', 'duplicates', 'duplicate_of', 'order', +); +my %keys_to_ignore = map { $_ => 1 } @keys_to_ignore; + sub compare_extra { my ($self, $other) = @_; @@ -163,18 +177,20 @@ sub compare_extra { my $new = $other->get_extra_metadata; my $both = { %$old, %$new }; - my @all_keys = sort keys %$both; + my @all_keys = grep { !$keys_to_ignore{$_} } sort keys %$both; my @s; foreach (@all_keys) { + $old->{$_} = join(', ', @{$old->{$_}}) if ref $old->{$_} eq 'ARRAY'; + $new->{$_} = join(', ', @{$new->{$_}}) if ref $new->{$_} eq 'ARRAY'; if ($old->{$_} && $new->{$_}) { push @s, string_diff("$_ = $old->{$_}", "$_ = $new->{$_}"); } elsif ($new->{$_}) { push @s, string_diff("", "$_ = $new->{$_}"); - } else { + } elsif ($old->{$_}) { push @s, string_diff("$_ = $old->{$_}", ""); } } - return join ', ', grep { $_ } @s; + return join '; ', grep { $_ } @s; } sub extra_diff { @@ -193,7 +209,7 @@ sub string_diff { $new = FixMyStreet::Template::html_filter($new); if ($options{single}) { - return unless $old; + return '' unless $old; $old = [ $old ]; $new = [ $new ]; } diff --git a/perllib/FixMyStreet/DB/Result/Problem.pm b/perllib/FixMyStreet/DB/Result/Problem.pm index 37563d327..ceb41b40f 100644 --- a/perllib/FixMyStreet/DB/Result/Problem.pm +++ b/perllib/FixMyStreet/DB/Result/Problem.pm @@ -193,6 +193,27 @@ __PACKAGE__->might_have( cascade_copy => 0, cascade_delete => 1 }, ); +# Add a possible join for the Contact object associated with +# this report (based on bodies_str and category). If the report +# was sent to multiple bodies, only returns the first. +__PACKAGE__->belongs_to( + contact => "FixMyStreet::DB::Result::Contact", + sub { + my $args = shift; + return { + "$args->{foreign_alias}.category" => { -ident => "$args->{self_alias}.category" }, + -and => [ + \[ "CAST($args->{foreign_alias}.body_id AS text) = (regexp_split_to_array($args->{self_alias}.bodies_str, ','))[1]" ], + ] + }; + }, + { + join_type => "LEFT", + on_delete => "NO ACTION", + on_update => "NO ACTION", + }, +); + __PACKAGE__->load_components("+FixMyStreet::DB::RABXColumn"); __PACKAGE__->rabx_column('extra'); __PACKAGE__->rabx_column('geocode'); @@ -407,30 +428,11 @@ sub confirm { sub category_display { my $self = shift; - my $contact = $self->category_row; + my $contact = $self->contact; return $self->category unless $contact; # Fallback; shouldn't happen, but some tests return $contact->category_display; } -=head2 category_row - -Returns the corresponding Contact object for this problem's category and body. -If the report was sent to multiple bodies, only returns the first. - -=cut - -sub category_row { - my $self = shift; - my $schema = $self->result_source->schema; - my $body_id = $self->bodies_str_ids->[0]; - return unless $body_id && $body_id =~ /^[0-9]+$/; - my $contact = $schema->resultset("Contact")->find({ - body_id => $body_id, - category => $self->category, - }); - return $contact; -} - sub bodies_str_ids { my $self = shift; return [] unless $self->bodies_str; @@ -629,13 +631,14 @@ meta data about the report. =cut sub meta_line { - my ( $problem, $c ) = @_; + my ( $problem, $user ) = @_; + my $cobrand = $problem->result_source->schema->cobrand; my $date_time = Utils::prettify_dt( $problem->confirmed ); my $meta = ''; my $category = $problem->category_display; - $category = $c->cobrand->call_hook(change_category_text => $category) || $category; + $category = $cobrand->call_hook(change_category_text => $category) || $category; if ( $problem->anonymous ) { if ( $problem->service and $category && $category ne _('Other') ) { @@ -654,8 +657,8 @@ sub meta_line { } else { my $problem_name = $problem->name; - if ($c->user_exists and - $c->user->has_permission_to('view_body_contribute_details', $problem->bodies_str_ids) and + if ($user and + $user->has_permission_to('view_body_contribute_details', $problem->bodies_str_ids) and $problem->name ne $problem->user->name) { $problem_name = sprintf('%s (%s)', $problem->name, $problem->user->name ); } @@ -690,12 +693,12 @@ sub nearest_address { } sub body { - my ( $problem, $c ) = @_; + my ( $problem, $link ) = @_; my $body; if ($problem->external_body) { if ($problem->cobrand eq 'zurich') { my $cache = $problem->result_source->schema->cache; - return $cache->{bodies}{$problem->external_body} //= $c->model('DB::Body')->find({ id => $problem->external_body }); + return $cache->{bodies}{$problem->external_body} //= FixMyStreet::DB->resultset('Body')->find({ id => $problem->external_body }); } else { $body = FixMyStreet::Template::html_filter($problem->external_body); } @@ -703,7 +706,7 @@ sub body { my $bodies = $problem->bodies; my @body_names = sort map { my $name = $_->name; - if ($c and FixMyStreet->config('AREA_LINKS_FROM_PROBLEMS')) { + if ($link and FixMyStreet->config('AREA_LINKS_FROM_PROBLEMS')) { '<a href="' . $_->url . '">' . FixMyStreet::Template::html_filter($name) . '</a>'; } else { FixMyStreet::Template::html_filter($name); @@ -777,7 +780,11 @@ alphabetical order of name. sub response_priorities { my $self = shift; - return $self->result_source->schema->resultset('ResponsePriority')->for_bodies($self->bodies_str_ids, $self->category); + my $rs = $self->result_source->schema->resultset('ResponsePriority')->for_bodies($self->bodies_str_ids, $self->category); + $rs->search([ + 'me.deleted' => 0, + 'me.id' => $self->response_priority_id, + ]); } =head2 defect_types @@ -808,9 +815,10 @@ sub can_display_external_id { # This can return HTML and is safe, so returns a FixMyStreet::Template::SafeString sub duration_string { - my ( $problem, $c ) = @_; - my $body = $c->cobrand->call_hook(link_to_council_cobrand => $problem) || $problem->body($c); - my $handler = $c->cobrand->call_hook(get_body_handler_for_problem => $problem); + my $problem = shift; + my $cobrand = $problem->result_source->schema->cobrand; + my $body = $cobrand->call_hook(link_to_council_cobrand => $problem) || $problem->body(1); + my $handler = $cobrand->call_hook(get_body_handler_for_problem => $problem); if ( $handler && $handler->call_hook('is_council_with_case_management') ) { my $s = sprintf(_('Received by %s moments later'), $body); return FixMyStreet::Template::SafeString->new($s); @@ -836,87 +844,6 @@ sub local_coords { } } -=head2 update_from_open311_service_request - - $p->update_from_open311_service_request( $request, $body, $system_user ); - -Updates the problem based on information in the passed in open311 request -(standard, not the extension that uses GetServiceRequestUpdates) . If the -request has an older update time than the problem's lastupdate time then -nothing happens. - -Otherwise a comment will be created if there is status update text in the -open311 request. If the open311 request has a state of closed then the problem -will be marked as fixed. - -NB: a comment will always be created if the problem is being marked as fixed. - -Fixed problems will not be re-opened by this method. - -=cut - -sub update_from_open311_service_request { - my ( $self, $request, $body, $system_user ) = @_; - - my ( $updated, $status_notes ); - - if ( ! ref $request->{updated_datetime} ) { - $updated = $request->{updated_datetime}; - } - - if ( ! ref $request->{status_notes} ) { - $status_notes = $request->{status_notes}; - } - - my $update = $self->new_related(comments => { - state => 'confirmed', - created => $updated || \'current_timestamp', - confirmed => \'current_timestamp', - text => $status_notes, - mark_open => 0, - mark_fixed => 0, - user => $system_user, - anonymous => 0, - name => $body->name, - }); - - my $w3c = DateTime::Format::W3CDTF->new; - my $req_time = $w3c->parse_datetime( $request->{updated_datetime} ); - - # set a timezone here as the $req_time will have one and if we don't - # use a timezone then the date comparisons are invalid. - # of course if local timezone is not the one that went into the data - # base then we're also in trouble - my $lastupdate = $self->lastupdate; - $lastupdate->set_time_zone( FixMyStreet->local_time_zone ); - - # update from open311 is older so skip - if ( $req_time < $lastupdate ) { - return 0; - } - - if ( $request->{status} eq 'closed' ) { - if ( $self->state ne 'fixed' ) { - $self->state('fixed'); - $update->mark_fixed(1); - - if ( !$status_notes ) { - # FIXME - better text here - $status_notes = _('Closed by council'); - } - } - } - - if ( $status_notes ) { - $update->text( $status_notes ); - $self->lastupdate( $req_time ); - $self->update; - $update->insert; - } - - return 1; -} - sub update_send_failed { my $self = shift; my $msg = shift; @@ -975,7 +902,8 @@ sub resend { } sub as_hashref { - my ($self, $c, $cols) = @_; + my ($self, $cols) = @_; + my $cobrand = $self->result_source->schema->cobrand; my $state_t = FixMyStreet::DB->resultset("State")->display($self->state); @@ -995,11 +923,11 @@ sub as_hashref { }; $out->{is_fixed} = $self->fixed_states->{ $self->state } ? 1 : 0 if !$cols || $cols->{is_fixed}; $out->{photos} = [ map { $_->{url} } @{$self->photos} ] if !$cols || $cols->{photos}; - $out->{meta} = $self->confirmed ? $self->meta_line( $c ) : '' if !$cols || $cols->{meta}; - $out->{created_pp} = $c->cobrand->prettify_dt( $self->created ) if !$cols || $cols->{created_pp}; + $out->{meta} = $self->confirmed ? $self->meta_line : '' if !$cols || $cols->{meta}; + $out->{created_pp} = $cobrand->prettify_dt( $self->created ) if !$cols || $cols->{created_pp}; if ($self->confirmed) { $out->{confirmed} = $self->confirmed if !$cols || $cols->{confirmed}; - $out->{confirmed_pp} = $c->cobrand->prettify_dt( $self->confirmed ) if !$cols || $cols->{confirmed_pp}; + $out->{confirmed_pp} = $cobrand->prettify_dt( $self->confirmed ) if !$cols || $cols->{confirmed_pp}; } return $out; } @@ -1052,10 +980,11 @@ has get_cobrand_logged => ( sub pin_data { - my ($self, $c, $page, %opts) = @_; - my $colour = $c->cobrand->pin_colour($self, $page); + my ($self, $page, %opts) = @_; + my $cobrand = $self->result_source->schema->cobrand; + my $colour = $cobrand->pin_colour($self, $page); my $title = $opts{private} ? $self->title : $self->title_safe; - $title = $c->cobrand->call_hook(pin_hover_title => $self, $title) || $title; + $title = $cobrand->call_hook(pin_hover_title => $self, $title) || $title; { latitude => $self->latitude, longitude => $self->longitude, @@ -1065,7 +994,7 @@ sub pin_data { problem => $self, draggable => $opts{draggable}, type => $opts{type}, - base_url => $c->cobrand->relative_url_for_report($self), + base_url => $cobrand->relative_url_for_report($self), } }; @@ -1196,17 +1125,6 @@ has duplicates => ( }, ); -has traffic_management_options => ( - is => 'ro', - lazy => 1, - default => sub { - my $self = shift; - my $cobrand = $self->get_cobrand_logged; - $cobrand = $cobrand->call_hook(get_body_handler_for_problem => $self) || $cobrand; - return $cobrand->traffic_management_options; - }, -); - has inspection_log_entry => ( is => 'ro', lazy => 1, diff --git a/perllib/FixMyStreet/DB/Result/User.pm b/perllib/FixMyStreet/DB/Result/User.pm index b0a05d0b7..e5be14abf 100644 --- a/perllib/FixMyStreet/DB/Result/User.pm +++ b/perllib/FixMyStreet/DB/Result/User.pm @@ -179,6 +179,11 @@ sub check_password { } } +sub access_token { + my $self = shift; + return $self->get_extra_metadata('access_token'); +} + around password => sub { my ($orig, $self) = (shift, shift); if (@_) { @@ -444,7 +449,7 @@ sub permissions { my $body_id = $problem->bodies_str; - return unless $self->belongs_to_body($body_id); + return {} unless $self->belongs_to_body($body_id); my @permissions = grep { $_->{body_id} == $self->from_body->id } @{$self->body_permissions}; return { map { $_->{permission} => 1 } @permissions }; diff --git a/perllib/FixMyStreet/DB/ResultSet/Comment.pm b/perllib/FixMyStreet/DB/ResultSet/Comment.pm index 034b86a40..ea38b3e14 100644 --- a/perllib/FixMyStreet/DB/ResultSet/Comment.pm +++ b/perllib/FixMyStreet/DB/ResultSet/Comment.pm @@ -4,12 +4,18 @@ use base 'DBIx::Class::ResultSet'; use strict; use warnings; +use Moo; +with 'FixMyStreet::Roles::FullTextSearch'; +__PACKAGE__->load_components('Helper::ResultSet::Me'); +sub text_search_columns { qw(id problem_id name text) } +sub text_search_nulls { qw(name) } +sub text_search_translate { '/.' } + sub to_body { my ($rs, $bodies) = @_; return FixMyStreet::DB::ResultSet::Problem::to_body($rs, $bodies, 1); } - sub timeline { my ( $rs ) = @_; diff --git a/perllib/FixMyStreet/DB/ResultSet/Contact.pm b/perllib/FixMyStreet/DB/ResultSet/Contact.pm index 801d20cc0..accdbb7de 100644 --- a/perllib/FixMyStreet/DB/ResultSet/Contact.pm +++ b/perllib/FixMyStreet/DB/ResultSet/Contact.pm @@ -5,7 +5,7 @@ use strict; use warnings; use POSIX qw(strcoll); -sub me { join('.', shift->current_source_alias, shift || q{}) } +__PACKAGE__->load_components('Helper::ResultSet::Me'); =head2 not_deleted @@ -90,13 +90,4 @@ sub summary_count { ); } -sub group_lookup { - my $rs = shift; - map { - my $group = $_->get_extra_metadata('group') || ''; - $group = join(',', ref $group ? @$group : $group); - $_->category => $group - } $rs->all; -} - 1; diff --git a/perllib/FixMyStreet/DB/ResultSet/DefectType.pm b/perllib/FixMyStreet/DB/ResultSet/DefectType.pm index 5b1247129..c4c11042f 100644 --- a/perllib/FixMyStreet/DB/ResultSet/DefectType.pm +++ b/perllib/FixMyStreet/DB/ResultSet/DefectType.pm @@ -12,7 +12,7 @@ sub join_table { } sub map_extras { - my ($rs, @ts) = @_; + my ($rs, $params, @ts) = @_; return map { my $meta = $_->get_extra_metadata(); my %extra = map { $_ => $meta->{$_} } keys %$meta; diff --git a/perllib/FixMyStreet/DB/ResultSet/Nearby.pm b/perllib/FixMyStreet/DB/ResultSet/Nearby.pm index 2ebe309e3..af1142c3a 100644 --- a/perllib/FixMyStreet/DB/ResultSet/Nearby.pm +++ b/perllib/FixMyStreet/DB/ResultSet/Nearby.pm @@ -17,16 +17,16 @@ sub nearby { } my $params = { - state => [ keys %{$args{states}} ], + 'problem.state' => [ keys %{$args{states}} ], }; - $params->{id} = { -not_in => $args{ids} } + $params->{problem_id} = { -not_in => $args{ids} } if $args{ids}; - $params->{category} = $args{categories} if $args{categories} && @{$args{categories}}; + $params->{'problem.category'} = $args{categories} if $args{categories} && @{$args{categories}}; $params->{$c->stash->{report_age_field}} = { '>=', \"current_timestamp-'$args{report_age}'::interval" } if $args{report_age}; - FixMyStreet::DB::ResultSet::Problem->non_public_if_possible($params, $c); + FixMyStreet::DB::ResultSet::Problem->non_public_if_possible($params, $c, 'problem'); $rs = $c->cobrand->problems_restriction($rs); @@ -34,11 +34,22 @@ sub nearby { $params = { %$params, %{$args{extra}} } if $args{extra}; my $attrs = { - prefetch => 'problem', + prefetch => { problem => [] }, bind => [ $args{latitude}, $args{longitude}, $args{distance} ], order_by => [ 'distance', { -desc => 'created' } ], rows => $args{limit}, }; + if ($c->user_exists) { + if ($c->user->from_body || $c->user->is_superuser) { + push @{$attrs->{prefetch}{problem}}, 'contact'; + } + if ($c->user->has_body_permission_to('planned_reports')) { + push @{$attrs->{prefetch}{problem}}, 'user_planned_reports'; + } + if ($c->user->has_body_permission_to('report_edit_priority') || $c->user->has_body_permission_to('report_inspect')) { + push @{$attrs->{prefetch}{problem}}, 'response_priority'; + } + } my @problems = mySociety::Locale::in_gb_locale { $rs->search( $params, $attrs )->all }; return \@problems; diff --git a/perllib/FixMyStreet/DB/ResultSet/Problem.pm b/perllib/FixMyStreet/DB/ResultSet/Problem.pm index e23cf78e1..a7c365c1e 100644 --- a/perllib/FixMyStreet/DB/ResultSet/Problem.pm +++ b/perllib/FixMyStreet/DB/ResultSet/Problem.pm @@ -8,6 +8,13 @@ use Memcached; use mySociety::Locale; use FixMyStreet::DB; +use Moo; +with 'FixMyStreet::Roles::FullTextSearch'; +__PACKAGE__->load_components('Helper::ResultSet::Me'); +sub text_search_columns { qw(id external_id bodies_str name title detail) } +sub text_search_nulls { qw(external_id bodies_str) } +sub text_search_translate { '/.' } + my $site_key; sub set_restriction { @@ -26,30 +33,31 @@ sub body_query { # Edits PARAMS in place to either hide non_public reports, or show them # if user is superuser (all) or inspector (correct body) sub non_public_if_possible { - my ($rs, $params, $c) = @_; + my ($rs, $params, $c, $table) = @_; + $table ||= 'me'; if ($c->user_exists) { my $only_non_public = $c->stash->{only_non_public} ? 1 : 0; if ($c->user->is_superuser) { # See all reports, no restriction - $params->{non_public} = 1 if $only_non_public; + $params->{"$table.non_public"} = 1 if $only_non_public; } elsif ($c->user->has_body_permission_to('report_inspect') || $c->user->has_body_permission_to('report_mark_private')) { if ($only_non_public) { $params->{'-and'} = [ - non_public => 1, + "$table.non_public" => 1, $rs->body_query($c->user->from_body->id), ]; } else { $params->{'-or'} = [ - non_public => 0, + "$table.non_public" => 0, $rs->body_query($c->user->from_body->id), ]; } } else { - $params->{non_public} = 0; + $params->{"$table.non_public"} = 0; } } else { - $params->{non_public} = 0; + $params->{"$table.non_public"} = 0; } } @@ -71,13 +79,26 @@ sub _cache_timeout { FixMyStreet->config('CACHE_TIMEOUT') // 3600; } +sub recent_completed { + my $rs = shift; + $rs->_recent_in_states('completed', [ + FixMyStreet::DB::Result::Problem->fixed_states(), + FixMyStreet::DB::Result::Problem->closed_states() + ]); +} + sub recent_fixed { my $rs = shift; - my $key = "recent_fixed:$site_key"; + $rs->_recent_in_states('fixed', [ FixMyStreet::DB::Result::Problem->fixed_states() ]); +} + +sub _recent_in_states { + my ($rs, $state_key, $states) = @_; + my $key = "recent_$state_key:$site_key"; my $result = Memcached::get($key); unless ($result) { $result = $rs->search( { - state => [ FixMyStreet::DB::Result::Problem->fixed_states() ], + state => $states, lastupdate => { '>', \"current_timestamp-'1 month'::interval" }, } )->count; Memcached::set($key, $result, _cache_timeout()); @@ -175,22 +196,33 @@ sub around_map { my ( $rs, $c, %p) = @_; my $attr = { order_by => $p{order}, + rows => $c->cobrand->reports_per_page, }; - $attr->{rows} = $c->cobrand->reports_per_page; + if ($c->user_exists) { + if ($c->user->from_body || $c->user->is_superuser) { + push @{$attr->{prefetch}}, 'contact'; + } + if ($c->user->has_body_permission_to('planned_reports')) { + push @{$attr->{prefetch}}, 'user_planned_reports'; + } + if ($c->user->has_body_permission_to('report_edit_priority') || $c->user->has_body_permission_to('report_inspect')) { + push @{$attr->{prefetch}}, 'response_priority'; + } + } unless ( $p{states} ) { $p{states} = FixMyStreet::DB::Result::Problem->visible_states(); } my $q = { - state => [ keys %{$p{states}} ], + 'me.state' => [ keys %{$p{states}} ], latitude => { '>=', $p{min_lat}, '<', $p{max_lat} }, longitude => { '>=', $p{min_lon}, '<', $p{max_lon} }, }; $q->{$c->stash->{report_age_field}} = { '>=', \"current_timestamp-'$p{report_age}'::interval" } if $p{report_age}; - $q->{category} = $p{categories} if $p{categories} && @{$p{categories}}; + $q->{'me.category'} = $p{categories} if $p{categories} && @{$p{categories}}; $rs->non_public_if_possible($q, $c); diff --git a/perllib/FixMyStreet/DB/ResultSet/ResponsePriority.pm b/perllib/FixMyStreet/DB/ResultSet/ResponsePriority.pm index 96f7cf7a0..af605afa6 100644 --- a/perllib/FixMyStreet/DB/ResultSet/ResponsePriority.pm +++ b/perllib/FixMyStreet/DB/ResultSet/ResponsePriority.pm @@ -12,8 +12,12 @@ sub join_table { } sub map_extras { - my ($rs, @ts) = @_; - return map { { id => $_->id, name => $_->name } } @ts; + my ($rs, $params, @ts) = @_; + my $current = $params->{problem} && $params->{problem}->response_priority_id || 0; + return + map { { id => $_->id, name => $_->name } } + grep { !$_->deleted || $_->id == $current } + @ts; } 1; diff --git a/perllib/FixMyStreet/DB/ResultSet/ResponseTemplate.pm b/perllib/FixMyStreet/DB/ResultSet/ResponseTemplate.pm index 46fcba153..88ecc2f94 100644 --- a/perllib/FixMyStreet/DB/ResultSet/ResponseTemplate.pm +++ b/perllib/FixMyStreet/DB/ResultSet/ResponseTemplate.pm @@ -14,7 +14,7 @@ sub name_column { } sub map_extras { - my ($rs, @ts) = @_; + my ($rs, $params, @ts) = @_; return map { my $out = { id => $_->text, name => $_->title }; $out->{state} = $_->state if $_->state; diff --git a/perllib/FixMyStreet/DB/ResultSet/User.pm b/perllib/FixMyStreet/DB/ResultSet/User.pm index 9a8a50559..baae024bf 100644 --- a/perllib/FixMyStreet/DB/ResultSet/User.pm +++ b/perllib/FixMyStreet/DB/ResultSet/User.pm @@ -5,6 +5,11 @@ use strict; use warnings; use Moo; +with 'FixMyStreet::Roles::FullTextSearch'; +__PACKAGE__->load_components('Helper::ResultSet::Me'); +sub text_search_columns { qw(id name email phone) } +sub text_search_nulls { qw(name email phone) } +sub text_search_translate { '@.' } # The database has a partial unique index on email (when email_verified is # true), and phone (when phone_verified is true). In the code, we can only diff --git a/perllib/FixMyStreet/Email.pm b/perllib/FixMyStreet/Email.pm index 3d7b48539..0cc6a880c 100644 --- a/perllib/FixMyStreet/Email.pm +++ b/perllib/FixMyStreet/Email.pm @@ -79,7 +79,9 @@ sub _render_template { } sub unique_verp_id { - sprintf('fms-%s@%s', generate_verp_token(@_), FixMyStreet->config('EMAIL_DOMAIN')); + my $parts = shift; + my $domain = shift || FixMyStreet->config('EMAIL_DOMAIN'); + sprintf('fms-%s@%s', generate_verp_token(@$parts), $domain); } sub _unique_id { diff --git a/perllib/FixMyStreet/Gaze.pm b/perllib/FixMyStreet/Gaze.pm index bccc81d8c..e2b2e0e08 100644 --- a/perllib/FixMyStreet/Gaze.pm +++ b/perllib/FixMyStreet/Gaze.pm @@ -3,6 +3,7 @@ package FixMyStreet::Gaze; use strict; use warnings; +use FixMyStreet; use mySociety::Gaze; sub get_radius_containing_population ($$) { @@ -24,4 +25,11 @@ sub get_radius_containing_population ($$) { return $dist; } +sub get_country_from_ip { + my ($ip) = @_; + return 'GB' if FixMyStreet->test_mode; + # uncoverable statement + return mySociety::Gaze::get_country_from_ip($ip); +} + 1; diff --git a/perllib/FixMyStreet/Geocode/Bing.pm b/perllib/FixMyStreet/Geocode/Bing.pm index 1d39d911f..8c5366d3d 100644 --- a/perllib/FixMyStreet/Geocode/Bing.pm +++ b/perllib/FixMyStreet/Geocode/Bing.pm @@ -7,6 +7,7 @@ package FixMyStreet::Geocode::Bing; use strict; +use warnings; use FixMyStreet::Geocode; use Utils; @@ -36,6 +37,8 @@ sub string { $url .= '&userMapView=' . join(',', @{$params->{bounds}}) if $params->{bounds}; $url .= '&userLocation=' . $params->{centre} if $params->{centre}; + $url .= '&userIp=127.0.0.1'; # So server location does not affect results + $url .= '&maxResults=10'; # Match what is said in the front end $url .= '&c=' . $params->{bing_culture} if $params->{bing_culture}; $c->stash->{geocoder_url} = $url; @@ -50,9 +53,28 @@ sub string { my $results = $js->{resourceSets}->[0]->{resources}; my ( $error, @valid_locations, $latitude, $longitude ); + # If there are any High/Medium confidence results, don't include Low ones + my $exclude_low; + foreach (@$results) { + my $confidence = $_->{confidence}; + if ($confidence eq 'High' || $confidence eq 'Medium') { + $exclude_low = 1; + last; + } + } + if ($exclude_low) { + @$results = grep { $_->{confidence} ne 'Low' } @$results; + } + foreach (@$results) { my $address = $_->{name}; - next if $params->{bing_country} && $_->{address}->{countryRegion} ne $params->{bing_country}; + if ($params->{bing_country}) { + next if $_->{address}->{countryRegion} ne $params->{bing_country}; + $address =~ s/, $params->{bing_country}$//; + } + if ($address !~ /$_->{address}->{locality}/) { + $address .= ", $_->{address}->{locality}"; + } # Getting duplicate, yet different, results from Bing sometimes next if @valid_locations diff --git a/perllib/FixMyStreet/Geocode/OSM.pm b/perllib/FixMyStreet/Geocode/OSM.pm index 20e653cf6..06162d74c 100644 --- a/perllib/FixMyStreet/Geocode/OSM.pm +++ b/perllib/FixMyStreet/Geocode/OSM.pm @@ -45,6 +45,7 @@ sub string { if $params->{bounds}; $query_params{countrycodes} = $params->{country} if $params->{country}; + $c->cobrand->call_hook(geocoder_munge_query_params => \%query_params); $url .= join('&', map { "$_=$query_params{$_}" } sort keys %query_params); $c->stash->{geocoder_url} = $url; diff --git a/perllib/FixMyStreet/ImageMagick.pm b/perllib/FixMyStreet/ImageMagick.pm index d9f643801..ec99fd877 100644 --- a/perllib/FixMyStreet/ImageMagick.pm +++ b/perllib/FixMyStreet/ImageMagick.pm @@ -64,7 +64,7 @@ sub shrink { my ($self, $size) = @_; return $self unless $self->image; my $err = $self->image->Scale(geometry => "$size>"); - throw Error::Simple("resize failed: $err") if "$err"; + die "resize failed: $err" if "$err"; $self->_set_width_and_height(); return $self->strip; } @@ -76,9 +76,9 @@ sub crop { $size //= '90x60'; return $self unless $self->image; my $err = $self->image->Resize( geometry => "$size^" ); - throw Error::Simple("resize failed: $err") if "$err"; + die "resize failed: $err" if "$err"; $err = $self->image->Extent( geometry => $size, gravity => 'Center' ); - throw Error::Simple("resize failed: $err") if "$err"; + die "resize failed: $err" if "$err"; $self->_set_width_and_height(); return $self->strip; } diff --git a/perllib/FixMyStreet/Map/Bing.pm b/perllib/FixMyStreet/Map/Bing.pm index 97a0d229f..17bdc3a53 100644 --- a/perllib/FixMyStreet/Map/Bing.pm +++ b/perllib/FixMyStreet/Map/Bing.pm @@ -29,14 +29,13 @@ sub get_quadkey { return $key; } -sub map_tile_base { - '', "//ecn.%s.tiles.virtualearth.net/tiles/r%s.png?g=6570"; -} +my $road_base = '//%s.ssl.ak.dynamic.tiles.virtualearth.net/comp/ch/%s?mkt=en-US&it=G,L&src=t&shading=hill&og=969&n=z'; +my $aerial_base = '//%s.ssl.ak.dynamic.tiles.virtualearth.net/comp/ch/%s?mkt=en-US&it=A,G,L&src=t&og=969&n=z'; sub map_tiles { my ( $self, %params ) = @_; my ( $x, $y, $z ) = ( $params{x_tile}, $params{y_tile}, $params{zoom_act} ); - my ($tile_sep, $tile_base) = $self->map_tile_base; + my $tile_base = $params{aerial} ? $aerial_base : $road_base; return [ sprintf($tile_base, 't0', $self->get_quadkey($x-1, $y-1, $z)), sprintf($tile_base, 't1', $self->get_quadkey($x, $y-1, $z)), diff --git a/perllib/FixMyStreet/Map/Bromley.pm b/perllib/FixMyStreet/Map/Bromley.pm index 518382fc0..29063778e 100644 --- a/perllib/FixMyStreet/Map/Bromley.pm +++ b/perllib/FixMyStreet/Map/Bromley.pm @@ -9,8 +9,6 @@ use base 'FixMyStreet::Map::FMS'; use strict; -sub map_tile_base { - '-', "//%stilma.mysociety.org/bromley/%d/%d/%d.png"; -} +sub map_tile_base { "bromley" } 1; diff --git a/perllib/FixMyStreet/Map/FMS.pm b/perllib/FixMyStreet/Map/FMS.pm index 126fc34bf..53d911a57 100644 --- a/perllib/FixMyStreet/Map/FMS.pm +++ b/perllib/FixMyStreet/Map/FMS.pm @@ -9,6 +9,8 @@ use base 'FixMyStreet::Map::Bing'; use strict; +use constant ZOOM_LEVELS => 6; + sub map_template { 'fms' } sub map_javascript { [ @@ -18,31 +20,30 @@ sub map_javascript { [ '/js/map-fms.js', ] } -sub map_tile_base { - '-', "//%stilma.mysociety.org/oml/%d/%d/%d.png"; -} +sub map_tile_base { "oml" } sub map_tiles { my ( $self, %params ) = @_; my ( $x, $y, $z ) = ( $params{x_tile}, $params{y_tile}, $params{zoom_act} ); my $ni = in_northern_ireland_box( $params{latitude}, $params{longitude} ); - if (!$ni && $z >= 16) { - my ($tile_sep, $tile_base) = $self->map_tile_base; + if ($params{aerial} || $ni || $z <= 11) { + return $self->SUPER::map_tiles(%params); + } elsif ($z >= 16) { + my $tile_base = '//%stilma.mysociety.org/' . $self->map_tile_base . '/%d/%d/%d.png'; return [ - sprintf($tile_base, 'a' . $tile_sep, $z, $x-1, $y-1), - sprintf($tile_base, 'b' . $tile_sep, $z, $x, $y-1), - sprintf($tile_base, 'c' . $tile_sep, $z, $x-1, $y), + sprintf($tile_base, 'a-', $z, $x-1, $y-1), + sprintf($tile_base, 'b-', $z, $x, $y-1), + sprintf($tile_base, 'c-', $z, $x-1, $y), sprintf($tile_base, '', $z, $x, $y), ]; - } else { + } elsif ($z > 11) { my $key = FixMyStreet->config('BING_MAPS_API_KEY'); - my $url = "g=6570"; - $url .= "&productSet=mmOS&key=$key" if $z > 11 && !$ni; + my $base = "//ecn.%s.tiles.virtualearth.net/tiles/r%s?g=8702&lbl=l1&productSet=mmOS&key=$key"; return [ - "//ecn.t0.tiles.virtualearth.net/tiles/r" . $self->get_quadkey($x-1, $y-1, $z) . ".png?$url", - "//ecn.t1.tiles.virtualearth.net/tiles/r" . $self->get_quadkey($x, $y-1, $z) . ".png?$url", - "//ecn.t2.tiles.virtualearth.net/tiles/r" . $self->get_quadkey($x-1, $y, $z) . ".png?$url", - "//ecn.t3.tiles.virtualearth.net/tiles/r" . $self->get_quadkey($x, $y, $z) . ".png?$url", + sprintf($base, "t0", $self->get_quadkey($x-1, $y-1, $z)), + sprintf($base, "t1", $self->get_quadkey($x, $y-1, $z)), + sprintf($base, "t2", $self->get_quadkey($x-1, $y, $z)), + sprintf($base, "t3", $self->get_quadkey($x, $y, $z)), ]; } } diff --git a/perllib/FixMyStreet/Map/Google.pm b/perllib/FixMyStreet/Map/Google.pm index c1fb05e43..dfebef5a3 100644 --- a/perllib/FixMyStreet/Map/Google.pm +++ b/perllib/FixMyStreet/Map/Google.pm @@ -10,8 +10,9 @@ use strict; use FixMyStreet::Gaze; use Utils; -use constant ZOOM_LEVELS => 6; +use constant ZOOM_LEVELS => 7; use constant MIN_ZOOM_LEVEL => 13; +use constant DEFAULT_ZOOM => 3; sub map_javascript { [ "http://maps.googleapis.com/maps/api/js?sensor=false", @@ -28,16 +29,16 @@ sub display_map { my $numZoomLevels = ZOOM_LEVELS; my $zoomOffset = MIN_ZOOM_LEVEL; - if ($params{any_zoom}) { - $numZoomLevels = 19; - $zoomOffset = 0; - } # Adjust zoom level dependent upon population density - my $dist = $c->stash->{distance} - || FixMyStreet::Gaze::get_radius_containing_population( $params{latitude}, $params{longitude} ); - my $default_zoom = $c->cobrand->default_map_zoom() ? $c->cobrand->default_map_zoom() : $numZoomLevels - 4; - $default_zoom = $numZoomLevels - 3 if $dist < 10; + my $default_zoom; + if (my $cobrand_default_zoom = $c->cobrand->default_map_zoom) { + $default_zoom = $cobrand_default_zoom; + } else { + my $dist = $c->stash->{distance} + || FixMyStreet::Gaze::get_radius_containing_population( $params{latitude}, $params{longitude} ); + $default_zoom = $dist < 10 ? $self->DEFAULT_ZOOM : $self->DEFAULT_ZOOM - 1; + } # Map centre may be overridden in the query string $params{latitude} = Utils::truncate_coordinate($c->get_param('lat') + 0) @@ -46,6 +47,12 @@ sub display_map { if defined $c->get_param('lon'); $params{zoomToBounds} = $params{any_zoom} && !defined $c->get_param('zoom'); + if ($params{any_zoom}) { + $numZoomLevels += $zoomOffset; + $default_zoom += $zoomOffset; + $zoomOffset = 0; + } + my $zoom = defined $c->get_param('zoom') ? $c->get_param('zoom') + 0 : $default_zoom; $zoom = $numZoomLevels - 1 if $zoom >= $numZoomLevels; $zoom = 0 if $zoom < 0; diff --git a/perllib/FixMyStreet/Map/MasterMap.pm b/perllib/FixMyStreet/Map/MasterMap.pm index d66234bbf..5edbb28fb 100644 --- a/perllib/FixMyStreet/Map/MasterMap.pm +++ b/perllib/FixMyStreet/Map/MasterMap.pm @@ -8,6 +8,7 @@ use base 'FixMyStreet::Map::FMS'; use strict; use constant ZOOM_LEVELS => 7; +use constant DEFAULT_ZOOM => 4; sub map_template { 'fms' } diff --git a/perllib/FixMyStreet/Map/OSM.pm b/perllib/FixMyStreet/Map/OSM.pm index ef465d7dc..082605568 100644 --- a/perllib/FixMyStreet/Map/OSM.pm +++ b/perllib/FixMyStreet/Map/OSM.pm @@ -11,8 +11,9 @@ use Math::Trig; use FixMyStreet::Gaze; use Utils; -use constant ZOOM_LEVELS => 6; +use constant ZOOM_LEVELS => 7; use constant MIN_ZOOM_LEVEL => 13; +use constant DEFAULT_ZOOM => 3; sub map_type { 'OpenLayers.Layer.OSM.Mapnik' } @@ -21,11 +22,13 @@ sub map_template { 'osm' } sub map_javascript { [ '/vendor/OpenLayers/OpenLayers.wfs.js', '/js/map-OpenLayers.js', + FixMyStreet->config('BING_MAPS_API_KEY') ? ('/js/map-bing-ol.js') : (), '/js/map-OpenStreetMap.js', ] } sub map_tiles { my ( $self, %params ) = @_; + return FixMyStreet::Map::Bing->map_tiles(%params) if $params{aerial}; my ( $x, $y, $z ) = ( $params{x_tile}, $params{y_tile}, $params{zoom_act} ); my $tile_url = $self->base_tile_url(); return [ @@ -59,6 +62,8 @@ sub display_map { if defined $c->get_param('lon'); $params{zoomToBounds} = $params{any_zoom} && !defined $c->get_param('zoom'); + $params{aerial} = $c->get_param("aerial") && FixMyStreet->config('BING_MAPS_API_KEY') ? 1 : 0; + my %data; $data{cobrand} = $c->cobrand; $data{distance} = $c->stash->{distance}; @@ -72,17 +77,24 @@ sub generate_map_data { my $numZoomLevels = $self->ZOOM_LEVELS; my $zoomOffset = $self->MIN_ZOOM_LEVEL; + + # Adjust zoom level dependent upon population density if cobrand hasn't + # specified a default zoom. + my $default_zoom; + if (my $cobrand_default_zoom = $data->{cobrand}->default_map_zoom) { + $default_zoom = $cobrand_default_zoom; + } else { + my $dist = $data->{distance} + || FixMyStreet::Gaze::get_radius_containing_population( $params{latitude}, $params{longitude} ); + $default_zoom = $dist < 10 ? $self->DEFAULT_ZOOM : $self->DEFAULT_ZOOM - 1; + } + if ($params{any_zoom}) { - $numZoomLevels = 19; + $numZoomLevels += $zoomOffset; + $default_zoom += $zoomOffset; $zoomOffset = 0; } - # Adjust zoom level dependent upon population density - my $dist = $data->{distance} - || FixMyStreet::Gaze::get_radius_containing_population( $params{latitude}, $params{longitude} ); - my $default_zoom = $data->{cobrand}->default_map_zoom() || ($numZoomLevels - 4); - $default_zoom = $numZoomLevels - 3 if $dist < 10; - my $zoom = $data->{zoom} || $default_zoom; $zoom = $numZoomLevels - 1 if $zoom >= $numZoomLevels; $zoom = 0 if $zoom < 0; diff --git a/perllib/FixMyStreet/Map/OSM/MapQuest.pm b/perllib/FixMyStreet/Map/OSM/MapQuest.pm deleted file mode 100644 index 8b24e1ba2..000000000 --- a/perllib/FixMyStreet/Map/OSM/MapQuest.pm +++ /dev/null @@ -1,34 +0,0 @@ -# FixMyStreet:Map::OSM::CycleMap -# OSM CycleMap maps on FixMyStreet. -# -# Copyright (c) 2010 UK Citizens Online Democracy. All rights reserved. -# Email: matthew@mysociety.org; WWW: http://www.mysociety.org/ - -package FixMyStreet::Map::OSM::MapQuest; -use base 'FixMyStreet::Map::OSM'; - -use strict; - -sub map_type { 'OpenLayers.Layer.OSM.MapQuestOpen' } - -sub map_tiles { - my ( $self, %params ) = @_; - my ( $x, $y, $z ) = ( $params{x_tile}, $params{y_tile}, $params{zoom_act} ); - my $tile_url = $self->base_tile_url(); - return [ - "https://otile1-s.$tile_url/$z/" . ($x - 1) . "/" . ($y - 1) . ".png", - "https://otile2-s.$tile_url/$z/$x/" . ($y - 1) . ".png", - "https://otile3-s.$tile_url/$z/" . ($x - 1) . "/$y.png", - "https://otile4-s.$tile_url/$z/$x/$y.png", - ]; -} - -sub base_tile_url { - return 'mqcdn.com/tiles/1.0.0/map/'; -} - -sub copyright { - 'Data, imagery and map information provided by <a href="https://www.mapquest.com/">MapQuest</a> <img src="https://developer.mapquest.com/sites/default/files/mapquest/osm/mq_logo.png" />, <a href="https://openstreetmap.org/">OpenStreetMap</a> and contributors, <a href="https://opendatacommons.org/licenses/odbl/">ODbL</a>' -} - -1; diff --git a/perllib/FixMyStreet/Map/OSM/StreetView.pm b/perllib/FixMyStreet/Map/OSM/StreetView.pm index 3281faa35..820a3b87f 100644 --- a/perllib/FixMyStreet/Map/OSM/StreetView.pm +++ b/perllib/FixMyStreet/Map/OSM/StreetView.pm @@ -9,6 +9,8 @@ use base 'FixMyStreet::Map::OSM'; use strict; +use constant ZOOM_LEVELS => 6; + sub map_type { '' } sub map_javascript { [ diff --git a/perllib/FixMyStreet/Queue/Item/Report.pm b/perllib/FixMyStreet/Queue/Item/Report.pm index e38987838..070b244bb 100644 --- a/perllib/FixMyStreet/Queue/Item/Report.pm +++ b/perllib/FixMyStreet/Queue/Item/Report.pm @@ -172,7 +172,7 @@ sub _create_reporters { my @dear; my %reporters = (); while (my $body = $bodies->next) { - my $sender_info = $self->cobrand->get_body_sender( $body, $row->category ); + my $sender_info = $self->cobrand_handler->get_body_sender( $body, $row ); my $sender = "FixMyStreet::SendReport::" . $sender_info->{method}; if ( ! exists $self->senders->{ $sender } ) { @@ -243,7 +243,7 @@ sub _send { sub _post_send { my ($self, $result) = @_; - my $send_confirmation_email = $self->cobrand_handler->report_sent_confirmation_email; + my $send_confirmation_email = $self->cobrand_handler->report_sent_confirmation_email($self->report); unless ($result) { $self->report->update( { whensent => \'current_timestamp', diff --git a/perllib/FixMyStreet/Reporting.pm b/perllib/FixMyStreet/Reporting.pm new file mode 100644 index 000000000..efd12718c --- /dev/null +++ b/perllib/FixMyStreet/Reporting.pm @@ -0,0 +1,388 @@ +package FixMyStreet::Reporting; + +use DateTime; +use Moo; +use Path::Tiny; +use Text::CSV; +use Types::Standard qw(ArrayRef CodeRef Enum HashRef InstanceOf Int Maybe Str); +use FixMyStreet::DB; + +# What are we reporting on + +has type => ( is => 'ro', isa => Enum['problems','updates'] ); +has on_problems => ( is => 'lazy', default => sub { $_[0]->type eq 'problems' } ); +has on_updates => ( is => 'lazy', default => sub { $_[0]->type eq 'updates' } ); + +# Filters to restrict the reporting to + +has body => ( is => 'ro', isa => Maybe[InstanceOf['FixMyStreet::DB::Result::Body']] ); +has wards => ( is => 'ro', isa => ArrayRef[Int], default => sub { [] } ); +has category => ( is => 'ro', isa => Maybe[Str] ); +has state => ( is => 'ro', isa => Maybe[Str] ); +has start_date => ( is => 'ro', + isa => Str, + default => sub { + my $days30 = DateTime->now(time_zone => FixMyStreet->time_zone || FixMyStreet->local_time_zone)->subtract(days => 30); + $days30->truncate( to => 'day' ); + $days30->strftime('%Y-%m-%d'); + } +); +has end_date => ( is => 'ro', isa => Maybe[Str] ); + +# Things needed for cobrand specific extra data or checks + +has cobrand => ( is => 'ro', default => sub { FixMyStreet::DB->schema->cobrand } ); # Which cobrand is asking, to get the right data / hooks / base URL +has user => ( is => 'ro', isa => Maybe[InstanceOf['FixMyStreet::DB::Result::User']] ); + +# Things created in the process, that can be manually overridden + +has objects_rs => ( is => 'rwp' ); # ResultSet of rows + +sub objects_attrs { + my ($self, $attrs) = @_; + my $rs = $self->objects_rs->search(undef, $attrs); + $self->_set_objects_rs($rs); + return $rs; +} + +# CSV header strings and column keys (looked up in the row's as_hashref, plus +# the following: user_name_display, acknowledged, fixed, closed, wards, +# local_coords_x, local_coords_y, url, subcategory, site_used, reported_as) +has csv_headers => ( is => 'rwp', isa => ArrayRef[Str], default => sub { [] } ); +has csv_columns => ( is => 'rwp', isa => ArrayRef[Str], default => sub { [] } ); + +sub modify_csv_header { + my ($self, %mapping) = @_; + $self->_set_csv_headers([ + map { $mapping{$_} || $_ } @{ $self->csv_headers }, + ]); +} + +sub splice_csv_column { + my ($self, $before, $column, $header) = @_; + + for (my $i = 0; $i < @{$self->csv_columns}; $i++) { + my $col = $self->csv_columns->[$i]; + if ($col eq $before) { + splice @{$self->csv_columns}, $i, 0, $column; + splice @{$self->csv_headers}, $i, 0, $header; + last; + } + } +} + +sub add_csv_columns { + my $self = shift; + for (my $i = 0; $i < @_; $i += 2) { + my $column = $_[$i]; + my $header = $_[$i+1]; + push @{$self->csv_columns}, $column; + push @{$self->csv_headers}, $header; + } +} + +# A function that is passed the report and returns a hashref of extra data to +# include that can be used by 'columns' +has csv_extra_data => ( is => 'rw', isa => CodeRef ); + +has filename => ( is => 'rw', isa => Str, lazy => 1, default => sub { + my $self = shift; + my %where = ( + category => $self->category, + state => $self->state, + ward => join(',', @{$self->wards}), + start_date => $self->start_date, + end_date => $self->end_date, + ); + $where{body} = $self->body->id if $self->body; + my $host = URI->new($self->cobrand->base_url)->host; + join '-', + $host, + $self->on_updates ? ('updates') : (), + map { + my $value = $where{$_}; + (my $nosp = $value || '') =~ s/ /-/g; + (defined $value and length $value) ? ($_, $nosp) : () + } sort keys %where +}); + +# Generation code + +sub construct_rs_filter { + my $self = shift; + + my $table_name = $self->on_updates ? 'problem' : 'me'; + + my %where; + $where{areas} = [ map { { 'like', "%,$_,%" } } @{$self->wards} ] + if @{$self->wards}; + $where{"$table_name.category"} = $self->category + if $self->category; + + if ( $self->state && FixMyStreet::DB::Result::Problem->fixed_states->{$self->state} ) { # Probably fixed - council + $where{"$table_name.state"} = [ FixMyStreet::DB::Result::Problem->fixed_states() ]; + } elsif ( $self->state ) { + $where{"$table_name.state"} = $self->state; + } else { + $where{"$table_name.state"} = [ FixMyStreet::DB::Result::Problem->visible_states() ]; + } + + my $range = FixMyStreet::DateRange->new( + start_date => $self->start_date, + end_date => $self->end_date, + formatter => FixMyStreet::DB->schema->storage->datetime_parser, + ); + $where{"$table_name.confirmed"} = $range->sql; + + my $rs = $self->on_updates ? $self->cobrand->updates : $self->cobrand->problems; + my $objects_rs = $rs->to_body($self->body)->search( \%where ); + $self->_set_objects_rs($objects_rs); + return { + params => \%where, + objects_rs => $objects_rs, + } +} + +sub csv_parameters { + my $self = shift; + if ($self->on_updates) { + $self->_csv_parameters_updates; + } else { + $self->_csv_parameters_problems; + } +} + +sub _csv_parameters_updates { + my $self = shift; + + $self->objects_attrs({ + join => 'problem', + order_by => ['me.confirmed', 'me.id'], + '+columns' => ['problem.bodies_str'], + cursor_page_size => 1000, + }); + $self->_set_csv_headers([ + 'Report ID', 'Update ID', 'Date', 'Status', 'Problem state', + 'Text', 'User Name', 'Reported As', + ]); + $self->_set_csv_columns([ + 'problem_id', 'id', 'confirmed', 'state', 'problem_state', + 'text', 'user_name_display', 'reported_as', + ]); + $self->cobrand->call_hook(dashboard_export_updates_add_columns => $self); +} + +sub _csv_parameters_problems { + my $self = shift; + + my $groups = $self->cobrand->enable_category_groups ? 1 : 0; + my $join = ['comments']; + my $columns = ['comments.id', 'comments.problem_state', 'comments.state', 'comments.confirmed', 'comments.mark_fixed']; + if ($groups) { + push @$join, 'contact'; + push @$columns, 'contact.id', 'contact.extra'; + } + $self->objects_attrs({ + join => $join, + collapse => 1, + '+columns' => $columns, + order_by => ['me.confirmed', 'me.id'], + cursor_page_size => 1000, + }); + $self->_set_csv_headers([ + 'Report ID', + 'Title', + 'Detail', + 'User Name', + 'Category', + $groups ? ('Subcategory') : (), + 'Created', + 'Confirmed', + 'Acknowledged', + 'Fixed', + 'Closed', + 'Status', + 'Latitude', 'Longitude', + 'Query', + 'Ward', + 'Easting', + 'Northing', + 'Report URL', + 'Site Used', + 'Reported As', + ]); + $self->_set_csv_columns([ + 'id', + 'title', + 'detail', + 'user_name_display', + 'category', + $groups ? ('subcategory') : (), + 'created', + 'confirmed', + 'acknowledged', + 'fixed', + 'closed', + 'state', + 'latitude', 'longitude', + 'postcode', + 'wards', + 'local_coords_x', + 'local_coords_y', + 'url', + 'site_used', + 'reported_as', + ]); + $self->cobrand->call_hook(dashboard_export_problems_add_columns => $self); +} + +=head2 generate_csv + +Generates a CSV output to a file handler provided + +=cut + +sub generate_csv { + my ($self, $handle) = @_; + + my $csv = Text::CSV->new({ binary => 1, eol => "\n" }); + $csv->print($handle, $self->csv_headers); + + my $fixed_states = FixMyStreet::DB::Result::Problem->fixed_states; + my $closed_states = FixMyStreet::DB::Result::Problem->closed_states; + + my %asked_for = map { $_ => 1 } @{$self->csv_columns}; + + my $children = $self->body ? $self->body->first_area_children : {}; + + my $objects = $self->objects_rs; + while ( my $obj = $objects->next ) { + my $hashref = $obj->as_hashref(\%asked_for); + + $hashref->{user_name_display} = $obj->anonymous + ? '(anonymous)' : $obj->name; + + if ($asked_for{acknowledged}) { + for my $comment ($obj->comments) { + my $problem_state = $comment->problem_state or next; + next unless $comment->state eq 'confirmed'; + next if $problem_state eq 'confirmed'; + $hashref->{acknowledged} //= $comment->confirmed; + $hashref->{action_scheduled} //= $problem_state eq 'action scheduled' ? $comment->confirmed : undef; + $hashref->{fixed} //= $fixed_states->{ $problem_state } || $comment->mark_fixed ? + $comment->confirmed : undef; + if ($closed_states->{ $problem_state }) { + $hashref->{closed} = $comment->confirmed; + last; + } + } + } + + if ($asked_for{wards}) { + $hashref->{wards} = join ', ', + map { $children->{$_}->{name} } + grep { $children->{$_} } + split ',', $hashref->{areas}; + } + + if ($obj->can('local_coords') && $asked_for{local_coords_x}) { + ($hashref->{local_coords_x}, $hashref->{local_coords_y}) = + $obj->local_coords; + } + + if ($asked_for{subcategory}) { + my $group = $obj->contact ? $obj->contact->groups : []; + $group = join(',', @$group); + if ($group) { + $hashref->{subcategory} = $obj->category; + $hashref->{category} = $group; + } + } + + my $base = $self->cobrand->base_url_for_report($obj->can('problem') ? $obj->problem : $obj); + $hashref->{url} = join '', $base, $obj->url; + + $hashref->{site_used} = $obj->can('service') ? ($obj->service || $obj->cobrand) : $obj->cobrand; + + $hashref->{reported_as} = $obj->get_extra_metadata('contributed_as') || ''; + + if (my $fn = $self->csv_extra_data) { + my $extra = $fn->($obj); + $hashref = { %$hashref, %$extra }; + } + + $csv->print($handle, [ + @{$hashref}{ + @{$self->csv_columns} + }, + ] ); + } +} + +# Output code + +sub cache_dir { + my $self = shift; + + my $cfg = FixMyStreet->config('PHOTO_STORAGE_OPTIONS'); + my $dir = $cfg ? $cfg->{UPLOAD_DIR} : FixMyStreet->config('UPLOAD_DIR'); + $dir = path($dir, "dashboard_csv")->absolute(FixMyStreet->path_to()); + my $subdir = $self->user ? $self->user->id : 0; + $dir = $dir->child($subdir); + $dir->mkpath; + $dir; +} + +sub kick_off_process { + my $self = shift; + + my $out = path($self->cache_dir, $self->filename . '.csv'); + my $file = path($out . '-part'); + return if $file->exists; + $file->touch; # So status page shows it even if process takes short while to spin up + + my $cmd = FixMyStreet->path_to('bin/csv-export'); + $cmd .= ' --cobrand ' . $self->cobrand->moniker; + $cmd .= " --out \Q$out\E"; + foreach (qw(type category state start_date end_date)) { + $cmd .= " --$_ " . quotemeta($self->$_) if $self->$_; + } + foreach (qw(body user)) { + $cmd .= " --$_ " . $self->$_->id if $self->$_; + } + $cmd .= " --wards " . join(',', map { quotemeta } @{$self->wards}) if @{$self->wards}; + $cmd .= ' &' unless FixMyStreet->test_mode; + + system($cmd); +} + +# Outputs relevant CSV HTTP headers, and then streams the CSV +sub generate_csv_http { + my ($self, $c) = @_; + $self->http_setup($c); + $self->generate_csv($c->response); +} + +sub http_setup { + my ($self, $c) = @_; + my $filename = $self->filename; + + $c->res->content_type('text/csv; charset=utf-8'); + $c->res->header('content-disposition' => "attachment; filename=\"${filename}.csv\""); + + # Emit a header (copying Drupal's naming) telling an intermediary (e.g. + # Varnish) not to buffer the output. Varnish will need to know this, e.g.: + # if (beresp.http.Surrogate-Control ~ "BigPipe/1.0") { + # set beresp.do_stream = true; + # set beresp.ttl = 0s; + # } + $c->res->header('Surrogate-Control' => 'content="BigPipe/1.0"'); + + # Tell nginx not to buffer this response + $c->res->header('X-Accel-Buffering' => 'no'); + + # Define an empty body so the web view doesn't get added at the end + $c->res->body(""); +} + +1; diff --git a/perllib/FixMyStreet/Roles/ConfirmOpen311.pm b/perllib/FixMyStreet/Roles/ConfirmOpen311.pm index 0845105f1..1663844a2 100644 --- a/perllib/FixMyStreet/Roles/ConfirmOpen311.pm +++ b/perllib/FixMyStreet/Roles/ConfirmOpen311.pm @@ -13,8 +13,8 @@ sub open311_config { $params->{multi_photos} = 1; } -sub open311_extra_data { - my ($self, $row, $h, $extra) = @_; +sub open311_extra_data_include { + my ($self, $row, $h) = @_; my $open311_only = [ { name => 'report_url', @@ -31,9 +31,7 @@ sub open311_extra_data { # service at the point we're sending the report over Open311. if (!$row->get_extra_field_value('site_code')) { if (my $site_code = $self->lookup_site_code($row)) { - push @$extra, - { name => 'site_code', - value => $site_code }; + $row->update_extra_field({ name => 'site_code', value => $site_code }); } } diff --git a/perllib/FixMyStreet/Roles/ContactExtra.pm b/perllib/FixMyStreet/Roles/ContactExtra.pm index e78d9b53f..9615b0f0c 100644 --- a/perllib/FixMyStreet/Roles/ContactExtra.pm +++ b/perllib/FixMyStreet/Roles/ContactExtra.pm @@ -45,8 +45,8 @@ sub by_categories { $_->$join_table == 0 # There's no category at all on this defect type/template/priority || (grep { $_->contact_id == $contact->get_column('id') } $_->$join_table) } @results; - @ts = $rs->map_extras(@ts); - $extras{$contact->category} = encode_json(\@ts); + @ts = $rs->map_extras(\%params, @ts); + $extras{$contact->category} = JSON::XS->new->encode(\@ts); } return \%extras; diff --git a/perllib/FixMyStreet/Roles/FullTextSearch.pm b/perllib/FixMyStreet/Roles/FullTextSearch.pm new file mode 100644 index 000000000..871b1d185 --- /dev/null +++ b/perllib/FixMyStreet/Roles/FullTextSearch.pm @@ -0,0 +1,29 @@ +package FixMyStreet::Roles::FullTextSearch; + +use Moo::Role; +use FixMyStreet; + +requires 'text_search_columns'; +requires 'text_search_nulls'; +requires 'text_search_translate'; + +sub search_text { + my ($rs, $query) = @_; + my %nulls = map { $_ => 1 } $rs->text_search_nulls; + my @cols = map { + my $col = $rs->me($_); + $nulls{$_} ? "coalesce($col, '')" : $col; + } $rs->text_search_columns; + my $vector = join(" || ' ' || ", @cols); + my $bind = '?'; + if (my $trans = $rs->text_search_translate) { + my $replace = ' ' x length $trans; + $vector = "translate($vector, '$trans', '$replace')"; + $bind = "translate(?, '$trans', '$replace')"; + } + my $config = FixMyStreet->config('DB_FULL_TEXT_SEARCH_CONFIG') || 'english'; + $rs->search(\[ "to_tsvector('$config', $vector) @@ plainto_tsquery('$config', $bind)", $query ]); +} + +1; + diff --git a/perllib/FixMyStreet/Script/Alerts.pm b/perllib/FixMyStreet/Script/Alerts.pm index cb1f022fa..fa90ede48 100644 --- a/perllib/FixMyStreet/Script/Alerts.pm +++ b/perllib/FixMyStreet/Script/Alerts.pm @@ -41,6 +41,7 @@ sub send() { $item_table.photo as item_photo, $item_table.problem_state as item_problem_state, $item_table.cobrand as item_cobrand, + $item_table.extra as item_extra, $head_table.* from alert, $item_table, $head_table where alert.parameter::integer = $head_table.id @@ -307,6 +308,10 @@ sub _send_aggregated_alert_email(%) { # Ignore phone-only users return unless $data{alert_user}->email_verified; + # Mark user as active as they're being sent an alert + $data{alert_user}->set_last_active; + $data{alert_user}->update; + my $email = $data{alert_user}->email; my ($domain) = $email =~ m{ @ (.*) \z }x; return if $data{schema}->resultset('Abuse')->search( { @@ -323,7 +328,7 @@ sub _send_aggregated_alert_email(%) { } ); $data{unsubscribe_url} = $cobrand->base_url( $data{cobrand_data} ) . '/A/' . $token->token; - my $sender = FixMyStreet::Email::unique_verp_id('alert', $data{alert_id}); + my $sender = FixMyStreet::Email::unique_verp_id([ 'alert', $data{alert_id} ], $cobrand->call_hook('verp_email_domain')); my $result = FixMyStreet::Email::send_cron( $data{schema}, "$data{template}.txt", diff --git a/perllib/FixMyStreet/Script/ArchiveOldEnquiries.pm b/perllib/FixMyStreet/Script/ArchiveOldEnquiries.pm index 7ba763515..7c183ecbc 100644 --- a/perllib/FixMyStreet/Script/ArchiveOldEnquiries.pm +++ b/perllib/FixMyStreet/Script/ArchiveOldEnquiries.pm @@ -141,7 +141,6 @@ sub close_problems { my $problems = shift; my $extra = { auto_closed_by_script => 1 }; - $extra->{is_superuser} = 1 if !$opts->{user_name}; my $cobrand; while (my $problem = $problems->next) { @@ -152,16 +151,9 @@ sub close_problems { $cobrand->set_lang_and_domain($problem->lang, 1); } - my $timestamp = \'current_timestamp'; my $comment = $problem->add_to_comments( { text => $opts->{closure_text} || '', - created => $timestamp, - confirmed => $timestamp, - user_id => $opts->{user}, - name => $opts->{user_name} || _('an administrator'), - mark_fixed => 0, - anonymous => 0, - state => 'confirmed', + user => FixMyStreet::DB->resultset("User")->find($opts->{user}), problem_state => $opts->{closed_state}, extra => $extra, } ); diff --git a/perllib/FixMyStreet/Script/Inactive.pm b/perllib/FixMyStreet/Script/Inactive.pm index 8dd524ce1..6b3372a2b 100644 --- a/perllib/FixMyStreet/Script/Inactive.pm +++ b/perllib/FixMyStreet/Script/Inactive.pm @@ -158,8 +158,14 @@ sub delete_reports { sub anonymize_users { my $self = shift; + my $body_users = FixMyStreet::DB->resultset("Body")->search({ + comment_user_id => { '!=' => undef }, + }, { + columns => 'comment_user_id', + }); my $users = FixMyStreet::DB->resultset("User")->search({ last_active => { '<', interval($self->anonymize) }, + id => { -not_in => $body_users->as_query }, email => { -not_like => 'removed-%@' . FixMyStreet->config('EMAIL_DOMAIN') }, }); @@ -179,6 +185,7 @@ sub email_inactive_users { }); while (my $user = $users->next) { next if $user->get_extra_metadata('inactive_email_sent'); + next unless $user->email && $user->email_verified; say "Emailing user #" . $user->id if $self->verbose; next if $self->dry_run; diff --git a/perllib/FixMyStreet/Script/TfL/AutoClose.pm b/perllib/FixMyStreet/Script/TfL/AutoClose.pm new file mode 100644 index 000000000..687a29e7f --- /dev/null +++ b/perllib/FixMyStreet/Script/TfL/AutoClose.pm @@ -0,0 +1,129 @@ +package FixMyStreet::Script::TfL::AutoClose; + +use v5.14; + +use Moo; +use CronFns; +use FixMyStreet; +use FixMyStreet::Cobrand; +use FixMyStreet::DB; +use Types::Standard qw(InstanceOf Maybe); + +has commit => ( is => 'ro', default => 0 ); + +has verbose => ( is => 'ro', default => 0 ); + +has body => ( + is => 'lazy', + isa => Maybe[InstanceOf['FixMyStreet::DB::Result::Body']], + default => sub { + my $body = FixMyStreet::DB->resultset('Body')->find({ name => 'TfL' }); + return $body; + } +); + +has days => ( + is => 'ro', + default => 28 +); + +sub close { + my $self = shift; + + die "Can't find body\n" unless $self->body; + warn "DRY RUN: use --commit to close reports\n" unless $self->commit; + my $categories = $self->categories; + $self->close_reports($categories); +} + +has newest => ( + is => 'lazy', + isa => InstanceOf['DateTime'], + default => sub { + my $self = shift; + my $days = $self->days * -1; + my $date = DateTime->now->add( days => $days )->truncate( to => 'day' ); + return $date; + } +); + +# get list of cateories that have a response template for the fixed +# state marked as auto-response. +sub categories { + my $self = shift; + + my $templates = FixMyStreet::DB->resultset('ResponseTemplate')->search({ + state => 'fixed - council', + auto_response => 1, + body_id => $self->body->id, + }); + + my %categories; + for my $template ( $templates->all ) { + map { $categories{$_->category} = $template; } $template->contacts->all; + } + + return \%categories; +} + +# find reports in relevant categories that have been set to action +# scheduled for 30 days. +sub close_reports { + my ($self, $categories) = @_; + + my $dtf = FixMyStreet::DB->schema->storage->datetime_parser; + + my $reports = FixMyStreet::DB->resultset('Problem')->search({ + category => { -in => [ keys %$categories ] }, + 'me.state' => 'action scheduled', + bodies_str => $self->body->id, + 'comments.state' => 'confirmed', + 'comments.problem_state' => 'action scheduled', + }, + { + group_by => 'me.id', + join => [ 'comments' ], + having => \[ 'MIN(comments.confirmed) < ?', $dtf->format_datetime($self->newest) ] + }); + + my $count = 0; + for my $r ( $reports->all ) { + my $comments = FixMyStreet::DB->resultset('Comment')->search( + { problem_id => $r->id }, + { order_by => 'confirmed' } + ); + my $earliest; + while ( my $c = $comments->next ) { + if ( $c->problem_state ne 'action scheduled' ) { + $earliest = undef; + next; + } + $earliest = $c->confirmed unless defined $earliest; + } + next unless defined $earliest && $earliest < $self->newest; + if ($self->commit) { + $r->update({ + state => 'fixed - council', + lastupdate => \'current_timestamp', + }); + my $c = FixMyStreet::DB->resultset('Comment')->new( + { + problem => $r, + text => $categories->{$r->category}->text, + state => 'confirmed', + problem_state => 'fixed - council', + user => $self->body->comment_user, + confirmed => \'current_timestamp' + } + ); + $c->insert; + } + $count++; + } + + say "$count reports closed" if $self->verbose; + + return 1; +} + +1; diff --git a/perllib/FixMyStreet/SendReport/Email.pm b/perllib/FixMyStreet/SendReport/Email.pm index 72cd42952..2d5e85f3e 100644 --- a/perllib/FixMyStreet/SendReport/Email.pm +++ b/perllib/FixMyStreet/SendReport/Email.pm @@ -16,9 +16,6 @@ sub build_recipient_list { my ($body_email, $state, $note) = ( $contact->email, $contact->state, $contact->note ); - $body_email = swandt_contact($row->latitude, $row->longitude) - if $body->name eq 'Somerset West and Taunton Council' && $body_email eq 'SPECIAL'; - unless ($state eq 'confirmed') { $all_confirmed = 0; $note = 'Body ' . $row->bodies_str . ' deleted' @@ -56,10 +53,11 @@ sub send_from { sub envelope_sender { my ($self, $row) = @_; + my $cobrand = $row->get_cobrand_logged; if ($row->user->email && $row->user->email_verified) { - return FixMyStreet::Email::unique_verp_id('report', $row->id); + return FixMyStreet::Email::unique_verp_id([ 'report', $row->id ], $cobrand->call_hook('verp_email_domain')); } - return $row->get_cobrand_logged->do_not_reply_email; + return $cobrand->do_not_reply_email; } sub send { @@ -128,21 +126,4 @@ sub email_list { return \@list; } -# SW&T has different contact addresses depending upon the old district -sub swandt_contact { - my $district = _get_district_for_contact(@_); - my $email; - $email = ['customerservices', 'westsomerset'] if $district == 2427; - $email = ['enquiries', 'tauntondeane'] if $district == 2429; - return join('@', $email->[0], $email->[1] . '.gov.uk'); -} - -sub _get_district_for_contact { - my ( $lat, $lon ) = @_; - my $district = - FixMyStreet::MapIt::call( 'point', "4326/$lon,$lat", type => 'DIS', generation => 34 ); - ($district) = keys %$district; - return $district; -} - 1; diff --git a/perllib/FixMyStreet/SendReport/Email/Highways.pm b/perllib/FixMyStreet/SendReport/Email/Highways.pm index 2bcd120d3..3ace07b6a 100644 --- a/perllib/FixMyStreet/SendReport/Email/Highways.pm +++ b/perllib/FixMyStreet/SendReport/Email/Highways.pm @@ -12,11 +12,14 @@ sub build_recipient_list { my $contact = $self->fetch_category($body, $row) or return; my $email = $contact->email; my $area_name = $row->get_extra_field_value('area_name') || ''; - if ($area_name eq 'Area 7') { - my $a7email = FixMyStreet->config('COBRAND_FEATURES') || {}; - $a7email = $a7email->{open311_email}->{highwaysengland}->{area_seven}; - $email = $a7email if $a7email; - } + + # config is read-only, so must step through one-by-one to prevent + # vivification + my $area_email = FixMyStreet->config('COBRAND_FEATURES') || {}; + $area_email = $area_email->{open311_email} || {}; + $area_email = $area_email->{highwaysengland} || {}; + $area_email = $area_email->{$area_name}; + $email = $area_email if $area_email; @{$self->to} = map { [ $_, $body->name ] } split /,/, $email; return 1; diff --git a/perllib/FixMyStreet/SendReport/Open311.pm b/perllib/FixMyStreet/SendReport/Open311.pm index e8e840ef5..e51bd76c9 100644 --- a/perllib/FixMyStreet/SendReport/Open311.pm +++ b/perllib/FixMyStreet/SendReport/Open311.pm @@ -41,13 +41,13 @@ sub send { # Try and fill in some ones that we've been asked for, but not asked the user for my $extra = $row->get_extra_fields(); - my ($include, $exclude) = $cobrand->call_hook(open311_extra_data => $row, $h, $extra, $contact); + my ($include, $exclude) = $cobrand->call_hook(open311_extra_data => $row, $h, $contact); my $original_extra = [ @$extra ]; push @$extra, @$include if $include; if ($exclude) { $exclude = join('|', @$exclude); - @$extra = grep { $_->{name} !~ /$exclude/ } @$extra; + @$extra = grep { $_->{name} !~ /$exclude/i } @$extra; } my $id_field = $contact->id_field; diff --git a/perllib/FixMyStreet/Template.pm b/perllib/FixMyStreet/Template.pm index 6317f7552..275089a35 100644 --- a/perllib/FixMyStreet/Template.pm +++ b/perllib/FixMyStreet/Template.pm @@ -7,10 +7,14 @@ use FixMyStreet; use mySociety::Locale; use Attribute::Handlers; use HTML::Scrubber; +use HTML::TreeBuilder; use FixMyStreet::Template::SafeString; use FixMyStreet::Template::Context; use FixMyStreet::Template::Stash; +use RABX; +use IO::String; + my %FILTERS; my %SUBS; @@ -141,6 +145,8 @@ sub html_paragraph : Filter('html_para') { sub sanitize { my $text = shift; + $text = $$text if UNIVERSAL::isa($text, 'FixMyStreet::Template::SafeString'); + my %allowed_tags = map { $_ => 1 } qw( p ul ol li br b i strong em ); my $scrubber = HTML::Scrubber->new( rules => [ @@ -155,4 +161,141 @@ sub sanitize { return $text; } +=head2 email_sanitize_text + +Intended for use in the _email_comment_list.txt template to allow HTML +in updates from staff/superusers. Sanitizes the HTML and then converts +it all to text. + +=cut + +sub email_sanitize_text : Fn('email_sanitize_text') { + my $update = shift; + + my $text = $update->{item_text}; + my $extra = $update->{item_extra}; + utf8::encode($extra) if $extra; + $extra = $extra ? RABX::wire_rd(new IO::String($extra)) : {}; + + my $staff = $extra->{is_superuser} || $extra->{is_body_user}; + + return $text unless $staff; + + $text = FixMyStreet::Template::sanitize($text); + + my $tree = HTML::TreeBuilder->new_from_content($text); + _sanitize_elt($tree); + + return $tree->as_text; +} + +my $list_type; +my $list_num; +my $sanitize_text_subs = { + b => [ '*', '*' ], + strong => [ '*', '*' ], + i => [ '_', '_' ], + em => [ '_', '_' ], + p => [ '', "\n\n" ], + li => [ '', "\n\n" ], +}; +sub _sanitize_elt { + my $elt = shift; + foreach ($elt->content_list) { + next unless ref $_; + $list_type = $_->tag, $list_num = 1 if $_->tag eq 'ol' || $_->tag eq 'ul'; + _sanitize_elt($_); + $_->replace_with("\n") if $_->tag eq 'br'; + $_->replace_with('[image: ', $_->attr('alt'), ']') if $_->tag eq 'img'; + $_->replace_with($_->as_text, ' [', $_->attr('href'), ']') if $_->tag eq 'a'; + $_->replace_with_content if $_->tag eq 'span' || $_->tag eq 'font'; + $_->replace_with_content if $_->tag eq 'ul' || $_->tag eq 'ol'; + if ($_->tag eq 'li') { + $sanitize_text_subs->{li}[0] = $list_type eq 'ol' ? "$list_num. " : '* '; + $list_num++; + } + if (my $sub = $sanitize_text_subs->{$_->tag}) { + $_->preinsert($sub->[0]); + $_->postinsert($sub->[1]); + $_->replace_with_content; + } + } +} + +=head2 email_sanitize_html + +Intended for use in the _email_comment_list.html template to allow HTML +in updates from staff/superusers. + +=cut + +sub email_sanitize_html : Fn('email_sanitize_html') { + my $update = shift; + + my $text = $update->{item_text}; + my $extra = $update->{item_extra}; + utf8::encode($extra) if $extra; + $extra = $extra ? RABX::wire_rd(new IO::String($extra)) : {}; + + my $staff = $extra->{is_superuser} || $extra->{is_body_user}; + + return _staff_html_markup($text, $staff); +} + +sub _staff_html_markup { + my ( $text, $staff ) = @_; + unless ($staff) { + return html_paragraph(add_links($text)); + } + + $text = sanitize($text); + + # Apply Markdown-style italics + $text =~ s{\*(\S.*?\S)\*}{<i>$1</i>}; + + # Mark safe so add_links doesn't escape everything. + $text = FixMyStreet::Template::SafeString->new($text); + + $text = add_links($text); + + # If the update already has block-level elements then don't wrap + # individual lines in <p> elements, as we assume the user knows what + # they're doing. + unless ($text =~ /<(p|ol|ul)>/) { + $text = html_paragraph($text); + } + + return $text; +} + +=head2 add_links + + [% text | add_links | html_para %] + +Add some links to some text (and thus HTML-escapes the other text). + +=cut + +sub add_links { + my $text = shift; + $text = conditional_escape($text); + $text =~ s/\r//g; + $text =~ s{(?<!["'])(https?://)([^\s]+)}{"<a href=\"$1$2\">$1" . _space_slash($2) . '</a>'}ge; + return FixMyStreet::Template::SafeString->new($text); +} + +sub _space_slash { + my $t = shift; + $t =~ s{/(?!$)}{/ }g; + return $t; +} + +sub title : Filter { + my $text = shift; + $text =~ s{(\w[\w']*)}{\u\L$1}g; + # Postcode special handling + $text =~ s{(\w?\w\d[\d\w]?\s*\d\w\w)}{\U$1}g; + return $text; +} + 1; diff --git a/perllib/FixMyStreet/TestAppProve.pm b/perllib/FixMyStreet/TestAppProve.pm index ec245e72c..d977c0a94 100644 --- a/perllib/FixMyStreet/TestAppProve.pm +++ b/perllib/FixMyStreet/TestAppProve.pm @@ -111,7 +111,7 @@ sub run { $prove->process_args(@ARGV); # If no arguments, test everything - $prove->argv(['t']) unless @{$prove->argv}; + $prove->argv(['t']) unless @{$prove->argv} || @state; # verbose if we have a single file $prove->verbose(1) if @{$prove->argv} and -f $prove->argv->[-1] && !$ENV{CI}; # we always want to recurse diff --git a/perllib/FixMyStreet/TestMech.pm b/perllib/FixMyStreet/TestMech.pm index 1b7fba1bd..7f7104d3d 100644 --- a/perllib/FixMyStreet/TestMech.pm +++ b/perllib/FixMyStreet/TestMech.pm @@ -276,6 +276,24 @@ sub get_text_body_from_email { return $body; } +sub get_html_body_from_email { + my ($mech, $email, $obj) = @_; + unless ($email) { + $email = $mech->get_email; + $mech->clear_emails_ok; + } + + my $body; + $email->walk_parts(sub { + my $part = shift; + return if $part->subparts; + return if $part->content_type !~ m{text/html}; + $body = $obj ? $part : $part->body_str; + ok $body, "Found HTML body"; + }); + return $body; +} + sub get_link_from_email { my ($mech, $email, $multiple, $mismatch) = @_; unless ($email) { @@ -340,12 +358,13 @@ arrayref of TEXTs. If none found return empty arrayref. sub page_errors { my $mech = shift; my $result = scraper { - process 'div.form-error, p.form-error, p.error, ul.error li', 'errors[]', 'TEXT'; + process 'div.form-error, p.form-error, p.error, ul.error li, .search-help__header', 'errors[]', 'TEXT'; } ->scrape( $mech->response ); my $err = $result->{errors} || []; my %seen = (); $err = [ grep { not $seen{$_}++ } @$err ]; + @$err = map { s/^\s+|\s+$//g; $_ } @$err; return $err; } @@ -590,29 +609,6 @@ sub get_ok_json { return decode_json( $res->content ); } -sub delete_body { - my $mech = shift; - my $body = shift; - - $mech->delete_problems_for_body($body->id); - $mech->delete_defect_type($_) for $body->defect_types; - $mech->delete_contact($_) for $body->contacts; - $mech->delete_user($_) for $body->users; - $_->delete for $body->response_templates; - $_->delete for $body->response_priorities; - $body->body_areas->delete; - $body->delete; -} - -sub delete_contact { - my $mech = shift; - my $contact = shift; - - $contact->contact_response_templates->delete_all; - $contact->contact_response_priorities->delete_all; - $contact->delete; -} - sub delete_problems_for_body { my $mech = shift; my $body = shift; @@ -627,14 +623,6 @@ sub delete_problems_for_body { } } -sub delete_defect_type { - my $mech = shift; - my $defect_type = shift; - - $defect_type->contact_defect_types->delete_all; - $defect_type->delete; -} - sub delete_response_template { my $mech = shift; my $response_template = shift; diff --git a/perllib/FixMyStreet/WorkingDays.pm b/perllib/FixMyStreet/WorkingDays.pm new file mode 100644 index 000000000..615b226c6 --- /dev/null +++ b/perllib/FixMyStreet/WorkingDays.pm @@ -0,0 +1,78 @@ +package FixMyStreet::WorkingDays; + +use Moo; + +=head1 FixMyStreet::WorkingDays + +Given a list of public holiday dates, creates an object that can be used to +add/subtract days from a date, only counting working days (excluding public +holidays and weekends). + +=over + +=cut + +has public_holidays => ( + is => 'ro', + coerce => sub { + return { map { $_ => 1 } @{$_[0]} }; + }, +); + +=item add_days + +Given a DateTime object and a number of days, returns a new DateTime object +that many working days (excluding public holidays and weekends) later. + +=cut + +sub add_days { + my ( $self, $dt, $days, $subtract ) = @_; + $dt = $dt->clone; + while ( $days > 0 ) { + $dt->add ( days => $subtract ? -1 : 1 ); + next if $self->is_public_holiday($dt); + next if $self->is_weekend($dt); + $days--; + } + return $dt; +} + +=item sub_days + +Given a DateTime object and a number of days, returns a new DateTime object +that many working days (excluding public holidays and weekends) earlier. + +=cut + +sub sub_days { + my $self = shift; + return $self->add_days(@_, 1); +} + +=item is_public_holiday + +Given a DateTime object, return true if it is a public holiday. + +=cut + +sub is_public_holiday { + my ($self, $dt) = @_; + return $self->public_holidays->{$dt->ymd}; +} + +=item is_weekend + +Given a DateTime object, return true if it is a weekend. + +=cut + +sub is_weekend { + my ($self, $dt) = @_; + return $dt->dow > 5; +} + +1; + +=back + |