diff options
Diffstat (limited to 'perllib/FixMyStreet')
70 files changed, 1966 insertions, 1451 deletions
diff --git a/perllib/FixMyStreet/App.pm b/perllib/FixMyStreet/App.pm index c9286b177..b8ce2e051 100644 --- a/perllib/FixMyStreet/App.pm +++ b/perllib/FixMyStreet/App.pm @@ -2,9 +2,6 @@ package FixMyStreet::App; use Moose; use namespace::autoclean; -# Should move away from Email::Send, but until then: -$Return::Value::NO_CLUCK = 1; - use Catalyst::Runtime 5.80; use FixMyStreet; use FixMyStreet::Cobrand; @@ -12,6 +9,7 @@ use Memcached; use mySociety::Email; use mySociety::Random qw(random_bytes); use FixMyStreet::Map; +use FixMyStreet::Email; use Utils; use Path::Class; @@ -319,7 +317,7 @@ sub send_email { ] }; - return if $c->is_abuser($vars->{to}); + return if FixMyStreet::Email::is_abuser($c->model('DB')->schema, $vars->{to}); # render the template my $content = $c->view('Email')->render( $c, $template, $vars ); @@ -347,7 +345,7 @@ sub send_email { ) }; if (my $attachments = $extra_stash_values->{attachments}) { - $email_text = munge_attachments($email_text, $attachments); + $email_text = FixMyStreet::Email::munge_attachments($email_text, $attachments); } # send the email @@ -356,107 +354,6 @@ sub send_email { return $email; } -sub send_email_cron { - my ( $c, $params, $env_from, $nomail, $cobrand, $lang_code ) = @_; - - my $sender = $c->config->{DO_NOT_REPLY_EMAIL}; - $env_from ||= $sender; - if (!$params->{From}) { - my $sender_name = $cobrand->contact_name; - $params->{From} = [ $sender, _($sender_name) ]; - } - - return 1 if $c->is_abuser($params->{To}); - - $params->{'Message-ID'} = sprintf('<fms-cron-%s-%s@%s>', time(), - unpack('h*', random_bytes(5, 1)), FixMyStreet->config('EMAIL_DOMAIN') - ); - - # This is all to set the path for the templates processor so we can override - # signature and site names in emails using templates in the old style emails. - # It's a bit involved as not everywhere we use it knows about the cobrand so - # we can't assume there will be one. - my $include_path = FixMyStreet->path_to( 'templates', 'email', 'default' )->stringify; - if ( $cobrand ) { - $include_path = - FixMyStreet->path_to( 'templates', 'email', $cobrand->moniker )->stringify . ':' - . $include_path; - if ( $lang_code ) { - $include_path = - FixMyStreet->path_to( 'templates', 'email', $cobrand->moniker, $lang_code )->stringify . ':' - . $include_path; - } - } - my $tt = Template->new({ - INCLUDE_PATH => $include_path - }); - my ($sig, $site_name); - $tt->process( 'signature.txt', $params, \$sig ); - $sig = Encode::decode('utf8', $sig); - $params->{_parameters_}->{signature} = $sig; - - $tt->process( 'site-name.txt', $params, \$site_name ); - $site_name = Utils::trim_text(Encode::decode('utf8', $site_name)); - $params->{_parameters_}->{site_name} = $site_name; - - $params->{_line_indent} = ''; - my $attachments = delete $params->{attachments}; - - my $email = mySociety::Locale::in_gb_locale { mySociety::Email::construct_email($params) }; - - $email = munge_attachments($email, $attachments) if $attachments; - - if ($nomail) { - print $email; - return 1; # Failure - } else { - my %model_args; - if (!FixMyStreet->test_mode && $env_from eq FixMyStreet->config('CONTACT_EMAIL')) { - $model_args{mailer} = 'FixMyStreet::EmailSend::ContactEmail'; - } - my $result = $c->model('EmailSend', %model_args)->send($email); - return $result ? 0 : 1; - } -} - -sub munge_attachments { - my ($message, $attachments) = @_; - # $attachments should be an array_ref of things that can be parsed to Email::MIME, - # for example - # [ - # body => $binary_data, - # attributes => { - # content_type => 'image/jpeg', - # encoding => 'base64', - # filename => '1234.1.jpeg', - # name => '1234.1.jpeg', - # }, - # ... - # ] - # - # XXX: mySociety::Email::construct_email isn't using a MIME library and - # requires more analysis to refactor, so for now, we'll simply parse the - # generated MIME and add attachments. - # - # (Yes, this means that the email is constructed by Email::Simple, munged - # manually by custom code, turned back into Email::Simple, and then munged - # with Email::MIME. What's your point?) - - require Email::MIME; - my $mime = Email::MIME->new($message); - $mime->parts_add([ map { Email::MIME->create(%$_)} @$attachments ]); - my $data = $mime->as_string; - - # unsure why Email::MIME adds \r\n. Possibly mail client should handle - # gracefully, BUT perhaps as the segment constructed by - # mySociety::Email::construct_email strips to \n, they seem not to. - # So we re-run the same regexp here to the added part. - $data =~ s/\r\n/\n/gs; - - return $data; -} - - =head2 uri_with $uri = $c->uri_with( ... ); @@ -535,58 +432,6 @@ sub render_fragment { $c->view('Web')->render($c, $template, $vars); } -=head2 get_photo_params - -Returns a hashref of details of any attached photo for use in templates. -Hashref contains height, width and url keys. - -=cut - -sub get_photo_params { - my ($self, $key) = @_; - - return {} unless $self->photo; - - $key = ($key eq 'id') ? '' : "/$key"; - - my $pre = "/photo$key/" . $self->id; - my $post = '.jpeg'; - my $photo = {}; - - if (length($self->photo) == 40) { - $post .= '?' . $self->photo; - $photo->{url_full} = "$pre.full$post"; - # XXX Can't use size here because {url} (currently 250px height) may be - # being used, but at this point it doesn't yet exist to find the width - # $str = FixMyStreet->config('UPLOAD_DIR') . $self->photo . '.jpeg'; - } else { - my $str = \$self->photo; - ( $photo->{width}, $photo->{height} ) = Image::Size::imgsize( $str ); - } - - $photo->{url} = "$pre$post"; - $photo->{url_tn} = "$pre.tn$post"; - $photo->{url_fp} = "$pre.fp$post"; - - return $photo; -} - -sub is_abuser { - my ($c, $to) = @_; - my $email; - if (ref($to) eq 'ARRAY') { - if (ref($to->[0]) eq 'ARRAY') { - $email = $to->[0][0]; - } else { - $email = $to->[0]; - } - } else { - $email = $to; - } - my ($domain) = $email =~ m{ @ (.*) \z }x; - return $c->model('DB::Abuse')->search( { email => [ $email, $domain ] } )->first; -} - =head2 get_param $param = $c->get_param('name'); diff --git a/perllib/FixMyStreet/App/Controller/About.pm b/perllib/FixMyStreet/App/Controller/About.pm new file mode 100755 index 000000000..78e548c5f --- /dev/null +++ b/perllib/FixMyStreet/App/Controller/About.pm @@ -0,0 +1,67 @@ +package FixMyStreet::App::Controller::About; +use Moose; +use namespace::autoclean; + +BEGIN { extends 'Catalyst::Controller'; } + +=head1 NAME + +FixMyStreet::App::Controller::About - Catalyst Controller + +=head1 DESCRIPTION + +About pages Catalyst Controller. + +=head1 METHODS + +=cut + +my %found; + +sub page : Path("/about") : Args(1) { + my ( $self, $c, $page ) = @_; + my $template = $c->forward('find_template'); + $c->detach('/page_error_404_not_found', []) unless $template; + $c->stash->{template} = $template; +} + +sub index : Path("/about") : Args(0) { + my ( $self, $c ) = @_; + $c->forward('page', [ 'about' ]); +} + +# We have multiple possibilities to try, and we want to cache where we find it +sub find_template : Private { + my ( $self, $c, $page ) = @_; + + return $found{$page} if !FixMyStreet->config('STAGING_SITE') && exists $found{$page}; + + my $lang_code = $c->stash->{lang_code}; + foreach my $dir_templates (@{$c->stash->{additional_template_paths}}, @{$c->view('Web')->paths}) { + foreach my $dir_static (static_dirs($page, $dir_templates)) { + foreach my $file ("$page-$lang_code.html", "$page.html") { + if (-e "$dir_templates/$dir_static/$file") { + $found{$page} = "$dir_static/$file"; + return $found{$page}; + } + } + } + } + # Cache that the page does not exist, so we don't look next time + $found{$page} = undef; + return $found{$page}; +} + +sub static_dirs { + my ($page, $dir_templates) = @_; + my @v = ("about"); + # If legacy directories exist, check for templates there too; + # The FAQ page used to be in its own directory + push @v, "static" if -d "$dir_templates/static"; + push @v, "faq" if -d "$dir_templates/faq" && $page =~ /faq/; + return @v; +} + +__PACKAGE__->meta->make_immutable; + +1; diff --git a/perllib/FixMyStreet/App/Controller/Admin.pm b/perllib/FixMyStreet/App/Controller/Admin.pm index a61032988..2bf215c56 100644 --- a/perllib/FixMyStreet/App/Controller/Admin.pm +++ b/perllib/FixMyStreet/App/Controller/Admin.pm @@ -84,14 +84,14 @@ sub index : Path : Args(0) { for ( FixMyStreet::DB::Result::Problem->visible_states() ); $c->stash->{total_problems_users} = $c->cobrand->problems->unique_users; - my $comments = $c->model('DB::Comment')->summary_count( $c->cobrand->body_restriction ); + my $comments = $c->cobrand->updates->summary_count; my %comment_counts = map { $_->state => $_->get_column('state_count') } $comments->all; $c->stash->{comments} = \%comment_counts; - my $alerts = $c->model('DB::Alert')->summary_count( $c->cobrand->restriction ); + my $alerts = $c->model('DB::Alert')->summary_report_alerts( $c->cobrand->restriction ); my %alert_counts = map { $_->confirmed => $_->get_column('confirmed_count') } $alerts->all; @@ -171,7 +171,7 @@ sub timeline : Path( 'timeline' ) : Args(0) { push @{$time{$_->whenanswered->epoch}}, { type => 'quesAnswered', date => $_->whenanswered, obj => $_ } if $_->whenanswered; } - my $updates = $c->model('DB::Comment')->timeline( $c->cobrand->body_restriction ); + my $updates = $c->cobrand->updates->timeline; foreach ($updates->all) { push @{$time{$_->created->epoch}}, { type => 'update', date => $_->created, obj => $_} ; @@ -622,9 +622,7 @@ sub reports : Path('reports') { } if (@$query) { - my $updates = $c->model('DB::Comment') - ->to_body($c->cobrand->body_restriction) - ->search( + my $updates = $c->cobrand->updates->search( { -or => $query, }, @@ -685,7 +683,7 @@ sub report_edit : Path('report_edit') : Args(1) { } if (my $rotate_photo_param = $self->_get_rotate_photo_param($c)) { - $self->rotate_photo($c, @$rotate_photo_param); + $self->rotate_photo($c, $problem, @$rotate_photo_param); if ( $c->cobrand->moniker eq 'zurich' ) { # Clicking the photo rotation buttons should do nothing # except for rotating the photo, so return the user @@ -792,11 +790,12 @@ sub report_edit : Path('report_edit') : Args(1) { } # Deal with photos - if ( $c->get_param('remove_photo') ) { - $problem->photo(undef); + my $remove_photo_param = $self->_get_remove_photo_param($c); + if ($remove_photo_param) { + $self->remove_photo($c, $problem, $remove_photo_param); } - if ( $c->get_param('remove_photo') || $new_state eq 'hidden' ) { + if ( $remove_photo_param || $new_state eq 'hidden' ) { unlink glob FixMyStreet->path_to( 'web', 'photo', $problem->id . '.*' ); } @@ -967,9 +966,7 @@ sub users: Path('users') : Args(0) { sub update_edit : Path('update_edit') : Args(1) { my ( $self, $c, $id ) = @_; - my $update = $c->model('DB::Comment') - ->to_body($c->cobrand->body_restriction) - ->search({ id => $id })->first; + my $update = $c->cobrand->updates->search({ id => $id })->first; $c->detach( '/page_error_404_not_found' ) unless $update; @@ -978,6 +975,11 @@ sub update_edit : Path('update_edit') : Args(1) { $c->stash->{update} = $update; + if (my $rotate_photo_param = $self->_get_rotate_photo_param($c)) { + $self->rotate_photo($c, $update, @$rotate_photo_param); + return 1; + } + $c->forward('check_email_for_abuse', [ $update->user->email ] ); if ( $c->get_param('banuser') ) { @@ -1007,13 +1009,14 @@ sub update_edit : Path('update_edit') : Args(1) { || $c->get_param('anonymous') ne $update->anonymous || $c->get_param('text') ne $update->text ) { $edited = 1; - } + } - if ( $c->get_param('remove_photo') ) { - $update->photo(undef); + my $remove_photo_param = $self->_get_remove_photo_param($c); + if ($remove_photo_param) { + $self->remove_photo($c, $update, $remove_photo_param); } - if ( $c->get_param('remove_photo') || $new_state eq 'hidden' ) { + if ( $remove_photo_param || $new_state eq 'hidden' ) { unlink glob FixMyStreet->path_to( 'web', 'photo', 'c', $update->id . '.*' ); } @@ -1076,16 +1079,18 @@ sub user_add : Path('user_edit') : Args(0) { $c->forward('get_token'); $c->forward('fetch_all_bodies'); - return 1 unless $c->get_param('submit'); + return unless $c->get_param('submit'); $c->forward('check_token'); - if ( $c->cobrand->moniker eq 'zurich' and $c->get_param('email') eq '' ) { + unless ($c->get_param('email')) { $c->stash->{field_errors}->{email} = _('Please enter a valid email'); - return 1; + return; + } + unless ($c->get_param('name')) { + $c->stash->{field_errors}->{name} = _('Please enter a name'); + return; } - - return unless $c->get_param('name') && $c->get_param('email'); my $user = $c->model('DB::User')->find_or_create( { name => $c->get_param('name'), @@ -1133,12 +1138,16 @@ sub user_edit : Path('user_edit') : Args(1) { $user->from_body( $c->get_param('body') || undef ); $user->flagged( $c->get_param('flagged') || 0 ); - if ( $c->cobrand->moniker eq 'zurich' and $user->email eq '' ) { + unless ($user->email) { $c->stash->{field_errors}->{email} = _('Please enter a valid email'); - return 1; + return; + } + unless ($user->name) { + $c->stash->{field_errors}->{name} = _('Please enter a name'); + return; } - $user->update; + $user->update; if ($edited) { $c->forward( 'log_edit', [ $id, 'user', 'edit' ] ); } @@ -1316,9 +1325,9 @@ Generate a token based on user and secret sub get_token : Private { my ( $self, $c ) = @_; - my $secret = $c->model('DB::Secret')->search()->first; + my $secret = $c->model('DB::Secret')->get; my $user = $c->forward('get_user'); - my $token = sha1_hex($user . $secret->secret); + my $token = sha1_hex($user . $secret); $c->stash->{token} = $token; return 1; @@ -1486,25 +1495,51 @@ sub _get_rotate_photo_param { my $key = first { /^rotate_photo/ } keys %{ $c->req->params } or return; my ($index) = $key =~ /(\d+)$/; my $direction = $c->get_param($key); - return [ $index || 0, $key, $direction ]; + return [ $index || 0, $direction ]; } sub rotate_photo : Private { - my ( $self, $c, $index, $key, $direction ) = @_; + my ( $self, $c, $object, $index, $direction ) = @_; return unless $direction eq _('Rotate Left') or $direction eq _('Rotate Right'); - my $problem = $c->stash->{problem}; - my $fileid = $problem->get_photoset($c)->rotate_image( + my $fileid = $object->get_photoset->rotate_image( $index, $direction eq _('Rotate Left') ? -90 : 90 ) or return; - $problem->update({ photo => $fileid }); + $object->update({ photo => $fileid }); return 1; } +=head2 remove_photo + +Remove a photo from a report + +=cut + +# Returns index of photo(s) to remove, if any +sub _get_remove_photo_param { + my ($self, $c) = @_; + + return 'ALL' if $c->get_param('remove_photo'); + + my @keys = map { /(\d+)$/ } grep { /^remove_photo_/ } keys %{ $c->req->params } or return; + return \@keys; +} + +sub remove_photo : Private { + my ($self, $c, $object, $keys) = @_; + if ($keys eq 'ALL') { + $object->photo(undef); + } else { + my $fileids = $object->get_photoset->remove_images($keys); + $object->photo($fileids); + } + return 1; +} + =head2 check_page_allowed Checks if the current catalyst action is in the list of allowed pages and diff --git a/perllib/FixMyStreet/App/Controller/Around.pm b/perllib/FixMyStreet/App/Controller/Around.pm index 4aa695ae5..1e6d9ec9e 100644 --- a/perllib/FixMyStreet/App/Controller/Around.pm +++ b/perllib/FixMyStreet/App/Controller/Around.pm @@ -6,6 +6,7 @@ BEGIN { extends 'Catalyst::Controller'; } use FixMyStreet::Map; use Encode; +use JSON::MaybeXS; use Utils; =head1 NAME @@ -306,7 +307,7 @@ sub ajax : Path('/ajax') { # JSON encode the response my $json = { pins => $pins }; $json->{current} = $on_map_list_html if $on_map_list_html; - my $body = JSON->new->utf8(1)->encode($json); + my $body = encode_json($json); $c->res->body($body); } @@ -350,8 +351,10 @@ sub _geocode : Private { } else { if ( ref($suggestions) eq 'ARRAY' ) { foreach (@$suggestions) { - push @addresses, decode_utf8($_->{address}); - push @locations, { address => decode_utf8($_->{address}), lat => $_->{latitude}, long => $_->{longitude} }; + my $address = $_->{address}; + $address = decode_utf8($address) if !utf8::is_utf8($address); + push @addresses, $address; + push @locations, { address => $address, lat => $_->{latitude}, long => $_->{longitude} }; } $response = { suggestions => \@addresses, locations => \@locations }; } else { @@ -363,9 +366,7 @@ sub _geocode : Private { $response = \@addresses; } - my $body = JSON->new->utf8(1)->encode( - $response - ); + my $body = encode_json($response); $c->res->body($body); } diff --git a/perllib/FixMyStreet/App/Controller/Auth.pm b/perllib/FixMyStreet/App/Controller/Auth.pm index 6de416c53..9e8fb29aa 100644 --- a/perllib/FixMyStreet/App/Controller/Auth.pm +++ b/perllib/FixMyStreet/App/Controller/Auth.pm @@ -7,7 +7,8 @@ BEGIN { extends 'Catalyst::Controller'; } use Email::Valid; use Net::Domain::TLD; use mySociety::AuthToken; -use JSON; +use JSON::MaybeXS; +use Net::Facebook::Oauth2; =head1 NAME @@ -36,6 +37,8 @@ sub general : Path : Args(0) { return unless $c->req->method eq 'POST'; # decide which action to take + $c->detach('facebook_sign_in') if $c->get_param('facebook_sign_in'); + my $clicked_password = $c->get_param('sign_in'); my $clicked_email = $c->get_param('email_sign_in'); my $data_password = $c->get_param('password_sign_in'); @@ -122,18 +125,19 @@ sub email_sign_in : Private { if $c->get_param('password_register'); my $user = $c->model('DB::User')->new( $user_params ); - my $token_obj = $c->model('DB::Token') # - ->create( - { - scope => 'email_sign_in', - data => { - email => $good_email, - r => $c->get_param('r'), - name => $c->get_param('name'), - password => $user->password, - } - } - ); + my $token_data = { + email => $good_email, + r => $c->get_param('r'), + name => $c->get_param('name'), + password => $user->password, + }; + $token_data->{facebook_id} = $c->session->{oauth}{facebook_id} + if $c->get_param('oauth_need_email') && $c->session->{oauth}{facebook_id}; + + my $token_obj = $c->model('DB::Token')->create({ + scope => 'email_sign_in', + data => $token_data, + }); $c->stash->{token} = $token_obj->token; $c->send_email( 'login.txt', { to => $good_email } ); @@ -175,6 +179,7 @@ sub token : Path('/M') : Args(1) { my $user = $c->model('DB::User')->find_or_create( { email => $data->{email} } ); $user->name( $data->{name} ) if $data->{name}; $user->password( $data->{password}, 1 ) if $data->{password}; + $user->facebook_id( $data->{facebook_id} ) if $data->{facebook_id}; $user->update; $c->authenticate( { email => $user->email }, 'no_password' ); @@ -182,6 +187,113 @@ sub token : Path('/M') : Args(1) { $c->detach( 'redirect_on_signin', [ $data->{r} ] ); } +=head2 facebook_sign_in + +Starts the Facebook authentication sequence. + +=cut + +sub fb : Private { + my ($self, $c) = @_; + Net::Facebook::Oauth2->new( + application_id => $c->config->{FACEBOOK_APP_ID}, + application_secret => $c->config->{FACEBOOK_APP_SECRET}, + callback => $c->uri_for('/auth/Facebook'), + ); +} + +sub facebook_sign_in : Private { + my( $self, $c ) = @_; + + my $fb = $c->forward('/auth/fb'); + my $url = $fb->get_authorization_url(scope => ['email']); + + my %oauth; + $oauth{return_url} = $c->get_param('r'); + $oauth{detach_to} = $c->stash->{detach_to}; + $oauth{detach_args} = $c->stash->{detach_args}; + $c->session->{oauth} = \%oauth; + $c->res->redirect($url); +} + +=head2 facebook_callback + +Handles the Facebook callback request and completes the authentication sequence. + +=cut + +sub facebook_callback: Path('/auth/Facebook') : Args(0) { + my( $self, $c ) = @_; + + if ( $c->get_param('error_code') ) { + $c->stash->{oauth_failure} = 1; + if ($c->session->{oauth}{detach_to}) { + $c->detach($c->session->{oauth}{detach_to}, $c->session->{oauth}{detach_args}); + } else { + $c->stash->{template} = 'auth/general.html'; + $c->detach; + } + } + + my $fb = $c->forward('/auth/fb'); + my $access_token; + eval { + $access_token = $fb->get_access_token(code => $c->get_param('code')); + }; + if ($@) { + ($c->stash->{message} = $@) =~ s/at [^ ]*Auth.pm.*//; + $c->stash->{template} = 'errors/generic.html'; + $c->detach; + } + + # save this token in session + $c->session->{oauth}{token} = $access_token; + + my $info = $fb->get('https://graph.facebook.com/me?fields=name,email')->as_hash(); + my $name = $info->{name}; + my $email = lc ($info->{email} || ""); + my $uid = $info->{id}; + + my $user; + if ($email) { + # We've got an ID and an email address + # Remove any existing mention of this ID + my $existing = $c->model('DB::User')->find( { facebook_id => $uid } ); + $existing->update( { facebook_id => undef } ) if $existing; + # Get or create a user, give it this Facebook ID + $user = $c->model('DB::User')->find_or_new( { email => $email } ); + $user->facebook_id($uid); + $user->name($name); + $user->in_storage() ? $user->update : $user->insert; + } else { + # We've got an ID, but no email + $user = $c->model('DB::User')->find( { facebook_id => $uid } ); + if ($user) { + # Matching Facebook ID in our database + $user->name($name); + $user->update; + } else { + # No matching ID, store ID for use later + $c->session->{oauth}{facebook_id} = $uid; + $c->stash->{oauth_need_email} = 1; + } + } + + # If we've got here with a full user, log in + if ($user) { + $c->authenticate( { email => $user->email }, 'no_password' ); + $c->stash->{login_success} = 1; + } + + if ($c->session->{oauth}{detach_to}) { + $c->detach($c->session->{oauth}{detach_to}, $c->session->{oauth}{detach_args}); + } elsif ($c->stash->{oauth_need_email}) { + $c->stash->{template} = 'auth/general.html'; + } else { + $c->detach( 'redirect_on_signin', [ $c->session->{oauth}{return_url} ] ); + } +} + =head2 redirect_on_signin Used after signing in to take the person back to where they were. @@ -275,7 +387,7 @@ sub ajax_sign_in : Path('ajax/sign_in') { $return->{error} = 1; } - my $body = JSON->new->utf8(1)->encode( $return ); + my $body = encode_json($return); $c->res->content_type('application/json; charset=utf-8'); $c->res->body($body); @@ -287,7 +399,7 @@ sub ajax_sign_out : Path('ajax/sign_out') { $c->logout(); - my $body = JSON->new->utf8(1)->encode( { signed_out => 1 } ); + my $body = encode_json( { signed_out => 1 } ); $c->res->content_type('application/json; charset=utf-8'); $c->res->body($body); @@ -305,7 +417,7 @@ sub ajax_check_auth : Path('ajax/check_auth') { $code = 200; } - my $body = JSON->new->utf8(1)->encode( $data ); + my $body = encode_json($data); $c->res->content_type('application/json; charset=utf-8'); $c->res->code($code); $c->res->body($body); diff --git a/perllib/FixMyStreet/App/Controller/Dashboard.pm b/perllib/FixMyStreet/App/Controller/Dashboard.pm index faddaa89e..9189b28e5 100644 --- a/perllib/FixMyStreet/App/Controller/Dashboard.pm +++ b/perllib/FixMyStreet/App/Controller/Dashboard.pm @@ -4,6 +4,7 @@ use namespace::autoclean; use DateTime; use File::Slurp; +use JSON::MaybeXS; BEGIN { extends 'Catalyst::Controller'; } @@ -40,7 +41,7 @@ sub example : Local : Args(0) { my $data = File::Slurp::read_file( FixMyStreet->path_to( 'data/dashboard.json' )->stringify ); - my $j = JSON->new->utf8->decode($data); + my $j = decode_json($data); if ( !$c->stash->{ward} && !$c->stash->{category} ) { $c->stash->{problems} = $j->{counts_all}; } else { diff --git a/perllib/FixMyStreet/App/Controller/FakeMapit.pm b/perllib/FixMyStreet/App/Controller/FakeMapit.pm index 253c75ba4..a4adadd09 100755 --- a/perllib/FixMyStreet/App/Controller/FakeMapit.pm +++ b/perllib/FixMyStreet/App/Controller/FakeMapit.pm @@ -1,6 +1,7 @@ package FixMyStreet::App::Controller::FakeMapit; use Moose; use namespace::autoclean; +use JSON::MaybeXS; BEGIN { extends 'Catalyst::Controller'; } @@ -22,7 +23,7 @@ my $area = { "name" => "Everywhere", "type" => "ZZZ", "id" => 161 }; sub output : Private { my ( $self, $c, $data ) = @_; - my $body = JSON->new->utf8(1)->encode( $data ); + my $body = encode_json($data); $c->res->content_type('application/json; charset=utf-8'); $c->res->body( $body ); } diff --git a/perllib/FixMyStreet/App/Controller/JSON.pm b/perllib/FixMyStreet/App/Controller/JSON.pm index 959ead245..d3cd33546 100644 --- a/perllib/FixMyStreet/App/Controller/JSON.pm +++ b/perllib/FixMyStreet/App/Controller/JSON.pm @@ -4,11 +4,10 @@ use namespace::autoclean; BEGIN { extends 'Catalyst::Controller'; } -use JSON; +use JSON::MaybeXS; use DateTime; use DateTime::Format::ISO8601; use List::MoreUtils 'uniq'; -use FixMyStreet::App; =head1 NAME @@ -81,7 +80,7 @@ sub problems : Local { $date_col = 'lastupdate'; } - my $dt_parser = FixMyStreet::App->model('DB')->schema->storage->datetime_parser; + my $dt_parser = $c->model('DB')->schema->storage->datetime_parser; my $one_day = DateTime::Duration->new( days => 1 ); my $query = { diff --git a/perllib/FixMyStreet/App/Controller/Location.pm b/perllib/FixMyStreet/App/Controller/Location.pm index ff90d3d60..6def423ce 100644 --- a/perllib/FixMyStreet/App/Controller/Location.pm +++ b/perllib/FixMyStreet/App/Controller/Location.pm @@ -95,7 +95,8 @@ sub determine_location_from_pc : Private { # $error doubles up to return multiple choices by being an array if ( ref($error) eq 'ARRAY' ) { foreach (@$error) { - my $a = decode_utf8($_->{address}); + my $a = $_->{address}; + $a = decode_utf8($a) if !utf8::is_utf8($a); $a =~ s/, United Kingdom//; $a =~ s/, UK//; $_->{address} = $a; diff --git a/perllib/FixMyStreet/App/Controller/My.pm b/perllib/FixMyStreet/App/Controller/My.pm index 3c4ce2cf7..8eb7f079e 100644 --- a/perllib/FixMyStreet/App/Controller/My.pm +++ b/perllib/FixMyStreet/App/Controller/My.pm @@ -36,6 +36,7 @@ sub my : Path : Args(0) { my $states = $c->stash->{filter_problem_states}; my $params = { state => [ keys %$states ], + user_id => $c->user->id, }; my $category = $c->get_param('filter_category'); @@ -44,9 +45,7 @@ sub my : Path : Args(0) { $c->stash->{filter_category} = $category; } - my $rs = $c->user->problems - ->to_body($c->cobrand->body_restriction) - ->search( $params, { + my $rs = $c->cobrand->problems->search( $params, { order_by => { -desc => 'confirmed' }, rows => 50 } )->page( $p_page ); @@ -77,7 +76,7 @@ sub my : Path : Args(0) { $c->stash->{updates} = \@updates; $c->stash->{updates_pager} = $rs->pager; - my @categories = $c->user->problems->search( undef, { + my @categories = $c->cobrand->problems->search( { user_id => $c->user->id }, { columns => [ 'category' ], distinct => 1, order_by => [ 'category' ], diff --git a/perllib/FixMyStreet/App/Controller/Open311.pm b/perllib/FixMyStreet/App/Controller/Open311.pm index 96066ca93..f35dc64a5 100644 --- a/perllib/FixMyStreet/App/Controller/Open311.pm +++ b/perllib/FixMyStreet/App/Controller/Open311.pm @@ -4,7 +4,7 @@ use utf8; use Moose; use namespace::autoclean; -use JSON; +use JSON::MaybeXS; use XML::Simple; use DateTime::Format::W3CDTF; diff --git a/perllib/FixMyStreet/App/Controller/Photo.pm b/perllib/FixMyStreet/App/Controller/Photo.pm index bc72f4bfb..bfb1c5535 100644 --- a/perllib/FixMyStreet/App/Controller/Photo.pm +++ b/perllib/FixMyStreet/App/Controller/Photo.pm @@ -4,11 +4,9 @@ use namespace::autoclean; BEGIN {extends 'Catalyst::Controller'; } -use DateTime::Format::HTTP; -use Digest::SHA qw(sha1_hex); +use JSON::MaybeXS; use File::Path; use File::Slurp; -use Path::Class; use FixMyStreet::App::Model::PhotoSet; use if !$ENV{TRAVIS}, 'Image::Magick'; @@ -35,16 +33,12 @@ sub during :LocalRegex('^([0-9a-f]{40})\.(temp|fulltemp)\.jpeg$') { my ( $self, $c ) = @_; my ( $hash, $size ) = @{ $c->req->captures }; - my $file = file( $c->config->{UPLOAD_DIR}, "$hash.jpeg" ); - my $photo = $file->slurp; + my $photoset = FixMyStreet::App::Model::PhotoSet->new({ + data_items => [ $hash ] + }); - if ( $size eq 'temp' ) { - if ( $c->cobrand->default_photo_resize ) { - $photo = _shrink( $photo, $c->cobrand->default_photo_resize ); - } else { - $photo = _shrink( $photo, '250x250' ); - } - } + $size = $size eq 'temp' ? 'default' : 'full'; + my $photo = $photoset->get_image_data(size => $size, default => $c->cobrand->default_photo_resize); $c->forward( 'output', [ $photo ] ); } @@ -61,15 +55,9 @@ sub index :LocalRegex('^(c/)?(\d+)(?:\.(\d+))?(?:\.(full|tn|fp))?\.jpeg$') { photo => { '!=', undef }, } ); } else { - # GoogleBot-Image is doing this for some reason? - if ( $id =~ m{ ^(\d+) \D .* $ }x ) { - return $c->res->redirect( $c->uri_with( { id => $1 } ), 301 ); - } - - $c->detach( 'no_photo' ) if $id =~ /\D/; ($item) = $c->cobrand->problems->search( { id => $id, - state => [ FixMyStreet::DB::Result::Problem->visible_states(), 'partial' ], + state => [ FixMyStreet::DB::Result::Problem->visible_states() ], photo => { '!=', undef }, } ); } @@ -79,29 +67,9 @@ sub index :LocalRegex('^(c/)?(\d+)(?:\.(\d+))?(?:\.(full|tn|fp))?\.jpeg$') { $c->detach( 'no_photo' ) unless $c->cobrand->allow_photo_display($item); # Should only be for reports, not updates my $photo; - if ($item->can('get_photoset')) { - $photo = $item->get_photoset( $c ) - ->get_image_data( num => $photo_number, size => $size ) + $photo = $item->get_photoset + ->get_image_data( num => $photo_number, size => $size, default => $c->cobrand->default_photo_resize ) or $c->detach( 'no_photo' ); - } else { - $photo = $item->photo; - # If photo field contains a hash - if (length($photo) == 40) { - my $file = file( $c->config->{UPLOAD_DIR}, "$photo.jpeg" ); - $photo = $file->slurp; - } - - if ( $size eq 'tn' ) { - $photo = _shrink( $photo, 'x100' ); - } elsif ( $size eq 'fp' ) { - $photo = _crop( $photo ); - } elsif ( $size eq 'full' ) { - } elsif ( $c->cobrand->default_photo_resize ) { - $photo = _shrink( $photo, $c->cobrand->default_photo_resize ); - } else { - $photo = _shrink( $photo, '250x250' ); - } - } $c->forward( 'output', [ $photo ] ); } @@ -122,32 +90,30 @@ sub no_photo : Private { $c->detach( '/page_error_404_not_found', [ 'No photo' ] ); } -# Shrinks a picture to the specified size, but keeping in proportion. -sub _shrink { - my ($photo, $size) = @_; - my $image = Image::Magick->new; - $image->BlobToImage($photo); - my $err = $image->Scale(geometry => "$size>"); - throw Error::Simple("resize failed: $err") if "$err"; - $image->Strip(); - my @blobs = $image->ImageToBlob(); - undef $image; - return $blobs[0]; -} +sub upload : Local { + my ( $self, $c ) = @_; + my @items = ( + ( map { + /^photo/ ? # photo, photo1, photo2 etc. + ($c->req->upload($_)) : () + } sort $c->req->upload), + ); + my $photoset = FixMyStreet::App::Model::PhotoSet->new({ + c => $c, + data_items => \@items, + }); + + my $fileid = $photoset->data; + my $out; + if ($c->stash->{photo_error} || !$fileid) { + $c->res->status(500); + $out = { error => $c->stash->{photo_error} || _('Unknown error') }; + } else { + $out = { id => $fileid }; + } -# Shrinks a picture to 90x60, cropping so that it is exactly that. -sub _crop { - my ($photo) = @_; - my $image = Image::Magick->new; - $image->BlobToImage($photo); - my $err = $image->Resize( geometry => "90x60^" ); - throw Error::Simple("resize failed: $err") if "$err"; - $err = $image->Extent( geometry => '90x60', gravity => 'Center' ); - throw Error::Simple("resize failed: $err") if "$err"; - $image->Strip(); - my @blobs = $image->ImageToBlob(); - undef $image; - return $blobs[0]; + $c->res->content_type('application/json; charset=utf-8'); + $c->res->body(encode_json($out)); } =head2 process_photo diff --git a/perllib/FixMyStreet/App/Controller/Report.pm b/perllib/FixMyStreet/App/Controller/Report.pm index d7cae05d4..5cc44bb8f 100644 --- a/perllib/FixMyStreet/App/Controller/Report.pm +++ b/perllib/FixMyStreet/App/Controller/Report.pm @@ -2,6 +2,8 @@ package FixMyStreet::App::Controller::Report; use Moose; use namespace::autoclean; +use JSON::MaybeXS; + BEGIN { extends 'Catalyst::Controller'; } =head1 NAME @@ -184,7 +186,7 @@ sub format_problem_for_display : Private { if ( $c->stash->{ajax} ) { $c->res->content_type('application/json; charset=utf-8'); - my $content = JSON->new->utf8(1)->encode( + my $content = encode_json( { report => $c->cobrand->problem_as_hashref( $problem, $c ), updates => $c->cobrand->updates_as_hashref( $problem, $c ), diff --git a/perllib/FixMyStreet/App/Controller/Report/New.pm b/perllib/FixMyStreet/App/Controller/Report/New.pm index 246facbee..66dc20a3a 100644 --- a/perllib/FixMyStreet/App/Controller/Report/New.pm +++ b/perllib/FixMyStreet/App/Controller/Report/New.pm @@ -12,7 +12,7 @@ use mySociety::MaPit; use Path::Class; use Utils; use mySociety::EmailUtil; -use JSON; +use JSON::MaybeXS; =head1 NAME @@ -149,6 +149,7 @@ sub report_new_ajax : Path('mobile') : Args(0) { } } ); if ( $report->confirmed ) { + $c->forward( 'create_reporter_alert' ); $c->stash->{ json_response } = { success => 1, report => $report->id }; } else { $c->stash->{token_url} = $c->uri_for_email( '/P', $token->token ); @@ -164,9 +165,7 @@ sub report_new_ajax : Path('mobile') : Args(0) { sub send_json_response : Private { my ( $self, $c ) = @_; - my $body = JSON->new->utf8(1)->encode( - $c->stash->{json_response}, - ); + my $body = encode_json($c->stash->{json_response}); $c->res->content_type('application/json; charset=utf-8'); $c->res->body($body); } @@ -178,9 +177,7 @@ sub report_form_ajax : Path('ajax') : Args(0) { # work out the location for this report and do some checks if ( ! $c->forward('determine_location') ) { - my $body = JSON->new->utf8(1)->encode( { - error => $c->stash->{location_error}, - } ); + my $body = encode_json({ error => $c->stash->{location_error} }); $c->res->content_type('application/json; charset=utf-8'); $c->res->body($body); return; @@ -197,7 +194,7 @@ sub report_form_ajax : Path('ajax') : Args(0) { my $extra_titles_list = $c->cobrand->title_list($c->stash->{all_areas}); - my $body = JSON->new->utf8(1)->encode( + my $body = encode_json( { councils_text => $councils_text, category => $category, @@ -216,11 +213,7 @@ sub category_extras_ajax : Path('category_extras') : Args(0) { $c->forward('initialize_report'); if ( ! $c->forward('determine_location') ) { - my $body = JSON->new->utf8(1)->encode( - { - error => _("Sorry, we could not find that location."), - } - ); + my $body = encode_json({ error => _("Sorry, we could not find that location.") }); $c->res->content_type('application/json; charset=utf-8'); $c->res->body($body); return 1; @@ -244,11 +237,7 @@ sub category_extras_ajax : Path('category_extras') : Args(0) { $category_extra = $c->render_fragment( 'report/new/category_extras.html'); } - my $body = JSON->new->utf8(1)->encode( - { - category_extra => $category_extra, - } - ); + my $body = encode_json({ category_extra => $category_extra }); $c->res->content_type('application/json; charset=utf-8'); $c->res->body($body); @@ -403,6 +392,12 @@ sub report_import : Path('/import') { return 1; } +sub oauth_callback : Private { + my ( $self, $c, $token_code ) = @_; + $c->stash->{oauth_report} = $token_code; + $c->detach('report_new'); +} + =head2 initialize_report Create the report and set up some basics in it. If there is a partial report @@ -426,26 +421,21 @@ sub initialize_report : Private { for (1) { # use as pseudo flow control - # did we find a token - last unless $partial; - # is it in the database my $token = $c->model("DB::Token") - ->find( { scope => 'partial', token => $partial } ) # + ->find( { scope => 'partial', token => $partial } ) || last; # can we get an id from it? - my $id = $token->data # - || last; + my $id = $token->data || last; # load the related problem - $report = $c->cobrand->problems # - ->search( { id => $id, state => 'partial' } ) # + $report = $c->cobrand->problems + ->search( { id => $id, state => 'partial' } ) ->first; if ($report) { - # log the problem creation user in to the site $c->authenticate( { email => $report->user->email }, 'no_password' ); @@ -453,27 +443,32 @@ sub initialize_report : Private { # save the token to delete at the end $c->stash->{partial_token} = $token if $report; - } - else { - + } else { # no point keeping it if it is done. $token->delete; } } } - if ( !$report ) { + if (!$report && $c->stash->{oauth_report}) { + my $auth_token = $c->forward( '/tokens/load_auth_token', + [ $c->stash->{oauth_report}, 'problem/social' ] ); + $report = $c->model("DB::Problem")->new($auth_token->data); + } - # If we didn't find a partial then create a new one + if ($report) { + # Stash the photo IDs for "already got" display + $c->stash->{upload_fileid} = $report->get_photoset->data; + } else { + # If we didn't find one otherwise, start with a blank report $report = $c->model('DB::Problem')->new( {} ); + } - # If we have a user logged in let's prefill some values for them. - if ( $c->user ) { - my $user = $c->user->obj; - $report->user($user); - $report->name( $user->name ); - } - + # If we have a user logged in let's prefill some values for them. + if (!$report->user && $c->user) { + my $user = $c->user->obj; + $report->user($user); + $report->name( $user->name ); } if ( $c->get_param('first_name') && $c->get_param('last_name') ) { @@ -859,12 +854,6 @@ sub process_report : Private { $bodies = join( ',', @{ $c->stash->{bodies_to_list} } ) || -1; $report->bodies_str( $bodies ); - my %extra; - $c->cobrand->process_extras( $c, undef, \%extra ); - if ( %extra ) { - $report->extra( \%extra ); - } - } elsif ( $report->category ) { # FIXME All contacts were fetched in setup_categories_and_bodies, @@ -936,7 +925,7 @@ sub process_report : Private { $report->non_public( 1 ); } - $c->cobrand->process_extras( $c, $contacts[0]->body_id, \@extra ); + $c->cobrand->process_open311_extras( $c, $contacts[0]->body_id, \@extra ); if ( @extra ) { $c->stash->{report_meta} = { map { $_->{name} => $_ } @extra }; @@ -955,6 +944,15 @@ sub process_report : Private { } + # Get a list of custom form fields we want and store them in extra metadata + foreach my $field ($c->cobrand->report_form_extras) { + my $form_name = $field->{name}; + my $value = $c->get_param($form_name) || ''; + $c->stash->{field_errors}->{$form_name} = _('This information is required') + if $field->{required} && !$value; + $report->set_extra_metadata( $form_name => $value ); + } + # set defaults that make sense $report->state('unconfirmed'); @@ -1005,6 +1003,13 @@ sub check_for_errors : Private { delete $field_errors{name}; } + # if using social login then we don't care about name and email errors + $c->stash->{is_social_user} = $c->get_param('facebook_sign_in') || $c->get_param('twitter_sign_in'); + if ( $c->stash->{is_social_user} ) { + delete $field_errors{name}; + delete $field_errors{email}; + } + # add the photo error if there is one. if ( my $photo_error = delete $c->stash->{photo_error} ) { $field_errors{photo} = $photo_error; @@ -1018,6 +1023,19 @@ sub check_for_errors : Private { return; } +# Store changes in token for when token is validated. +sub tokenize_user : Private { + my ($self, $c, $report) = @_; + $c->stash->{token_data} = { + name => $report->user->name, + phone => $report->user->phone, + password => $report->user->password, + title => $report->user->title, + }; + $c->stash->{token_data}{facebook_id} = $c->session->{oauth}{facebook_id} + if $c->get_param('oauth_need_email') && $c->session->{oauth}{facebook_id}; +} + =head2 save_user_and_report Save the user and the report. @@ -1029,7 +1047,41 @@ before or they are currently logged in. Otherwise discard any changes. sub save_user_and_report : Private { my ( $self, $c ) = @_; - my $report = $c->stash->{report}; + my $report = $c->stash->{report}; + + # If there was a photo add that + if ( my $fileid = $c->stash->{upload_fileid} ) { + $report->photo($fileid); + } + + # Set a default if possible + $report->category( _('Other') ) unless $report->category; + + # Set unknown to DB unknown + $report->bodies_str( undef ) if $report->bodies_str eq '-1'; + + # if there is a Message Manager message ID, pass it back to the client view + if ($c->cobrand->moniker eq 'fixmybarangay' && $c->get_param('external_source_id') =~ /^\d+$/) { + $c->stash->{external_source_id} = $c->get_param('external_source_id'); + $report->external_source_id( $c->get_param('external_source_id') ); + $report->external_source( $c->config->{MESSAGE_MANAGER_URL} ) ; + } + + if ( $c->stash->{is_social_user} ) { + my $token = $c->model("DB::Token")->create( { + scope => 'problem/social', + data => { $report->get_inflated_columns }, + } ); + + $c->stash->{detach_to} = '/report/new/oauth_callback'; + $c->stash->{detach_args} = [$token->token]; + + if ( $c->get_param('facebook_sign_in') ) { + $c->detach('/auth/facebook_sign_in'); + } elsif ( $c->get_param('twitter_sign_in') ) { + $c->detach('/auth/twitter_sign_in'); + } + } # Save or update the user if appropriate if ( $c->cobrand->never_confirm_reports ) { @@ -1039,15 +1091,10 @@ sub save_user_and_report : Private { $report->user->insert(); } $report->confirm(); + } elsif ( !$report->user->in_storage ) { # User does not exist. - # Store changes in token for when token is validated. - $c->stash->{token_data} = { - name => $report->user->name, - phone => $report->user->phone, - password => $report->user->password, - title => $report->user->title, - }; + $c->forward('tokenize_user', [ $report ]); $report->user->name( undef ); $report->user->phone( undef ); $report->user->password( '', 1 ); @@ -1064,35 +1111,11 @@ sub save_user_and_report : Private { } else { # User exists and we are not logged in as them. - # Store changes in token for when token is validated. - $c->stash->{token_data} = { - name => $report->user->name, - phone => $report->user->phone, - password => $report->user->password, - title => $report->user->title, - }; + $c->forward('tokenize_user', [ $report ]); $report->user->discard_changes(); $c->log->info($report->user->id . ' exists, but is not logged in for this report'); } - # If there was a photo add that too - if ( my $fileid = $c->stash->{upload_fileid} ) { - $report->photo($fileid); - } - - # Set a default if possible - $report->category( _('Other') ) unless $report->category; - - # Set unknown to DB unknown - $report->bodies_str( undef ) if $report->bodies_str eq '-1'; - - # if there is a Message Manager message ID, pass it back to the client view - if ($c->cobrand->moniker eq 'fixmybarangay' && $c->get_param('external_source_id') =~ /^\d+$/) { - $c->stash->{external_source_id} = $c->get_param('external_source_id'); - $report->external_source_id( $c->get_param('external_source_id') ); - $report->external_source( $c->config->{MESSAGE_MANAGER_URL} ) ; - } - # save the report; $report->in_storage ? $report->update : $report->insert(); @@ -1118,8 +1141,8 @@ sub generate_map : Private { my $longitude = $c->stash->{longitude}; # Don't do anything if the user skipped the map + $c->stash->{page} = 'new'; if ( $c->stash->{report}->used_map ) { - $c->stash->{page} = 'new'; FixMyStreet::Map::display_map( $c, latitude => $latitude, diff --git a/perllib/FixMyStreet/App/Controller/Report/Update.pm b/perllib/FixMyStreet/App/Controller/Report/Update.pm index 445723fec..8d6bc2019 100644 --- a/perllib/FixMyStreet/App/Controller/Report/Update.pm +++ b/perllib/FixMyStreet/App/Controller/Report/Update.pm @@ -20,12 +20,16 @@ Creates an update to a report sub report_update : Path : Args(0) { my ( $self, $c ) = @_; - $c->forward( '/report/load_problem_or_display_error', [ $c->get_param('id') ] ); + $c->forward('initialize_update'); + $c->forward('load_problem'); + $c->forward('check_form_submitted') + or $c->go( '/report/display', [ $c->stash->{problem}->id ] ); + $c->forward('process_update'); $c->forward('process_user'); $c->forward('/photo/process_photo'); $c->forward('check_for_errors') - or $c->go( '/report/display', [ $c->get_param('id') ] ); + or $c->go( '/report/display', [ $c->stash->{problem}->id ] ); $c->forward('save_update'); $c->forward('redirect_or_confirm_creation'); @@ -151,19 +155,46 @@ sub process_user : Private { return 1; } -=head2 process_update +=head2 oauth_callback -Take the submitted params and create a new update item. Does not save -anything to the database. +Called when we successfully login via OAuth. Stores the token so we can look up +what we have so far. -NB: relies on their being a problem and update_user in the stash. May -want to move adding these elsewhere +=cut + +sub oauth_callback : Private { + my ( $self, $c, $token_code ) = @_; + $c->stash->{oauth_update} = $token_code; + $c->detach('report_update'); +} + +=head2 initialize_update + +Create an initial update object, either empty or from stored OAuth data. =cut -sub process_update : Private { +sub initialize_update : Private { my ( $self, $c ) = @_; + my $update; + if ($c->stash->{oauth_update}) { + my $auth_token = $c->forward( '/tokens/load_auth_token', + [ $c->stash->{oauth_update}, 'update/social' ] ); + $update = $c->model("DB::Comment")->new($auth_token->data); + } + + if ($update) { + $c->stash->{upload_fileid} = $update->get_photoset->data; + } else { + $update = $c->model('DB::Comment')->new({ + state => 'unconfirmed', + cobrand => $c->cobrand->moniker, + cobrand_data => '', + lang => $c->stash->{lang_code}, + }); + } + if ( $c->get_param('first_name') && $c->get_param('last_name') ) { my $first_name = $c->get_param('first_name'); my $last_name = $c->get_param('last_name'); @@ -173,6 +204,48 @@ sub process_update : Private { $c->stash->{last_name} = $last_name; } + $c->stash->{update} = $update; +} + +=head2 load_problem + +Our update could be prefilled, or we could be submitting a form containing an +ID. Look up the relevant report either way. + +=cut + +sub load_problem : Private { + my ( $self, $c ) = @_; + + my $update = $c->stash->{update}; + # Problem ID could come from existing update in token, or from query parameter + my $problem_id = $update->problem_id || $c->get_param('id'); + $c->forward( '/report/load_problem_or_display_error', [ $problem_id ] ); + $update->problem($c->stash->{problem}); +} + +=head2 check_form_submitted + +This makes sure we only proceed to processing if we've had the form submitted +(we may have come here via an OAuth login, for example). + +=cut + +sub check_form_submitted : Private { + my ( $self, $c ) = @_; + return $c->get_param('submit_update') || ''; +} + +=head2 process_update + +Take the submitted params and updates our update item. Does not save +anything to the database. + +=cut + +sub process_update : Private { + my ( $self, $c ) = @_; + my %params = map { $_ => $c->get_param($_) } ( 'update', 'name', 'fixed', 'state', 'reopen' ); @@ -184,20 +257,12 @@ sub process_update : Private { $params{reopen} = 0 unless $c->user && $c->user->id == $c->stash->{problem}->user->id; - my $update = $c->model('DB::Comment')->new( - { - text => $params{update}, - name => $name, - problem => $c->stash->{problem}, - state => 'unconfirmed', - mark_fixed => $params{fixed} ? 1 : 0, - mark_open => $params{reopen} ? 1 : 0, - cobrand => $c->cobrand->moniker, - cobrand_data => '', - lang => $c->stash->{lang_code}, - anonymous => $anonymous, - } - ); + my $update = $c->stash->{update}; + $update->text($params{update}); + $update->name($name); + $update->mark_fixed($params{fixed} ? 1 : 0); + $update->mark_open($params{reopen} ? 1 : 0); + $update->anonymous($anonymous); if ( $params{state} ) { $params{state} = 'fixed - council' @@ -221,9 +286,9 @@ sub process_update : Private { my @extra; # Next function fills this, but we don't need it here. - # This is just so that the error checkign for these extra fields runs. + # This is just so that the error checking for these extra fields runs. # TODO Use extra here as it is used on reports. - $c->cobrand->process_extras( $c, $update->problem->bodies_str, \@extra ); + $c->cobrand->process_open311_extras( $c, $update->problem->bodies_str, \@extra ); if ( $c->get_param('fms_extra_title') ) { my %extras = (); @@ -241,7 +306,6 @@ sub process_update : Private { $c->log->debug( 'name is ' . $c->get_param('name') ); - $c->stash->{update} = $update; $c->stash->{add_alert} = $c->get_param('add_alert'); return 1; @@ -283,6 +347,13 @@ sub check_for_errors : Private { %{ $c->stash->{update}->check_for_errors }, ); + # if using social login then we don't care about name and email errors + $c->stash->{is_social_user} = $c->get_param('facebook_sign_in') || $c->get_param('twitter_sign_in'); + if ( $c->stash->{is_social_user} ) { + delete $field_errors{name}; + delete $field_errors{email}; + } + if ( my $photo_error = delete $c->stash->{photo_error} ) { $field_errors{photo} = $photo_error; } @@ -302,6 +373,17 @@ sub check_for_errors : Private { return; } +# Store changes in token for when token is validated. +sub tokenize_user : Private { + my ($self, $c, $update) = @_; + $c->stash->{token_data} = { + name => $update->user->name, + password => $update->user->password, + }; + $c->stash->{token_data}{facebook_id} = $c->session->{oauth}{facebook_id} + if $c->get_param('oauth_need_email') && $c->session->{oauth}{facebook_id}; +} + =head2 save_update Save the update and the user as appropriate. @@ -313,6 +395,27 @@ sub save_update : Private { my $update = $c->stash->{update}; + # If there was a photo add that too + if ( my $fileid = $c->stash->{upload_fileid} ) { + $update->photo($fileid); + } + + if ( $c->stash->{is_social_user} ) { + my $token = $c->model("DB::Token")->create( { + scope => 'update/social', + data => { $update->get_inflated_columns }, + } ); + + $c->stash->{detach_to} = '/report/update/oauth_callback'; + $c->stash->{detach_args} = [$token->token]; + + if ( $c->get_param('facebook_sign_in') ) { + $c->detach('/auth/facebook_sign_in'); + } elsif ( $c->get_param('twitter_sign_in') ) { + $c->detach('/auth/twitter_sign_in'); + } + } + if ( $c->cobrand->never_confirm_updates ) { if ( $update->user->in_storage() ) { $update->user->update(); @@ -322,11 +425,7 @@ sub save_update : Private { $update->confirm(); } elsif ( !$update->user->in_storage ) { # User does not exist. - # Store changes in token for when token is validated. - $c->stash->{token_data} = { - name => $update->user->name, - password => $update->user->password, - }; + $c->forward('tokenize_user', [ $update ]); $update->user->name( undef ); $update->user->password( '', 1 ); $update->user->insert; @@ -338,19 +437,10 @@ sub save_update : Private { $update->confirm; } else { # User exists and we are not logged in as them. - # Store changes in token for when token is validated. - $c->stash->{token_data} = { - name => $update->user->name, - password => $update->user->password, - }; + $c->forward('tokenize_user', [ $update ]); $update->user->discard_changes(); } - # If there was a photo add that too - if ( my $fileid = $c->stash->{upload_fileid} ) { - $update->photo($fileid); - } - if ( $update->in_storage ) { $update->update; } diff --git a/perllib/FixMyStreet/App/Controller/Reports.pm b/perllib/FixMyStreet/App/Controller/Reports.pm index 407fc625e..027b0d5a4 100644 --- a/perllib/FixMyStreet/App/Controller/Reports.pm +++ b/perllib/FixMyStreet/App/Controller/Reports.pm @@ -3,6 +3,7 @@ use Moose; use namespace::autoclean; use File::Slurp; +use JSON::MaybeXS; use List::MoreUtils qw(any); use POSIX qw(strcoll); use RABX; @@ -68,7 +69,7 @@ sub index : Path : Args(0) { my $data = File::Slurp::read_file( FixMyStreet->path_to( '../data/all-reports.json' )->stringify ); - my $j = JSON->new->utf8->decode($data); + my $j = decode_json($data); $c->stash->{fixed} = $j->{fixed}; $c->stash->{open} = $j->{open}; }; diff --git a/perllib/FixMyStreet/App/Controller/Rss.pm b/perllib/FixMyStreet/App/Controller/Rss.pm index b817fe326..6047f063b 100755 --- a/perllib/FixMyStreet/App/Controller/Rss.pm +++ b/perllib/FixMyStreet/App/Controller/Rss.pm @@ -218,7 +218,7 @@ sub query_main : Private { . ($alert_type->head_table ? $alert_type->head_table . '_id=? and ' : '') . $alert_type->item_where . ' order by ' . $alert_type->item_order; - my $rss_limit = mySociety::Config::get('RSS_LIMIT'); + my $rss_limit = FixMyStreet->config('RSS_LIMIT'); $query .= " limit $rss_limit" unless $c->stash->{type} =~ /^all/; my $q = $c->model('DB::Alert')->result_source->storage->dbh->prepare($query); @@ -260,9 +260,9 @@ sub add_row : Private { $row->{confirmed} =~ s/^(\d+)/ordinal($1)/e if $c->stash->{lang_code} eq 'en-gb'; } - (my $title = _($alert_type->item_title)) =~ s/{{(.*?)}}/$row->{$1}/g; - (my $link = $alert_type->item_link) =~ s/{{(.*?)}}/$row->{$1}/g; - (my $desc = _($alert_type->item_description)) =~ s/{{(.*?)}}/$row->{$1}/g; + (my $title = _($alert_type->item_title)) =~ s/\{\{(.*?)}}/$row->{$1}/g; + (my $link = $alert_type->item_link) =~ s/\{\{(.*?)}}/$row->{$1}/g; + (my $desc = _($alert_type->item_description)) =~ s/\{\{(.*?)}}/$row->{$1}/g; my $base_url = $c->cobrand->base_url_for_report($row); my $url = $base_url . $link; @@ -317,9 +317,9 @@ sub add_parameters : Private { $row->{$_} = $c->stash->{title_params}->{$_}; } - (my $title = _($alert_type->head_title)) =~ s/{{(.*?)}}/$row->{$1}/g; - (my $link = $alert_type->head_link) =~ s/{{(.*?)}}/$row->{$1}/g; - (my $desc = _($alert_type->head_description)) =~ s/{{(.*?)}}/$row->{$1}/g; + (my $title = _($alert_type->head_title)) =~ s/\{\{(.*?)}}/$row->{$1}/g; + (my $link = $alert_type->head_link) =~ s/\{\{(.*?)}}/$row->{$1}/g; + (my $desc = _($alert_type->head_description)) =~ s/\{\{(.*?)}}/$row->{$1}/g; $c->stash->{rss}->channel( title => ent($title), diff --git a/perllib/FixMyStreet/App/Controller/Static.pm b/perllib/FixMyStreet/App/Controller/Static.pm index d91a07fea..5054809c7 100755 --- a/perllib/FixMyStreet/App/Controller/Static.pm +++ b/perllib/FixMyStreet/App/Controller/Static.pm @@ -10,56 +10,23 @@ FixMyStreet::App::Controller::Static - Catalyst Controller =head1 DESCRIPTION -Static pages Catalyst Controller. FAQ does some smarts to choose the correct -template depending on language, will need extending at some point. +Old static pages Catalyst Controller. =head1 METHODS =cut -sub about : Global : Args(0) { - my ( $self, $c ) = @_; - - my $lang_code = $c->stash->{lang_code}; - my $template = "static/about-$lang_code.html"; - $c->stash->{template} = $template; -} - -sub privacy : Global : Args(0) { +sub about_redirect : Private { my ( $self, $c ) = @_; + $c->res->redirect( $c->uri_for_action('/about/page', [ $c->action->name ] )); } -sub faq : Global : Args(0) { - my ( $self, $c ) = @_; - - # There should be a faq template for each language in a cobrand or default. - # This is because putting the FAQ translations into the PO files is - # overkill. - - # We rely on the list of languages for the site being restricted so that there - # will be a faq template for that language/cobrand combo. - - my $lang_code = $c->stash->{lang_code}; - my $template = "faq/faq-$lang_code.html"; - $c->stash->{template} = $template; -} - -sub fun : Global : Args(0) { - my ( $self, $c ) = @_; - # don't need to do anything here - should just pass through. -} - -sub posters : Global : Args(0) { - my ( $self, $c ) = @_; -} - -sub iphone : Global : Args(0) { - my ( $self, $c ) = @_; -} - -sub council : Global : Args(0) { - my ( $self, $c ) = @_; -} +sub faq : Global : Args(0) { $_[1]->forward('/about/page', ['faq']) } +sub privacy : Global : Args(0) { $_[1]->detach('about_redirect') } +sub fun : Global : Args(0) { $_[1]->detach('about_redirect') } +sub posters : Global : Args(0) { $_[1]->detach('about_redirect') } +sub iphone : Global : Args(0) { $_[1]->detach('about_redirect') } +sub council : Global : Args(0) { $_[1]->detach('about_redirect') } sub unresponsive : Global : Args(0) { my ( $self, $c ) = @_; diff --git a/perllib/FixMyStreet/App/Controller/Status.pm b/perllib/FixMyStreet/App/Controller/Status.pm index 907fe5456..931c7bd47 100755 --- a/perllib/FixMyStreet/App/Controller/Status.pm +++ b/perllib/FixMyStreet/App/Controller/Status.pm @@ -3,7 +3,7 @@ use Moose; use namespace::autoclean; use HTTP::Negotiate; -use JSON; +use JSON::MaybeXS; BEGIN { extends 'Catalyst::Controller'; } @@ -27,6 +27,9 @@ sub index_json : Path('/status.json') : Args(0) { sub index : Path : Args(0) { my ($self, $c, $format) = @_; + # Workaround that the admin summary page is only displayed to Zurich + # superusers. It doesn't have anything sensitive + $c->stash->{admin_type} = 'super'; # Fetch summary stats from admin front page $c->forward('/admin/index'); diff --git a/perllib/FixMyStreet/App/Controller/Tokens.pm b/perllib/FixMyStreet/App/Controller/Tokens.pm index ba15162ce..eb35fd152 100644 --- a/perllib/FixMyStreet/App/Controller/Tokens.pm +++ b/perllib/FixMyStreet/App/Controller/Tokens.pm @@ -34,6 +34,7 @@ sub confirm_problem : Path('/P') { title => 'Title of Report', bodies_str => 'True', url => '/report/123', + service => $c->get_param('service'), }; return; } @@ -104,6 +105,7 @@ sub confirm_problem : Path('/P') { $problem->user->phone( $data->{phone} ) if $data->{phone}; $problem->user->password( $data->{password}, 1 ) if $data->{password}; $problem->user->title( $data->{title} ) if $data->{title}; + $problem->user->facebook_id( $data->{facebook_id} ) if $data->{facebook_id}; $problem->user->update; } $c->authenticate( { email => $problem->user->email }, 'no_password' ); @@ -229,6 +231,7 @@ sub confirm_update : Path('/C') { if ( $data->{name} || $data->{password} ) { $comment->user->name( $data->{name} ) if $data->{name}; $comment->user->password( $data->{password}, 1 ) if $data->{password}; + $comment->user->facebook_id( $data->{facebook_id} ) if $data->{facebook_id}; $comment->user->update; } @@ -323,11 +326,7 @@ sub load_auth_token : Private { } ); - unless ( $token ) { - $c->stash->{template} = 'errors/generic.html'; - $c->stash->{message} = _("I'm afraid we couldn't validate that token. If you've copied the URL from an email, please check that you copied it exactly.\n"); - $c->detach; - } + $c->detach('token_too_old') unless $token; return $token; } diff --git a/perllib/FixMyStreet/App/Model/DB.pm b/perllib/FixMyStreet/App/Model/DB.pm index f9e43172f..ac1f98dc9 100644 --- a/perllib/FixMyStreet/App/Model/DB.pm +++ b/perllib/FixMyStreet/App/Model/DB.pm @@ -8,7 +8,7 @@ use FixMyStreet; __PACKAGE__->config( schema_class => 'FixMyStreet::DB', - connect_info => FixMyStreet->dbic_connect_info, + connect_info => sub { FixMyStreet::DB->storage->dbh }, ); =head1 NAME diff --git a/perllib/FixMyStreet/App/Model/EmailSend.pm b/perllib/FixMyStreet/App/Model/EmailSend.pm index 475026267..93751d4a6 100644 --- a/perllib/FixMyStreet/App/Model/EmailSend.pm +++ b/perllib/FixMyStreet/App/Model/EmailSend.pm @@ -4,67 +4,16 @@ use base 'Catalyst::Model::Factory'; use strict; use warnings; -use FixMyStreet; -use Email::Send; - =head1 NAME FixMyStreet::App::Model::EmailSend =head1 DESCRIPTION -Thin wrapper around Email::Send - configuring it correctly acording to our config. - -If the config value 'SMTP_SMARTHOST' is set then email is routed via SMTP to -that. Otherwise it is sent using a 'sendmail' like binary on the local system. - -And finally if if FixMyStreet->test_mode returns true then emails are not sent -at all but are stored in memory for the test suite to inspect (using -Email::Send::Test). +Catalyst Model wrapper around FixMyStreet::EmailSend =cut -my $args = undef; - -if ( FixMyStreet->test_mode ) { - - # Email::Send::Test - $args = { mailer => 'Test', }; -} -elsif ( my $smtp_host = FixMyStreet->config('SMTP_SMARTHOST') ) { - - # Email::Send::SMTP - my $type = FixMyStreet->config('SMTP_TYPE') || ''; - my $port = FixMyStreet->config('SMTP_PORT') || ''; - my $username = FixMyStreet->config('SMTP_USERNAME') || ''; - my $password = FixMyStreet->config('SMTP_PASSWORD') || ''; - - unless ($port) { - $port = 25; - $port = 465 if $type eq 'ssl'; - $port = 587 if $type eq 'tls'; - } - - my $mailer_args = [ - Host => $smtp_host, - Port => $port, - ]; - push @$mailer_args, ssl => 1 if $type eq 'ssl'; - push @$mailer_args, tls => 1 if $type eq 'tls'; - push @$mailer_args, username => $username, password => $password - if $username && $password; - $args = { - mailer => 'FixMyStreet::EmailSend::DoNotReply', - mailer_args => $mailer_args, - }; -} -else { - - # Email::Send::Sendmail - $args = { mailer => 'Sendmail' }; -} - __PACKAGE__->config( - class => 'Email::Send', - args => $args, + class => 'FixMyStreet::EmailSend', ); diff --git a/perllib/FixMyStreet/App/Model/PhotoSet.pm b/perllib/FixMyStreet/App/Model/PhotoSet.pm index b18460821..54457bae9 100644 --- a/perllib/FixMyStreet/App/Model/PhotoSet.pm +++ b/perllib/FixMyStreet/App/Model/PhotoSet.pm @@ -14,27 +14,32 @@ has c => ( is => 'ro', ); +# The attached report, for using its ID has object => ( is => 'ro', ); -has data => ( # generic data from DB field +# If a PhotoSet is generated from a database row, db_data is set, which then +# fills data_items -> ids -> data. If it is generated during creation, +# data_items is set, which then similarly fills ids -> data. + +has db_data => ( # generic data from DB field + is => 'ro', +); + +has data => ( # String of photo hashes is => 'ro', lazy => 1, default => sub { - # yes, this is a little circular: data -> data_items -> items -> data - # e.g. if not provided, then we're presumably uploading/etc., so calculate from - # the stored cached fileids - # (obviously if you provide none of these, then you'll get an infinite loop) my $self = shift; - my $data = join ',', map { $_->[0] } $self->all_images; + my $data = join ',', $self->all_ids; return $data; } ); -has data_items => ( # either a) split from data or b) provided by photo upload +has data_items => ( # either a) split from db_data or b) provided by photo upload isa => 'ArrayRef', - is => 'rw', + is => 'ro', traits => ['Array'], lazy => 1, handles => { @@ -42,8 +47,7 @@ has data_items => ( # either a) split from data or b) provided by photo upload }, default => sub { my $self = shift; - my $data = $self->data - or return []; + my $data = $self->db_data or return []; return [$data] if (_jpeg_magic($data)); @@ -56,7 +60,7 @@ has upload_dir => ( lazy => 1, default => sub { my $self = shift; - my $cache_dir = path( $self->c->config->{UPLOAD_DIR} ); + my $cache_dir = path( FixMyStreet->config('UPLOAD_DIR') ); $cache_dir->mkpath; unless ( -d $cache_dir && -w $cache_dir ) { warn "Can't find/write to photo cache directory '$cache_dir'"; @@ -72,33 +76,29 @@ sub _jpeg_magic { # and \x{49}\x{49} (Tiff, 3 results in live DB) ? } -=head2 C<images>, C<num_images>, C<get_raw_image_data>, C<all_images> +=head2 C<ids>, C<num_images>, C<get_id>, C<all_ids> -C<$photoset-E<GT>images> is an AoA containing the filed and the binary image data. +C<$photoset-E<GT>ids> is an arrayref containing the fileid data. - [ - [ $fileid1, $binary_data ], - [ $fileid2, $binary_data ], - ... - ] + [ $fileid1, $fileid2, ... ] Various accessors are provided onto it: num_images: count - get_raw_image_data ($index): return the [$fileid, $binary_data] tuple - all_images: return AoA as an array (e.g. rather than arrayref) + get_id ($index): return the correct id + all_ids: array of elements, rather than arrayref =cut -has images => ( # AoA of [$fileid, $binary_data] tuples +has ids => ( # Arrayref of $fileid tuples (always, so post upload/raw data processing) isa => 'ArrayRef', - is => 'rw', + is => 'ro', traits => ['Array'], lazy => 1, handles => { num_images => 'count', - get_raw_image_data => 'get', - all_images => 'elements', + get_id => 'get', + all_ids => 'elements', }, default => sub { my $self = shift; @@ -159,7 +159,7 @@ has images => ( # AoA of [$fileid, $binary_data] tuples my $fileid = $self->get_fileid($photo_blob); my $file = $self->get_file($fileid); $upload->copy_to( $file ); - return [$fileid, $photo_blob]; + return $fileid; } if (_jpeg_magic($part)) { @@ -167,21 +167,18 @@ has images => ( # AoA of [$fileid, $binary_data] tuples my $fileid = $self->get_fileid($photo_blob); my $file = $self->get_file($fileid); $file->spew_raw($photo_blob); - return [$fileid, $photo_blob]; + return $fileid; } if (length($part) == 40) { my $fileid = $part; my $file = $self->get_file($fileid); if ($file->exists) { - my $photo = $file->slurp_raw; - [$fileid, $photo]; - } - else { + $fileid; + } else { warn "File $fileid doesn't exist"; (); } - } - else { + } else { warn sprintf "Received bad photo hash of length %d", length($part); (); } @@ -201,15 +198,23 @@ sub get_file { return path( $cache_dir, "$fileid.jpeg" ); } +sub get_raw_image_data { + my ($self, $index) = @_; + my $fileid = $self->get_id($index); + my $file = $self->get_file($fileid); + if ($file->exists) { + my $photo = $file->slurp_raw; + return $photo; + } +} + sub get_image_data { my ($self, %args) = @_; my $num = $args{num} || 0; - my $data = $self->get_raw_image_data( $num ) + my $photo = $self->get_raw_image_data( $num ) or return; - my ($fileid, $photo) = @$data; - my $size = $args{size}; if ( $size eq 'tn' ) { $photo = _shrink( $photo, 'x100' ); @@ -218,7 +223,7 @@ sub get_image_data { } elsif ( $size eq 'full' ) { # do nothing } else { - $photo = _shrink( $photo, $self->c->cobrand->default_photo_resize || '250x250' ); + $photo = _shrink( $photo, $args{default} || '250x250' ); } return $photo; @@ -235,18 +240,37 @@ sub delete_cached { ); } +sub remove_images { + my ($self, $ids) = @_; + + my @images = $self->all_ids; + my $dec = 0; + for (sort { $a <=> $b } @$ids) { + splice(@images, $_ + $dec, 1); + --$dec; + } + + my $new_set = (ref $self)->new({ + data_items => \@images, + object => $self->object, + }); + + $self->delete_cached(); + + return $new_set->data; # e.g. new comma-separated fileid +} + sub rotate_image { my ($self, $index, $direction) = @_; - my @images = $self->all_images; + my @images = $self->all_ids; return if $index > $#images; - my @items = map $_->[0], @images; - $items[$index] = _rotate_image( $images[$index][1], $direction ); + my $image_data = $self->get_raw_image_data($index); + $images[$index] = _rotate_image( $image_data, $direction ); my $new_set = (ref $self)->new({ - data_items => \@items, - c => $self->c, + data_items => \@images, object => $self->object, }); @@ -268,11 +292,6 @@ sub _rotate_image { } - - - -# NB: These 2 subs stolen from A::C::Photo, should be purged from there! -# # Shrinks a picture to the specified size, but keeping in proportion. sub _shrink { my ($photo, $size) = @_; diff --git a/perllib/FixMyStreet/Cobrand.pm b/perllib/FixMyStreet/Cobrand.pm index ff7d7f943..9f61635d8 100644 --- a/perllib/FixMyStreet/Cobrand.pm +++ b/perllib/FixMyStreet/Cobrand.pm @@ -8,7 +8,7 @@ use warnings; use FixMyStreet; use Carp; -use Moose; +use Package::Stash; use Module::Pluggable sub_name => '_cobrands', @@ -77,11 +77,12 @@ sub available_cobrand_classes { sub class { my $avail = shift; return $avail->{class} if $avail->{class}; - my $moniker = $avail->{moniker}; - Class::MOP::Class->create("FixMyStreet::Cobrand::$moniker" => ( - superclasses => [ 'FixMyStreet::Cobrand::Default' ], - )); - return "FixMyStreet::Cobrand::$moniker"; + my $moniker = "FixMyStreet::Cobrand::$avail->{moniker}"; + my $class = bless {}, $moniker; + my $stash = Package::Stash->new($moniker); + my $isa = $stash->get_or_add_symbol('@ISA'); + @{$isa} = ('FixMyStreet::Cobrand::Default'); + return $moniker; } =head2 get_class_for_host diff --git a/perllib/FixMyStreet/Cobrand/Bromley.pm b/perllib/FixMyStreet/Cobrand/Bromley.pm index 687843a2a..c9f9a98be 100644 --- a/perllib/FixMyStreet/Cobrand/Bromley.pm +++ b/perllib/FixMyStreet/Cobrand/Bromley.pm @@ -81,9 +81,9 @@ sub ask_ever_reported { return 0; } -sub process_extras { +sub process_open311_extras { my $self = shift; - $self->SUPER::process_extras( @_, [ 'first_name', 'last_name' ] ); + $self->SUPER::process_open311_extras( @_, [ 'first_name', 'last_name' ] ); } sub contact_email { diff --git a/perllib/FixMyStreet/Cobrand/Default.pm b/perllib/FixMyStreet/Cobrand/Default.pm index 9541f2601..b3b830f06 100644 --- a/perllib/FixMyStreet/Cobrand/Default.pm +++ b/perllib/FixMyStreet/Cobrand/Default.pm @@ -47,24 +47,45 @@ sub country { =head1 problems -Returns a ResultSet of Problems, restricted to a subset if we're on a cobrand -that only wants some of the data. +Returns a ResultSet of Problems, potentially restricted to a subset if we're on +a cobrand that only wants some of the data. =cut sub problems { my $self = shift; - return $self->{c}->model('DB::Problem'); + return $self->problems_restriction($self->{c}->model('DB::Problem')); } -=head1 body_restriction +=head1 updates -Return an extra query parameter to restrict reports to those sent to a -particular body. +Returns a ResultSet of Comments, potentially restricted to a subset if we're on +a cobrand that only wants some of the data. =cut -sub body_restriction {} +sub updates { + my $self = shift; + return $self->updates_restriction($self->{c}->model('DB::Comment')); +} + +=head1 problems_restriction/updates_restriction + +Used to restricts reports and updates in a cobrand in a particular way. Do +nothing by default. + +=cut + +sub problems_restriction { + my ($self, $rs) = @_; + return $rs; +} + +sub updates_restriction { + my ($self, $rs) = @_; + return $rs; +} + sub site_key { return 0; } @@ -682,7 +703,7 @@ sub get_body_sender { if ( $body->can_be_devolved ) { # look up via category - my $config = FixMyStreet::App->model("DB::Contact")->search( { body_id => $body->id, category => $category } )->first; + my $config = $body->result_source->schema->resultset("Contact")->search( { body_id => $body->id, category => $category } )->first; if ( $config->send_method ) { return { method => $config->send_method, config => $config }; } else { @@ -733,7 +754,18 @@ If set to an arrayref, will plot those area ID(s) from mapit on all the /around sub areas_on_around { []; } -sub process_extras {} +=head2 + +A list of extra fields we wish to save to the database in the 'extra' column of +problems based on variables passed in by the form. Return a list of hashrefs +of values we wish to save, e.g. +( { name => 'address', required => 1 }, { name => 'passport', required => 0 } ) + +=cut + +sub report_form_extras {} + +sub process_open311_extras {} =head 2 pin_colour diff --git a/perllib/FixMyStreet/Cobrand/EmptyHomes.pm b/perllib/FixMyStreet/Cobrand/EmptyHomes.pm index 0b02f90c4..995c39c85 100644 --- a/perllib/FixMyStreet/Cobrand/EmptyHomes.pm +++ b/perllib/FixMyStreet/Cobrand/EmptyHomes.pm @@ -132,16 +132,8 @@ sub council_rss_alert_options { return ( \@options, @reported_to_options ? \@reported_to_options : undef ); } -sub process_extras { - my $self = shift; - my $ctx = shift; - my $body_id = shift; - my $extra = shift; - - my $value = $ctx->get_param('address') || ''; - $ctx->stash->{field_errors}->{address} = _('This information is required') - unless $value; - $extra->{address} = $value; +sub report_form_extras { + ( { name => 'address', required => 1 } ) } sub front_stats_data { diff --git a/perllib/FixMyStreet/Cobrand/FixMyStreet.pm b/perllib/FixMyStreet/Cobrand/FixMyStreet.pm index f21d38ff8..159f2f5db 100644 --- a/perllib/FixMyStreet/Cobrand/FixMyStreet.pm +++ b/perllib/FixMyStreet/Cobrand/FixMyStreet.pm @@ -59,5 +59,26 @@ sub get_country_for_ip_address { return mySociety::Gaze::get_country_from_ip($ip); } +sub report_form_extras { + ( { name => 'gender', required => 0 }, { name => 'variant', required => 0 } ) +} + +sub ask_gender_question { + my $self = shift; + + return 1 unless $self->{c}->user; + + my $reports = $self->{c}->model('DB::Problem')->search({ + user_id => $self->{c}->user->id, + extra => { like => '%gender%' } + }, { order_by => { -desc => 'id' } }); + + while (my $report = $reports->next) { + my $gender = $report->get_extra_metadata('gender'); + return 0 if $gender =~ /female|male|other|unknown/; + } + return 1; +} + 1; diff --git a/perllib/FixMyStreet/Cobrand/SeeSomething.pm b/perllib/FixMyStreet/Cobrand/SeeSomething.pm index 9ae15fd8d..22750aafa 100644 --- a/perllib/FixMyStreet/Cobrand/SeeSomething.pm +++ b/perllib/FixMyStreet/Cobrand/SeeSomething.pm @@ -10,11 +10,6 @@ sub council_name { return 'See Something Say Something'; } sub council_url { return 'seesomething'; } sub area_types { [ 'MTD' ] } -sub body_restriction { - my $self = shift; - return $self->council_id; -} - sub area_check { my ( $self, $params, $context ) = @_; diff --git a/perllib/FixMyStreet/Cobrand/UK.pm b/perllib/FixMyStreet/Cobrand/UK.pm index 2473f386f..c33b39aac 100644 --- a/perllib/FixMyStreet/Cobrand/UK.pm +++ b/perllib/FixMyStreet/Cobrand/UK.pm @@ -1,6 +1,7 @@ package FixMyStreet::Cobrand::UK; use base 'FixMyStreet::Cobrand::Default'; +use JSON::MaybeXS; use mySociety::MaPit; use mySociety::VotingArea; @@ -26,7 +27,7 @@ sub disambiguate_location { }; } -sub process_extras { +sub process_open311_extras { my $self = shift; my $ctx = shift; my $body_id = shift; diff --git a/perllib/FixMyStreet/Cobrand/UKCouncils.pm b/perllib/FixMyStreet/Cobrand/UKCouncils.pm index 074da0915..b4b91b7dd 100644 --- a/perllib/FixMyStreet/Cobrand/UKCouncils.pm +++ b/perllib/FixMyStreet/Cobrand/UKCouncils.pm @@ -22,11 +22,6 @@ sub path_to_web_templates { ]; } -sub body_restriction { - my $self = shift; - return $self->council_id; -} - sub site_key { my $self = shift; return $self->council_url; @@ -36,14 +31,19 @@ sub restriction { return { cobrand => shift->moniker }; } -sub problems { - my $self = shift; - return $self->{c}->model('DB::Problem')->to_body($self->council_id); +sub problems_restriction { + my ($self, $rs) = @_; + return $rs->to_body($self->council_id); +} + +sub updates_restriction { + my ($self, $rs) = @_; + return $rs->to_body($self->council_id); } sub base_url { my $self = shift; - my $base_url = mySociety::Config::get('BASE_URL'); + my $base_url = FixMyStreet->config('BASE_URL'); my $u = $self->council_url; if ( $base_url !~ /$u/ ) { # council cobrands are not https so transform to http as well @@ -113,7 +113,7 @@ sub owns_problem { if (ref $report eq 'HASH') { return unless $report->{bodies_str}; @bodies = split /,/, $report->{bodies_str}; - @bodies = FixMyStreet::App->model('DB::Body')->search({ id => \@bodies })->all; + @bodies = FixMyStreet::DB->resultset('Body')->search({ id => \@bodies })->all; } else { # Object @bodies = values %{$report->bodies}; } diff --git a/perllib/FixMyStreet/Cobrand/Zurich.pm b/perllib/FixMyStreet/Cobrand/Zurich.pm index 6da3e566c..c7e3c4d45 100644 --- a/perllib/FixMyStreet/Cobrand/Zurich.pm +++ b/perllib/FixMyStreet/Cobrand/Zurich.pm @@ -850,7 +850,7 @@ sub admin_report_edit { # Add new update from status_update if (my $update = $c->get_param('status_update')) { - FixMyStreet::App->model('DB::Comment')->create( { + $c->model('DB::Comment')->create( { text => $update, user => $c->user->obj, state => 'unconfirmed', @@ -1010,27 +1010,25 @@ sub _admin_send_email { } sub munge_sendreport_params { - my ($self, $c, $row, $h, $params) = @_; + my ($self, $row, $h, $params) = @_; if ($row->state =~ /^(closed|investigating)$/ && $row->get_extra_metadata('publish_photo')) { # we attach images to reports sent to external bodies - my $photoset = $row->get_photoset($c); - my @images = $photoset->all_images + my $photoset = $row->get_photoset(); + my $num = $photoset->num_images or return; - my $index = 0; my $id = $row->id; my @attachments = map { - my $i = $index++; { - body => $_->[1], + body => $photoset->get_raw_image_data($_), attributes => { - filename => "$id.$i.jpeg", + filename => "$id.$_.jpeg", content_type => 'image/jpeg', encoding => 'base64', # quoted-printable ends up with newlines corrupting binary data - name => "$id.$i.jpeg", + name => "$id.$_.jpeg", }, } - } @images; + } (0..$num-1); $params->{attachments} = \@attachments; } } @@ -1117,9 +1115,41 @@ sub admin_stats { ] } ); - my $body = "Report ID,Created,Sent to Agency,Last Updated,E,N,Category,Status,UserID,External Body,Time Spent,Title,Detail,Media URL,Interface Used,Council Response\n"; + my @fields = ( + 'Report ID', + 'Created', + 'Sent to Agency', + 'Last Updated', + 'E', + 'N', + 'Category', + 'Status', + 'Closure Status', + 'UserID', + 'External Body', + 'Time Spent', + 'Title', + 'Detail', + 'Media URL', + 'Interface Used', + 'Council Response', + 'Strasse', + 'Mast-Nr.', + 'Haus-Nr.', + 'Hydranten-Nr.', + ); + + my $body = ""; require Text::CSV; my $csv = Text::CSV->new({ binary => 1 }); + + if ($csv->combine(@fields)) { + $body .= $csv->string . "\n"; + } + else { + $body .= sprintf "{{error emitting CSV line: %s}}\n", $csv->error_diag; + } + while ( my $report = $problems->next ) { my $external_body; my $body_name = ""; @@ -1129,13 +1159,18 @@ sub admin_stats { my $detail = $report->detail; my $public_response = $report->get_extra_metadata('public_response') || ''; + my $metas = $report->get_extra_fields(); + my %extras; + foreach my $field (@$metas) { + $extras{$field->{name}} = $field->{value}; + } # replace newlines with HTML <br/> element $detail =~ s{\r?\n}{ <br/> }g; $public_response =~ s{\r?\n}{ <br/> }g if $public_response; # Assemble photo URL, if report has a photo - my $media_url = $report->get_photo_params->{url} ? ($c->cobrand->base_url . $report->get_photo_params->{url}) : ''; + my $media_url = @{$report->photos} ? ($c->cobrand->base_url . $report->photos->[0]->{url}) : ''; my @columns = ( $report->id, @@ -1143,7 +1178,9 @@ sub admin_stats { $report->whensent, $report->lastupdate, $report->local_coords, $report->category, - $report->state, $report->user_id, + $report->state, + $report->get_extra_metadata('closure_status') || '', + $report->user_id, $body_name, $report->get_column('sum_time_spent') || 0, $report->title, @@ -1151,6 +1188,10 @@ sub admin_stats { $media_url, $report->service || 'Web interface', $public_response, + $extras{'strasse'} || '', + $extras{'mast_nr'} || '', + $extras{'haus_nr'} || '', + $extras{'hydranten_nr'} || '' ); if ($csv->combine(@columns)) { $body .= $csv->string . "\n"; diff --git a/perllib/FixMyStreet/DB.pm b/perllib/FixMyStreet/DB.pm index a1767abe9..d920c809f 100644 --- a/perllib/FixMyStreet/DB.pm +++ b/perllib/FixMyStreet/DB.pm @@ -15,5 +15,8 @@ __PACKAGE__->load_namespaces; # Created by DBIx::Class::Schema::Loader v0.07017 @ 2012-03-08 17:19:55 # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:CjFpUvon7KggFM7OF7VK/w -# You can replace this text with custom code or comments, and it will be preserved on regeneration +use FixMyStreet; + +__PACKAGE__->connection(FixMyStreet->dbic_connect_info); + 1; diff --git a/perllib/FixMyStreet/DB/Result/Alert.pm b/perllib/FixMyStreet/DB/Result/Alert.pm index 35cce8368..2a52a7bca 100644 --- a/perllib/FixMyStreet/DB/Result/Alert.pm +++ b/perllib/FixMyStreet/DB/Result/Alert.pm @@ -70,7 +70,7 @@ __PACKAGE__->belongs_to( # You can replace this text with custom code or comments, and it will be preserved on regeneration -use Moose; +use Moo; use namespace::clean -except => [ 'meta' ]; with 'FixMyStreet::Roles::Abuser'; @@ -113,7 +113,4 @@ sub disable { return 1; } -# need the inline_constuctor bit as we don't inherit from Moose -__PACKAGE__->meta->make_immutable( inline_constructor => 0 ); - 1; diff --git a/perllib/FixMyStreet/DB/Result/Comment.pm b/perllib/FixMyStreet/DB/Result/Comment.pm index 0caaa8968..5734ff8d5 100644 --- a/perllib/FixMyStreet/DB/Result/Comment.pm +++ b/perllib/FixMyStreet/DB/Result/Comment.pm @@ -96,8 +96,7 @@ __PACKAGE__->belongs_to( __PACKAGE__->load_components("+FixMyStreet::DB::RABXColumn"); __PACKAGE__->rabx_column('extra'); -use Image::Size; -use Moose; +use Moo; use namespace::clean -except => [ 'meta' ]; with 'FixMyStreet::Roles::Abuser'; @@ -148,15 +147,44 @@ sub confirm { $self->confirmed( \'current_timestamp' ); } -=head2 get_photo_params +=head2 get_photoset -Returns a hashref of details of any attached photo for use in templates. +Return a PhotoSet object for all photos attached to this field + + my $photoset = $obj->get_photoset; + print $photoset->num_images; + return $photoset->get_image_data(num => 0, size => 'full'); =cut -sub get_photo_params { +sub get_photoset { + my ($self) = @_; + my $class = 'FixMyStreet::App::Model::PhotoSet'; + eval "use $class"; + return $class->new({ + db_data => $self->photo, + object => $self, + }); +} + +sub photos { my $self = shift; - return FixMyStreet::App::get_photo_params($self, 'c'); + my $photoset = $self->get_photoset; + my $i = 0; + my $id = $self->id; + my @photos = map { + my $format = 'jpeg'; + my $cachebust = substr($_, 0, 8); + { + id => $_, + url_temp => "/photo/$_.temp.$format", + url_temp_full => "/photo/$_.fulltemp.$format", + url => "/photo/c/$id.$i.$format?$cachebust", + url_full => "/photo/c/$id.$i.full.$format?$cachebust", + idx => $i++, + } + } $photoset->all_ids; + return \@photos; } =head2 meta_problem_state @@ -214,7 +242,4 @@ __PACKAGE__->might_have( { cascade_copy => 0, cascade_delete => 1 }, ); -# we need the inline_constructor bit as we don't inherit from Moose -__PACKAGE__->meta->make_immutable( inline_constructor => 0 ); - 1; diff --git a/perllib/FixMyStreet/DB/Result/Contact.pm b/perllib/FixMyStreet/DB/Result/Contact.pm index 2fbb0716d..dab5432c6 100644 --- a/perllib/FixMyStreet/DB/Result/Contact.pm +++ b/perllib/FixMyStreet/DB/Result/Contact.pm @@ -63,12 +63,9 @@ __PACKAGE__->belongs_to( __PACKAGE__->load_components("+FixMyStreet::DB::RABXColumn"); __PACKAGE__->rabx_column('extra'); -use Moose; +use Moo; use namespace::clean -except => [ 'meta' ]; with 'FixMyStreet::Roles::Extra'; -# we need the inline_constructor bit as we don't inherit from Moose -__PACKAGE__->meta->make_immutable( inline_constructor => 0 ); - 1; diff --git a/perllib/FixMyStreet/DB/Result/Nearby.pm b/perllib/FixMyStreet/DB/Result/Nearby.pm index d3d228788..adeba703a 100644 --- a/perllib/FixMyStreet/DB/Result/Nearby.pm +++ b/perllib/FixMyStreet/DB/Result/Nearby.pm @@ -6,7 +6,7 @@ use strict; use warnings; use base 'DBIx::Class::Core'; -use Moose; +use Moo; use namespace::clean -except => [ 'meta' ]; __PACKAGE__->table( 'NONE' ); @@ -27,7 +27,4 @@ __PACKAGE__->belongs_to( __PACKAGE__->result_source_instance ->name( \'problem_find_nearby(?,?,?)' ); -# we need the inline_constructor bit as we don't inherit from Moose -__PACKAGE__->meta->make_immutable( inline_constructor => 0 ); - 1; diff --git a/perllib/FixMyStreet/DB/Result/Problem.pm b/perllib/FixMyStreet/DB/Result/Problem.pm index 3b7f8bcfd..2a90d0bec 100644 --- a/perllib/FixMyStreet/DB/Result/Problem.pm +++ b/perllib/FixMyStreet/DB/Result/Problem.pm @@ -157,8 +157,7 @@ __PACKAGE__->load_components("+FixMyStreet::DB::RABXColumn"); __PACKAGE__->rabx_column('extra'); __PACKAGE__->rabx_column('geocode'); -use Image::Size; -use Moose; +use Moo; use namespace::clean -except => [ 'meta' ]; use Utils; @@ -363,7 +362,8 @@ around lastupdate => $stz; around service => sub { my ( $orig, $self ) = ( shift, shift ); - my $s = $self->$orig(@_); + # service might be undef if e.g. unsaved code report + my $s = $self->$orig(@_) || ""; $s =~ s/_/ /g; return $s; }; @@ -462,7 +462,7 @@ sub bodies($) { my $self = shift; return {} unless $self->bodies_str; my $bodies = $self->bodies_str_ids; - my @bodies = FixMyStreet::App->model('DB::Body')->search({ id => $bodies })->all; + my @bodies = $self->result_source->schema->resultset('Body')->search({ id => $bodies })->all; return { map { $_->id => $_ } @bodies }; } @@ -477,21 +477,9 @@ sub url { return "/report/" . $self->id; } -=head2 get_photo_params - -Returns a hashref of details of the attached photo, if any, for use in templates. - -NB: this method doesn't currently support multiple photos gracefully. - -Use get_photoset($c) instead to do the right thing with reports with 0, 1, or more photos. - -=cut - -sub get_photo_params { - # use Carp 'cluck'; - # cluck "get_photo_params called"; # TEMPORARY die to make sure I've done right thing with Zurich templates - my $self = shift; - return FixMyStreet::App::get_photo_params($self, 'id'); +sub admin_url { + my ($self, $cobrand) = @_; + return $cobrand->admin_base_url . '/report_edit/' . $self->id; } =head2 is_open @@ -628,7 +616,7 @@ sub body { $body = join( _(' and '), map { my $name = $_->name; - if ($c and mySociety::Config::get('AREA_LINKS_FROM_PROBLEMS')) { + if ($c and FixMyStreet->config('AREA_LINKS_FROM_PROBLEMS')) { '<a href="' . $_->url($c) . '">' . $name . '</a>'; } else { $name; @@ -648,7 +636,7 @@ order of title. sub response_templates { my $problem = shift; - return FixMyStreet::App->model('DB::ResponseTemplate')->search( + return $problem->result_source->schema->resultset('ResponseTemplate')->search( { body_id => $problem->bodies_str_ids }, @@ -754,20 +742,17 @@ sub update_from_open311_service_request { $status_notes = $request->{status_notes}; } - my $update = FixMyStreet::App->model('DB::Comment')->new( - { - problem_id => $self->id, - state => 'confirmed', - created => $updated || \'current_timestamp', - confirmed => \'current_timestamp', - text => $status_notes, - mark_open => 0, - mark_fixed => 0, - user => $system_user, - anonymous => 0, - name => $body->name, - } - ); + my $update = $self->new_related(comments => { + state => 'confirmed', + created => $updated || \'current_timestamp', + confirmed => \'current_timestamp', + text => $status_notes, + mark_open => 0, + mark_fixed => 0, + user => $system_user, + anonymous => 0, + name => $body->name, + }); my $w3c = DateTime::Format::W3CDTF->new; my $req_time = $w3c->parse_datetime( $request->{updated_datetime} ); @@ -833,7 +818,7 @@ sub as_hashref { state_t => _( $self->state ), used_map => $self->used_map, is_fixed => $self->fixed_states->{ $self->state } ? 1 : 0, - photo => $self->get_photo_params, + photos => [ map { $_->{url} } @{$self->photos} ], meta => $self->confirmed ? $self->meta_line( $c ) : '', confirmed_pp => $self->confirmed ? $c->cobrand->prettify_dt( $self->confirmed ): '', created_pp => $c->cobrand->prettify_dt( $self->created ), @@ -855,23 +840,44 @@ sub latest_moderation_log_entry { Return a PhotoSet object for all photos attached to this field - my $photoset = $obj->get_photoset( $c ); + my $photoset = $obj->get_photoset; print $photoset->num_images; return $photoset->get_image_data(num => 0, size => 'full'); =cut sub get_photoset { - my ($self, $c) = @_; + my ($self) = @_; my $class = 'FixMyStreet::App::Model::PhotoSet'; eval "use $class"; return $class->new({ - c => $c, - data => $self->photo, + db_data => $self->photo, object => $self, }); } +sub photos { + my $self = shift; + my $photoset = $self->get_photoset; + my $i = 0; + my $id = $self->id; + my @photos = map { + my $format = 'jpeg'; + my $cachebust = substr($_, 0, 8); + { + id => $_, + url_temp => "/photo/$_.temp.$format", + url_temp_full => "/photo/$_.fulltemp.$format", + url => "/photo/$id.$i.$format?$cachebust", + url_full => "/photo/$id.$i.full.$format?$cachebust", + url_tn => "/photo/$id.$i.tn.$format?$cachebust", + url_fp => "/photo/$id.$i.fp.$format?$cachebust", + idx => $i++, + } + } $photoset->all_ids; + return \@photos; +} + __PACKAGE__->has_many( "admin_log_entries", "FixMyStreet::DB::Result::AdminLog", @@ -894,7 +900,4 @@ sub get_time_spent { return $admin_logs ? $admin_logs->get_column('sum_time_spent') : 0; } -# we need the inline_constructor bit as we don't inherit from Moose -__PACKAGE__->meta->make_immutable( inline_constructor => 0 ); - 1; diff --git a/perllib/FixMyStreet/DB/Result/Questionnaire.pm b/perllib/FixMyStreet/DB/Result/Questionnaire.pm index 6f2941546..30f2ab7ce 100644 --- a/perllib/FixMyStreet/DB/Result/Questionnaire.pm +++ b/perllib/FixMyStreet/DB/Result/Questionnaire.pm @@ -43,7 +43,7 @@ __PACKAGE__->belongs_to( # Created by DBIx::Class::Schema::Loader v0.07035 @ 2013-09-10 17:11:54 # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:oL1Hk4/bNG14CY74GA75SA -use Moose; +use Moo; use namespace::clean -except => [ 'meta' ]; my $stz = sub { diff --git a/perllib/FixMyStreet/DB/Result/User.pm b/perllib/FixMyStreet/DB/Result/User.pm index f08b666c8..6bce415a6 100644 --- a/perllib/FixMyStreet/DB/Result/User.pm +++ b/perllib/FixMyStreet/DB/Result/User.pm @@ -32,9 +32,15 @@ __PACKAGE__->add_columns( { data_type => "boolean", default_value => \"false", is_nullable => 0 }, "title", { data_type => "text", is_nullable => 1 }, + "twitter_id", + { data_type => "bigint", is_nullable => 1 }, + "facebook_id", + { data_type => "bigint", is_nullable => 1 }, ); __PACKAGE__->set_primary_key("id"); __PACKAGE__->add_unique_constraint("users_email_key", ["email"]); +__PACKAGE__->add_unique_constraint("users_facebook_id_key", ["facebook_id"]); +__PACKAGE__->add_unique_constraint("users_twitter_id_key", ["twitter_id"]); __PACKAGE__->has_many( "admin_logs", "FixMyStreet::DB::Result::AdminLog", @@ -84,8 +90,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.07035 @ 2014-07-29 13:54:07 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Y41/jGp93IxSpyJ/o6Q1gQ +# Created by DBIx::Class::Schema::Loader v0.07035 @ 2015-12-09 16:02:08 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:hCq6ZDZfV/6iiu3HFhPPOg __PACKAGE__->add_columns( "password" => { @@ -98,6 +104,16 @@ __PACKAGE__->add_columns( use mySociety::EmailUtil; +sub latest_anonymity { + my $self = shift; + my $p = $self->problems->search(undef, { order_by => { -desc => 'id' } } )->first; + my $c = $self->comments->search(undef, { order_by => { -desc => 'id' } } )->first; + my $p_created = $p ? $p->created->epoch : 0; + my $c_created = $c ? $c->created->epoch : 0; + my $obj = $p_created >= $c_created ? $p : $c; + return $obj ? $obj->anonymous : 0; +} + =head2 check_for_errors $error_hashref = $user->check_for_errors(); diff --git a/perllib/FixMyStreet/DB/ResultSet/Alert.pm b/perllib/FixMyStreet/DB/ResultSet/Alert.pm index bb1c61141..c61053fff 100644 --- a/perllib/FixMyStreet/DB/ResultSet/Alert.pm +++ b/perllib/FixMyStreet/DB/ResultSet/Alert.pm @@ -8,7 +8,7 @@ sub timeline_created { my ( $rs, $restriction ) = @_; my $prefetch = - FixMyStreet::App->model('DB')->schema->storage->sql_maker->quote_char ? + $rs->result_source->storage->sql_maker->quote_char ? [ qw/alert_type user/ ] : [ qw/alert_type/ ]; @@ -35,6 +35,13 @@ sub timeline_disabled { ); } +# Return summary for alerts on reports (so excluding alerts on updates) +sub summary_report_alerts { + my ( $rs, $restriction ) = @_; + $rs = $rs->search({ alert_type => { '!=', 'new_updates' } }); + return $rs->summary_count($restriction); +} + sub summary_count { my ( $rs, $restriction ) = @_; @@ -47,4 +54,5 @@ sub summary_count { } ); } + 1; diff --git a/perllib/FixMyStreet/DB/ResultSet/AlertType.pm b/perllib/FixMyStreet/DB/ResultSet/AlertType.pm index 25c727e25..a801600ea 100644 --- a/perllib/FixMyStreet/DB/ResultSet/AlertType.pm +++ b/perllib/FixMyStreet/DB/ResultSet/AlertType.pm @@ -4,272 +4,10 @@ use base 'DBIx::Class::ResultSet'; use strict; use warnings; -use mySociety::DBHandle qw(dbh); -use mySociety::Gaze; -use mySociety::Locale; -use mySociety::MaPit; -use IO::String; -use RABX; - -# Child must have confirmed, id, email, state(!) columns -# If parent/child, child table must also have name and text -# and foreign key to parent must be PARENT_id sub email_alerts ($) { my ( $rs ) = @_; - - my $q = $rs->search( { ref => { -not_like => '%local_problems%' } } ); - while (my $alert_type = $q->next) { - my $ref = $alert_type->ref; - my $head_table = $alert_type->head_table; - my $item_table = $alert_type->item_table; - my $query = 'select alert.id as alert_id, alert.user_id as alert_user_id, alert.lang as alert_lang, alert.cobrand as alert_cobrand, - alert.cobrand_data as alert_cobrand_data, alert.parameter as alert_parameter, alert.parameter2 as alert_parameter2, '; - if ($head_table) { - $query .= " - $item_table.id as item_id, $item_table.text as item_text, - $item_table.name as item_name, $item_table.anonymous as item_anonymous, - $item_table.confirmed as item_confirmed, - $head_table.* - from alert, $item_table, $head_table - where alert.parameter::integer = $head_table.id - and $item_table.${head_table}_id = $head_table.id - "; - } else { - $query .= " $item_table.*, - $item_table.id as item_id - from alert, $item_table - where 1 = 1"; - } - $query .= " - and alert_type='$ref' and whendisabled is null and $item_table.confirmed >= whensubscribed - and $item_table.confirmed >= current_timestamp - '7 days'::interval - and (select whenqueued from alert_sent where alert_sent.alert_id = alert.id and alert_sent.parameter::integer = $item_table.id) is null - and $item_table.user_id <> alert.user_id - and " . $alert_type->item_where . " - and alert.confirmed = 1 - order by alert.id, $item_table.confirmed"; - # XXX Ugh - needs work - $query =~ s/\?/alert.parameter/ if ($query =~ /\?/); - $query =~ s/\?/alert.parameter2/ if ($query =~ /\?/); - - $query = dbh()->prepare($query); - $query->execute(); - my $last_alert_id; - my %data = ( template => $alert_type->template, data => '' ); - while (my $row = $query->fetchrow_hashref) { - - my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($row->{alert_cobrand})->new(); - $cobrand->set_lang_and_domain( $row->{alert_lang}, 1, FixMyStreet->path_to('locale')->stringify ); - - # Cobranded and non-cobranded messages can share a database. In this case, the conf file - # should specify a vhost to send the reports for each cobrand, so that they don't get sent - # more than once if there are multiple vhosts running off the same database. The email_host - # call checks if this is the host that sends mail for this cobrand. - next unless $cobrand->email_host; - - # this is for the new_updates alerts - next if $row->{non_public} and $row->{user_id} != $row->{alert_user_id}; - - FixMyStreet::App->model('DB::AlertSent')->create( { - alert_id => $row->{alert_id}, - parameter => $row->{item_id}, - } ); - if ($last_alert_id && $last_alert_id != $row->{alert_id}) { - _send_aggregated_alert_email(%data); - %data = ( template => $alert_type->template, data => '' ); - } - - # create problem status message for the templates - if ( FixMyStreet::DB::Result::Problem::fixed_states()->{$row->{state}} ) { - $data{state_message} = _("This report is currently marked as fixed."); - } elsif ( FixMyStreet::DB::Result::Problem::closed_states()->{$row->{state}} ) { - $data{state_message} = _("This report is currently marked as closed.") - } else { - $data{state_message} = _("This report is currently marked as open."); - } - - my $url = $cobrand->base_url_for_report($row); - # this is currently only for new_updates - if ($row->{item_text}) { - if ( $cobrand->moniker ne 'zurich' && $row->{alert_user_id} == $row->{user_id} ) { - # This is an alert to the same user who made the report - make this a login link - # Don't bother with Zurich which has no accounts - my $user = FixMyStreet::App->model('DB::User')->find( { - id => $row->{alert_user_id} - } ); - $data{alert_email} = $user->email; - my $token_obj = FixMyStreet::App->model('DB::Token')->create( { - scope => 'alert_to_reporter', - data => { - id => $row->{id}, - } - } ); - $data{problem_url} = $url . "/R/" . $token_obj->token; - } else { - $data{problem_url} = $url . "/report/" . $row->{id}; - } - $data{data} .= $row->{item_name} . ' : ' if $row->{item_name} && !$row->{item_anonymous}; - if ( $cobrand->include_time_in_update_alerts ) { - my $parser = DateTime::Format::Pg->new(); - my $dt = $parser->parse_timestamp( $row->{item_confirmed} ); - # We need to always set this otherwise we end up with the DateTime - # object being in the floating timezone in which case applying a - # subsequent timezone set will have no effect. - # this is basically recreating the code from the inflate wrapper - # in the database model. - FixMyStreet->set_time_zone($dt); - $data{data} .= $cobrand->prettify_dt( $dt, 'alert' ) . "\n\n"; - } - $data{data} .= $row->{item_text} . "\n\n------\n\n"; - # this is ward and council problems - } else { - $data{data} .= $url . "/report/" . $row->{id} . " - $row->{title}\n\n"; - if ( exists $row->{geocode} && $row->{geocode} && $ref =~ /ward|council/ ) { - my $nearest_st = _get_address_from_gecode( $row->{geocode} ); - $data{data} .= $nearest_st if $nearest_st; - } - $data{data} .= "\n\n------\n\n"; - } - if (!$data{alert_user_id}) { - %data = (%data, %$row); - if ($ref eq 'area_problems' || $ref eq 'council_problems' || $ref eq 'ward_problems') { - my $va_info = mySociety::MaPit::call('area', $row->{alert_parameter}); - $data{area_name} = $va_info->{name}; - } - if ($ref eq 'ward_problems') { - my $va_info = mySociety::MaPit::call('area', $row->{alert_parameter2}); - $data{ward_name} = $va_info->{name}; - } - } - $data{cobrand} = $row->{alert_cobrand}; - $data{cobrand_data} = $row->{alert_cobrand_data}; - $data{lang} = $row->{alert_lang}; - $last_alert_id = $row->{alert_id}; - } - if ($last_alert_id) { - _send_aggregated_alert_email(%data); - } - } - - # Nearby done separately as the table contains the parameters - my $template = $rs->find( { ref => 'local_problems' } )->template; - my $query = FixMyStreet::App->model('DB::Alert')->search( { - alert_type => 'local_problems', - whendisabled => undef, - confirmed => 1 - }, { - order_by => 'id' - } ); - while (my $alert = $query->next) { - my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($alert->cobrand)->new(); - next unless $cobrand->email_host; - next if $alert->is_from_abuser; - - my $longitude = $alert->parameter; - my $latitude = $alert->parameter2; - my $d = mySociety::Gaze::get_radius_containing_population($latitude, $longitude, 200000); - # Convert integer to GB locale string (with a ".") - $d = mySociety::Locale::in_gb_locale { - sprintf("%f", int($d*10+0.5)/10); - }; - my $states = "'" . join( "', '", FixMyStreet::DB::Result::Problem::visible_states() ) . "'"; - my %data = ( template => $template, data => '', alert_id => $alert->id, alert_email => $alert->user->email, lang => $alert->lang, cobrand => $alert->cobrand, cobrand_data => $alert->cobrand_data ); - my $q = "select problem.id, problem.bodies_str, problem.postcode, problem.geocode, problem.title from problem_find_nearby(?, ?, ?) as nearby, problem, users - where nearby.problem_id = problem.id - and problem.user_id = users.id - and problem.state in ($states) - and problem.non_public = 'f' - and problem.confirmed >= ? and problem.confirmed >= current_timestamp - '7 days'::interval - and (select whenqueued from alert_sent where alert_sent.alert_id = ? and alert_sent.parameter::integer = problem.id) is null - and users.email <> ? - order by confirmed desc"; - $q = dbh()->prepare($q); - $q->execute($latitude, $longitude, $d, $alert->whensubscribed, $alert->id, $alert->user->email); - while (my $row = $q->fetchrow_hashref) { - FixMyStreet::App->model('DB::AlertSent')->create( { - alert_id => $alert->id, - parameter => $row->{id}, - } ); - my $url = $cobrand->base_url_for_report($row); - $data{data} .= $url . "/report/" . $row->{id} . " - $row->{title}\n\n"; - if ( exists $row->{geocode} && $row->{geocode} ) { - my $nearest_st = _get_address_from_gecode( $row->{geocode} ); - $data{data} .= $nearest_st if $nearest_st; - } - $data{data} .= "\n\n------\n\n"; - } - _send_aggregated_alert_email(%data) if $data{data}; - } -} - -sub _send_aggregated_alert_email(%) { - my %data = @_; - - my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($data{cobrand})->new(); - - $cobrand->set_lang_and_domain( $data{lang}, 1, FixMyStreet->path_to('locale')->stringify ); - - if (!$data{alert_email}) { - my $user = FixMyStreet::App->model('DB::User')->find( { - id => $data{alert_user_id} - } ); - $data{alert_email} = $user->email; - } - - my ($domain) = $data{alert_email} =~ m{ @ (.*) \z }x; - return if FixMyStreet::App->model('DB::Abuse')->search( { - email => [ $data{alert_email}, $domain ] - } )->first; - - my $token = FixMyStreet::App->model("DB::Token")->new_result( { - scope => 'alert', - data => { - id => $data{alert_id}, - type => 'unsubscribe', - email => $data{alert_email}, - } - } ); - $data{unsubscribe_url} = $cobrand->base_url( $data{cobrand_data} ) . '/A/' . $token->token; - - my $template = FixMyStreet->get_email_template($cobrand->moniker, $data{lang}, "$data{template}.txt"); - - my $result = FixMyStreet::App->send_email_cron( - { - _template_ => $template, - _parameters_ => \%data, - To => $data{alert_email}, - }, - undef, - 0, - $cobrand, - $data{lang} - ); - - unless ($result) { - $token->insert(); - } else { - print "Failed to send alert $data{alert_id}!"; - } -} - -sub _get_address_from_gecode { - my $geocode = shift; - - return '' unless defined $geocode; - utf8::encode($geocode) if utf8::is_utf8($geocode); - my $h = new IO::String($geocode); - my $data = RABX::wire_rd($h); - - my $str = ''; - - my $address = $data->{resourceSets}[0]{resources}[0]{address}; - my @address; - push @address, $address->{addressLine} if $address->{addressLine} && $address->{addressLine} ne 'Street'; - push @address, $address->{locality} if $address->{locality}; - $str .= sprintf(_("Nearest road to the pin placed on the map (automatically generated by Bing Maps): %s\n\n"), - join( ', ', @address ) ) if @address; - - return $str; + require FixMyStreet::Script::Alerts; + FixMyStreet::Script::Alerts::send(@_); } 1; diff --git a/perllib/FixMyStreet/DB/ResultSet/Comment.pm b/perllib/FixMyStreet/DB/ResultSet/Comment.pm index 1b6afb819..9f254a018 100644 --- a/perllib/FixMyStreet/DB/ResultSet/Comment.pm +++ b/perllib/FixMyStreet/DB/ResultSet/Comment.pm @@ -5,20 +5,20 @@ use strict; use warnings; sub to_body { - my ($rs, $body_restriction) = @_; - return FixMyStreet::DB::ResultSet::Problem::to_body($rs, $body_restriction); + my ($rs, $bodies) = @_; + return FixMyStreet::DB::ResultSet::Problem::to_body($rs, $bodies, 1); } sub timeline { - my ( $rs, $body_restriction ) = @_; + my ( $rs ) = @_; my $prefetch = - FixMyStreet::App->model('DB')->schema->storage->sql_maker->quote_char ? + $rs->result_source->storage->sql_maker->quote_char ? [ qw/user/ ] : []; - return $rs->to_body($body_restriction)->search( + return $rs->search( { state => 'confirmed', created => { '>=', \"current_timestamp-'7 days'::interval" }, @@ -30,17 +30,13 @@ sub timeline { } sub summary_count { - my ( $rs, $body_restriction ) = @_; + my ( $rs ) = @_; my $params = { group_by => ['me.state'], select => [ 'me.state', { count => 'me.id' } ], as => [qw/state state_count/], }; - if ($body_restriction) { - $rs = $rs->to_body($body_restriction); - $params->{join} = 'problem'; - } return $rs->search(undef, $params); } diff --git a/perllib/FixMyStreet/DB/ResultSet/Nearby.pm b/perllib/FixMyStreet/DB/ResultSet/Nearby.pm index a6b00ef7b..9db1c6525 100644 --- a/perllib/FixMyStreet/DB/ResultSet/Nearby.pm +++ b/perllib/FixMyStreet/DB/ResultSet/Nearby.pm @@ -4,6 +4,11 @@ use base 'DBIx::Class::ResultSet'; use strict; use warnings; +sub to_body { + my ($rs, $bodies) = @_; + return FixMyStreet::DB::ResultSet::Problem::to_body($rs, $bodies, 1); +} + sub nearby { my ( $rs, $c, $dist, $ids, $limit, $mid_lat, $mid_lon, $interval, $category, $states ) = @_; @@ -21,7 +26,7 @@ sub nearby { if $ids; $params->{category} = $category if $category; - $rs = FixMyStreet::DB::ResultSet::Problem::to_body($rs, $c->cobrand->body_restriction); + $rs = $c->cobrand->problems_restriction($rs); my $attrs = { prefetch => 'problem', diff --git a/perllib/FixMyStreet/DB/ResultSet/Problem.pm b/perllib/FixMyStreet/DB/ResultSet/Problem.pm index e9f5d0f8e..488030928 100644 --- a/perllib/FixMyStreet/DB/ResultSet/Problem.pm +++ b/perllib/FixMyStreet/DB/ResultSet/Problem.pm @@ -4,14 +4,9 @@ use base 'DBIx::Class::ResultSet'; use strict; use warnings; -use CronFns; - -use Utils; -use mySociety::Config; -use mySociety::MaPit; - -use FixMyStreet::App; -use FixMyStreet::SendReport; +use Memcached; +use mySociety::Locale; +use FixMyStreet::DB; my $site_key; @@ -21,13 +16,15 @@ sub set_restriction { } sub to_body { - my ($rs, $bodies) = @_; + my ($rs, $bodies, $join) = @_; return $rs unless $bodies; unless (ref $bodies eq 'ARRAY') { $bodies = [ map { ref $_ ? $_->id : $_ } $bodies ]; } + $join = { join => 'problem' } if $join; $rs = $rs->search( - \[ "regexp_split_to_array(bodies_str, ',') && ?", [ {} => $bodies ] ] + \[ "regexp_split_to_array(bodies_str, ',') && ?", [ {} => $bodies ] ], + $join ); return $rs; } @@ -173,7 +170,7 @@ sub timeline { my ( $rs ) = @_; my $prefetch = - FixMyStreet::App->model('DB')->schema->storage->sql_maker->quote_char ? + $rs->result_source->storage->sql_maker->quote_char ? [ qw/user/ ] : []; @@ -235,302 +232,10 @@ sub categories_summary { return \%categories; } -sub get_admin_url { - my ($rs, $cobrand, $row) = @_; - return $cobrand->admin_base_url . '/report_edit/' . $row->id; -} - sub send_reports { my ( $rs, $site_override ) = @_; - - # Set up site, language etc. - my ($verbose, $nomail, $debug_mode) = CronFns::options(); - - my $base_url = mySociety::Config::get('BASE_URL'); - my $site = $site_override || CronFns::site($base_url); - - my $states = [ 'confirmed', 'fixed' ]; - $states = [ 'unconfirmed', 'confirmed', 'in progress', 'planned', 'closed', 'investigating' ] if $site eq 'zurich'; - my $unsent = $rs->search( { - state => $states, - whensent => undef, - bodies_str => { '!=', undef }, - } ); - my (%notgot, %note); - - my $send_report = FixMyStreet::SendReport->new(); - my $senders = $send_report->get_senders; - - my $debug_unsent_count = 0; - debug_print("starting to loop through unsent problem reports...") if $debug_mode; - while (my $row = $unsent->next) { - - my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($row->cobrand)->new(); - - if ($debug_mode) { - $debug_unsent_count++; - print "\n"; - debug_print("state=" . $row->state . ", bodies_str=" . $row->bodies_str . ($row->cobrand? ", cobrand=" . $row->cobrand : ""), $row->id); - } - - # Cobranded and non-cobranded messages can share a database. In this case, the conf file - # should specify a vhost to send the reports for each cobrand, so that they don't get sent - # more than once if there are multiple vhosts running off the same database. The email_host - # call checks if this is the host that sends mail for this cobrand. - if (! $cobrand->email_host()) { - debug_print("skipping because this host does not send reports for cobrand " . $cobrand->moniker, $row->id) if $debug_mode; - next; - } - - $cobrand->set_lang_and_domain($row->lang, 1); - if ( $row->is_from_abuser) { - $row->update( { state => 'hidden' } ); - debug_print("hiding because its sender is flagged as an abuser", $row->id) if $debug_mode; - next; - } elsif ( $row->title =~ /app store test/i ) { - $row->update( { state => 'hidden' } ); - debug_print("hiding because it is an app store test message", $row->id) if $debug_mode; - next; - } - - # Template variables for the email - my $email_base_url = $cobrand->base_url_for_report($row); - my %h = map { $_ => $row->$_ } qw/id title detail name category latitude longitude used_map/; - map { $h{$_} = $row->user->$_ || '' } qw/email phone/; - $h{confirmed} = DateTime::Format::Pg->format_datetime( $row->confirmed->truncate (to => 'second' ) ) - if $row->confirmed; - - $h{query} = $row->postcode; - $h{url} = $email_base_url . $row->url; - $h{admin_url} = $rs->get_admin_url($cobrand, $row); - $h{phone_line} = $h{phone} ? _('Phone:') . " $h{phone}\n\n" : ''; - if ($row->photo) { - $h{has_photo} = _("This web page also contains a photo of the problem, provided by the user.") . "\n\n"; - $h{image_url} = $email_base_url . '/photo/' . $row->id . '.full.jpeg'; - } else { - $h{has_photo} = ''; - $h{image_url} = ''; - } - $h{fuzzy} = $row->used_map ? _('To view a map of the precise location of this issue') - : _('The user could not locate the problem on a map, but to see the area around the location they entered'); - $h{closest_address} = ''; - - if ( $row->used_map ) { - $h{closest_address} = $cobrand->find_closest( $h{latitude}, $h{longitude}, $row ); - } - - if ( $cobrand->allow_anonymous_reports && - $row->user->email eq $cobrand->anonymous_account->{'email'} - ) { - $h{anonymous_report} = 1; - $h{user_details} = _('This report was submitted anonymously'); - } else { - $h{user_details} = sprintf(_('Name: %s'), $row->name) . "\n\n"; - $h{user_details} .= sprintf(_('Email: %s'), $row->user->email) . "\n\n"; - } - - $h{easting_northing} = ''; - - if ($cobrand->can('process_additional_metadata_for_email')) { - $cobrand->process_additional_metadata_for_email($row, \%h); - } - - my $bodies = FixMyStreet::App->model("DB::Body")->search( - { id => $row->bodies_str_ids }, - { order_by => 'name' }, - ); - - my $missing; - if ($row->bodies_missing) { - my @missing = FixMyStreet::App->model("DB::Body")->search( - { id => [ split /,/, $row->bodies_missing ] }, - { order_by => 'name' } - )->get_column('name')->all; - $missing = join(' / ', @missing) if @missing; - } - - my @dear; - my %reporters = (); - while (my $body = $bodies->next) { - my $sender_info = $cobrand->get_body_sender( $body, $row->category ); - my $sender = "FixMyStreet::SendReport::" . $sender_info->{method}; - - if ( ! exists $senders->{ $sender } ) { - warn sprintf "No such sender [ $sender ] for body %s ( %d )", $body->name, $body->id; - next; - } - $reporters{ $sender } ||= $sender->new(); - - if ( $reporters{ $sender }->should_skip( $row ) ) { - debug_print("skipped by sender " . $sender_info->{method} . " (might be due to previous failed attempts?)", $row->id) if $debug_mode; - } else { - debug_print("OK, adding recipient body " . $body->id . ":" . $body->name . ", " . $body->send_method, $row->id) if $debug_mode; - push @dear, $body->name; - $reporters{ $sender }->add_body( $body, $sender_info->{config} ); - } - - # If we are in the UK include eastings and northings, and nearest stuff - if ( $cobrand->country eq 'GB' && !$h{easting} ) { - my $coordsyst = 'G'; - my $first_area = $body->body_areas->first->area_id; - my $area_info = mySociety::MaPit::call('area', $first_area); - $coordsyst = 'I' if $area_info->{type} eq 'LGD'; - - ( $h{easting}, $h{northing} ) = Utils::convert_latlon_to_en( $h{latitude}, $h{longitude}, $coordsyst ); - - # email templates don't have conditionals so we need to format this here - $h{easting_northing} = "Easting/Northing"; - $h{easting_northing} .= " (IE)" if $coordsyst eq 'I'; - $h{easting_northing} .= ": $h{easting}/$h{northing}\n\n"; - } - } - - unless ( keys %reporters ) { - die 'Report not going anywhere for ID ' . $row->id . '!'; - } - - unless (@dear) { - debug_print("can't send because sender count is zero", $row->id) if $debug_mode; - next; - } - - if ($h{category} eq _('Other')) { - $h{category_footer} = _('this type of local problem'); - $h{category_line} = ''; - } else { - $h{category_footer} = "'" . $h{category} . "'"; - $h{category_line} = sprintf(_("Category: %s"), $h{category}) . "\n\n"; - } - - if ( $row->subcategory ) { - $h{subcategory_line} = sprintf(_("Subcategory: %s"), $row->subcategory) . "\n\n"; - } else { - $h{subcategory_line} = "\n\n"; - } - - $h{bodies_name} = join(_(' and '), @dear); - if ($h{category} eq _('Other')) { - $h{multiple} = @dear>1 ? "[ " . _("This email has been sent to both councils covering the location of the problem, as the user did not categorise it; please ignore it if you're not the correct council to deal with the issue, or let us know what category of problem this is so we can add it to our system.") . " ]\n\n" - : ''; - } else { - $h{multiple} = @dear>1 ? "[ " . _("This email has been sent to several councils covering the location of the problem, as the category selected is provided for all of them; please ignore it if you're not the correct council to deal with the issue.") . " ]\n\n" - : ''; - } - $h{missing} = ''; - if ($missing) { - $h{missing} = '[ ' - . sprintf(_('We realise this problem might be the responsibility of %s; however, we don\'t currently have any contact details for them. If you know of an appropriate contact address, please do get in touch.'), $missing) - . " ]\n\n"; - } - - if (mySociety::Config::get('STAGING_SITE') && !mySociety::Config::get('SEND_REPORTS_ON_STAGING')) { - # on a staging server send emails to ourselves rather than the bodies - %reporters = map { $_ => $reporters{$_} } grep { /FixMyStreet::SendReport::(Email|EmptyHomes)/ } keys %reporters; - unless (%reporters) { - %reporters = ( 'FixMyStreet::SendReport::Email' => FixMyStreet::SendReport::Email->new() ); - } - } - - # Multiply results together, so one success counts as a success. - my $result = -1; - - for my $sender ( keys %reporters ) { - debug_print("sending using " . $sender, $row->id) if $debug_mode; - $result *= $reporters{ $sender }->send( $row, \%h ); - if ( $reporters{ $sender }->unconfirmed_counts) { - foreach my $e (keys %{ $reporters{ $sender }->unconfirmed_counts } ) { - foreach my $c (keys %{ $reporters{ $sender }->unconfirmed_counts->{$e} }) { - $notgot{$e}{$c} += $reporters{ $sender }->unconfirmed_counts->{$e}{$c}; - } - } - %note = ( - %note, - %{ $reporters{ $sender }->unconfirmed_notes } - ); - } - } - - unless ($result) { - $row->update( { - whensent => \'current_timestamp', - lastupdate => \'current_timestamp', - } ); - if ( $cobrand->report_sent_confirmation_email && !$h{anonymous_report}) { - _send_report_sent_email( $row, \%h, $nomail, $cobrand ); - } - debug_print("send successful: OK", $row->id) if $debug_mode; - } else { - my @errors; - for my $sender ( keys %reporters ) { - unless ( $reporters{ $sender }->success ) { - push @errors, $reporters{ $sender }->error; - } - } - $row->update_send_failed( join( '|', @errors ) ); - debug_print("send FAILED: " . join( '|', @errors ), $row->id) if $debug_mode; - } - } - if ($debug_mode) { - print "\n"; - if ($debug_unsent_count) { - debug_print("processed all unsent reports (total: $debug_unsent_count)"); - } else { - debug_print("no unsent reports were found (must have whensent=null and suitable bodies_str & state) -- nothing to send"); - } - } - - if ($verbose || $debug_mode) { - print "Council email addresses that need checking:\n" if keys %notgot; - foreach my $e (keys %notgot) { - foreach my $c (keys %{$notgot{$e}}) { - print " " . $notgot{$e}{$c} . " problem, to $e category $c (" . $note{$e}{$c}. ")\n"; - } - } - my $sending_errors = ''; - my $unsent = $rs->search( { - state => [ 'confirmed', 'fixed' ], - whensent => undef, - bodies_str => { '!=', undef }, - send_fail_count => { '>', 0 } - } ); - while (my $row = $unsent->next) { - my $base_url = mySociety::Config::get('BASE_URL'); - $sending_errors .= "* " . $base_url . "/report/" . $row->id . ", failed " - . $row->send_fail_count . " times, last at " . $row->send_fail_timestamp - . ", reason " . $row->send_fail_reason . "\n"; - } - if ($sending_errors) { - print "The following reports had problems sending:\n$sending_errors"; - } - } -} - -sub _send_report_sent_email { - my $row = shift; - my $h = shift; - my $nomail = shift; - my $cobrand = shift; - - my $template = FixMyStreet->get_email_template($row->cobrand, $row->lang, 'confirm_report_sent.txt'); - - FixMyStreet::App->send_email_cron( - { - _template_ => $template, - _parameters_ => $h, - To => $row->user->email, - From => [ mySociety::Config::get('CONTACT_EMAIL'), $cobrand->contact_name ], - }, - mySociety::Config::get('CONTACT_EMAIL'), - $nomail, - $cobrand - ); -} - -sub debug_print { - my $msg = shift; - my $id = shift || ''; - $id = "report $id: " if $id; - print "[] $id$msg\n"; + require FixMyStreet::Script::Reports; + FixMyStreet::Script::Reports::send($site_override); } 1; diff --git a/perllib/FixMyStreet/DB/ResultSet/Questionnaire.pm b/perllib/FixMyStreet/DB/ResultSet/Questionnaire.pm index bf1c68c49..6a1ce0d78 100644 --- a/perllib/FixMyStreet/DB/ResultSet/Questionnaire.pm +++ b/perllib/FixMyStreet/DB/ResultSet/Questionnaire.pm @@ -3,111 +3,11 @@ use base 'DBIx::Class::ResultSet'; use strict; use warnings; -use Encode; -use Utils; sub send_questionnaires { my ( $rs, $params ) = @_; - $rs->send_questionnaires_period( '4 weeks', $params ); - $rs->send_questionnaires_period( '26 weeks', $params ) - if $params->{site} eq 'emptyhomes'; -} - -sub send_questionnaires_period { - my ( $rs, $period, $params ) = @_; - - # Select all problems that need a questionnaire email sending - my $q_params = { - state => [ FixMyStreet::DB::Result::Problem::visible_states() ], - whensent => [ - '-and', - { '!=', undef }, - { '<', \"current_timestamp - '$period'::interval" }, - ], - send_questionnaire => 1, - }; - # FIXME Do these a bit better... - if ($params->{site} eq 'emptyhomes' && $period eq '4 weeks') { - $q_params->{'(select max(whensent) from questionnaire where me.id=problem_id)'} = undef; - } elsif ($params->{site} eq 'emptyhomes' && $period eq '26 weeks') { - $q_params->{'(select max(whensent) from questionnaire where me.id=problem_id)'} = { '!=', undef }; - } else { - $q_params->{'-or'} = [ - '(select max(whensent) from questionnaire where me.id=problem_id)' => undef, - '(select max(whenanswered) from questionnaire where me.id=problem_id)' => { '<', \"current_timestamp - '$period'::interval" } - ]; - } - - my $unsent = FixMyStreet::App->model('DB::Problem')->search( $q_params, { - order_by => { -desc => 'confirmed' } - } ); - - while (my $row = $unsent->next) { - - my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($row->cobrand)->new(); - $cobrand->set_lang_and_domain($row->lang, 1); - - # Not all cobrands send questionnaires - next unless $cobrand->send_questionnaires; - next if $row->is_from_abuser; - - # Cobranded and non-cobranded messages can share a database. In this case, the conf file - # should specify a vhost to send the reports for each cobrand, so that they don't get sent - # more than once if there are multiple vhosts running off the same database. The email_host - # call checks if this is the host that sends mail for this cobrand. - next unless $cobrand->email_host; - - my $template; - if ($params->{site} eq 'emptyhomes') { - ($template = $period) =~ s/ //; - $template = Utils::read_file( FixMyStreet->path_to( "templates/email/emptyhomes/" . $row->lang . "/questionnaire-$template.txt" )->stringify ); - } else { - $template = FixMyStreet->get_email_template($cobrand->moniker, $row->lang, 'questionnaire.txt'); - } - - my %h = map { $_ => $row->$_ } qw/name title detail category/; - $h{created} = Utils::prettify_duration( time() - $row->confirmed->epoch, 'week' ); - - my $questionnaire = FixMyStreet::App->model('DB::Questionnaire')->create( { - problem_id => $row->id, - whensent => \'current_timestamp', - } ); - - # We won't send another questionnaire unless they ask for it (or it was - # the first EHA questionnaire. - $row->send_questionnaire( 0 ) - if $params->{site} ne 'emptyhomes' || $period eq '26 weeks'; - - my $token = FixMyStreet::App->model("DB::Token")->new_result( { - scope => 'questionnaire', - data => $questionnaire->id, - } ); - $h{url} = $cobrand->base_url($row->cobrand_data) . '/Q/' . $token->token; - - print "Sending questionnaire " . $questionnaire->id . ", problem " - . $row->id . ", token " . $token->token . " to " - . $row->user->email . "\n" - if $params->{verbose}; - - my $result = FixMyStreet::App->send_email_cron( - { - _template_ => $template, - _parameters_ => \%h, - To => [ [ $row->user->email, $row->name ] ], - }, - undef, - $params->{nomail}, - $cobrand - ); - unless ($result) { - print " ...success\n" if $params->{verbose}; - $row->update(); - $token->insert(); - } else { - print " ...failed\n" if $params->{verbose}; - $questionnaire->delete; - } - } + require FixMyStreet::Script::Questionnaires; + FixMyStreet::Script::Questionnaires::send($params); } sub timeline { diff --git a/perllib/FixMyStreet/DB/ResultSet/Secret.pm b/perllib/FixMyStreet/DB/ResultSet/Secret.pm new file mode 100644 index 000000000..971584b9a --- /dev/null +++ b/perllib/FixMyStreet/DB/ResultSet/Secret.pm @@ -0,0 +1,12 @@ +package FixMyStreet::DB::ResultSet::Secret; +use base 'DBIx::Class::ResultSet'; + +use strict; +use warnings; + +sub get { + my $rs = shift; + return $rs->first->secret; +} + +1; diff --git a/perllib/FixMyStreet/Email.pm b/perllib/FixMyStreet/Email.pm index 4a2784787..1787c32da 100644 --- a/perllib/FixMyStreet/Email.pm +++ b/perllib/FixMyStreet/Email.pm @@ -1,7 +1,15 @@ package FixMyStreet::Email; +use Encode; +use Template; +use Digest::HMAC_SHA1 qw(hmac_sha1_hex); +use mySociety::Email; +use mySociety::Locale; +use mySociety::Random qw(random_bytes); use Utils::Email; use FixMyStreet; +use FixMyStreet::DB; +use FixMyStreet::EmailSend; sub test_dmarc { my $email = shift; @@ -9,4 +17,145 @@ sub test_dmarc { return Utils::Email::test_dmarc($email); } +sub hash_from_id { + my ($type, $id) = @_; + my $secret = FixMyStreet::DB->resultset('Secret')->get; + # Make sure the ID is stringified, a number is treated differently + return substr(hmac_sha1_hex("$type-$id", $secret), 0, 8); +} + +sub generate_verp_token { + my ($type, $id) = @_; + my $hash = hash_from_id($type, $id); + return "$type-$id-$hash"; +} + +sub check_verp_token { + my ($token) = @_; + $token = lc($token); + $token =~ s#[./_]##g; + + my ($type, $id, $hash) = $token =~ /(report|alert)-([a-z0-9]+)-([a-z0-9]+)/; + return unless $type; + + $hash =~ tr/lo/10/; + return unless hash_from_id($type, $id) eq $hash; + + return ($type, $id); +} + +sub is_abuser { + my ($schema, $to) = @_; + + my $email; + if (ref($to) eq 'ARRAY') { + if (ref($to->[0]) eq 'ARRAY') { + $email = $to->[0][0]; + } else { + $email = $to->[0]; + } + } else { + $email = $to; + } + + my ($domain) = $email =~ m{ @ (.*) \z }x; + return $schema->resultset('Abuse')->search( { email => [ $email, $domain ] } )->first; +} + +sub send_cron { + my ( $schema, $params, $env_from, $nomail, $cobrand, $lang_code ) = @_; + + my $sender = FixMyStreet->config('DO_NOT_REPLY_EMAIL'); + $env_from ||= $sender; + if (!$params->{From}) { + my $sender_name = $cobrand->contact_name; + $params->{From} = [ $sender, _($sender_name) ]; + } + + return 1 if is_abuser($schema, $params->{To}); + + $params->{'Message-ID'} = sprintf('<fms-cron-%s-%s@%s>', time(), + unpack('h*', random_bytes(5, 1)), FixMyStreet->config('EMAIL_DOMAIN') + ); + + # This is all to set the path for the templates processor so we can override + # signature and site names in emails using templates in the old style emails. + # It's a bit involved as not everywhere we use it knows about the cobrand so + # we can't assume there will be one. + my $include_path = FixMyStreet->path_to( 'templates', 'email', 'default' )->stringify; + if ( $cobrand ) { + $include_path = + FixMyStreet->path_to( 'templates', 'email', $cobrand->moniker )->stringify . ':' + . $include_path; + if ( $lang_code ) { + $include_path = + FixMyStreet->path_to( 'templates', 'email', $cobrand->moniker, $lang_code )->stringify . ':' + . $include_path; + } + } + my $tt = Template->new({ + INCLUDE_PATH => $include_path + }); + my ($sig, $site_name); + $tt->process( 'signature.txt', $params, \$sig ); + $sig = Encode::decode('utf8', $sig); + $params->{_parameters_}->{signature} = $sig; + + $tt->process( 'site-name.txt', $params, \$site_name ); + $site_name = Utils::trim_text(Encode::decode('utf8', $site_name)); + $params->{_parameters_}->{site_name} = $site_name; + + $params->{_line_indent} = ''; + my $attachments = delete $params->{attachments}; + + my $email = mySociety::Locale::in_gb_locale { mySociety::Email::construct_email($params) }; + + $email = munge_attachments($email, $attachments) if $attachments; + + if ($nomail) { + print $email; + return 1; # Failure + } else { + my $result = FixMyStreet::EmailSend->new({ env_from => $env_from })->send($email); + return $result ? 0 : 1; + } +} + +sub munge_attachments { + my ($message, $attachments) = @_; + # $attachments should be an array_ref of things that can be parsed to Email::MIME, + # for example + # [ + # body => $binary_data, + # attributes => { + # content_type => 'image/jpeg', + # encoding => 'base64', + # filename => '1234.1.jpeg', + # name => '1234.1.jpeg', + # }, + # ... + # ] + # + # XXX: mySociety::Email::construct_email isn't using a MIME library and + # requires more analysis to refactor, so for now, we'll simply parse the + # generated MIME and add attachments. + # + # (Yes, this means that the email is constructed by Email::Simple, munged + # manually by custom code, turned back into Email::Simple, and then munged + # with Email::MIME. What's your point?) + + require Email::MIME; + my $mime = Email::MIME->new($message); + $mime->parts_add([ map { Email::MIME->create(%$_)} @$attachments ]); + my $data = $mime->as_string; + + # unsure why Email::MIME adds \r\n. Possibly mail client should handle + # gracefully, BUT perhaps as the segment constructed by + # mySociety::Email::construct_email strips to \n, they seem not to. + # So we re-run the same regexp here to the added part. + $data =~ s/\r\n/\n/gs; + + return $data; +} + 1; diff --git a/perllib/FixMyStreet/EmailSend.pm b/perllib/FixMyStreet/EmailSend.pm new file mode 100644 index 000000000..09f434931 --- /dev/null +++ b/perllib/FixMyStreet/EmailSend.pm @@ -0,0 +1,78 @@ +package FixMyStreet::EmailSend; + +use strict; +use warnings; + +BEGIN { + # Should move away from Email::Send, but until then: + $Return::Value::NO_CLUCK = 1; +} + +use FixMyStreet; +use Email::Send; + +=head1 NAME + +FixMyStreet::EmailSend + +=head1 DESCRIPTION + +Thin wrapper around Email::Send - configuring it correctly according to our config. + +If the config value 'SMTP_SMARTHOST' is set then email is routed via SMTP to +that. Otherwise it is sent using a 'sendmail' like binary on the local system. + +And finally if if FixMyStreet->test_mode returns true then emails are not sent +at all but are stored in memory for the test suite to inspect (using +Email::Send::Test). + +=cut + +my $args = undef; + +if ( FixMyStreet->test_mode ) { + # Email::Send::Test + $args = { mailer => 'Test', }; +} elsif ( my $smtp_host = FixMyStreet->config('SMTP_SMARTHOST') ) { + # Email::Send::SMTP + my $type = FixMyStreet->config('SMTP_TYPE') || ''; + my $port = FixMyStreet->config('SMTP_PORT') || ''; + my $username = FixMyStreet->config('SMTP_USERNAME') || ''; + my $password = FixMyStreet->config('SMTP_PASSWORD') || ''; + + unless ($port) { + $port = 25; + $port = 465 if $type eq 'ssl'; + $port = 587 if $type eq 'tls'; + } + + my $mailer_args = [ + Host => $smtp_host, + Port => $port, + ]; + push @$mailer_args, ssl => 1 if $type eq 'ssl'; + push @$mailer_args, tls => 1 if $type eq 'tls'; + push @$mailer_args, username => $username, password => $password + if $username && $password; + $args = { + mailer => 'FixMyStreet::EmailSend::Variable', + mailer_args => $mailer_args, + }; +} else { + # Email::Send::Sendmail + $args = { mailer => 'Sendmail' }; +} + +sub new { + my ($cls, $hash) = @_; + $hash ||= {}; + my %args = ( %$args, %$hash ); + + my $sender = delete($args{env_from}); + if ($sender) { + $args{mailer_args} = [ @{$args{mailer_args}} ] if $args{mailer_args}; + push @{$args{mailer_args}}, env_from => $sender; + } + + return Email::Send->new(\%args); +} diff --git a/perllib/FixMyStreet/EmailSend/ContactEmail.pm b/perllib/FixMyStreet/EmailSend/ContactEmail.pm deleted file mode 100644 index 28bcc983b..000000000 --- a/perllib/FixMyStreet/EmailSend/ContactEmail.pm +++ /dev/null @@ -1,9 +0,0 @@ -package FixMyStreet::EmailSend::ContactEmail; -use base Email::Send::SMTP; - -sub get_env_sender { - my $sender = FixMyStreet->config('CONTACT_EMAIL'); - return $sender; -} - -1; diff --git a/perllib/FixMyStreet/EmailSend/DoNotReply.pm b/perllib/FixMyStreet/EmailSend/DoNotReply.pm deleted file mode 100644 index d1368f00f..000000000 --- a/perllib/FixMyStreet/EmailSend/DoNotReply.pm +++ /dev/null @@ -1,9 +0,0 @@ -package FixMyStreet::EmailSend::DoNotReply; -use base Email::Send::SMTP; - -sub get_env_sender { - my $sender = FixMyStreet->config('DO_NOT_REPLY_EMAIL'); - return $sender; -} - -1; diff --git a/perllib/FixMyStreet/EmailSend/Variable.pm b/perllib/FixMyStreet/EmailSend/Variable.pm new file mode 100644 index 000000000..4ba56dd41 --- /dev/null +++ b/perllib/FixMyStreet/EmailSend/Variable.pm @@ -0,0 +1,17 @@ +package FixMyStreet::EmailSend::Variable; +use base Email::Send::SMTP; +use FixMyStreet; + +my $sender; + +sub send { + my ($class, $message, %args) = @_; + $sender = delete($args{env_from}) || FixMyStreet->config('DO_NOT_REPLY_EMAIL'); + $class->SUPER::send($message, %args); +} + +sub get_env_sender { + $sender; +} + +1; diff --git a/perllib/FixMyStreet/Geocode.pm b/perllib/FixMyStreet/Geocode.pm index aac52fbaa..4470ecbca 100644 --- a/perllib/FixMyStreet/Geocode.pm +++ b/perllib/FixMyStreet/Geocode.pm @@ -11,6 +11,7 @@ use Digest::MD5 qw(md5_hex); use Encode; use File::Slurp; use File::Path (); +use JSON::MaybeXS; use LWP::Simple qw($ua); use URI::Escape; use FixMyStreet::Geocode::Bing; diff --git a/perllib/FixMyStreet/Geocode/Bing.pm b/perllib/FixMyStreet/Geocode/Bing.pm index d7db10ae6..a846f3348 100644 --- a/perllib/FixMyStreet/Geocode/Bing.pm +++ b/perllib/FixMyStreet/Geocode/Bing.pm @@ -8,6 +8,7 @@ package FixMyStreet::Geocode::Bing; use strict; +use FixMyStreet::Geocode; use Utils; # string STRING CONTEXT @@ -71,7 +72,7 @@ sub reverse { my ( $latitude, $longitude, $bing_culture ) = @_; # Get nearest road-type thing from Bing - my $key = mySociety::Config::get('BING_MAPS_API_KEY', ''); + my $key = FixMyStreet->config('BING_MAPS_API_KEY', ''); if ($key) { my $url = "http://dev.virtualearth.net/REST/v1/Locations/$latitude,$longitude?key=$key"; $url .= '&c=' . $bing_culture if $bing_culture; diff --git a/perllib/FixMyStreet/Geocode/Google.pm b/perllib/FixMyStreet/Geocode/Google.pm index 5261bb7e4..e64d02c4c 100644 --- a/perllib/FixMyStreet/Geocode/Google.pm +++ b/perllib/FixMyStreet/Geocode/Google.pm @@ -53,7 +53,7 @@ sub string { next unless $c->cobrand->geocoded_string_check( $address ); ( $longitude, $latitude ) = map { Utils::truncate_coordinate($_) } - ($_->{geometry}{location}{lat}, $_->{geometry}{location}{lng}); + ($_->{geometry}{location}{lng}, $_->{geometry}{location}{lat}); push (@$error, { address => $address, latitude => $latitude, diff --git a/perllib/FixMyStreet/Map/OSM/TonerLite.pm b/perllib/FixMyStreet/Map/OSM/TonerLite.pm index 543cd6002..b0d12c453 100644 --- a/perllib/FixMyStreet/Map/OSM/TonerLite.pm +++ b/perllib/FixMyStreet/Map/OSM/TonerLite.pm @@ -1,5 +1,3 @@ -#!/usr/bin/perl -# # FixMyStreet:Map::OSM::TonerLite # OSM TonerLite maps on FixMyStreet. # diff --git a/perllib/FixMyStreet/Roles/Abuser.pm b/perllib/FixMyStreet/Roles/Abuser.pm index b9e951305..fc76565ca 100644 --- a/perllib/FixMyStreet/Roles/Abuser.pm +++ b/perllib/FixMyStreet/Roles/Abuser.pm @@ -1,6 +1,6 @@ package FixMyStreet::Roles::Abuser; -use Moose::Role; +use Moo::Role; =head2 is_from_abuser diff --git a/perllib/FixMyStreet/Roles/Extra.pm b/perllib/FixMyStreet/Roles/Extra.pm index f815a3e9a..19fc91873 100644 --- a/perllib/FixMyStreet/Roles/Extra.pm +++ b/perllib/FixMyStreet/Roles/Extra.pm @@ -1,5 +1,5 @@ package FixMyStreet::Roles::Extra; -use Moose::Role; +use Moo::Role; =head1 NAME @@ -9,12 +9,9 @@ FixMyStreet::Roles::Extra - role for accessing {extra} field This is to applied to a DB class like Problem or Contacts that has a rich {extra} field: - use Moose; + use Moo; with 'FixMyStreet::Roles::Extra'; -(NB: there is actually a little more boilerplate, because DBIC doesn't actually -inherit from Moose, see ::Problem for an example.) - Then: $contact->set_extra_fields( diff --git a/perllib/FixMyStreet/Script/Alerts.pm b/perllib/FixMyStreet/Script/Alerts.pm new file mode 100644 index 000000000..e799a5446 --- /dev/null +++ b/perllib/FixMyStreet/Script/Alerts.pm @@ -0,0 +1,298 @@ +package FixMyStreet::Script::Alerts; + +use strict; +use warnings; + +use DateTime::Format::Pg; +use IO::String; + +use mySociety::DBHandle qw(dbh); +use mySociety::Gaze; +use mySociety::Locale; +use mySociety::MaPit; +use RABX; + +use FixMyStreet::Cobrand; +use FixMyStreet::DB; +use FixMyStreet::Email; + +FixMyStreet->configure_mysociety_dbhandle; + +# Child must have confirmed, id, email, state(!) columns +# If parent/child, child table must also have name and text +# and foreign key to parent must be PARENT_id +sub send() { + my $rs = FixMyStreet::DB->resultset('AlertType'); + my $schema = $rs->result_source->schema; + + my $q = $rs->search( { ref => { -not_like => '%local_problems%' } } ); + while (my $alert_type = $q->next) { + my $ref = $alert_type->ref; + my $head_table = $alert_type->head_table; + my $item_table = $alert_type->item_table; + my $query = 'select alert.id as alert_id, alert.user_id as alert_user_id, alert.lang as alert_lang, alert.cobrand as alert_cobrand, + alert.cobrand_data as alert_cobrand_data, alert.parameter as alert_parameter, alert.parameter2 as alert_parameter2, '; + if ($head_table) { + $query .= " + $item_table.id as item_id, $item_table.text as item_text, + $item_table.name as item_name, $item_table.anonymous as item_anonymous, + $item_table.confirmed as item_confirmed, + $head_table.* + from alert, $item_table, $head_table + where alert.parameter::integer = $head_table.id + and $item_table.${head_table}_id = $head_table.id + "; + } else { + $query .= " $item_table.*, + $item_table.id as item_id + from alert, $item_table + where 1 = 1"; + } + $query .= " + and alert_type='$ref' and whendisabled is null and $item_table.confirmed >= whensubscribed + and $item_table.confirmed >= current_timestamp - '7 days'::interval + and (select whenqueued from alert_sent where alert_sent.alert_id = alert.id and alert_sent.parameter::integer = $item_table.id) is null + and $item_table.user_id <> alert.user_id + and " . $alert_type->item_where . " + and alert.confirmed = 1 + order by alert.id, $item_table.confirmed"; + # XXX Ugh - needs work + $query =~ s/\?/alert.parameter/ if ($query =~ /\?/); + $query =~ s/\?/alert.parameter2/ if ($query =~ /\?/); + + $query = dbh()->prepare($query); + $query->execute(); + my $last_alert_id; + my %data = ( template => $alert_type->template, data => '', schema => $schema ); + while (my $row = $query->fetchrow_hashref) { + + my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($row->{alert_cobrand})->new(); + $cobrand->set_lang_and_domain( $row->{alert_lang}, 1, FixMyStreet->path_to('locale')->stringify ); + + # Cobranded and non-cobranded messages can share a database. In this case, the conf file + # should specify a vhost to send the reports for each cobrand, so that they don't get sent + # more than once if there are multiple vhosts running off the same database. The email_host + # call checks if this is the host that sends mail for this cobrand. + next unless $cobrand->email_host; + + # this is for the new_updates alerts + next if $row->{non_public} and $row->{user_id} != $row->{alert_user_id}; + + $schema->resultset('AlertSent')->create( { + alert_id => $row->{alert_id}, + parameter => $row->{item_id}, + } ); + if ($last_alert_id && $last_alert_id != $row->{alert_id}) { + _send_aggregated_alert_email(%data); + %data = ( template => $alert_type->template, data => '', schema => $schema ); + } + + # create problem status message for the templates + if ( FixMyStreet::DB::Result::Problem::fixed_states()->{$row->{state}} ) { + $data{state_message} = _("This report is currently marked as fixed."); + } elsif ( FixMyStreet::DB::Result::Problem::closed_states()->{$row->{state}} ) { + $data{state_message} = _("This report is currently marked as closed.") + } else { + $data{state_message} = _("This report is currently marked as open."); + } + + my $url = $cobrand->base_url_for_report($row); + # this is currently only for new_updates + if ($row->{item_text}) { + if ( $cobrand->moniker ne 'zurich' && $row->{alert_user_id} == $row->{user_id} ) { + # This is an alert to the same user who made the report - make this a login link + # Don't bother with Zurich which has no accounts + my $user = $schema->resultset('User')->find( { + id => $row->{alert_user_id} + } ); + $data{alert_email} = $user->email; + my $token_obj = $schema->resultset('Token')->create( { + scope => 'alert_to_reporter', + data => { + id => $row->{id}, + } + } ); + $data{problem_url} = $url . "/R/" . $token_obj->token; + } else { + $data{problem_url} = $url . "/report/" . $row->{id}; + } + $data{data} .= $row->{item_name} . ' : ' if $row->{item_name} && !$row->{item_anonymous}; + if ( $cobrand->include_time_in_update_alerts ) { + my $parser = DateTime::Format::Pg->new(); + my $dt = $parser->parse_timestamp( $row->{item_confirmed} ); + # We need to always set this otherwise we end up with the DateTime + # object being in the floating timezone in which case applying a + # subsequent timezone set will have no effect. + # this is basically recreating the code from the inflate wrapper + # in the database model. + FixMyStreet->set_time_zone($dt); + $data{data} .= $cobrand->prettify_dt( $dt, 'alert' ) . "\n\n"; + } + $data{data} .= $row->{item_text} . "\n\n------\n\n"; + # this is ward and council problems + } else { + $data{data} .= $url . "/report/" . $row->{id} . " - $row->{title}\n\n"; + if ( exists $row->{geocode} && $row->{geocode} && $ref =~ /ward|council/ ) { + my $nearest_st = _get_address_from_gecode( $row->{geocode} ); + $data{data} .= $nearest_st if $nearest_st; + } + $data{data} .= "\n\n------\n\n"; + } + if (!$data{alert_user_id}) { + %data = (%data, %$row); + if ($ref eq 'area_problems' || $ref eq 'council_problems' || $ref eq 'ward_problems') { + my $va_info = mySociety::MaPit::call('area', $row->{alert_parameter}); + $data{area_name} = $va_info->{name}; + } + if ($ref eq 'ward_problems') { + my $va_info = mySociety::MaPit::call('area', $row->{alert_parameter2}); + $data{ward_name} = $va_info->{name}; + } + } + $data{cobrand} = $row->{alert_cobrand}; + $data{cobrand_data} = $row->{alert_cobrand_data}; + $data{lang} = $row->{alert_lang}; + $last_alert_id = $row->{alert_id}; + } + if ($last_alert_id) { + _send_aggregated_alert_email(%data); + } + } + + # Nearby done separately as the table contains the parameters + my $template = $rs->find( { ref => 'local_problems' } )->template; + my $query = $schema->resultset('Alert')->search( { + alert_type => 'local_problems', + whendisabled => undef, + confirmed => 1 + }, { + order_by => 'id' + } ); + while (my $alert = $query->next) { + my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($alert->cobrand)->new(); + next unless $cobrand->email_host; + next if $alert->is_from_abuser; + + my $longitude = $alert->parameter; + my $latitude = $alert->parameter2; + my $d = mySociety::Gaze::get_radius_containing_population($latitude, $longitude, 200000); + # Convert integer to GB locale string (with a ".") + $d = mySociety::Locale::in_gb_locale { + sprintf("%f", int($d*10+0.5)/10); + }; + my $states = "'" . join( "', '", FixMyStreet::DB::Result::Problem::visible_states() ) . "'"; + my %data = ( + template => $template, + data => '', + alert_id => $alert->id, + alert_email => $alert->user->email, + lang => $alert->lang, + cobrand => $alert->cobrand, + cobrand_data => $alert->cobrand_data, + schema => $schema, + ); + my $q = "select problem.id, problem.bodies_str, problem.postcode, problem.geocode, problem.title from problem_find_nearby(?, ?, ?) as nearby, problem, users + where nearby.problem_id = problem.id + and problem.user_id = users.id + and problem.state in ($states) + and problem.non_public = 'f' + and problem.confirmed >= ? and problem.confirmed >= current_timestamp - '7 days'::interval + and (select whenqueued from alert_sent where alert_sent.alert_id = ? and alert_sent.parameter::integer = problem.id) is null + and users.email <> ? + order by confirmed desc"; + $q = dbh()->prepare($q); + $q->execute($latitude, $longitude, $d, $alert->whensubscribed, $alert->id, $alert->user->email); + while (my $row = $q->fetchrow_hashref) { + $schema->resultset('AlertSent')->create( { + alert_id => $alert->id, + parameter => $row->{id}, + } ); + my $url = $cobrand->base_url_for_report($row); + $data{data} .= $url . "/report/" . $row->{id} . " - $row->{title}\n\n"; + if ( exists $row->{geocode} && $row->{geocode} ) { + my $nearest_st = _get_address_from_gecode( $row->{geocode} ); + $data{data} .= $nearest_st if $nearest_st; + } + $data{data} .= "\n\n------\n\n"; + } + _send_aggregated_alert_email(%data) if $data{data}; + } +} + +sub _send_aggregated_alert_email(%) { + my %data = @_; + + my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($data{cobrand})->new(); + + $cobrand->set_lang_and_domain( $data{lang}, 1, FixMyStreet->path_to('locale')->stringify ); + + if (!$data{alert_email}) { + my $user = $data{schema}->resultset('User')->find( { + id => $data{alert_user_id} + } ); + $data{alert_email} = $user->email; + } + + my ($domain) = $data{alert_email} =~ m{ @ (.*) \z }x; + return if $data{schema}->resultset('Abuse')->search( { + email => [ $data{alert_email}, $domain ] + } )->first; + + my $token = $data{schema}->resultset("Token")->new_result( { + scope => 'alert', + data => { + id => $data{alert_id}, + type => 'unsubscribe', + email => $data{alert_email}, + } + } ); + $data{unsubscribe_url} = $cobrand->base_url( $data{cobrand_data} ) . '/A/' . $token->token; + + my $template = FixMyStreet->get_email_template($cobrand->moniker, $data{lang}, "$data{template}.txt"); + + my $sender = sprintf('<fms-%s@%s>', + FixMyStreet::Email::generate_verp_token('alert', $data{alert_id}), + FixMyStreet->config('EMAIL_DOMAIN') + ); + + my $result = FixMyStreet::Email::send_cron( + $data{schema}, + { + _template_ => $template, + _parameters_ => \%data, + To => $data{alert_email}, + }, + $sender, + 0, + $cobrand, + $data{lang} + ); + + unless ($result) { + $token->insert(); + } else { + print "Failed to send alert $data{alert_id}!"; + } +} + +sub _get_address_from_gecode { + my $geocode = shift; + + return '' unless defined $geocode; + utf8::encode($geocode) if utf8::is_utf8($geocode); + my $h = new IO::String($geocode); + my $data = RABX::wire_rd($h); + + my $str = ''; + + my $address = $data->{resourceSets}[0]{resources}[0]{address}; + my @address; + push @address, $address->{addressLine} if $address->{addressLine} && $address->{addressLine} ne 'Street'; + push @address, $address->{locality} if $address->{locality}; + $str .= sprintf(_("Nearest road to the pin placed on the map (automatically generated by Bing Maps): %s\n\n"), + join( ', ', @address ) ) if @address; + + return $str; +} + +1; diff --git a/perllib/FixMyStreet/Script/Questionnaires.pm b/perllib/FixMyStreet/Script/Questionnaires.pm new file mode 100644 index 000000000..2d676f15d --- /dev/null +++ b/perllib/FixMyStreet/Script/Questionnaires.pm @@ -0,0 +1,117 @@ +package FixMyStreet::Script::Questionnaires; + +use strict; +use warnings; +use Utils; +use FixMyStreet::DB; +use FixMyStreet::Email; +use FixMyStreet::Cobrand; + +sub send { + my ( $params ) = @_; + send_questionnaires_period( '4 weeks', $params ); + send_questionnaires_period( '26 weeks', $params ) + if $params->{site} eq 'emptyhomes'; +} + +sub send_questionnaires_period { + my ( $period, $params ) = @_; + + my $rs = FixMyStreet::DB->resultset('Questionnaire'); + + # Select all problems that need a questionnaire email sending + my $q_params = { + state => [ FixMyStreet::DB::Result::Problem::visible_states() ], + whensent => [ + '-and', + { '!=', undef }, + { '<', \"current_timestamp - '$period'::interval" }, + ], + send_questionnaire => 1, + }; + # FIXME Do these a bit better... + if ($params->{site} eq 'emptyhomes' && $period eq '4 weeks') { + $q_params->{'(select max(whensent) from questionnaire where me.id=problem_id)'} = undef; + } elsif ($params->{site} eq 'emptyhomes' && $period eq '26 weeks') { + $q_params->{'(select max(whensent) from questionnaire where me.id=problem_id)'} = { '!=', undef }; + } else { + $q_params->{'-or'} = [ + '(select max(whensent) from questionnaire where me.id=problem_id)' => undef, + '(select max(whenanswered) from questionnaire where me.id=problem_id)' => { '<', \"current_timestamp - '$period'::interval" } + ]; + } + + my $unsent = FixMyStreet::DB->resultset('Problem')->search( $q_params, { + order_by => { -desc => 'confirmed' } + } ); + + while (my $row = $unsent->next) { + + my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($row->cobrand)->new(); + $cobrand->set_lang_and_domain($row->lang, 1); + + # Not all cobrands send questionnaires + next unless $cobrand->send_questionnaires; + next if $row->is_from_abuser; + + # Cobranded and non-cobranded messages can share a database. In this case, the conf file + # should specify a vhost to send the reports for each cobrand, so that they don't get sent + # more than once if there are multiple vhosts running off the same database. The email_host + # call checks if this is the host that sends mail for this cobrand. + next unless $cobrand->email_host; + + my $template; + if ($params->{site} eq 'emptyhomes') { + ($template = $period) =~ s/ //; + $template = Utils::read_file( FixMyStreet->path_to( "templates/email/emptyhomes/" . $row->lang . "/questionnaire-$template.txt" )->stringify ); + } else { + $template = FixMyStreet->get_email_template($cobrand->moniker, $row->lang, 'questionnaire.txt'); + } + + my %h = map { $_ => $row->$_ } qw/name title detail category/; + $h{created} = Utils::prettify_duration( time() - $row->confirmed->epoch, 'week' ); + + my $questionnaire = $rs->create( { + problem_id => $row->id, + whensent => \'current_timestamp', + } ); + + # We won't send another questionnaire unless they ask for it (or it was + # the first EHA questionnaire. + $row->send_questionnaire( 0 ) + if $params->{site} ne 'emptyhomes' || $period eq '26 weeks'; + + my $token = FixMyStreet::DB->resultset("Token")->new_result( { + scope => 'questionnaire', + data => $questionnaire->id, + } ); + $h{url} = $cobrand->base_url($row->cobrand_data) . '/Q/' . $token->token; + + print "Sending questionnaire " . $questionnaire->id . ", problem " + . $row->id . ", token " . $token->token . " to " + . $row->user->email . "\n" + if $params->{verbose}; + + my $result = FixMyStreet::Email::send_cron( + $rs->result_source->schema, + { + _template_ => $template, + _parameters_ => \%h, + To => [ [ $row->user->email, $row->name ] ], + }, + undef, + $params->{nomail}, + $cobrand + ); + unless ($result) { + print " ...success\n" if $params->{verbose}; + $row->update(); + $token->insert(); + } else { + print " ...failed\n" if $params->{verbose}; + $questionnaire->delete; + } + } +} + +1; diff --git a/perllib/FixMyStreet/Script/Reports.pm b/perllib/FixMyStreet/Script/Reports.pm new file mode 100644 index 000000000..e1eb5e2c5 --- /dev/null +++ b/perllib/FixMyStreet/Script/Reports.pm @@ -0,0 +1,315 @@ +package FixMyStreet::Script::Reports; + +use strict; +use warnings; + +use CronFns; +use DateTime::Format::Pg; + +use Utils; +use Utils::OpenStreetMap; +use mySociety::MaPit; + +use FixMyStreet; +use FixMyStreet::Cobrand; +use FixMyStreet::DB; +use FixMyStreet::Email; +use FixMyStreet::SendReport; + +sub send(;$) { + my ($site_override) = @_; + my $rs = FixMyStreet::DB->resultset('Problem'); + + # Set up site, language etc. + my ($verbose, $nomail, $debug_mode) = CronFns::options(); + + my $base_url = FixMyStreet->config('BASE_URL'); + my $site = $site_override || CronFns::site($base_url); + + my $states = [ 'confirmed', 'fixed' ]; + $states = [ 'unconfirmed', 'confirmed', 'in progress', 'planned', 'closed', 'investigating' ] if $site eq 'zurich'; + my $unsent = $rs->search( { + state => $states, + whensent => undef, + bodies_str => { '!=', undef }, + } ); + my (%notgot, %note); + + my $send_report = FixMyStreet::SendReport->new(); + my $senders = $send_report->get_senders; + + my $debug_unsent_count = 0; + debug_print("starting to loop through unsent problem reports...") if $debug_mode; + while (my $row = $unsent->next) { + + my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($row->cobrand)->new(); + + if ($debug_mode) { + $debug_unsent_count++; + print "\n"; + debug_print("state=" . $row->state . ", bodies_str=" . $row->bodies_str . ($row->cobrand? ", cobrand=" . $row->cobrand : ""), $row->id); + } + + # Cobranded and non-cobranded messages can share a database. In this case, the conf file + # should specify a vhost to send the reports for each cobrand, so that they don't get sent + # more than once if there are multiple vhosts running off the same database. The email_host + # call checks if this is the host that sends mail for this cobrand. + if (! $cobrand->email_host()) { + debug_print("skipping because this host does not send reports for cobrand " . $cobrand->moniker, $row->id) if $debug_mode; + next; + } + + $cobrand->set_lang_and_domain($row->lang, 1); + if ( $row->is_from_abuser) { + $row->update( { state => 'hidden' } ); + debug_print("hiding because its sender is flagged as an abuser", $row->id) if $debug_mode; + next; + } elsif ( $row->title =~ /app store test/i ) { + $row->update( { state => 'hidden' } ); + debug_print("hiding because it is an app store test message", $row->id) if $debug_mode; + next; + } + + # Template variables for the email + my $email_base_url = $cobrand->base_url_for_report($row); + my %h = map { $_ => $row->$_ } qw/id title detail name category latitude longitude used_map/; + map { $h{$_} = $row->user->$_ || '' } qw/email phone/; + $h{confirmed} = DateTime::Format::Pg->format_datetime( $row->confirmed->truncate (to => 'second' ) ) + if $row->confirmed; + + $h{query} = $row->postcode; + $h{url} = $email_base_url . $row->url; + $h{admin_url} = $row->admin_url($cobrand); + $h{phone_line} = $h{phone} ? _('Phone:') . " $h{phone}\n\n" : ''; + if ($row->photo) { + $h{has_photo} = _("This web page also contains a photo of the problem, provided by the user.") . "\n\n"; + $h{image_url} = $email_base_url . '/photo/' . $row->id . '.full.jpeg'; + } else { + $h{has_photo} = ''; + $h{image_url} = ''; + } + $h{fuzzy} = $row->used_map ? _('To view a map of the precise location of this issue') + : _('The user could not locate the problem on a map, but to see the area around the location they entered'); + $h{closest_address} = ''; + + $h{osm_url} = Utils::OpenStreetMap::short_url($h{latitude}, $h{longitude}); + if ( $row->used_map ) { + $h{closest_address} = $cobrand->find_closest( $h{latitude}, $h{longitude}, $row ); + $h{osm_url} .= '?m'; + } + + if ( $cobrand->allow_anonymous_reports && + $row->user->email eq $cobrand->anonymous_account->{'email'} + ) { + $h{anonymous_report} = 1; + $h{user_details} = _('This report was submitted anonymously'); + } else { + $h{user_details} = sprintf(_('Name: %s'), $row->name) . "\n\n"; + $h{user_details} .= sprintf(_('Email: %s'), $row->user->email) . "\n\n"; + } + + $h{easting_northing} = ''; + + if ($cobrand->can('process_additional_metadata_for_email')) { + $cobrand->process_additional_metadata_for_email($row, \%h); + } + + my $bodies = FixMyStreet::DB->resultset('Body')->search( + { id => $row->bodies_str_ids }, + { order_by => 'name' }, + ); + + my $missing; + if ($row->bodies_missing) { + my @missing = FixMyStreet::DB->resultset("Body")->search( + { id => [ split /,/, $row->bodies_missing ] }, + { order_by => 'name' } + )->get_column('name')->all; + $missing = join(' / ', @missing) if @missing; + } + + my @dear; + my %reporters = (); + my $skip = 0; + while (my $body = $bodies->next) { + my $sender_info = $cobrand->get_body_sender( $body, $row->category ); + my $sender = "FixMyStreet::SendReport::" . $sender_info->{method}; + + if ( ! exists $senders->{ $sender } ) { + warn sprintf "No such sender [ $sender ] for body %s ( %d )", $body->name, $body->id; + next; + } + $reporters{ $sender } ||= $sender->new(); + + if ( $reporters{ $sender }->should_skip( $row ) ) { + $skip = 1; + debug_print("skipped by sender " . $sender_info->{method} . " (might be due to previous failed attempts?)", $row->id) if $debug_mode; + } else { + debug_print("OK, adding recipient body " . $body->id . ":" . $body->name . ", " . $body->send_method, $row->id) if $debug_mode; + push @dear, $body->name; + $reporters{ $sender }->add_body( $body, $sender_info->{config} ); + } + + # If we are in the UK include eastings and northings, and nearest stuff + if ( $cobrand->country eq 'GB' && !$h{easting} ) { + my $coordsyst = 'G'; + my $first_area = $body->body_areas->first->area_id; + my $area_info = mySociety::MaPit::call('area', $first_area); + $coordsyst = 'I' if $area_info->{type} eq 'LGD'; + + ( $h{easting}, $h{northing} ) = Utils::convert_latlon_to_en( $h{latitude}, $h{longitude}, $coordsyst ); + + # email templates don't have conditionals so we need to format this here + $h{easting_northing} = "Easting/Northing"; + $h{easting_northing} .= " (IE)" if $coordsyst eq 'I'; + $h{easting_northing} .= ": $h{easting}/$h{northing}\n\n"; + } + } + + unless ( keys %reporters ) { + die 'Report not going anywhere for ID ' . $row->id . '!'; + } + + next if $skip; + + if ($h{category} eq _('Other')) { + $h{category_footer} = _('this type of local problem'); + $h{category_line} = ''; + } else { + $h{category_footer} = "'" . $h{category} . "'"; + $h{category_line} = sprintf(_("Category: %s"), $h{category}) . "\n\n"; + } + + if ( $row->subcategory ) { + $h{subcategory_line} = sprintf(_("Subcategory: %s"), $row->subcategory) . "\n\n"; + } else { + $h{subcategory_line} = "\n\n"; + } + + $h{bodies_name} = join(_(' and '), @dear); + if ($h{category} eq _('Other')) { + $h{multiple} = @dear>1 ? "[ " . _("This email has been sent to both councils covering the location of the problem, as the user did not categorise it; please ignore it if you're not the correct council to deal with the issue, or let us know what category of problem this is so we can add it to our system.") . " ]\n\n" + : ''; + } else { + $h{multiple} = @dear>1 ? "[ " . _("This email has been sent to several councils covering the location of the problem, as the category selected is provided for all of them; please ignore it if you're not the correct council to deal with the issue.") . " ]\n\n" + : ''; + } + $h{missing} = ''; + if ($missing) { + $h{missing} = '[ ' + . sprintf(_('We realise this problem might be the responsibility of %s; however, we don\'t currently have any contact details for them. If you know of an appropriate contact address, please do get in touch.'), $missing) + . " ]\n\n"; + } + + if (FixMyStreet->config('STAGING_SITE') && !FixMyStreet->config('SEND_REPORTS_ON_STAGING')) { + # on a staging server send emails to ourselves rather than the bodies + %reporters = map { $_ => $reporters{$_} } grep { /FixMyStreet::SendReport::(Email|EmptyHomes)/ } keys %reporters; + unless (%reporters) { + %reporters = ( 'FixMyStreet::SendReport::Email' => FixMyStreet::SendReport::Email->new() ); + } + } + + # Multiply results together, so one success counts as a success. + my $result = -1; + + for my $sender ( keys %reporters ) { + debug_print("sending using " . $sender, $row->id) if $debug_mode; + $result *= $reporters{ $sender }->send( $row, \%h ); + if ( $reporters{ $sender }->unconfirmed_counts) { + foreach my $e (keys %{ $reporters{ $sender }->unconfirmed_counts } ) { + foreach my $c (keys %{ $reporters{ $sender }->unconfirmed_counts->{$e} }) { + $notgot{$e}{$c} += $reporters{ $sender }->unconfirmed_counts->{$e}{$c}; + } + } + %note = ( + %note, + %{ $reporters{ $sender }->unconfirmed_notes } + ); + } + } + + unless ($result) { + $row->update( { + whensent => \'current_timestamp', + lastupdate => \'current_timestamp', + } ); + if ( $cobrand->report_sent_confirmation_email && !$h{anonymous_report}) { + _send_report_sent_email( $row, \%h, $nomail, $cobrand ); + } + debug_print("send successful: OK", $row->id) if $debug_mode; + } else { + my @errors; + for my $sender ( keys %reporters ) { + unless ( $reporters{ $sender }->success ) { + push @errors, $reporters{ $sender }->error; + } + } + $row->update_send_failed( join( '|', @errors ) ); + debug_print("send FAILED: " . join( '|', @errors ), $row->id) if $debug_mode; + } + } + if ($debug_mode) { + print "\n"; + if ($debug_unsent_count) { + debug_print("processed all unsent reports (total: $debug_unsent_count)"); + } else { + debug_print("no unsent reports were found (must have whensent=null and suitable bodies_str & state) -- nothing to send"); + } + } + + if ($verbose || $debug_mode) { + print "Council email addresses that need checking:\n" if keys %notgot; + foreach my $e (keys %notgot) { + foreach my $c (keys %{$notgot{$e}}) { + print " " . $notgot{$e}{$c} . " problem, to $e category $c (" . $note{$e}{$c}. ")\n"; + } + } + my $sending_errors = ''; + my $unsent = $rs->search( { + state => [ 'confirmed', 'fixed' ], + whensent => undef, + bodies_str => { '!=', undef }, + send_fail_count => { '>', 0 } + } ); + while (my $row = $unsent->next) { + my $base_url = FixMyStreet->config('BASE_URL'); + $sending_errors .= "* " . $base_url . "/report/" . $row->id . ", failed " + . $row->send_fail_count . " times, last at " . $row->send_fail_timestamp + . ", reason " . $row->send_fail_reason . "\n"; + } + if ($sending_errors) { + print "The following reports had problems sending:\n$sending_errors"; + } + } +} + +sub _send_report_sent_email { + my $row = shift; + my $h = shift; + my $nomail = shift; + my $cobrand = shift; + + my $template = FixMyStreet->get_email_template($row->cobrand, $row->lang, 'confirm_report_sent.txt'); + + FixMyStreet::Email::send_cron( + $row->result_source->schema, + { + _template_ => $template, + _parameters_ => $h, + To => $row->user->email, + From => [ FixMyStreet->config('CONTACT_EMAIL'), $cobrand->contact_name ], + }, + FixMyStreet->config('CONTACT_EMAIL'), + $nomail, + $cobrand + ); +} + +sub debug_print { + my $msg = shift; + my $id = shift || ''; + $id = "report $id: " if $id; + print "[] $id$msg\n"; +} + +1; diff --git a/perllib/FixMyStreet/SendReport.pm b/perllib/FixMyStreet/SendReport.pm index 9967b0663..40ec4caf2 100644 --- a/perllib/FixMyStreet/SendReport.pm +++ b/perllib/FixMyStreet/SendReport.pm @@ -1,20 +1,21 @@ package FixMyStreet::SendReport; -use Moose; +use Moo; +use MooX::Types::MooseLike::Base qw(:all); use Module::Pluggable sub_name => 'senders', search_path => __PACKAGE__, require => 1; -has 'body_config' => ( is => 'rw', isa => 'HashRef', default => sub { {} } ); -has 'bodies' => ( is => 'rw', isa => 'ArrayRef', default => sub { [] } ); -has 'to' => ( is => 'rw', isa => 'ArrayRef', default => sub { [] } ); -has 'bcc' => ( is => 'rw', isa => 'ArrayRef', default => sub { [] } ); -has 'success' => ( is => 'rw', isa => 'Bool', default => 0 ); -has 'error' => ( is => 'rw', isa => 'Str', default => '' ); -has 'unconfirmed_counts' => ( 'is' => 'rw', isa => 'HashRef', default => sub { {} } ); -has 'unconfirmed_notes' => ( 'is' => 'rw', isa => 'HashRef', default => sub { {} } ); +has 'body_config' => ( is => 'rw', isa => HashRef, default => sub { {} } ); +has 'bodies' => ( is => 'rw', isa => ArrayRef, default => sub { [] } ); +has 'to' => ( is => 'rw', isa => ArrayRef, default => sub { [] } ); +has 'bcc' => ( is => 'rw', isa => ArrayRef, default => sub { [] } ); +has 'success' => ( is => 'rw', isa => Bool, default => 0 ); +has 'error' => ( is => 'rw', isa => Str, default => '' ); +has 'unconfirmed_counts' => ( 'is' => 'rw', isa => HashRef, default => sub { {} } ); +has 'unconfirmed_notes' => ( 'is' => 'rw', isa => HashRef, default => sub { {} } ); sub should_skip { diff --git a/perllib/FixMyStreet/SendReport/EastHants.pm b/perllib/FixMyStreet/SendReport/EastHants.pm index 44bc084b1..3eb8ffcfa 100644 --- a/perllib/FixMyStreet/SendReport/EastHants.pm +++ b/perllib/FixMyStreet/SendReport/EastHants.pm @@ -1,6 +1,6 @@ package FixMyStreet::SendReport::EastHants; -use Moose; +use Moo; BEGIN { extends 'FixMyStreet::SendReport'; } @@ -28,7 +28,7 @@ EOF } sub send { - return if mySociety::Config::get('STAGING_SITE'); + return if FixMyStreet->config('STAGING_SITE'); my ( $self, $row, $h ) = @_; diff --git a/perllib/FixMyStreet/SendReport/Email.pm b/perllib/FixMyStreet/SendReport/Email.pm index bac408510..7e5c10469 100644 --- a/perllib/FixMyStreet/SendReport/Email.pm +++ b/perllib/FixMyStreet/SendReport/Email.pm @@ -1,6 +1,6 @@ package FixMyStreet::SendReport::Email; -use Moose; +use Moo; use FixMyStreet::Email; BEGIN { extends 'FixMyStreet::SendReport'; } @@ -11,7 +11,7 @@ sub build_recipient_list { my $all_confirmed = 1; foreach my $body ( @{ $self->bodies } ) { - my $contact = FixMyStreet::App->model("DB::Contact")->find( { + my $contact = $row->result_source->schema->resultset("Contact")->find( { deleted => 0, body_id => $body->id, category => $row->category @@ -75,7 +75,7 @@ sub send { my $recips = $self->build_recipient_list( $row, $h ); # on a staging server send emails to ourselves rather than the bodies - if (mySociety::Config::get('STAGING_SITE') && !mySociety::Config::get('SEND_REPORTS_ON_STAGING') && !FixMyStreet->test_mode) { + if (FixMyStreet->config('STAGING_SITE') && !FixMyStreet->config('SEND_REPORTS_ON_STAGING') && !FixMyStreet->test_mode) { $recips = 1; @{$self->to} = [ $row->user->email, $self->to->[0][1] || $row->name ]; } @@ -94,23 +94,21 @@ sub send { From => $self->send_from( $row ), }; - my $app = FixMyStreet::App->new( cobrand => $cobrand ); - - $cobrand->munge_sendreport_params($app, $row, $h, $params) if $cobrand->can('munge_sendreport_params'); + $cobrand->munge_sendreport_params($row, $h, $params) if $cobrand->can('munge_sendreport_params'); $params->{Bcc} = $self->bcc if @{$self->bcc}; + my $sender = sprintf('<fms-%s@%s>', + FixMyStreet::Email::generate_verp_token('report', $row->id), + FixMyStreet->config('EMAIL_DOMAIN') + ); + if (FixMyStreet::Email::test_dmarc($params->{From}[0])) { $params->{'Reply-To'} = [ $params->{From} ]; - $params->{From} = [ mySociety::Config::get('CONTACT_EMAIL'), $params->{From}[1] ]; + $params->{From} = [ $sender, $params->{From}[1] ]; } - my $result = $app->send_email_cron( - $params, - mySociety::Config::get('CONTACT_EMAIL'), - $nomail, - $cobrand - ); + my $result = FixMyStreet::Email::send_cron($row->result_source->schema, $params, $sender, $nomail, $cobrand); unless ($result) { $self->success(1); diff --git a/perllib/FixMyStreet/SendReport/EmptyHomes.pm b/perllib/FixMyStreet/SendReport/EmptyHomes.pm index ce69aaac3..b5faf8ddc 100644 --- a/perllib/FixMyStreet/SendReport/EmptyHomes.pm +++ b/perllib/FixMyStreet/SendReport/EmptyHomes.pm @@ -1,6 +1,6 @@ package FixMyStreet::SendReport::EmptyHomes; -use Moose; +use Moo; use namespace::autoclean; use mySociety::MaPit; @@ -12,7 +12,7 @@ sub build_recipient_list { my $all_confirmed = 1; foreach my $body ( @{ $self->bodies } ) { - my $contact = FixMyStreet::App->model("DB::Contact")->find( { + my $contact = $row->result_source->schema->resultset("Contact")->find( { deleted => 0, body_id => $body->id, category => 'Empty property', @@ -34,11 +34,11 @@ sub build_recipient_list { my $area_info = mySociety::MaPit::call('area', $body->body_areas->first->area_id); my $country = $area_info->{country}; if ($country eq 'W') { - push @{$self->bcc}, 'wales@' . mySociety::Config::get('EMAIL_DOMAIN'); + push @{$self->bcc}, 'wales@' . FixMyStreet->config('EMAIL_DOMAIN'); } elsif ($country eq 'S') { - push @{$self->bcc}, 'scotland@' . mySociety::Config::get('EMAIL_DOMAIN'); + push @{$self->bcc}, 'scotland@' . FixMyStreet->config('EMAIL_DOMAIN'); } else { - push @{$self->bcc}, 'eha@' . mySociety::Config::get('EMAIL_DOMAIN'); + push @{$self->bcc}, 'eha@' . FixMyStreet->config('EMAIL_DOMAIN'); } } diff --git a/perllib/FixMyStreet/SendReport/Noop.pm b/perllib/FixMyStreet/SendReport/Noop.pm index f2e0a3bdb..60edda373 100644 --- a/perllib/FixMyStreet/SendReport/Noop.pm +++ b/perllib/FixMyStreet/SendReport/Noop.pm @@ -1,6 +1,6 @@ package FixMyStreet::SendReport::Noop; -use Moose; +use Moo; BEGIN { extends 'FixMyStreet::SendReport'; } diff --git a/perllib/FixMyStreet/SendReport/Open311.pm b/perllib/FixMyStreet/SendReport/Open311.pm index fa216466e..4844aa2e9 100644 --- a/perllib/FixMyStreet/SendReport/Open311.pm +++ b/perllib/FixMyStreet/SendReport/Open311.pm @@ -1,12 +1,10 @@ package FixMyStreet::SendReport::Open311; -use Moose; +use Moo; use namespace::autoclean; BEGIN { extends 'FixMyStreet::SendReport'; } -use FixMyStreet::App; -use mySociety::Config; use DateTime::Format::W3CDTF; use Open311; use Readonly; @@ -85,7 +83,7 @@ sub send { } # FIXME: we've already looked this up before - my $contact = FixMyStreet::App->model("DB::Contact")->find( { + my $contact = $row->result_source->schema->resultset("Contact")->find( { deleted => 0, body_id => $body->id, category => $row->category diff --git a/perllib/FixMyStreet/SendReport/Refused.pm b/perllib/FixMyStreet/SendReport/Refused.pm index d71fc5c2c..c6c1a660d 100644 --- a/perllib/FixMyStreet/SendReport/Refused.pm +++ b/perllib/FixMyStreet/SendReport/Refused.pm @@ -1,6 +1,6 @@ package FixMyStreet::SendReport::Refused; -use Moose; +use Moo; BEGIN { extends 'FixMyStreet::SendReport::Noop'; } diff --git a/perllib/FixMyStreet/SendReport/Zurich.pm b/perllib/FixMyStreet/SendReport/Zurich.pm index 2838440cb..a8730bbe4 100644 --- a/perllib/FixMyStreet/SendReport/Zurich.pm +++ b/perllib/FixMyStreet/SendReport/Zurich.pm @@ -1,6 +1,6 @@ package FixMyStreet::SendReport::Zurich; -use Moose; +use Moo; BEGIN { extends 'FixMyStreet::SendReport::Email'; } @@ -14,7 +14,7 @@ sub build_recipient_list { # Wunsch set, but external_message hasn't yet been filled in. TODO should # we instead be holding off sending?) if ( $row->external_body ) { - $body = FixMyStreet::App->model("DB::Body")->find( { id => $row->external_body } ); + $body = $row->result_source->schema->resultset("Body")->find( { id => $row->external_body } ); $h->{bodies_name} = $body->name; $h->{external_message} = $row->get_extra_metadata('external_message') || ''; } @@ -29,7 +29,7 @@ sub build_recipient_list { my $parent = $body->parent; if ($parent && !$parent->parent) { # Division, might have an individual contact email address - my $contact = FixMyStreet::App->model("DB::Contact")->find( { + my $contact = $row->result_source->schema->resultset("Contact")->find( { body_id => $body->id, category => $row->category } ); @@ -72,7 +72,7 @@ sub send_from { if ( $row->external_body ) { my $body = @{ $self->bodies }[0]; my $body_email = $body->endpoint; - my $contact = FixMyStreet::App->model("DB::Contact")->find( { + my $contact = $body->result_source->schema->resultset("Contact")->find( { body_id => $body->id, category => $row->category } ); diff --git a/perllib/FixMyStreet/TestMech.pm b/perllib/FixMyStreet/TestMech.pm index cc5f9dd71..f3ee7787b 100644 --- a/perllib/FixMyStreet/TestMech.pm +++ b/perllib/FixMyStreet/TestMech.pm @@ -14,7 +14,7 @@ use Test::More; use Web::Scraper; use Carp; use Email::Send::Test; -use JSON; +use JSON::MaybeXS; =head1 NAME @@ -66,7 +66,7 @@ sub create_user_ok { my ($email) = @_; my $user = - FixMyStreet::App->model('DB::User') + FixMyStreet::DB->resultset('User') ->find_or_create( { email => $email } ); ok $user, "found/created user for $email"; @@ -147,7 +147,7 @@ sub delete_user { my $user = ref $email_or_user ? $email_or_user - : FixMyStreet::App->model('DB::User') + : FixMyStreet::DB->resultset('User') ->find( { email => $email_or_user } ); # If no user found we can't delete them @@ -234,8 +234,8 @@ sub get_first_email { my $mech = shift; my $email = shift or do { fail 'No email retrieved'; return }; my $email_as_string = $email->as_string; - ok $email_as_string =~ s{\s+Date:\s+\S.*?$}{}xmsg, "Found and stripped out date"; - ok $email_as_string =~ s{\s+Message-ID:\s+\S.*?$}{}xmsg, "Found and stripped out message ID (contains epoch)"; + ok $email_as_string =~ s{^Date:\s+\S.*?\r?\n}{}xmsg, "Found and stripped out date"; + ok $email_as_string =~ s{^Message-ID:\s+\S.*?\r?\n}{}xmsg, "Found and stripped out message ID (contains epoch)"; return $email_as_string; } @@ -567,7 +567,7 @@ sub delete_problems_for_body { my $mech = shift; my $body = shift; - my $reports = FixMyStreet::App->model('DB::Problem')->search( { bodies_str => $body } ); + my $reports = FixMyStreet::DB->resultset('Problem')->search( { bodies_str => $body } ); if ( $reports ) { for my $r ( $reports->all ) { $r->comments->delete; @@ -587,7 +587,7 @@ sub create_contact_ok { note => 'Created for test', @_ ); - my $contact = FixMyStreet::App->model('DB::Contact')->find_or_create( \%contact_params ); + my $contact = FixMyStreet::DB->resultset('Contact')->find_or_create( \%contact_params ); ok $contact, 'found/created contact ' . $contact->category;; return $contact; } @@ -596,7 +596,7 @@ sub create_body_ok { my $self = shift; my ( $area_id, $name, %extra ) = @_; - my $body = FixMyStreet::App->model('DB::Body'); + my $body = FixMyStreet::DB->resultset('Body'); my $params = { name => $name }; if ($extra{id}) { $body = $body->update_or_create({ %$params, id => $extra{id} }, { key => 'primary' }); @@ -606,7 +606,7 @@ sub create_body_ok { ok $body, "found/created body $name"; $body->body_areas->delete; - FixMyStreet::App->model('DB::BodyArea')->find_or_create({ + FixMyStreet::DB->resultset('BodyArea')->find_or_create({ area_id => $area_id, body_id => $body->id, }); @@ -621,7 +621,7 @@ sub create_problems_for_body { my $dt = $params->{dt} || DateTime->now(); my $user = $params->{user} || - FixMyStreet::App->model('DB::User') + FixMyStreet::DB->resultset('User') ->find_or_create( { email => 'test@example.com', name => 'Test User' } ); delete $params->{user}; @@ -656,7 +656,7 @@ sub create_problems_for_body { my %report_params = ( %$default_params, %$params ); my $problem = - FixMyStreet::App->model('DB::Problem')->create( \%report_params ); + FixMyStreet::DB->resultset('Problem')->create( \%report_params ); push @problems, $problem; $count--; |