4 files changed, 246 insertions, 38 deletions

diff --git a/perllib/FixMyStreet/Script/ArchiveOldEnquiries.pm b/perllib/FixMyStreet/Script/ArchiveOldEnquiries.pm
index 5d1d45379..03bc511a0 100644
--- a/perllib/FixMyStreet/Script/ArchiveOldEnquiries.pm
+++ b/perllib/FixMyStreet/Script/ArchiveOldEnquiries.pm
@@ -14,10 +14,6 @@ use FixMyStreet::Email;
 
 my $opts = {
     commit => 0,
-    body => '2237',
-    cobrand => 'oxfordshire',
-    closure_cutoff => "2015-01-01 00:00:00",
-    email_cutoff => "2016-01-01 00:00:00",
 };
 
 sub query {
@@ -84,11 +80,7 @@ sub archive {
     });
 
     printf("Closing %d old reports, without sending emails: ", $problems_to_close->count);
-
-    if ( $opts->{commit} ) {
-        $problems_to_close->update({ state => 'closed', send_questionnaire => 0 });
-    }
-
+    close_problems($problems_to_close);
     printf("done.\n")
 }
 
@@ -132,10 +124,46 @@ sub send_email_and_close {
 
     unless ( $email_error ) {
         printf("done.\n    Closing reports: ");
-
-        $problems->update({ state => 'closed', send_questionnaire => 0 });
+        close_problems($problems);
         printf("done.\n");
     } else {
         printf("error! Not closing reports for this user.\n")
     }
 }
+
+sub close_problems {
+    return unless $opts->{commit};
+
+    my $problems = shift;
+    while (my $problem = $problems->next) {
+        my $timestamp = \'current_timestamp';
+        my $comment = $problem->add_to_comments( {
+            text => '',
+            created => $timestamp,
+            confirmed => $timestamp,
+            user_id => $opts->{user},
+            name => _('an administrator'),
+            mark_fixed => 0,
+            anonymous => 0,
+            state => 'confirmed',
+            problem_state => 'closed',
+            extra => { is_superuser => 1 },
+        } );
+        $problem->update({ state => 'closed', send_questionnaire => 0 });
+
+        # Stop any alerts being sent out about this closure.
+        my @alerts = FixMyStreet::DB->resultset('Alert')->search( {
+            alert_type => 'new_updates',
+            parameter  => $problem->id,
+            confirmed  => 1,
+        } );
+
+        for my $alert (@alerts) {
+            my $alerts_sent = FixMyStreet::DB->resultset('AlertSent')->find_or_create( {
+                alert_id  => $alert->id,
+                parameter => $comment->id,
+            } );
+        }
+
+    }
+}
diff --git a/perllib/FixMyStreet/Script/Inactive.pm b/perllib/FixMyStreet/Script/Inactive.pm
new file mode 100644
index 000000000..0468d2a52
--- /dev/null
+++ b/perllib/FixMyStreet/Script/Inactive.pm
@@ -0,0 +1,174 @@
+package FixMyStreet::Script::Inactive;
+
+use v5.14;
+use warnings;
+
+use Moo;
+use CronFns;
+use FixMyStreet;
+use FixMyStreet::Cobrand;
+use FixMyStreet::DB;
+use FixMyStreet::Email;
+
+has anonymize => ( is => 'ro' );
+has close => ( is => 'ro' );
+has email => ( is => 'ro' );
+has verbose => ( is => 'ro' );
+has dry_run => ( is => 'ro' );
+
+sub BUILDARGS {
+    my ($cls, %args) = @_;
+    $args{dry_run} = delete $args{'dry-run'};
+    return \%args;
+}
+
+has cobrand => (
+    is => 'lazy',
+    default => sub {
+        my $base_url = FixMyStreet->config('BASE_URL');
+        my $site = CronFns::site($base_url);
+        my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($site)->new;
+        $cobrand->set_lang_and_domain(undef, 1);
+        $cobrand;
+    },
+);
+
+has anonymous_user => (
+    is => 'lazy',
+    default => sub {
+        FixMyStreet::DB->resultset("User")->find_or_create({
+            email => 'removed-automatically@' . FixMyStreet->config('EMAIL_DOMAIN'),
+        });
+    }
+);
+
+sub users {
+    my $self = shift;
+
+    say "DRY RUN" if $self->dry_run;
+    $self->anonymize_users;
+    $self->email_inactive_users if $self->email;
+}
+
+sub reports {
+    my $self = shift;
+
+    say "DRY RUN" if $self->dry_run;
+    $self->anonymize_reports if $self->anonymize;
+    $self->close_updates if $self->close;
+}
+
+sub close_updates {
+    my $self = shift;
+
+    my $problems = FixMyStreet::DB->resultset("Problem")->search({
+        lastupdate => { '<', interval($self->close) },
+        state => [ FixMyStreet::DB::Result::Problem->closed_states(), FixMyStreet::DB::Result::Problem->fixed_states() ],
+        extra => [ undef, { -not_like => '%closed_updates%' } ],
+    });
+
+    while (my $problem = $problems->next) {
+        say "Closing updates on problem #" . $problem->id if $self->verbose;
+        next if $self->dry_run;
+        $problem->set_extra_metadata( closed_updates => 1 );
+        $problem->update;
+    }
+}
+
+sub anonymize_reports {
+    my $self = shift;
+
+    # Need to look though them all each time, in case any new updates/alerts
+    my $problems = FixMyStreet::DB->resultset("Problem")->search({
+        lastupdate => { '<', interval($self->anonymize) },
+        state => [
+            FixMyStreet::DB::Result::Problem->closed_states(),
+            FixMyStreet::DB::Result::Problem->fixed_states(),
+            FixMyStreet::DB::Result::Problem->hidden_states(),
+        ],
+    });
+
+    while (my $problem = $problems->next) {
+        say "Anonymizing problem #" . $problem->id if $self->verbose;
+        next if $self->dry_run;
+
+        # Remove personal data from the report
+        $problem->update({
+            user => $self->anonymous_user,
+            name => '',
+            anonymous => 1,
+            send_questionnaire => 0,
+        }) if $problem->user != $self->anonymous_user;
+
+        # Remove personal data from the report's updates
+        $problem->comments->search({
+            user_id => { '!=' => $self->anonymous_user->id },
+        })->update({
+            user_id => $self->anonymous_user->id,
+            name => '',
+            anonymous => 1,
+        });
+
+        # Remove alerts - could just delete, but of interest how many there were, perhaps?
+        FixMyStreet::DB->resultset('Alert')->search({
+            user_id => { '!=' => $self->anonymous_user->id },
+            alert_type => 'new_updates',
+            parameter => $problem->id,
+        })->update({
+            user_id => $self->anonymous_user->id,
+            whendisabled => \'current_timestamp',
+        });
+    }
+}
+
+sub anonymize_users {
+    my $self = shift;
+
+    my $users = FixMyStreet::DB->resultset("User")->search({
+        last_active => { '<', interval($self->anonymize) },
+    });
+
+    while (my $user = $users->next) {
+        say "Anonymizing user #" . $user->id if $self->verbose;
+        next if $self->dry_run;
+        $user->anonymize_account;
+    }
+}
+
+sub email_inactive_users {
+    my $self = shift;
+
+    my $users = FixMyStreet::DB->resultset("User")->search({
+       last_active => [ -and => { '<', interval($self->email) },
+           { '>=', interval($self->anonymize) } ],
+    });
+    while (my $user = $users->next) {
+        next if $user->get_extra_metadata('inactive_email_sent');
+
+        say "Emailing user #" . $user->id if $self->verbose;
+        next if $self->dry_run;
+        FixMyStreet::Email::send_cron(
+            $user->result_source->schema,
+            'inactive-account.txt',
+            {
+                email_from => $self->email,
+                anonymize_from => $self->anonymize,
+                user => $user,
+                url => $self->cobrand->base_url_with_lang . '/my',
+            },
+            { To => [ $user->email, $user->name ] },
+            undef, 0, $self->cobrand,
+        );
+
+        $user->set_extra_metadata('inactive_email_sent', 1);
+        $user->update;
+    }
+}
+
+sub interval {
+    my $interval = shift;
+    my $s = "current_timestamp - '$interval months'::interval";
+    return \$s;
+}
+
+1;
diff --git a/perllib/FixMyStreet/Script/Reports.pm b/perllib/FixMyStreet/Script/Reports.pm
index 8e4a4aec1..578d966d6 100644
--- a/perllib/FixMyStreet/Script/Reports.pm
+++ b/perllib/FixMyStreet/Script/Reports.pm
@@ -29,7 +29,7 @@ sub send(;$) {
     my $site = $site_override || CronFns::site($base_url);
 
     my $states = [ FixMyStreet::DB::Result::Problem::open_states() ];
-    $states = [ 'unconfirmed', 'confirmed', 'in progress', 'planned', 'closed', 'investigating' ] if $site eq 'zurich';
+    $states = [ 'submitted', 'confirmed', 'in progress', 'feedback pending', 'external', 'wish' ] if $site eq 'zurich';
     my $unsent = $rs->search( {
         state => $states,
         whensent => undef,
@@ -78,6 +78,7 @@ sub send(;$) {
         my $email_base_url = $cobrand->base_url_for_report($row);
         my %h = map { $_ => $row->$_ } qw/id title detail name category latitude longitude used_map/;
         $h{report} = $row;
+        $h{cobrand} = $cobrand;
         map { $h{$_} = $row->user->$_ || '' } qw/email phone/;
         $h{confirmed} = DateTime::Format::Pg->format_datetime( $row->confirmed->truncate (to => 'second' ) )
             if $row->confirmed;
@@ -88,6 +89,8 @@ sub send(;$) {
         if ($row->photo) {
             $h{has_photo} = _("This web page also contains a photo of the problem, provided by the user.") . "\n\n";
             $h{image_url} = $email_base_url . $row->photos->[0]->{url_full};
+            my @all_images = map { $email_base_url . $_->{url_full} } @{ $row->photos };
+            $h{all_image_urls} = \@all_images;
         } else {
             $h{has_photo} = '';
             $h{image_url} = '';
@@ -178,16 +181,8 @@ sub send(;$) {
 
         if ($h{category} eq _('Other')) {
             $h{category_footer} = _('this type of local problem');
-            $h{category_line} = '';
         } else {
             $h{category_footer} = "'" . $h{category} . "'";
-            $h{category_line} = sprintf(_("Category: %s"), $h{category}) . "\n\n";
-        }
-
-        if ( $row->subcategory ) {
-            $h{subcategory_line} = sprintf(_("Subcategory: %s"), $row->subcategory) . "\n\n";
-        } else {
-            $h{subcategory_line} = "\n\n";
         }
 
         $h{bodies_name} = join(_(' and '), @dear);
@@ -300,6 +295,9 @@ sub _send_report_sent_email {
     # Don't send 'report sent' text
     return unless $row->user->email_verified;
 
+    my $contributed_as = $row->get_extra_metadata('contributed_as') || '';
+    return if $contributed_as eq 'body' || $contributed_as eq 'anonymous_user';
+
     FixMyStreet::Email::send_cron(
         $row->result_source->schema,
         'confirm_report_sent.txt',
@@ -308,7 +306,7 @@ sub _send_report_sent_email {
             To => $row->user->email,
             From => [ $cobrand->contact_email, $cobrand->contact_name ],
         },
-        $cobrand->contact_email,
+        undef,
         $nomail,
         $cobrand,
         $row->lang,
diff --git a/perllib/FixMyStreet/Script/UpdateAllReports.pm b/perllib/FixMyStreet/Script/UpdateAllReports.pm
index d6f3eb64b..21d8d28a0 100755
--- a/perllib/FixMyStreet/Script/UpdateAllReports.pm
+++ b/perllib/FixMyStreet/Script/UpdateAllReports.pm
@@ -6,23 +6,24 @@ use warnings;
 use FixMyStreet;
 use FixMyStreet::Cobrand;
 use FixMyStreet::DB;
+use CronFns;
 
 use List::MoreUtils qw(zip);
 use List::Util qw(sum);
 
+my $site = CronFns::site(FixMyStreet->config('BASE_URL'));
+
 my $fourweeks = 4*7*24*60*60;
 
 # Age problems from when they're confirmed, except on Zurich
 # where they appear as soon as they're created.
 my $age_column = 'confirmed';
-if ( FixMyStreet->config('BASE_URL') =~ /zurich|zueri/ ) {
-    $age_column = 'created';
-}
+$age_column = 'created' if $site eq 'zurich';
 
 my $dtf = FixMyStreet::DB->schema->storage->datetime_parser;
 
-my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker('default')->new;
-FixMyStreet::DB->schema->cobrand($cobrand);
+my $cobrand_cls = FixMyStreet::Cobrand->get_class_for_moniker($site)->new;
+FixMyStreet::DB->schema->cobrand($cobrand_cls);
 
 sub generate {
     my $include_areas = shift;
@@ -34,7 +35,7 @@ sub generate {
         },
         {
             columns => [
-                'id', 'bodies_str', 'state', 'areas', 'cobrand',
+                'id', 'bodies_str', 'state', 'areas', 'cobrand', 'category',
                 { duration => { extract => "epoch from current_timestamp-lastupdate" } },
                 { age      => { extract => "epoch from current_timestamp-$age_column"  } },
             ]
@@ -43,13 +44,26 @@ sub generate {
     $problems = $problems->cursor; # Raw DB cursor for speed
 
     my ( %fixed, %open );
-    my @cols = ( 'id', 'bodies_str', 'state', 'areas', 'cobrand', 'duration', 'age' );
+    my %stats = (
+        fixed => \%fixed,
+        open => \%open,
+    );
+    my @cols = ( 'id', 'bodies_str', 'state', 'areas', 'cobrand', 'category', 'duration', 'age' );
     while ( my @problem = $problems->next ) {
         my %problem = zip @cols, @problem;
-        my @bodies;
-        my @areas;
+        my @bodies = split( /,/, $problem{bodies_str} );
         my $cobrand = $problem{cobrand};
+
+        if (my $type = $cobrand_cls->call_hook(dashboard_categorize_problem => \%problem)) {
+            foreach my $body ( @bodies ) {
+                $stats{$type}{$body}++;
+                $stats{$cobrand}{$type}{$body}++;
+            }
+            next;
+        }
+
         my $duration_str = ( $problem{duration} > 2 * $fourweeks ) ? 'old' : 'new';
+
         my $type = ( $problem{duration} > 2 * $fourweeks )
             ? 'unknown'
             : ($problem{age} > $fourweeks ? 'older' : 'new');
@@ -57,9 +71,6 @@ sub generate {
                FixMyStreet::DB::Result::Problem->fixed_states()->{$problem{state}}
             || FixMyStreet::DB::Result::Problem->closed_states()->{$problem{state}};
 
-        # Add to bodies it was sent to
-        @bodies = split( /,/, $problem{bodies_str} );
-
         foreach my $body ( @bodies ) {
             if ( $problem_fixed ) {
                 # Fixed problems are either old or new
@@ -73,7 +84,7 @@ sub generate {
         }
 
         if ( $include_areas ) {
-            @areas = grep { $_ } split( /,/, $problem{areas} );
+            my @areas = grep { $_ } split( /,/, $problem{areas} );
             foreach my $area ( @areas ) {
                 if ( $problem_fixed ) {
                     $fixed{areas}{$area}{$duration_str}++;
@@ -84,10 +95,7 @@ sub generate {
         }
     }
 
-    return {
-        fixed => \%fixed,
-        open  => \%open,
-    };
+    return \%stats;
 }
 
 sub end_period {