22 files changed, 448 insertions, 36 deletions

diff --git a/bin/browser-tests b/bin/browser-tests
index c7e5df27c..bfbe4e51d 100755
--- a/bin/browser-tests
+++ b/bin/browser-tests
@@ -7,11 +7,23 @@ use lib '.'; # For the mock MapIt module
 use Getopt::Long ':config' => qw(pass_through auto_help);
 
 my ($run_server, $run_cypress, $vagrant, $wsl, $node, $config_file);
-my ($cobrand, $coords, $area_id, $name, $mapit_url);
+my ($cobrand, $coords, $area_id, $name, $mapit_url, $coverage);
 
 BEGIN {
     $config_file = 'conf/general.yml-example';
-    $cobrand = [ 'borsetshire', 'fixmystreet', 'northamptonshire', 'bathnes', 'buckinghamshire', 'hounslow', 'isleofwight', 'peterborough', 'tfl' ];
+    $cobrand = [qw(
+        bathnes
+        borsetshire
+        buckinghamshire
+        fixmystreet
+        hackney
+        hounslow
+        isleofwight
+        northamptonshire
+        oxfordshire
+        peterborough
+        tfl
+    )];
     $coords = '51.532851,-2.284277';
     $area_id = 2608;
     $name = 'Borsetshire';
@@ -19,6 +31,7 @@ BEGIN {
     $node = 'C:\Program Files\nodejs\node.exe';
 
     GetOptions(
+        'coverage' => \$coverage,
         'config=s' => \$config_file,
         'server' => \$run_server,
         'cypress' => \$run_cypress,
@@ -38,6 +51,11 @@ BEGIN {
         exit 1;
     }
 
+    if ($coverage && (system('git', 'diff', '--quiet', 'web') >> 8)) {
+        print 'Do not run coverage with changes in web, they will be lost';
+        exit 1;
+    }
+
     if (!$run_server && !$run_cypress) {
         # If asked for neither, run both
         $run_server = $run_cypress = 1;
@@ -75,6 +93,35 @@ if ($vagrant) {
     exit;
 }
 
+BEGIN {
+    # setenv.pl above unloads File:: modules but we need them
+    use File::Path qw(remove_tree);
+}
+
+sub coverage_setup {
+    # Add instrumentation to all JS files under web/
+    if (system('nyc', 'instrument', '--exclude', 'vendor', '--compact', 'false', 'web', 'webO') >> 8) {
+        print 'Could not instrument JS files - are @cypress/code-coverage and nyc installed?';
+        exit 1;
+    }
+
+    # Move the instrumented files on top of the originals
+    while (glob("webO/js/*.js webO/cobrands/*/*.js")) {
+        (my $new = $_) =~ s/webO/web/;
+        rename $_, $new;
+    }
+
+    remove_tree('webO', { safe => 1 }); # Remove anything else left
+}
+
+sub coverage_teardown {
+    remove_tree('.nyc_output', '.cypress/coverage', { safe => 1 }); # Remove old data and incorrect report
+    rename '.cypress/.nyc_output', './.nyc_output'; # Move to top level so nyc can find JS files
+    system('git', 'checkout', 'web'); # Remove instrumented JS files
+    system('nyc', 'report', '--reporter=lcov'); # Generate correct report
+    print "The JS coverage report is at coverage/lcov-report/index.html\n";
+}
+
 sub run {
     my $cmd = shift @ARGV;
     die "Must specify a cypress command\n" unless $cmd || !$run_cypress;
@@ -113,8 +160,12 @@ sub run {
         if ($wsl) {
             @cypress = ('cmd.exe', '/c', $node, $wsl);
         }
-        my $exit = system(@cypress, $cmd, '--config', 'pluginsFile=false,supportFile=false', '--project', '.cypress', @ARGV);
+        my @config = $coverage ? () : ('--config', 'pluginsFile=false,supportFile=false');
+        my $exit = system(@cypress, $cmd, @config, '--project', '.cypress', @ARGV);
         kill 'TERM', $pid if $pid;
+
+        coverage_teardown() if $coverage;
+
         exit $exit >> 8;
     } else {
         # Child, run the server on port 3001
@@ -128,6 +179,7 @@ sub run {
     }
 }
 
+coverage_setup() if $coverage;
 run();
 
 
@@ -150,7 +202,7 @@ browser-tests [running options] [fixture options] [cypress options]
    --help           this help message
 
  Fixture option:
-   --cobrand        Cobrand(s) to use, default is fixmystreet,northamptonshire,bathnes,buckinghamshire,isleofwight,peterborough,tfl
+   --cobrand        Cobrand(s) to use, default is fixmystreet,northamptonshire,bathnes,buckinghamshire,isleofwight,peterborough,tfl,hackney,oxfordshire
    --coords         Default co-ordinates for created reports
    --area_id        Area ID to use for created body
    --name           Name to use for created body
diff --git a/bin/csv-export b/bin/csv-export
new file mode 100755
index 000000000..29ca3388f
--- /dev/null
+++ b/bin/csv-export
@@ -0,0 +1,77 @@
+#!/usr/bin/env perl
+
+# csv-export
+# Offline creation of CSV export, first take
+
+use v5.14;
+use warnings;
+
+BEGIN {
+    use File::Basename qw(dirname);
+    use File::Spec;
+    my $d = dirname(File::Spec->rel2abs($0));
+    require "$d/../setenv.pl";
+}
+
+use open ':std', ':encoding(UTF-8)';
+use Getopt::Long::Descriptive;
+use Path::Tiny;
+use CronFns;
+use FixMyStreet::Cobrand;
+use FixMyStreet::DB;
+use FixMyStreet::Reporting;
+
+my $site = CronFns::site(FixMyStreet->config('BASE_URL'));
+CronFns::language($site);
+
+my ($opts, $usage) = describe_options(
+    '%c %o',
+    ['cobrand=s', 'which cobrand is asking for the data', { required => 1 }],
+    ['type=s', 'whether to export problems or updates', { required => 1 }],
+    ['out=s', 'where to output CSV data'],
+
+    ['body=i', 'Body ID to restrict export to'],
+    ['wards=s', 'Ward area IDs to restrict export to'],
+    ['category=s', 'Category to restrict export to'],
+    ['state=s', 'State to restrict export to'],
+    ['start_date=s', 'Start date for export (default 30 days ago)'],
+    ['end_date=s', 'End date for export'],
+
+    ['user=i', 'user ID which requested this export'],
+    ['verbose|v', 'more verbose output'],
+    ['help|h', "print usage message and exit" ],
+);
+$usage->die if $opts->help;
+
+my $use_stdout = !$opts->out || $opts->out eq '-';
+my ($file, $fh);
+if ($use_stdout) {
+    $fh = *STDOUT;
+} else {
+    $file = path($opts->out . '-part');
+    $fh = $file->openw_utf8;
+}
+
+my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($opts->cobrand);
+FixMyStreet::DB->schema->cobrand($cobrand);
+
+my $user = FixMyStreet::DB->resultset("User")->find($opts->user) if $opts->user;
+my $body = FixMyStreet::DB->resultset("Body")->find($opts->body) if $opts->body;
+my $wards = $opts->wards ? [split',', $opts->wards] : [];
+
+my $reporting = FixMyStreet::Reporting->new(
+    type => $opts->type,
+    user => $user,
+    category => $opts->category,
+    state => $opts->state,
+    wards => $wards,
+    body => $body,
+    $opts->start_date ? (start_date => $opts->start_date) : (),
+    end_date => $opts->end_date,
+);
+$reporting->construct_rs_filter;
+$reporting->csv_parameters;
+$reporting->generate_csv($fh);
+unless ($use_stdout) {
+    $file->move($opts->out);
+}
diff --git a/bin/docker.preinit b/bin/docker.preinit
index 459e89de2..501878185 100644
--- a/bin/docker.preinit
+++ b/bin/docker.preinit
@@ -34,6 +34,7 @@ fi
 su $FMS_DB_USER -c "${FMS_ROOT}/bin/docker-cobrand"
 
 # Ensure things are up to date - schema, CSS, etc
+${FMS_ROOT}/bin/install_packages docker
 su $FMS_DB_USER -c "${FMS_ROOT}/script/update"
 
 # Update reports
diff --git a/bin/export-import-data b/bin/export-import-data
index b73b4097e..5d7a0eaa7 100755
--- a/bin/export-import-data
+++ b/bin/export-import-data
@@ -20,6 +20,7 @@ my ($opt, $usage) = describe_options(
     [ 'name=s', "Name of body" ],
     [ 'import=s', "File to import" ],
     [ 'commit', "Actually commit changes to the database" ],
+    [ 'categories=s', "pipe-separated list of categories to export contacts for and filter templates by" ],
     [ 'help', "print usage message and exit", { shortcircuit => 1 } ],
 );
 print($usage->text), exit if $opt->help;
@@ -28,6 +29,7 @@ die "Please specify a file to import\n" if $opt->import && (! -e $opt->import ||
 
 my $J = JSON::MaybeXS->new(utf8 => 1, pretty => 1, canonical => 1);
 my $body = FixMyStreet::DB->resultset("Body")->find({ name => $opt->name }) or die "Cannot find body " . $opt->name . "\n";
+my @categories = split(/\|/, ($opt->{categories} || ''));
 
 if ($opt->import) {
     import($opt->import);
@@ -38,7 +40,31 @@ if ($opt->import) {
 sub export {
     my %out;
 
-    for ($body->response_templates->all) {
+    if (@categories) {
+        my @contacts = $body->contacts->search({
+            category => { -in => \@categories },
+        })->all;
+        die "Categories mismatch" unless scalar @categories == scalar @contacts;
+        for (@contacts) {
+            push @{$out{contacts}}, {
+                category => $_->category,
+                email => $_->email,
+                state => $_->state,
+                non_public => $_->non_public ? JSON->true : JSON->false,
+                extra => $_->extra,
+            };
+        }
+    }
+
+    my $templates = $body->response_templates;
+    if (@categories) {
+        $templates = $templates->search({
+            'contact.category' => { -in => \@categories }
+        }, {
+            join => { 'contact_response_templates' => 'contact' }
+        });
+    }
+    for ($templates->all) {
         push @{$out{templates}}, {
             title => $_->title,
             text => $_->text,
@@ -81,6 +107,25 @@ sub import {
     my $db = FixMyStreet::DB->schema->storage;
     $db->txn_begin;
 
+    foreach (@{$out->{contacts}}) {
+        my $existing = $body->contacts->search({ category => $_->{category} })->single;
+        if ($existing) {
+            warn "Category $_->{category} already exists, skipping";
+            next;
+        }
+        my $contact = $body->contacts->new({
+            note => "Imported from $file",
+            editor => 'export-import-data',
+            whenedited => \'current_timestamp',
+            category => $_->{category},
+            email => $_->{email},
+            state => $_->{state},
+            non_public => $_->{non_public},
+            extra => $_->{extra},
+        });
+        $contact->insert;
+    }
+
     foreach (@{$out->{templates}}) {
         my $existing = $body->response_templates->search({ title => $_->{title} })->single;
         if ($existing) {
diff --git a/bin/fixmystreet.com/banes-close-reports b/bin/fixmystreet.com/banes-close-reports
index bba4c88e0..79c1c44b9 100755
--- a/bin/fixmystreet.com/banes-close-reports
+++ b/bin/fixmystreet.com/banes-close-reports
@@ -50,7 +50,6 @@ my $q = FixMyStreet::DB->resultset("Problem")->search({
 # Provide some variables to the archiving script
 FixMyStreet::Script::ArchiveOldEnquiries::update_options({
     user => $body->comment_user->id,
-    user_name => $body->comment_user->name,
     closure_text => CLOSURE_TEXT,
     retain_alerts => 1,
     commit => $opts->commit,
diff --git a/bin/fixmystreet.com/bromley-echo b/bin/fixmystreet.com/bromley-echo
new file mode 100755
index 000000000..01fdec110
--- /dev/null
+++ b/bin/fixmystreet.com/bromley-echo
@@ -0,0 +1,33 @@
+#!/usr/bin/env perl
+
+# bromley-echo
+# Call the Echo API in parallel
+
+use v5.14;
+use warnings;
+
+BEGIN {
+    use File::Basename qw(dirname);
+    use File::Spec;
+    my $d = dirname(File::Spec->rel2abs($0));
+    require "$d/../../setenv.pl";
+}
+
+use Getopt::Long::Descriptive;
+use JSON::MaybeXS;
+use Storable;
+use FixMyStreet::Cobrand::Bromley;
+
+my ($opts, $usage) = describe_options(
+    '%c %o',
+    ['out=s', 'where to output CSV data'],
+    ['calls=s', 'JSON of what API calls to make'],
+    ['help|h', "print usage message and exit" ],
+);
+$usage->die if $opts->help;
+
+my $cobrand = FixMyStreet::Cobrand::Bromley->new;
+
+my $calls = decode_json($opts->calls);
+$calls = $cobrand->_parallel_api_calls(@$calls);
+Storable::store($calls, $opts->out);
diff --git a/bin/fixmystreet.com/bromley-fetch-waste b/bin/fixmystreet.com/bromley-fetch-waste
new file mode 100755
index 000000000..392905c83
--- /dev/null
+++ b/bin/fixmystreet.com/bromley-fetch-waste
@@ -0,0 +1,24 @@
+#!/usr/bin/env perl
+
+use v5.14;
+use warnings;
+
+BEGIN {
+    use File::Basename qw(dirname);
+    use File::Spec;
+    my $d = dirname(File::Spec->rel2abs($0));
+    require "$d/../../setenv.pl";
+}
+
+use Getopt::Long::Descriptive;
+use FixMyStreet::Cobrand::Bromley;
+
+my ($opts, $usage) = describe_options(
+    '%c %o',
+    ['verbose|v', 'more verbose output'],
+    ['help|h', "print usage message and exit" ],
+);
+$usage->die if $opts->help;
+
+my $cobrand = FixMyStreet::Cobrand::Bromley->new;
+$cobrand->waste_fetch_events($opts->verbose);
diff --git a/bin/fixmystreet.com/buckinghamshire-flytipping b/bin/fixmystreet.com/buckinghamshire-flytipping
index a312f9fbe..65e87d30e 100755
--- a/bin/fixmystreet.com/buckinghamshire-flytipping
+++ b/bin/fixmystreet.com/buckinghamshire-flytipping
@@ -79,7 +79,6 @@ sub find_problems {
     # Provide some variables to the archiving script
     FixMyStreet::Script::ArchiveOldEnquiries::update_options({
         user => $body->comment_user->id,
-        user_name => $body->comment_user->name,
         closure_text => $template->text,
         retain_alerts => $retain_alerts,
         commit => $opts->commit,
diff --git a/bin/fixmystreet.com/fixture b/bin/fixmystreet.com/fixture
index 1062eb16b..59de8e222 100755
--- a/bin/fixmystreet.com/fixture
+++ b/bin/fixmystreet.com/fixture
@@ -99,15 +99,35 @@ if ($opt->test_fixtures) {
         { area_id => 2257, categories => ['Flytipping', 'Graffiti'], name => 'Chiltern District Council' },
         { area_id => 2397, categories => [ 'Graffiti' ], name => 'Northampton Borough Council' },
         { area_id => 2483, categories => [ 'Potholes', 'Other' ], name => 'Hounslow Borough Council' },
+        { area_id => 2508, categories => [ 'Potholes', 'Other' ], name => 'Hackney Council' },
         { area_id => 2636, categories => [ 'Potholes', 'Private', 'Extra' ], name => 'Isle of Wight Council' },
-        { area_id => 2566, categories => [ 'Fallen branch' ], name => 'Peterborough City Council' },
+        { area_id => 2566, categories => [ 'Fallen branch', 'Light Out', 'Light Dim', 'Fallen Tree', 'Damaged Tree' ], name => 'Peterborough City Council' },
         { area_id => 2498, categories => [ 'Incorrect timetable', 'Glass broken', 'Mobile Crane Operation' ], name => 'TfL' },
+        { area_id => 2237, categories => [ 'Flytipping', 'Roads', 'Parks' ], name => 'Oxfordshire County Council' },
     ) {
         $bodies->{$_->{area_id}} = FixMyStreet::DB::Factory::Body->find_or_create($_);
         my $cats = join(', ', @{$_->{categories}});
         say "Created body $_->{name} for MapIt area ID $_->{area_id}, categories $cats";
     }
 
+    for my $cat_name ('Fallen Tree', 'Damaged Tree') {
+        my $cat = FixMyStreet::DB->resultset('Contact')->find({
+            body => $bodies->{2566},
+            category => $cat_name,
+        });
+        $cat->set_extra_metadata( group => 'Trees' );
+        $cat->update;
+    }
+
+    for my $cat_name ('Light Out', 'Light Dim') {
+        my $cat = FixMyStreet::DB->resultset('Contact')->find({
+            body => $bodies->{2566},
+            category => $cat_name,
+        });
+        $cat->set_extra_metadata( group => 'Street lighting' );
+        $cat->update;
+    }
+
     my $child_cat = FixMyStreet::DB->resultset("Contact")->find({
         body => $bodies->{2234},
         category => 'Very Urgent',
@@ -260,6 +280,7 @@ $priority->add_to_contacts($body->contacts->first);
 say "Created users, all with password 'password':";
 my %users;
 my $perms_inspector = ['report_inspect', 'planned_reports'];
+my $perms_inspector_with_instruct = [@$perms_inspector, 'report_instruct'];
 my $perms_cs = [
     'contribute_as_body', 'contribute_as_another_user',
     'moderate', 'view_body_contribute_details',
@@ -271,6 +292,7 @@ my $perms_cs_full = [
 ];
 foreach (
     { name => 'Inspector Gadget', email => 'inspector@example.org', email_verified => 1, body => $body, permissions => $perms_inspector },
+    { name => 'Inspector Instructor', email => 'inspector-instructor@example.org', email_verified => 1, body => $body, permissions => $perms_inspector_with_instruct },
     { name => 'Harriet Helpful', email_verified => 1, email => 'cs@example.org', body => $body, permissions => $perms_cs },
     { name => 'Andrew Agreeable', email_verified => 1, email => 'cs_full@example.org', body => $body, permissions => $perms_cs_full },
     { name => 'Super User', email_verified => 1, email => 'super@example.org', body => $body, permissions => [
diff --git a/bin/gettext-nget-patch b/bin/gettext-nget-patch
index 5d30cc922..decd18755 100755
--- a/bin/gettext-nget-patch
+++ b/bin/gettext-nget-patch
@@ -28,7 +28,7 @@ find( sub {
         }
     }
     close FP;
-}, 'templates', 'perllib');
+}, 'templates/email', 'templates/web/base', 'templates/web/zurich', 'perllib');
 
 foreach (sort { $a->{s} cmp $b->{s} } values %out) {
     print <<EOF;
@@ -41,4 +41,3 @@ msgstr[0] ""
 msgstr[1] ""
 EOF
 }
-
diff --git a/bin/import_categories b/bin/import_categories
index 490e2187f..23a1089f5 100644
--- a/bin/import_categories
+++ b/bin/import_categories
@@ -46,11 +46,10 @@ if (!$opt->commit) {
 my $config = decode_json(path($ARGV[0])->slurp_utf8);
 
 my $body = FixMyStreet::DB->resultset('Body')->find({ name => $opt->body });
+die "Couldn't find body " . $opt->body unless $body;
 
 $body->contacts->delete_all if $opt->delete;
 
-die "Couldn't find body" unless $body;
-
 my $groups = $config->{groups};
 if ($groups) {
     for my $group (keys %$groups) {
@@ -72,16 +71,18 @@ sub make_categories {
             category => $cat->{category}
         });
         $child_cat->email($cat->{email});
+        $child_cat->send_method($cat->{devolved}) if $cat->{devolved};
         $child_cat->state('confirmed');
         $child_cat->editor($0);
         $child_cat->whenedited(\'current_timestamp');
         $child_cat->note($child_cat->in_storage ? 'Updated by import_categories' : 'Created by import_categories');
         say colored("WARNING", 'red') . " " . $child_cat->category . " already exists" if $child_cat->in_storage and $child_cat->category ne 'Other (TfL)';
         $child_cat->extra(undef) if $child_cat->in_storage;
+        $child_cat->set_extra_metadata(open311_protect => 1) if $cat->{open311_protect};
 
         if ($group) {
             my $groups = $child_cat->groups;
-            my %groups = map { $_ => 1} @$groups;
+            my %groups = map { $_ => 1 } grep { $_ } @$groups;
             $groups{$group} = 1;
             my @groups = keys %groups;
             $child_cat->set_extra_metadata(group => \@groups);
diff --git a/bin/install_packages b/bin/install_packages
index 7b4bfd8e1..c7f1b9446 100755
--- a/bin/install_packages
+++ b/bin/install_packages
@@ -3,7 +3,7 @@ set -e
 
 cd "$(dirname "${BASH_SOURCE[0]}")/.."
 
-PACKAGE_FILE=conf/packages
+PACKAGE_FILE=conf/packages.generic
 
 [ -n "$1" ] && PACKAGE_FILE="conf/packages.${1}"
 
diff --git a/bin/northamptonshire/update_ids_for_v2 b/bin/northamptonshire/update_ids_for_v2
new file mode 100755
index 000000000..2876bb0d8
--- /dev/null
+++ b/bin/northamptonshire/update_ids_for_v2
@@ -0,0 +1,65 @@
+#!/usr/bin/env perl
+
+# The migration from Alloy V1 to V2 causes the IDs to change. This takes a CSV
+# with a mapping of old ids to new ones and applies it to the external_id, saving
+# the old one in extra in case it's relevant.
+
+use strict;
+use warnings;
+use v5.14;
+
+BEGIN {
+    use File::Basename qw(dirname);
+    use File::Spec;
+    my $d = dirname(File::Spec->rel2abs($0));
+    require "$d/../../setenv.pl";
+}
+
+use FixMyStreet::DB;
+
+use Getopt::Long;
+use Text::CSV;
+
+my $NEW_ID = 'alloy_item_id';
+my $OLD_ID = 'res_id';
+my $FMS_ID = 'fms_id';
+
+my ($commit, $file);
+GetOptions(
+    'commit' => \$commit,
+    'file=s' => \$file,
+);
+
+if (!$commit) {
+    say "*** DRY RUN ***";
+}
+
+my $csv = Text::CSV->new({ binary => 1 });
+open my $fh, "<:encoding(utf-8)", $file or die "Failed to open $file: $!\n";
+$csv->header($fh);
+
+my $count = 0;
+my $northants = FixMyStreet::DB->resultset("Body")->find({ name => 'Northamptonshire County Council' });
+if ($northants) {
+    while (my $report = $csv->getline_hr( $fh ) ) {
+        next unless $report->{$OLD_ID};
+        my $p = FixMyStreet::DB->resultset('Problem')->to_body( $northants->id )->search({ id => $report->{$FMS_ID} })->first;
+        next unless $p;
+        if ( $p->external_id ne $report->{$OLD_ID} ) {
+            warn "mismatch for " . $report->{$FMS_ID} . "\n";
+            next;
+        }
+
+        if ($commit) {
+            $p->set_extra_metadata('old_external_id', $p->external_id);
+            $p->external_id($report->{$NEW_ID});
+            $p->update;
+        }
+        say "Processed $count" if $count % 100 == 0;
+        $count++;
+    }
+
+    say "updated $count rows";
+} else {
+    say STDERR "Could not find Northamptonshire";
+}
diff --git a/bin/one-off-update-staff b/bin/one-off-update-staff
new file mode 100755
index 000000000..d94d6b961
--- /dev/null
+++ b/bin/one-off-update-staff
@@ -0,0 +1,38 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+
+BEGIN {
+    use File::Basename qw(dirname);
+    use File::Spec;
+    my $d = dirname(File::Spec->rel2abs($0));
+    require "$d/../setenv.pl";
+}
+
+use FixMyStreet::DB;
+
+my $rs = FixMyStreet::DB->resultset("Comment")->search({
+    'user.from_body' => { '!=', undef },
+    'user.is_superuser' => 0,
+    'me.extra' => [ undef, { -not_like => '%is_body_user%' } ],
+}, {
+    "+columns" => ["user.from_body"],
+    join => 'user',
+});
+while (my $row = $rs->next) {
+    my $id = $row->user->{_column_data}->{from_body}; # Avoid DB lookups
+    $row->set_extra_metadata( is_body_user => $id );
+    $row->update;
+}
+
+$rs = FixMyStreet::DB->resultset("Comment")->search({
+    'user.is_superuser' => 1,
+    'me.extra' => [ undef, { -not_like => '%is_superuser%' } ],
+}, {
+    join => 'user',
+});
+while (my $row = $rs->next) {
+    $row->set_extra_metadata( is_superuser => 1 );
+    $row->update;
+}
diff --git a/bin/open311-update-reports b/bin/open311-update-reports
index 2d384b813..b9b4d594d 100755
--- a/bin/open311-update-reports
+++ b/bin/open311-update-reports
@@ -8,6 +8,7 @@
 
 use strict;
 use warnings;
+use v5.14;
 
 BEGIN {
     use File::Basename qw(dirname);
@@ -16,21 +17,21 @@ BEGIN {
     require "$d/../setenv.pl";
 }
 
+use Getopt::Long::Descriptive;
 use Open311::GetUpdates;
-use FixMyStreet;
-use FixMyStreet::DB;
 
-# FIXME - make this configurable and/or better
-my $system_user = FixMyStreet::DB->resultset('User')->find_or_create(
-    {
-        email => FixMyStreet->config('CONTACT_EMAIL'),
-        name => 'System User',
-    }
+my ($opts, $usage) = describe_options(
+    '%c %o',
+    ['body|b:s', 'body name to only fetch this body' ],
+    ['verbose|v', 'more verbose output'],
+    ['help|h', "print usage message and exit" ],
 );
+$usage->die if $opts->help;
 
-my $body_list = FixMyStreet::DB->resultset('Body');
+my %params = (
+    verbose => $opts->verbose,
+    body => $opts->body,
+);
 
-my $update = Open311::GetUpdates->new(
-    body_list => $body_list,
-    system_user  => $system_user
-)->get_updates;
+my $updates = Open311::GetUpdates->new(%params);
+$updates->fetch;
diff --git a/bin/send-daemon b/bin/send-daemon
index dee9e949f..ae54c2564 100755
--- a/bin/send-daemon
+++ b/bin/send-daemon
@@ -125,7 +125,6 @@ sub look_for_update {
     my ($body) = grep { $bodies->{$_} } @{$comment->problem->bodies_str_ids};
     $body = $bodies->{$body};
 
-    $updates->construct_open311($body);
     $updates->process_update($body, $comment);
 }
 
diff --git a/bin/send-reports-failure-summary b/bin/send-failure-summary
index c7a5b4086..1a435dc7c 100755
--- a/bin/send-reports-failure-summary
+++ b/bin/send-failure-summary
@@ -1,9 +1,8 @@
 #!/usr/bin/env perl
 
-# send-reports-failure-summary:
-# Prints a summary of report sending failures
+# send-failure-summary:
+# Prints a summary of report/update sending failures
 
-use strict;
 use warnings;
 use v5.14;
 
@@ -15,6 +14,12 @@ BEGIN {
 }
 
 use FixMyStreet::Script::Reports;
+use Open311::PostServiceRequestUpdates;
 
+# report summary
 my $manager = FixMyStreet::Script::Reports->new;
 $manager->end_summary_failures;
+
+# updates summary
+my $updates = Open311::PostServiceRequestUpdates->new;
+$updates->summary_failures;
diff --git a/bin/site-specific-install.sh b/bin/site-specific-install.sh
index af52dbdbc..ae9497aef 100644
--- a/bin/site-specific-install.sh
+++ b/bin/site-specific-install.sh
@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # Set this to the version we want to check out
-VERSION=${VERSION_OVERRIDE:-v3.0.1}
+VERSION=${VERSION_OVERRIDE:-v3.1}
 
 PARENT_SCRIPT_URL=https://github.com/mysociety/commonlib/blob/master/bin/install-site.sh
 
@@ -54,8 +54,6 @@ fi
 
 install_website_packages
 
-su -l -c "touch '$DIRECTORY/admin-htpasswd'" "$UNIX_USER"
-
 if [ $INSTALL_DB = true ]; then
     add_postgresql_user
 fi
diff --git a/bin/tfl/auto-close-reports b/bin/tfl/auto-close-reports
new file mode 100755
index 000000000..00d310811
--- /dev/null
+++ b/bin/tfl/auto-close-reports
@@ -0,0 +1,48 @@
+#!/usr/bin/env perl
+
+# Closes reports that meet the following criteria:
+#  * status is action scheduled
+#  * in a category with a fixed - council auto response template
+#  * over n days in action scheduled state (n is an argument)
+#
+# Reports matching these criteria are marked as fixed - council and the
+# relevant response template text is added as a comment.
+
+use warnings;
+use v5.14;
+use utf8;
+
+BEGIN {
+    use File::Basename qw(dirname);
+    use File::Spec;
+    my $d = dirname(File::Spec->rel2abs($0));
+    require "$d/../../setenv.pl";
+}
+
+use FixMyStreet;
+use Getopt::Long;
+use FixMyStreet::Script::TfL::AutoClose;
+
+my %h;
+GetOptions(\%h, 'verbose|v', 'days|d', 'help|h', 'commit|c');
+pod2usage(0) if $h{help};
+
+FixMyStreet::Script::TfL::AutoClose->new(%h)->close;
+
+__END__
+
+=head1 NAME
+
+auto-close-reports - set action_scheduled reports to fixed
+
+=head1 SYNOPSIS
+
+auto-close-reports --commit
+
+ Options:
+   --commit      Actually close any reports
+   --days        Number of days before autoclosing
+   --verbose     Output how many reports have been closed
+   --help        This help message
+
+=cut
diff --git a/bin/tfl/import_categories b/bin/tfl/import_categories
index bb48ddc4e..d3b99547c 100755
--- a/bin/tfl/import_categories
+++ b/bin/tfl/import_categories
@@ -65,7 +65,7 @@ for my $group (keys %$groups) {
         $child_cat->note($child_cat->in_storage ? 'Updated by import_categories' : 'Created by import_categories');
         say colored("WARNING", 'red') . " " . $child_cat->category . " already exists" if $child_cat->in_storage and $child_cat->category ne 'Other (TfL)';
         my $groups = $child_cat->groups;
-        my %groups = map { $_ => 1} @$groups;
+        my %groups = map { $_ => 1 } grep { $_ } @$groups;
         $groups{$group} = 1;
         my @groups = keys %groups;
         $child_cat->extra(undef) if $child_cat->in_storage;
diff --git a/bin/update-schema b/bin/update-schema
index 8f31085c9..1cfb59f52 100755
--- a/bin/update-schema
+++ b/bin/update-schema
@@ -78,6 +78,8 @@ print "= Dry run =\n" unless $commit;
 
 my $upgrade = !$downgrade;
 
+my $db_fts_config = FixMyStreet->config('DB_FULL_TEXT_SEARCH_CONFIG') || 'english';
+
 sub get_statements {
     my $path = shift;
     open(my $FP, '<', $path) or die $!;
@@ -91,6 +93,7 @@ sub get_statements {
         $in_function = 1 if /create (or replace )?function/i;
         $in_function = 0 if /language (sql|'plpgsql')/i;
         if ($s =~ /;/ && !$in_function) {
+            $s =~ s/DB_FULL_TEXT_SEARCH_CONFIG/$db_fts_config/g;
             push @statements, $s;
             $s = '';
         }
@@ -212,6 +215,8 @@ else {
 # (assuming schema change files are never half-applied, which should be the case)
 sub get_db_version {
     return 'EMPTY' if ! table_exists('problem');
+    return '0074' if index_exists('users_fulltext_idx');
+    return '0073' if index_exists('problem_fulltext_idx');
     return '0072' if constraint_contains('contacts_state_check', 'staff');
     return '0071' if table_exists('manifest_theme');
     return '0070' if column_like('alert_type', "ref='new_problems'", 'head_title', '{{SITE_NAME}}');
@@ -319,7 +324,7 @@ sub constraint_exists {
 # Returns true if a check constraint contains a certain string
 sub constraint_contains {
     my ( $constraint, $check ) = @_;
-    my ($consrc) = $db->dbh->selectrow_array('select consrc from pg_constraint where conname = ?', {}, $constraint);
+    my ($consrc) = $db->dbh->selectrow_array('select pg_get_expr(conbin, conrelid) from pg_constraint where conname = ?', {}, $constraint);
     return unless $consrc;
     return $consrc =~ /$check/;
 }
diff --git a/bin/zurich/overdue-alert b/bin/zurich/overdue-alert
index 7689c172f..b052d94d3 100755
--- a/bin/zurich/overdue-alert
+++ b/bin/zurich/overdue-alert
@@ -45,6 +45,7 @@ sub loop_through {
         state => $states,
         created => { '<',  $date_threshold },
         bodies_str => { '!=', undef },
+        non_public => 0,
     } );
 
     my %to_send = ();