Merge tag 'v3.1' into fiksgatami-devfiksgatami-dev

27 files changed, 1237 insertions, 579 deletions

diff --git a/perllib/FixMyStreet/App/Controller/Admin.pm b/perllib/FixMyStreet/App/Controller/Admin.pm
index 038cba9e5..8d6a41b9f 100644
--- a/perllib/FixMyStreet/App/Controller/Admin.pm
+++ b/perllib/FixMyStreet/App/Controller/Admin.pm
@@ -71,13 +71,16 @@ sub index : Path : Args(0) {
     }
 
     my @unsent = $c->cobrand->problems->search( {
-        state => [ FixMyStreet::DB::Result::Problem::open_states() ],
+        'me.state' => [ FixMyStreet::DB::Result::Problem::open_states() ],
         whensent => undef,
         bodies_str => { '!=', undef },
         # Ignore very recent ones that probably just haven't been sent yet
         confirmed => { '<', \"current_timestamp - '5 minutes'::interval" },
     },
     {
+        '+columns' => ['user.email'],
+        prefetch => 'contact',
+        join => 'user',
         order_by => 'confirmed',
     } )->all;
     $c->stash->{unsent_reports} = \@unsent;
@@ -301,7 +304,14 @@ sub add_flags : Private {
 sub flagged : Path('flagged') : Args(0) {
     my ( $self, $c ) = @_;
 
-    my $problems = $c->cobrand->problems->search( { flagged => 1 } );
+    my $problems = $c->cobrand->problems->search(
+        { 'me.flagged' => 1 },
+        {
+            '+columns' => ['user.email'],
+            join => 'user',
+            prefetch => 'contact',
+        }
+    );
 
     # pass in as array ref as using same template as search_reports
     # which has to use an array ref for sql quoting reasons
diff --git a/perllib/FixMyStreet/App/Controller/Admin/Bodies.pm b/perllib/FixMyStreet/App/Controller/Admin/Bodies.pm
index 6ae068cd9..31a717a9c 100644
--- a/perllib/FixMyStreet/App/Controller/Admin/Bodies.pm
+++ b/perllib/FixMyStreet/App/Controller/Admin/Bodies.pm
@@ -151,8 +151,7 @@ sub category : Chained('body') : PathPart('') {
 
     my $history = $c->model('DB::ContactsHistory')->search(
         {
-            body_id => $c->stash->{body_id},
-            category => $c->stash->{contact}->category
+            contact_id => $c->stash->{contact}->id,
         },
         {
             order_by => ['contacts_history_id']
@@ -241,17 +240,27 @@ sub update_contact : Private {
     if ($current_contact && $contact->id && $contact->id != $current_contact->id) {
         $errors{category} = _('You cannot rename a category to an existing category');
     } elsif ($current_contact && !$contact->id) {
-        # Changed name
         $contact = $current_contact;
-        $c->model('DB::Problem')->to_body($c->stash->{body_id})->search({ category => $current_category })->update({ category => $category });
-        $contact->category($category);
+        # Set the flag here so we can run the editable test on it
+        $contact->set_extra_metadata(open311_protect => $c->get_param('open311_protect'));
+        if (!$contact->category_uneditable) {
+            # Changed name
+            $c->model('DB::Problem')->to_body($c->stash->{body_id})->search({ category => $current_category })->update({ category => $category });
+            $contact->category($category);
+        }
     }
 
     my $email = $c->get_param('email');
-    $email =~ s/\s+//g;
     my $send_method = $c->get_param('send_method') || $contact->body->send_method || "";
+    if ($send_method eq 'Open311') {
+        $email =~ s/^\s+|\s+$//g;
+    } else {
+        $email =~ s/\s+//g;
+    }
     my $email_unchanged = $contact->email && $email && $contact->email eq $email;
-    unless ( $send_method eq 'Open311' || $email_unchanged ) {
+    my $cobrand = $contact->body->get_cobrand_handler;
+    my $cobrand_valid = $cobrand && $cobrand->call_hook(validate_contact_email => $email);
+    unless ( $send_method eq 'Open311' || $email_unchanged || $cobrand_valid ) {
         $errors{email} = _('Please enter a valid email') unless is_valid_email_list($email) || $email eq 'REFUSED';
     }
 
@@ -267,15 +276,19 @@ sub update_contact : Private {
     $contact->send_method( $c->get_param('send_method') );
 
     # Set flags in extra to the appropriate values
-    if ( $c->get_param('photo_required') ) {
-        $contact->set_extra_metadata_if_undefined(  photo_required => 1 );
-    } else {
-        $contact->unset_extra_metadata( 'photo_required' );
+    foreach (qw(photo_required open311_protect updates_disallowed reopening_disallowed assigned_users_only anonymous_allowed)) {
+        if ( $c->get_param($_) ) {
+            $contact->set_extra_metadata( $_ => 1 );
+        } else {
+            $contact->unset_extra_metadata($_);
+        }
     }
-    if ( $c->get_param('open311_protect') ) {
-        $contact->set_extra_metadata( open311_protect => 1 );
-    } else {
-        $contact->unset_extra_metadata( 'open311_protect' );
+    if ( $c->user->is_superuser ) {
+        if ( $c->get_param('hardcoded') ) {
+            $contact->set_extra_metadata( hardcoded => 1 );
+        } else {
+            $contact->unset_extra_metadata('hardcoded');
+        }
     }
     if ( my @group = $c->get_param_list('group') ) {
         @group = grep { $_ } @group;
diff --git a/perllib/FixMyStreet/App/Controller/Admin/Reports.pm b/perllib/FixMyStreet/App/Controller/Admin/Reports.pm
index 7300fe676..20801e0cf 100644
--- a/perllib/FixMyStreet/App/Controller/Admin/Reports.pm
+++ b/perllib/FixMyStreet/App/Controller/Admin/Reports.pm
@@ -4,6 +4,7 @@ use namespace::autoclean;
 
 BEGIN { extends 'Catalyst::Controller'; }
 
+use utf8;
 use List::MoreUtils 'uniq';
 use FixMyStreet::SMS;
 use Utils;
@@ -50,6 +51,8 @@ sub index : Path {
 
     return if $c->cobrand->call_hook(report_search_query => $query, $p_page, $u_page, $order);
 
+    my $problems = $c->cobrand->problems;
+
     if (my $search = $c->get_param('search')) {
         $search = $self->trim($search);
 
@@ -62,9 +65,6 @@ sub index : Path {
 
         $c->stash->{searched} = $search;
 
-        my $search_n = 0;
-        $search_n = int($search) if $search =~ /^\d+$/;
-
         my $like_search = "%$search%";
 
         my $parsed = FixMyStreet::SMS->parse_username($search);
@@ -92,24 +92,18 @@ sub index : Path {
                 'me.external_id' => { like => "%$1%" }
             ];
         } else {
-            $query->{'-or'} = [
-                'me.id' => $search_n,
-                'user.email' => { ilike => $like_search },
-                'user.phone' => { ilike => $like_search },
-                'me.external_id' => { ilike => $like_search },
-                'me.name' => { ilike => $like_search },
-                'me.title' => { ilike => $like_search },
-                detail => { ilike => $like_search },
-                bodies_str => { like => $like_search },
-                cobrand_data => { like => $like_search },
-            ];
+            $problems = $problems->search_text($search);
+            # The below is added so that PostgreSQL does not try and use other indexes
+            # besides the full text search. It should have no impact on results shown.
+            $order = [ $order, { -desc => "me.id" }, { -desc => "me.created" } ];
         }
 
-        my $problems = $c->cobrand->problems->search(
+        $problems = $problems->search(
             $query,
             {
                 join => 'user',
                 '+columns' => 'user.email',
+                prefetch => 'contact',
                 rows => 50,
                 order_by => $order,
             }
@@ -118,6 +112,8 @@ sub index : Path {
         $c->stash->{problems} = [ $problems->all ];
         $c->stash->{problems_pager} = $problems->pager;
 
+        my $updates = $c->cobrand->updates;
+        $order = { -desc => 'me.id' };
         if ($valid_email) {
             $query = [
                 'user.email' => { ilike => $like_search },
@@ -132,30 +128,25 @@ sub index : Path {
                 'me.problem_id' => int($1),
             ];
         } elsif ($search =~ /^area:(\d+)$/) {
-            $query = [];
+            $query = 0;
         } else {
-            $query = [
-                'me.id' => $search_n,
-                'problem.id' => $search_n,
-                'user.email' => { ilike => $like_search },
-                'user.phone' => { ilike => $like_search },
-                'me.name' => { ilike => $like_search },
-                text => { ilike => $like_search },
-                'me.cobrand_data' => { ilike => $like_search },
-            ];
+            $updates = $updates->search_text($search);
+            $order = [ $order, { -desc => "me.created" } ];
+            $query = 1;
         }
 
-        if (@$query) {
-            my $updates = $c->cobrand->updates->search(
-                {
-                    -or => $query,
-                },
+        $query = { -or => $query } if ref $query;
+
+        if ($query) {
+            $query = undef unless ref $query;
+            $updates = $updates->search(
+                $query,
                 {
                     '+columns' => ['user.email'],
                     join => 'user',
                     prefetch => [qw/problem/],
                     rows => 50,
-                    order_by => { -desc => 'me.id' }
+                    order_by => $order,
                 }
             )->page( $u_page );
             $c->stash->{updates} = [ $updates->all ];
@@ -164,9 +155,15 @@ sub index : Path {
 
     } else {
 
-        my $problems = $c->cobrand->problems->search(
+        $problems = $problems->search(
             $query,
-            { order_by => $order, rows => 50 }
+            {
+                '+columns' => ['user.email'],
+                join => 'user',
+                prefetch => 'contact',
+                order_by => $order,
+                rows => 50
+            }
         )->page( $p_page );
         $c->stash->{problems} = [ $problems->all ];
         $c->stash->{problems_pager} = $problems->pager;
@@ -227,6 +224,26 @@ sub edit : Path('/admin/report_edit') : Args(1) {
             push @fields, { name => 'Duplicates', val => join( ',', @{ $problem->get_extra_metadata('duplicates') } ) };
             delete $extra->{duplicates};
         }
+
+        if ( $extra->{contributed_by} ) {
+            my $u = $c->cobrand->users->find({id => $extra->{contributed_by}});
+            if ( $u ) {
+                my $uri = $c->uri_for_action('admin/users/index', { search => $u->email } );
+                push @fields, {
+                    name => _('Created By'),
+                    val => FixMyStreet::Template::SafeString->new( "<a href=\"$uri\">@{[$u->name]} (@{[$u->email]})</a>" )
+                };
+                if ( $u->from_body ) {
+                    push @fields, { name => _('Created Body'), val => $u->from_body->name };
+                } elsif ( $u->is_superuser ) {
+                    push @fields, { name => _('Created Body'), val => _('Superuser') };
+                }
+            } else {
+                push @fields, { name => 'contributed_by', val => $extra->{contributed_by} };
+            }
+            delete $extra->{contributed_by};
+        }
+
         for my $key ( keys %$extra ) {
             push @fields, { name => $key, val => $extra->{$key} };
         }
@@ -341,24 +358,10 @@ sub edit : Path('/admin/report_edit') : Args(1) {
         if ( $problem->state ne $old_state ) {
             $c->forward( '/admin/log_edit', [ $id, 'problem', 'state_change' ] );
 
-            my $name = $c->user->moderating_user_name;
-            my $extra = { is_superuser => 1 };
-            if ($c->user->from_body) {
-                delete $extra->{is_superuser};
-                $extra->{is_body_user} = $c->user->from_body->id;
-            }
-            my $timestamp = \'current_timestamp';
             $problem->add_to_comments( {
                 text => $c->stash->{update_text} || '',
-                created => $timestamp,
-                confirmed => $timestamp,
-                user_id => $c->user->id,
-                name => $name,
-                mark_fixed => 0,
-                anonymous => 0,
-                state => 'confirmed',
+                user => $c->user->obj,
                 problem_state => $problem->state,
-                extra => $extra
             } );
         }
         $c->forward( '/admin/log_edit', [ $id, 'problem', 'edit' ] );
@@ -417,13 +420,7 @@ sub edit_category : Private {
         } else {
             $problem->add_to_comments({
                 text => $update_text,
-                created => \'current_timestamp',
-                confirmed => \'current_timestamp',
-                user_id => $c->user->id,
-                name => $c->user->from_body ? $c->user->from_body->name : $c->user->name,
-                state => 'confirmed',
-                mark_fixed => 0,
-                anonymous => 0,
+                user => $c->user->obj,
             });
         }
         $c->forward( '/admin/log_edit', [ $problem->id, 'problem', 'category_change' ] );
diff --git a/perllib/FixMyStreet/App/Controller/Admin/Templates.pm b/perllib/FixMyStreet/App/Controller/Admin/Templates.pm
index efff1b488..9fb401e2b 100644
--- a/perllib/FixMyStreet/App/Controller/Admin/Templates.pm
+++ b/perllib/FixMyStreet/App/Controller/Admin/Templates.pm
@@ -69,7 +69,7 @@ sub edit : Path : Args(2) {
         category => $_->category_display,
         active => $active_contacts{$_->id},
         email => $_->email,
-        group => $_->get_extra_metadata('group') // '',
+        group => $_->groups,
     } } @live_contacts;
     $c->stash->{contacts} = \@all_contacts;
     $c->forward('/report/stash_category_groups', [ \@all_contacts, 1 ]) if $c->cobrand->enable_category_groups;
@@ -100,7 +100,10 @@ sub edit : Path : Args(2) {
 
             $template->text( $c->get_param('text') );
             $template->state( $c->get_param('state') );
-            $template->external_status_code( $c->get_param('external_status_code') );
+
+            my $ext_code = $c->cobrand->call_hook('admin_templates_external_status_code_hook');
+            $ext_code ||= $c->get_param('external_status_code');
+            $template->external_status_code($ext_code);
 
             if ( $template->state && $template->external_status_code ) {
                 $c->stash->{errors} ||= {};
diff --git a/perllib/FixMyStreet/App/Controller/Admin/Triage.pm b/perllib/FixMyStreet/App/Controller/Admin/Triage.pm
index 428c35073..c5bb6628d 100644
--- a/perllib/FixMyStreet/App/Controller/Admin/Triage.pm
+++ b/perllib/FixMyStreet/App/Controller/Admin/Triage.pm
@@ -90,10 +90,8 @@ sub setup_categories : Private {
         delete $c->stash->{categories_hash};
         my %category_groups = ();
         for my $category (@{$c->stash->{end_options}}) {
-            my $group = $category->{group} // $category->get_extra_metadata('group') // [''];
-            # this could be an array ref or a string
-            my @groups = ref $group eq 'ARRAY' ? @$group : ($group);
-            push( @{$category_groups{$_}}, $category ) for @groups;
+            my $groups = $category->groups;
+            push( @{$category_groups{$_}}, $category ) for @$groups;
         }
         my @category_groups = ();
         for my $group ( grep { $_ ne _('Other') } sort keys %category_groups ) {
@@ -119,27 +117,14 @@ sub update : Private {
         $c->stash->{problem}->update( { state => 'confirmed' } );
         $c->forward( '/admin/log_edit', [ $problem->id, 'problem', 'triage' ] );
 
-        my $name = $c->user->moderating_user_name;
-        my $extra = { is_superuser => 1 };
-        if ($c->user->from_body) {
-            delete $extra->{is_superuser};
-            $extra->{is_body_user} = $c->user->from_body->id;
-        }
-
+        my $extra;
         $extra->{triage_report} = 1;
         $extra->{holding_category} = $current_category;
         $extra->{new_category} = $new_category;
 
-        my $timestamp = \'current_timestamp';
         my $comment = $problem->add_to_comments( {
             text => "Report triaged from $current_category to $new_category",
-            created => $timestamp,
-            confirmed => $timestamp,
-            user_id => $c->user->id,
-            name => $name,
-            mark_fixed => 0,
-            anonymous => 0,
-            state => 'confirmed',
+            user => $c->user->obj,
             problem_state => $problem->state,
             extra => $extra,
             whensent => \'current_timestamp',
diff --git a/perllib/FixMyStreet/App/Controller/Admin/Users.pm b/perllib/FixMyStreet/App/Controller/Admin/Users.pm
index 046e19126..7ebfb9bbd 100644
--- a/perllib/FixMyStreet/App/Controller/Admin/Users.pm
+++ b/perllib/FixMyStreet/App/Controller/Admin/Users.pm
@@ -29,17 +29,29 @@ sub index :Path : Args(0) {
 
     if ($c->req->method eq 'POST') {
         my @uids = $c->get_param_list('uid');
-        my @role_ids = $c->get_param_list('roles');
         my $user_rs = FixMyStreet::DB->resultset("User")->search({ id => \@uids });
-        foreach my $user ($user_rs->all) {
-            $user->admin_user_body_permissions->delete;
-            $user->user_roles->search({
-                role_id => { -not_in => \@role_ids },
-            })->delete;
-            foreach my $role (@role_ids) {
-                $user->user_roles->find_or_create({
-                    role_id => $role,
+        if ( $c->get_param('remove-staff') ) {
+            foreach my $user ($user_rs->all) {
+                $user->update({
+                    from_body => undef,
+                    email_verified => 0,
+                    phone_verified => 0,
                 });
+                $user->user_roles->delete;
+                $user->admin_user_body_permissions->delete;
+            }
+        } else {
+            my @role_ids = $c->get_param_list('roles');
+            foreach my $user ($user_rs->all) {
+                $user->admin_user_body_permissions->delete;
+                $user->user_roles->search({
+                    role_id => { -not_in => \@role_ids },
+                })->delete;
+                foreach my $role (@role_ids) {
+                    $user->user_roles->find_or_create({
+                        role_id => $role,
+                    });
+                }
             }
         }
         $c->stash->{status_message} = _('Updated!');
@@ -47,28 +59,13 @@ sub index :Path : Args(0) {
 
     my $search = $c->get_param('search');
     my $role = $c->get_param('role');
+    my $users = $c->cobrand->users;
     if ($search || $role) {
-        my $users = $c->cobrand->users;
-        my $isearch;
         if ($search) {
             $search = $self->trim($search);
             $search =~ s/^<(.*)>$/$1/; # In case email wrapped in <...>
             $c->stash->{searched} = $search;
-
-            $isearch = '%' . $search . '%';
-            my $search_n = 0;
-            $search_n = int($search) if $search =~ /^\d+$/;
-
-            $users = $users->search(
-                {
-                    -or => [
-                        email => { ilike => $isearch },
-                        phone => { ilike => $isearch },
-                        name => { ilike => $isearch },
-                        from_body => $search_n,
-                    ]
-                }
-            );
+            $users = $users->search_text($search);
         }
         if ($role) {
             $c->stash->{role_selected} = $role;
@@ -78,25 +75,23 @@ sub index :Path : Args(0) {
                 join => 'user_roles',
             });
         }
-
-        my @users = $users->all;
-        $c->stash->{users} = [ @users ];
-        if ($search) {
-            $c->forward('/admin/add_flags', [ { email => { ilike => $isearch } } ]);
-        }
-
     } else {
         $c->forward('/auth/get_csrf_token');
         $c->forward('/admin/fetch_all_bodies');
         $c->cobrand->call_hook('admin_user_edit_extra_data');
 
         # Admin users by default
-        my $users = $c->cobrand->users->search(
-            { from_body => { '!=', undef } },
-            { order_by => 'name' }
-        );
-        my @users = $users->all;
-        $c->stash->{users} = \@users;
+        $users = $users->search({ from_body => { '!=', undef } });
+    }
+
+    $users = $users->search(undef, {
+        prefetch => 'from_body',
+        order_by => [ \"me.name = ''", 'me.name' ],
+    });
+    my @users = $users->all;
+    $c->stash->{users} = \@users;
+    if ($search) {
+        $c->forward('/admin/add_flags', [ { email => { ilike => "%$search%" } } ]);
     }
 
     my $rs;
@@ -373,6 +368,11 @@ sub edit : Chained('user') : PathPart('') : Args(0) {
             my @live_contact_ids = map { $_->id } @live_contacts;
             my @new_contact_ids = grep { $c->get_param("contacts[$_]") } @live_contact_ids;
             $user->set_extra_metadata('categories', \@new_contact_ids);
+            if ($c->get_param('assigned_categories_only')) {
+                $user->set_extra_metadata(assigned_categories_only => 1);
+            } else {
+                $user->unset_extra_metadata('assigned_categories_only');
+            }
         } else {
             $user->unset_extra_metadata('categories');
         }
@@ -396,7 +396,7 @@ sub edit : Chained('user') : PathPart('') : Args(0) {
             id => $_->id,
             category => $_->category,
             active => $active_contacts{$_->id},
-            group => $_->get_extra_metadata('group') // '',
+            group => $_->groups,
         } } @live_contacts;
         $c->stash->{contacts} = \@all_contacts;
         $c->forward('/report/stash_category_groups', [ \@all_contacts, 1 ]) if $c->cobrand->enable_category_groups;
diff --git a/perllib/FixMyStreet/App/Controller/Around.pm b/perllib/FixMyStreet/App/Controller/Around.pm
index af50f1883..803bfad58 100644
--- a/perllib/FixMyStreet/App/Controller/Around.pm
+++ b/perllib/FixMyStreet/App/Controller/Around.pm
@@ -240,12 +240,8 @@ sub check_and_stash_category : Private {
     $c->stash->{bodies_ids} = [ map { $_->id } @bodies];
     $c->{stash}->{list_of_names_as_string} = $csv->string;
 
-    my $where  = { body_id => [ keys %bodies ], };
-
-    my $cobrand_where = $c->cobrand->call_hook('munge_around_category_where', $where );
-    if ( $cobrand_where ) {
-       $where = $cobrand_where;
-    }
+    my $where = { body_id => [ keys %bodies ], };
+    $c->cobrand->call_hook('munge_around_category_where', $where);
 
     my @categories = $c->model('DB::Contact')->not_deleted->search(
         $where,
@@ -254,6 +250,9 @@ sub check_and_stash_category : Private {
             distinct => 1
         }
     )->all_sorted;
+    # Ensure only uniquely named categories are shown
+    my %seen;
+    @categories = grep { !$seen{$_->category_display}++ } @categories;
     $c->stash->{filter_categories} = \@categories;
     my %categories_mapped = map { $_->category => 1 } @categories;
     $c->forward('/report/stash_category_groups', [ \@categories ]) if $c->cobrand->enable_category_groups;
@@ -262,6 +261,8 @@ sub check_and_stash_category : Private {
     my %valid_categories = map { $_ => 1 } grep { $_ && $categories_mapped{$_} } @$categories;
     $c->stash->{filter_category} = \%valid_categories;
     $c->cobrand->call_hook('munge_around_filter_category_list');
+
+    $c->forward('/report/assigned_users_only', [ \@categories ]);
 }
 
 sub map_features : Private {
@@ -293,7 +294,7 @@ sub map_features : Private {
         @pins = map {
             # Here we might have a DB::Problem or a DB::Result::Nearby, we always want the problem.
             my $p = (ref $_ eq 'FixMyStreet::DB::Result::Nearby') ? $_->problem : $_;
-            $p->pin_data($c, 'around');
+            $p->pin_data('around');
         } @$on_map, @$nearby;
     }
 
diff --git a/perllib/FixMyStreet/App/Controller/Auth.pm b/perllib/FixMyStreet/App/Controller/Auth.pm
index cecfa318c..0d0c2240a 100644
--- a/perllib/FixMyStreet/App/Controller/Auth.pm
+++ b/perllib/FixMyStreet/App/Controller/Auth.pm
@@ -448,6 +448,9 @@ sub check_csrf_token : Private {
         unless $time
             && $time > time() - 3600
             && $token eq $gen_token;
+
+    # Also check recaptcha if needed
+    $c->cobrand->call_hook('check_recaptcha');
 }
 
 sub no_csrf_token : Private {
@@ -457,7 +460,7 @@ sub no_csrf_token : Private {
 
 =item common_password
 
-Returns 1/0 depending on if password is common or not.
+Returns 1/0 depending on if password is common/breached or not.
 
 =cut
 
@@ -466,10 +469,8 @@ sub common_password : Local : Args(0) {
 
     my $password = $c->get_param('password_register');
 
-    my $return = JSON->true;
-    if (!$c->cobrand->call_hook('bypass_password_checks') && found($password)) {
-        $return = _('Please choose a less commonly-used password');
-    }
+    my $pass = $c->forward('test_password', [ $password ]);
+    my $return = $pass ? JSON->true : $c->stash->{field_errors}->{password_register};
 
     my $body = JSON->new->utf8->allow_nonref->encode($return);
     $c->res->content_type('application/json; charset=utf-8');
@@ -488,22 +489,50 @@ sub test_password : Private {
 
     return 1 if $c->cobrand->call_hook('bypass_password_checks');
 
-    my @errors;
-
+    my $error;
     my $min_length = $c->cobrand->password_minimum_length;
-    push @errors, sprintf(_('Please make sure your password is at least %d characters long'), $min_length)
-        if length($password) < $min_length;
-
-    push @errors, _('Please choose a less commonly-used password')
-        if found($password);
+    if (length($password) < $min_length) {
+        $error = sprintf(_('Please make sure your password is at least %d characters long'), $min_length);
+    } elsif (found($password)) {
+        $error = _('Please choose a less commonly-used password');
+    } elsif (hibp($password)) {
+        $error = _('That password has appeared in a known third-party data breach (<a href="https://haveibeenpwned.com/Passwords" target="_blank">more information</a>); please choose another');
+    }
 
-    if (@errors) {
-        $c->stash->{field_errors}->{password_register} = join('<br>', @errors);
+    if ($error) {
+        $c->stash->{field_errors}->{password_register} = $error;
         return 0;
     }
     return 1;
 }
 
+=item hibp
+
+Returns true if we should check Have I Been Pwned and the check
+comes back positive for a password that has been breached.
+
+=cut
+
+use Encode qw(encode);
+use Digest::SHA qw(sha1_hex);
+use LWP::Simple;
+use Unicode::Normalize;
+
+sub hibp : Private {
+    my $password = shift;
+
+    return 0 unless FixMyStreet->config('CHECK_HAVEIBEENPWNED');
+    my $sha1 = uc sha1_hex(encode('UTF-8', NFD($password)));
+    my $url = 'https://api.pwnedpasswords.com/range/' . substr($sha1, 0, 5);
+    my $response = LWP::Simple::get($url);
+    my $remainder = substr($sha1, 5);
+    foreach my $line (split /\r\n/, $response) {
+        my ($part, $count) = split /:/, $line;
+        return $count if $part eq $remainder;
+    }
+    return 0;
+}
+
 =head2 sign_out
 
 Log the user out. Tell them we've done so.
diff --git a/perllib/FixMyStreet/App/Controller/Auth/Profile.pm b/perllib/FixMyStreet/App/Controller/Auth/Profile.pm
index a89c6f539..a5dc5d3e7 100644
--- a/perllib/FixMyStreet/App/Controller/Auth/Profile.pm
+++ b/perllib/FixMyStreet/App/Controller/Auth/Profile.pm
@@ -188,9 +188,10 @@ sub generate_token : Path('/auth/generate_token') {
 
         if ($c->get_param('generate_token')) {
             my $token = mySociety::AuthToken::random_token();
-            $c->user->set_extra_metadata('access_token', $token);
+            my $u = FixMyStreet::DB->resultset("User")->new({ password => $token });
+            $c->user->set_extra_metadata('access_token', $u->password);
             $c->user->update;
-            $c->stash->{token_generated} = 1;
+            $c->stash->{token_generated} = $c->user->id . '-' . $token;
         }
 
         my $action = $c->get_param('2fa_action') || '';
@@ -224,7 +225,7 @@ sub generate_token : Path('/auth/generate_token') {
     }
 
     $c->stash->{has_2fa} = $has_2fa ? 1 : 0;
-    $c->stash->{existing_token} = $c->user->get_extra_metadata('access_token');
+    $c->stash->{existing_token} = $c->user->get_extra_metadata('access_token') ? 1 : 0;
 }
 
 __PACKAGE__->meta->make_immutable;
diff --git a/perllib/FixMyStreet/App/Controller/Auth/Social.pm b/perllib/FixMyStreet/App/Controller/Auth/Social.pm
index 06e67573f..ce94fe256 100644
--- a/perllib/FixMyStreet/App/Controller/Auth/Social.pm
+++ b/perllib/FixMyStreet/App/Controller/Auth/Social.pm
@@ -6,7 +6,7 @@ BEGIN { extends 'Catalyst::Controller'; }
 
 use Net::Facebook::Oauth2;
 use Net::Twitter::Lite::WithAPIv1_1;
-use OIDC::Lite::Client::WebServer::Azure;
+use OIDC::Lite::Client::WebServer::AuthCodeFlow;
 use URI::Escape;
 
 use mySociety::AuthToken;
@@ -167,7 +167,7 @@ sub oidc : Private {
 
     my $config = $c->cobrand->feature('oidc_login');
 
-    OIDC::Lite::Client::WebServer::Azure->new(
+    OIDC::Lite::Client::WebServer::AuthCodeFlow->new(
         id               => $config->{client_id},
         secret           => $config->{secret},
         authorize_uri    => $config->{auth_uri},
@@ -179,7 +179,9 @@ sub oidc_sign_in : Private {
     my ( $self, $c ) = @_;
 
     $c->detach( '/page_error_403_access_denied', [] ) if FixMyStreet->config('SIGNUPS_DISABLED');
-    $c->detach( '/page_error_400_bad_request', [] ) unless $c->cobrand->feature('oidc_login');
+
+    my $cfg = $c->cobrand->feature('oidc_login');
+    $c->detach( '/page_error_400_bad_request', [] ) unless $cfg;
 
     my $oidc = $c->forward('oidc');
     my $nonce = $self->generate_nonce();
@@ -190,6 +192,15 @@ sub oidc_sign_in : Private {
         extra        => {
             response_mode => 'form_post',
             nonce         => $nonce,
+            # auth_extra_params provides a way to pass custom parameters
+            # to the OIDC endpoint for the intial authentication request.
+            # This allows, for example, a custom scope to be used,
+            # or the `hd` parameter which customises the appearance of
+            # the login form.
+            # This is primarily useful for Google G Suite authentication - see
+            # available parameters here:
+            # https://developers.google.com/identity/protocols/oauth2/openid-connect#authenticationuriparameters
+            %{ $cfg->{auth_extra_params} || {} } ,
         },
     );
 
@@ -201,14 +212,14 @@ sub oidc_sign_in : Private {
 
     # The OIDC endpoint may require a specific URI to be called to log the user
     # out when they log out of FMS.
-    if ( my $redirect_uri = $c->cobrand->feature('oidc_login')->{logout_uri} ) {
+    if ( my $redirect_uri = $cfg->{logout_uri} ) {
         $redirect_uri .= "?post_logout_redirect_uri=";
         $redirect_uri .= URI::Escape::uri_escape( $c->uri_for('/auth/sign_out') );
         $oauth{logout_redirect_uri} = $redirect_uri;
     }
 
     # The OIDC endpoint may provide a specific URI for changing the user's password.
-    if ( my $password_change_uri = $c->cobrand->feature('oidc_login')->{password_change_uri} ) {
+    if ( my $password_change_uri = $cfg->{password_change_uri} ) {
         $oauth{change_password_uri} = $oidc->uri_to_redirect(
             uri          => $password_change_uri,
             redirect_uri => $c->uri_for('/auth/OIDC'),
@@ -279,6 +290,7 @@ sub oidc_callback: Path('/auth/OIDC') : Args(0) {
     eval {
         $id_token = $oidc->get_access_token(
             code => $c->get_param('code'),
+            redirect_uri => $c->uri_for('/auth/OIDC')
         );
     };
     if ($@) {
@@ -294,10 +306,18 @@ sub oidc_callback: Path('/auth/OIDC') : Args(0) {
     # check that the nonce matches what we set in the user session
     $c->detach('/page_error_500_internal_error', ['invalid id_token']) unless $id_token->payload->{nonce} eq $c->session->{oauth}{nonce};
 
+    if (my $domains = $c->cobrand->feature('oidc_login')->{allowed_domains}) {
+        # Check that the hd payload is present in the token and matches the
+        # list of allowed domains from the config
+        my $hd = $id_token->payload->{hd};
+        my %allowed_domains = map { $_ => 1} @$domains;
+        $c->detach('oauth_failure') unless $allowed_domains{$hd};
+    }
+
     # Some claims need parsing into a friendlier format
-    # XXX check how much of this is Westminster/Azure-specific
-    my $name = join(" ", $id_token->payload->{given_name}, $id_token->payload->{family_name});
+    my $name = $id_token->payload->{name} || join(" ", $id_token->payload->{given_name}, $id_token->payload->{family_name});
     my $email = $id_token->payload->{email};
+
     # WCC Azure provides a single email address as an array for some reason
     my $emails = $id_token->payload->{emails};
     if ($emails && @$emails) {
diff --git a/perllib/FixMyStreet/App/Controller/Contact.pm b/perllib/FixMyStreet/App/Controller/Contact.pm
index 9ce89a9e2..f919cbeff 100644
--- a/perllib/FixMyStreet/App/Controller/Contact.pm
+++ b/perllib/FixMyStreet/App/Controller/Contact.pm
@@ -199,7 +199,7 @@ sub prepare_params_for_email : Private {
     my $base_url = $c->cobrand->base_url();
     my $admin_url = $c->cobrand->admin_base_url;
 
-    my $user = $c->cobrand->users->find( { email => $c->stash->{em} } );
+    my $user = $c->cobrand->users->find( { email => lc $c->stash->{em} } );
     if ( $user ) {
         $c->stash->{user_admin_url} = $admin_url . '/users/' . $user->id;
         $c->stash->{user_reports_admin_url} = $admin_url . '/reports?search=' . $user->email;
diff --git a/perllib/FixMyStreet/App/Controller/Contact/Enquiry.pm b/perllib/FixMyStreet/App/Controller/Contact/Enquiry.pm
index 5b1c4980f..da5b9906e 100644
--- a/perllib/FixMyStreet/App/Controller/Contact/Enquiry.pm
+++ b/perllib/FixMyStreet/App/Controller/Contact/Enquiry.pm
@@ -49,15 +49,9 @@ sub submit : Path('submit') : Args(0) {
     $c->set_param('pc', '');
     $c->set_param('skipped', 1);
 
-    $c->forward('/report/new/initialize_report');
-    $c->forward('/report/new/check_for_category');
-    $c->forward('/auth/check_csrf_token');
-    $c->forward('/report/new/process_report');
-    $c->forward('/report/new/process_user');
-    $c->forward('handle_uploads');
-    $c->forward('/photo/process_photo');
-    $c->go('index', [ 1 ]) unless $c->forward('/report/new/check_for_errors');
-    $c->forward('/report/new/save_user_and_report');
+    $c->forward('/report/new/non_map_creation', [ [ '/contact/enquiry/handle_uploads' ] ])
+        or $c->go('index', [ 1 ]);
+
     $c->forward('confirm_report');
     $c->stash->{success} = 1;
 
diff --git a/perllib/FixMyStreet/App/Controller/Dashboard.pm b/perllib/FixMyStreet/App/Controller/Dashboard.pm
index ad6c9ba98..5400a6209 100644
--- a/perllib/FixMyStreet/App/Controller/Dashboard.pm
+++ b/perllib/FixMyStreet/App/Controller/Dashboard.pm
@@ -6,9 +6,9 @@ use DateTime;
 use Encode;
 use JSON::MaybeXS;
 use Path::Tiny;
-use Text::CSV;
 use Time::Piece;
 use FixMyStreet::DateRange;
+use FixMyStreet::Reporting;
 
 BEGIN { extends 'Catalyst::Controller'; }
 
@@ -135,13 +135,25 @@ sub index : Path : Args(0) {
     $c->stash->{end_date} = $c->get_param('end_date');
     $c->stash->{q_state} = $c->get_param('state') || '';
 
-    $c->forward('construct_rs_filter', [ $c->get_param('updates') ]);
-
-    if ( $c->get_param('export') ) {
-        if ($c->get_param('updates')) {
-            $c->forward('export_as_csv_updates');
-        } else {
-            $c->forward('export_as_csv');
+    my $reporting = $c->forward('construct_rs_filter', [ $c->get_param('updates') ]);
+
+    if ( my $export = $c->get_param('export') ) {
+        $reporting->csv_parameters;
+        if ($export == 1) {
+            # Existing method, generate and serve
+            $reporting->generate_csv_http($c);
+        } elsif ($export == 2) {
+            # New offline method
+            $reporting->kick_off_process;
+            my ($redirect, $code) = ('/dashboard/status', 303);
+            if (Catalyst::Authentication::Credential::AccessToken->get_token($c)) {
+                # Client knows to re-request until ready
+                $redirect = '/dashboard/csv/' . $reporting->filename . '.csv';
+                $c->res->body('');
+                $code = 202;
+            }
+            $c->res->redirect($redirect, $code);
+            $c->detach;
         }
     } else {
         $c->forward('generate_grouped_data');
@@ -152,37 +164,19 @@ sub index : Path : Args(0) {
 sub construct_rs_filter : Private {
     my ($self, $c, $updates) = @_;
 
-    my %where;
-    $where{areas} = [ map { { 'like', "%,$_,%" } } @{$c->stash->{ward}} ]
-        if @{$c->stash->{ward}};
-    $where{category} = $c->stash->{category}
-        if $c->stash->{category};
-
-    my $table_name = $updates ? 'problem' : 'me';
-
-    my $state = $c->stash->{q_state};
-    if ( FixMyStreet::DB::Result::Problem->fixed_states->{$state} ) { # Probably fixed - council
-        $where{"$table_name.state"} = [ FixMyStreet::DB::Result::Problem->fixed_states() ];
-    } elsif ( $state ) {
-        $where{"$table_name.state"} = $state;
-    } else {
-        $where{"$table_name.state"} = [ FixMyStreet::DB::Result::Problem->visible_states() ];
-    }
-
-    my $days30 = DateTime->now(time_zone => FixMyStreet->time_zone || FixMyStreet->local_time_zone)->subtract(days => 30);
-    $days30->truncate( to => 'day' );
-
-    my $range = FixMyStreet::DateRange->new(
+    my $reporting = FixMyStreet::Reporting->new(
+        type => $updates ? 'updates' : 'problems',
+        category => $c->stash->{category},
+        state => $c->stash->{q_state},
+        wards => $c->stash->{ward},
+        body => $c->stash->{body} || undef,
         start_date => $c->stash->{start_date},
-        start_default => $days30,
         end_date => $c->stash->{end_date},
-        formatter => $c->model('DB')->storage->datetime_parser,
+        user => $c->user_exists ? $c->user->obj : undef,
     );
-    $where{"$table_name.confirmed"} = $range->sql;
 
-    $c->stash->{params} = \%where;
-    my $rs = $updates ? $c->cobrand->updates : $c->cobrand->problems;
-    $c->stash->{objects_rs} = $rs->to_body($c->stash->{body})->search( \%where );
+    $c->stash($reporting->construct_rs_filter);
+    return $reporting;
 }
 
 sub generate_grouped_data : Private {
@@ -297,210 +291,67 @@ sub generate_summary_figures {
     }
 }
 
-sub generate_body_response_time : Private {
-    my ( $self, $c ) = @_;
-
-    my $avg = $c->stash->{body}->calculate_average($c->cobrand->call_hook("body_responsiveness_threshold"));
-    $c->stash->{body_average} = $avg ? int($avg / 60 / 60 / 24 + 0.5) : 0;
-}
-
-sub csv_filename {
-    my ($self, $c, $updates) = @_;
-    my %where = (
-        category => $c->stash->{category},
-        state => $c->stash->{q_state},
-        ward => join(',', @{$c->stash->{ward}}),
-    );
-    $where{body} = $c->stash->{body}->id if $c->stash->{body};
-    join '-',
-        $c->req->uri->host,
-        $updates ? ('updates') : (),
-        map {
-            my $value = $where{$_};
-            (defined $value and length $value) ? ($_, $value) : ()
-        } sort keys %where
-};
-
-sub export_as_csv_updates : Private {
+sub status : Local : Args(0) {
     my ($self, $c) = @_;
 
-    my $csv = $c->stash->{csv} = {
-        objects => $c->stash->{objects_rs}->search_rs({}, {
-            order_by => ['me.confirmed', 'me.id'],
-            '+columns' => ['problem.bodies_str'],
-            cursor_page_size => 1000,
-        }),
-        headers => [
-            'Report ID', 'Update ID', 'Date', 'Status', 'Problem state',
-            'Text', 'User Name', 'Reported As',
-        ],
-        columns => [
-            'problem_id', 'id', 'confirmed', 'state', 'problem_state',
-            'text', 'user_name_display', 'reported_as',
-        ],
-        filename => $self->csv_filename($c, 1),
-    };
-    $c->cobrand->call_hook("dashboard_export_updates_add_columns");
-    $c->forward('generate_csv');
-}
-
-sub export_as_csv : Private {
-    my ($self, $c) = @_;
+    my $body = $c->stash->{body} = $c->forward('check_page_allowed');
+    $c->stash->{body_name} = $body->name if $body;
 
-    my $csv = $c->stash->{csv} = {
-        objects => $c->stash->{objects_rs}->search_rs({}, {
-            join => 'comments',
-            '+columns' => ['comments.problem_state', 'comments.state', 'comments.confirmed', 'comments.mark_fixed'],
-            order_by => ['me.confirmed', 'me.id'],
-            cursor_page_size => 1000,
-        }),
-        headers => [
-            'Report ID',
-            'Title',
-            'Detail',
-            'User Name',
-            'Category',
-            'Created',
-            'Confirmed',
-            'Acknowledged',
-            'Fixed',
-            'Closed',
-            'Status',
-            'Latitude', 'Longitude',
-            'Query',
-            'Ward',
-            'Easting',
-            'Northing',
-            'Report URL',
-            'Site Used',
-            'Reported As',
-        ],
-        columns => [
-            'id',
-            'title',
-            'detail',
-            'user_name_display',
-            'category',
-            'created',
-            'confirmed',
-            'acknowledged',
-            'fixed',
-            'closed',
-            'state',
-            'latitude', 'longitude',
-            'postcode',
-            'wards',
-            'local_coords_x',
-            'local_coords_y',
-            'url',
-            'site_used',
-            'reported_as',
-        ],
-        filename => $self->csv_filename($c, 0),
-    };
-    $c->cobrand->call_hook("dashboard_export_problems_add_columns");
-    $c->forward('generate_csv');
+    my $reporting = FixMyStreet::Reporting->new(
+        user => $c->user_exists ? $c->user->obj : undef,
+    );
+    my $dir = $reporting->cache_dir;
+    my @data;
+    foreach ($dir->children) {
+        my $stat = $_->stat;
+        my $name = $_->basename;
+        my $finished = $name =~ /part$/ ? 0 : 1;
+        $name =~ s/-part$//;
+        push @data, {
+            ctime => $stat->ctime,
+            size => $stat->size,
+            name => $name,
+            finished => $finished,
+        };
+    }
+    @data = sort { $b->{ctime} <=> $a->{ctime} } @data;
+    $c->stash->{rows} = \@data;
 }
 
-=head2 generate_csv
-
-Generates a CSV output, given a 'csv' stash hashref containing:
-* filename: filename to be used in output
-* problems: a resultset of the rows to output
-* headers: an arrayref of the header row strings
-* columns: an arrayref of the columns (looked up in the row's as_hashref, plus
-the following: user_name_display, acknowledged, fixed, closed, wards,
-local_coords_x, local_coords_y, url).
-* extra_data: If present, a function that is passed the report and returns a
-hashref of extra data to include that can be used by 'columns'.
-
-=cut
+sub csv : Local : Args(1) {
+    my ($self, $c, $filename) = @_;
 
-sub generate_csv : Private {
-    my ($self, $c) = @_;
+    $c->authenticate(undef, "access_token");
 
-    my $filename = $c->stash->{csv}->{filename};
-    $c->res->content_type('text/csv; charset=utf-8');
-    $c->res->header('content-disposition' => "attachment; filename=\"${filename}.csv\"");
-
-    # Emit a header (copying Drupal's naming) telling an intermediary (e.g.
-    # Varnish) not to buffer the output. Varnish will need to know this, e.g.:
-    #   if (beresp.http.Surrogate-Control ~ "BigPipe/1.0") {
-    #     set beresp.do_stream = true;
-    #     set beresp.ttl = 0s;
-    #   }
-    $c->res->header('Surrogate-Control' => 'content="BigPipe/1.0"');
-
-    # Tell nginx not to buffer this response
-    $c->res->header('X-Accel-Buffering' => 'no');
-
-    # Define an empty body so the web view doesn't get added at the end
-    $c->res->body("");
-
-    # Old parameter renaming
-    $c->stash->{csv}->{objects} //= $c->stash->{csv}->{problems};
-
-    my $csv = Text::CSV->new({ binary => 1, eol => "\n" });
-    $csv->print($c->response, $c->stash->{csv}->{headers});
-
-    my $fixed_states = FixMyStreet::DB::Result::Problem->fixed_states;
-    my $closed_states = FixMyStreet::DB::Result::Problem->closed_states;
-
-    my %asked_for = map { $_ => 1 } @{$c->stash->{csv}->{columns}};
-
-    my $objects = $c->stash->{csv}->{objects};
-    while ( my $obj = $objects->next ) {
-        my $hashref = $obj->as_hashref($c, \%asked_for);
-
-        $hashref->{user_name_display} = $obj->anonymous
-            ? '(anonymous)' : $obj->name;
-
-        if ($asked_for{acknowledged}) {
-            for my $comment ($obj->comments) {
-                my $problem_state = $comment->problem_state or next;
-                next unless $comment->state eq 'confirmed';
-                next if $problem_state eq 'confirmed';
-                $hashref->{acknowledged} //= $comment->confirmed;
-                $hashref->{fixed} //= $fixed_states->{ $problem_state } || $comment->mark_fixed ?
-                    $comment->confirmed : undef;
-                if ($closed_states->{ $problem_state }) {
-                    $hashref->{closed} = $comment->confirmed;
-                    last;
-                }
-            }
-        }
-
-        if ($asked_for{wards}) {
-            $hashref->{wards} = join ', ',
-              map { $c->stash->{children}->{$_}->{name} }
-              grep {$c->stash->{children}->{$_} }
-              split ',', $hashref->{areas};
-        }
+    my $body = $c->stash->{body} = $c->forward('check_page_allowed');
 
-        if ($obj->can('local_coords') && $asked_for{local_coords_x}) {
-            ($hashref->{local_coords_x}, $hashref->{local_coords_y}) =
-                $obj->local_coords;
-        }
-        if ($obj->can('url')) {
-            my $base = $c->cobrand->base_url_for_report($obj->can('problem') ? $obj->problem : $obj);
-            $hashref->{url} = join '', $base, $obj->url;
+    (my $basename = $filename) =~ s/\.csv$//;
+    my $reporting = FixMyStreet::Reporting->new(
+        user => $c->user_exists ? $c->user->obj : undef,
+        filename => $basename,
+    );
+    my $dir = $reporting->cache_dir;
+    my $csv = path($dir, $filename);
+
+    if (!$csv->exists) {
+        if (path($dir, "$filename-part")->exists && Catalyst::Authentication::Credential::AccessToken->get_token($c)) {
+            $c->res->body('');
+            $c->res->status(202);
+            $c->detach;
+        } else {
+            $c->detach( '/page_error_404_not_found', [] ) unless $csv->exists;
         }
+    }
 
-        $hashref->{site_used} = $obj->can('service') ? ($obj->service || $obj->cobrand) : $obj->cobrand;
-
-        $hashref->{reported_as} = $obj->get_extra_metadata('contributed_as') || '';
+    $reporting->http_setup($c);
+    $c->res->body($csv->openr_raw);
+}
 
-        if (my $fn = $c->stash->{csv}->{extra_data}) {
-            my $extra = $fn->($obj);
-            $hashref = { %$hashref, %$extra };
-        }
+sub generate_body_response_time : Private {
+    my ( $self, $c ) = @_;
 
-        $csv->print($c->response, [
-            @{$hashref}{
-                @{$c->stash->{csv}->{columns}}
-            },
-        ] );
-    }
+    my $avg = $c->stash->{body}->calculate_average($c->cobrand->call_hook("body_responsiveness_threshold"));
+    $c->stash->{body_average} = $avg ? int($avg / 60 / 60 / 24 + 0.5) : 0;
 }
 
 sub heatmap : Local : Args(0) {
@@ -534,7 +385,7 @@ sub heatmap : Local : Args(0) {
     if ($c->get_param('ajax')) {
         my @pins;
         while ( my $problem = $problems->next ) {
-            push @pins, $problem->pin_data($c, 'reports');
+            push @pins, $problem->pin_data('reports');
         }
         $c->stash->{pins} = \@pins;
         $c->detach('/reports/ajax', [ 'dashboard/heatmap-list.html' ]);
@@ -544,7 +395,8 @@ sub heatmap : Local : Args(0) {
     $c->stash->{children} = $children;
     $c->stash->{ward_hash} = { map { $_->{id} => 1 } @{$c->stash->{wards}} } if $c->stash->{wards};
 
-    $c->forward('/reports/setup_categories_and_map');
+    $c->forward('/reports/setup_categories');
+    $c->forward('/reports/setup_map');
 }
 
 sub heatmap_filters :Private {
@@ -591,7 +443,15 @@ sub heatmap_sidebar :Private {
         order_by => 'lastupdate',
     })->all ];
 
-    my $params = { map { my $n = $_; s/me\./problem\./; $_ => $where->{$n} } keys %$where };
+    my $params = { map {
+        my $v = $where->{$_};
+        if (ref $v eq 'HASH') {
+            $v = { map { my $vv = $v->{$_}; s/me\./problem\./; $_ => $vv } keys %$v };
+        } else {
+            s/me\./problem\./;
+        }
+        $_ => $v;
+    } keys %$where };
     my $body = $c->stash->{body};
 
     my @user;
diff --git a/perllib/FixMyStreet/App/Controller/JSON.pm b/perllib/FixMyStreet/App/Controller/JSON.pm
index ccc5b31dc..4657fcf2c 100644
--- a/perllib/FixMyStreet/App/Controller/JSON.pm
+++ b/perllib/FixMyStreet/App/Controller/JSON.pm
@@ -7,7 +7,6 @@ BEGIN { extends 'Catalyst::Controller'; }
 use JSON::MaybeXS;
 use DateTime;
 use DateTime::Format::ISO8601;
-use List::MoreUtils 'uniq';
 use FixMyStreet::DateRange;
 
 =head1 NAME
@@ -102,6 +101,7 @@ sub problems : Local {
     } );
 
     foreach my $problem (@problems) {
+        $c->cobrand->call_hook(munge_problem_list => $problem);
         $problem->name( '' ) if $problem->anonymous == 1;
         $problem->service( 'Web interface' ) if $problem->service eq '';
         my $body_names = $problem->body_names;
diff --git a/perllib/FixMyStreet/App/Controller/Location.pm b/perllib/FixMyStreet/App/Controller/Location.pm
index 416fb942a..3869ef8d3 100644
--- a/perllib/FixMyStreet/App/Controller/Location.pm
+++ b/perllib/FixMyStreet/App/Controller/Location.pm
@@ -6,6 +6,7 @@ BEGIN {extends 'Catalyst::Controller'; }
 
 use Encode;
 use FixMyStreet::Geocode;
+use Geo::OLC;
 use Try::Tiny;
 use Utils;
 
@@ -74,6 +75,30 @@ sub determine_location_from_pc : Private {
             map { Utils::truncate_coordinate($_) } ($1, $2);
         return $c->forward( 'check_location' );
     }
+
+    if (Geo::OLC::is_full($pc)) {
+        my $ref = Geo::OLC::decode($pc);
+        ($c->stash->{latitude}, $c->stash->{longitude}) =
+            map { Utils::truncate_coordinate($_) } @{$ref->{center}};
+        return $c->forward( 'check_location' );
+    }
+
+    if ($pc =~ /^\s*([2-9CFGHJMPQRVWX]{4,6}\+[2-9CFGHJMPQRVWX]{2,3})\s+(.*)$/i) {
+        my ($code, $rest) = ($1, $2);
+        my ($lat, $lon, $error) = FixMyStreet::Geocode::lookup($rest, $c);
+        if (ref($error) eq 'ARRAY') { # Just take the first result
+            $lat = $error->[0]{latitude};
+            $lon = $error->[0]{longitude};
+        }
+        if (defined $lat && defined $lon) {
+            $code = Geo::OLC::recover_nearest($code, $lat, $lon);
+            my $ref = Geo::OLC::decode($code);
+            ($c->stash->{latitude}, $c->stash->{longitude}) =
+                map { Utils::truncate_coordinate($_) } @{$ref->{center}};
+            return $c->forward( 'check_location' );
+        }
+    }
+
     if ( $c->cobrand->country eq 'GB' && $pc =~ /^([A-Z])([A-Z])([\d\s]{4,})$/i) {
         if (my $convert = gridref_to_latlon( $1, $2, $3 )) {
             ($c->stash->{latitude}, $c->stash->{longitude}) =
diff --git a/perllib/FixMyStreet/App/Controller/Moderate.pm b/perllib/FixMyStreet/App/Controller/Moderate.pm
index f4143f0b4..c936f13c0 100644
--- a/perllib/FixMyStreet/App/Controller/Moderate.pm
+++ b/perllib/FixMyStreet/App/Controller/Moderate.pm
@@ -340,13 +340,8 @@ sub moderate_state : Private {
         $problem->state($new_state);
         $problem->add_to_comments( {
             text => $c->stash->{moderation_reason},
-            created => \'current_timestamp',
-            confirmed => \'current_timestamp',
-            user_id => $c->user->id,
-            name => $c->user->from_body ? $c->user->from_body->name : $c->user->name,
-            state => 'confirmed',
-            mark_fixed => 0,
-            anonymous => $c->user->from_body ? 0 : 1,
+            user => $c->user->obj,
+            anonymous => $c->user->is_superuser || $c->user->from_body ? 0 : 1,
             problem_state => $new_state,
         } );
         return 'state';
diff --git a/perllib/FixMyStreet/App/Controller/My.pm b/perllib/FixMyStreet/App/Controller/My.pm
index 3328caac0..52a3a8cef 100644
--- a/perllib/FixMyStreet/App/Controller/My.pm
+++ b/perllib/FixMyStreet/App/Controller/My.pm
@@ -135,13 +135,14 @@ sub get_problems : Private {
     my $problems = [];
 
     my $states = $c->stash->{filter_problem_states};
+    my $table = $c->action eq 'my/planned' ? 'report' : 'me';
     my $params = {
-        state => [ keys %$states ],
+        "$table.state" => [ keys %$states ],
     };
 
     my $categories = [ $c->get_param_list('filter_category', 1) ];
     if ( @$categories ) {
-        $params->{category} = $categories;
+        $params->{"$table.category"} = $categories;
         $c->stash->{filter_category} = { map { $_ => 1 } @$categories };
     }
 
@@ -149,13 +150,14 @@ sub get_problems : Private {
     $rows = 5000 if $c->stash->{sort_key} eq 'shortlist'; # Want all reports
 
     my $rs = $c->stash->{problems_rs}->search( $params, {
+        prefetch => 'contact',
         order_by => $c->stash->{sort_order},
         rows => $rows,
     } )->include_comment_counts->page( $p_page );
 
     while ( my $problem = $rs->next ) {
         $c->stash->{has_content}++;
-        push @$pins, $problem->pin_data($c, 'my', private => 1);
+        push @$pins, $problem->pin_data('my', private => 1);
         push @$problems, $problem;
     }
 
@@ -186,15 +188,28 @@ sub get_updates : Private {
 sub setup_page_data : Private {
     my ($self, $c) = @_;
 
+    my $table = $c->action eq 'my/planned' ? 'report' : 'me';
     my @categories = $c->stash->{problems_rs}->search({
-        state => [ FixMyStreet::DB::Result::Problem->visible_states() ],
+        "$table.state" => [ FixMyStreet::DB::Result::Problem->visible_states() ],
     }, {
-        columns => [ 'category', 'bodies_str', 'extra' ],
+        join => 'contact',
+        columns => [ "$table.category", 'contact.extra', 'contact.category' ],
         distinct => 1,
-        order_by => [ 'category' ],
+        order_by => [ "$table.category" ],
     } )->all;
+    # Ensure only uniquely named categories are shown
+    my %seen;
+    @categories = grep { !$seen{$_->category_display}++ } @categories;
     $c->stash->{filter_categories} = \@categories;
-    $c->forward('/report/stash_category_groups', [ \@categories ]) if $c->cobrand->enable_category_groups;
+
+    if ($c->cobrand->enable_category_groups) {
+        my @contacts = map { {
+            category => $_->category,
+            category_display => $_->category_display,
+            group => [''],
+        } } @categories;
+        $c->forward('/report/stash_category_groups', [ \@contacts ]);
+    }
 
     my $pins = $c->stash->{pins};
     FixMyStreet::Map::display_map(
diff --git a/perllib/FixMyStreet/App/Controller/Offline.pm b/perllib/FixMyStreet/App/Controller/Offline.pm
index adb3de14d..af05c585f 100644
--- a/perllib/FixMyStreet/App/Controller/Offline.pm
+++ b/perllib/FixMyStreet/App/Controller/Offline.pm
@@ -22,6 +22,7 @@ Offline pages Catalyst Controller - service worker handling
 
 sub service_worker : Path("/service-worker.js") {
     my ($self, $c) = @_;
+    $c->res->headers->header('Cache-Control' => 'max-age=0');
     $c->res->content_type('application/javascript');
 }
 
diff --git a/perllib/FixMyStreet/App/Controller/Open311.pm b/perllib/FixMyStreet/App/Controller/Open311.pm
index b4b5d5e3a..73a91a62a 100644
--- a/perllib/FixMyStreet/App/Controller/Open311.pm
+++ b/perllib/FixMyStreet/App/Controller/Open311.pm
@@ -210,7 +210,7 @@ sub output_requests : Private {
     };
 
     # Look up categories for this council or councils
-    my $problems = $c->cobrand->problems->search( $criteria, $attr );
+    my $problems = $c->stash->{rs}->search( $criteria, $attr );
 
     my %statusmap = (
         map( { $_ => 'open' } FixMyStreet::DB::Result::Problem->open_states() ),
@@ -220,6 +220,8 @@ sub output_requests : Private {
 
     my @problemlist;
     while ( my $problem = $problems->next ) {
+        $c->cobrand->call_hook(munge_problem_list => $problem);
+
         my $id = $problem->id;
 
         $problem->service( 'Web interface' ) unless $problem->service;
@@ -323,24 +325,8 @@ sub get_requests : Private {
         if ( 'status' eq $param ) {
             $value = {
                 'open' => [ FixMyStreet::DB::Result::Problem->open_states() ],
-                'closed' => [ FixMyStreet::DB::Result::Problem->fixed_states(), 'closed' ],
+                'closed' => [ FixMyStreet::DB::Result::Problem->fixed_states(), FixMyStreet::DB::Result::Problem->closed_states() ],
             }->{$value};
-        } elsif ( 'agency_responsible' eq $param ) {
-            my @valuelist;
-            for my $agency (split(/\|/, $value)) {
-                unless ($agency =~ m/^(\d+)$/) {
-                    $c->detach( 'error', [
-                        sprintf(_('Invalid agency_responsible value %s'),
-                            $value)
-                    ] );
-                }
-                my $agencyid = $1;
-                # FIXME This seem to match the wrong entries
-                # some times.  Not sure when or why
-                my $re = "(\\y$agencyid\\y|^$agencyid\\y|\\y$agencyid\$)";
-                push(@valuelist, $re);
-            }
-            $value = \@valuelist;
         } elsif ( 'has_photo' eq $param ) {
             $value = undef;
             $op = '!=' if 'true' eq $value;
@@ -363,6 +349,11 @@ sub get_requests : Private {
         $criteria->{confirmed} = { '<', $c->get_param('end_date') };
     }
 
+    $c->stash->{rs} = $c->cobrand->problems;
+    if (my $bodies = $c->get_param('agency_responsible')) {
+        $c->stash->{rs} = $c->stash->{rs}->to_body([ split(/\|/, $bodies) ]);
+    }
+
     if ('rss' eq $c->stash->{format}) {
         $c->stash->{type} = 'new_problems';
         $c->forward( '/rss/lookup_type' );
@@ -384,7 +375,7 @@ sub rss_query : Private {
         rows => $limit
     };
 
-    my $problems = $c->cobrand->problems->search( $criteria, $attr );
+    my $problems = $c->stash->{rs}->search( $criteria, $attr );
     $c->stash->{problems} = $problems;
 }
 
@@ -411,6 +402,7 @@ sub get_request : Private {
         id => $id,
         non_public => 0,
     };
+    $c->stash->{rs} = $c->cobrand->problems;
     $c->forward( 'output_requests', [ $criteria ] );
 }
 
diff --git a/perllib/FixMyStreet/App/Controller/Open311/Updates.pm b/perllib/FixMyStreet/App/Controller/Open311/Updates.pm
index 105400a8a..8881a1b87 100644
--- a/perllib/FixMyStreet/App/Controller/Open311/Updates.pm
+++ b/perllib/FixMyStreet/App/Controller/Open311/Updates.pm
@@ -3,7 +3,6 @@ package FixMyStreet::App::Controller::Open311::Updates;
 use utf8;
 use Moose;
 use namespace::autoclean;
-use Open311;
 use Open311::GetServiceRequestUpdates;
 
 BEGIN { extends 'Catalyst::Controller'; }
@@ -31,7 +30,6 @@ sub receive : Regex('^open311/v2/servicerequestupdates.(xml|json)$') : Args(0) {
         $body = $c->model('DB::Body')->find({ id => $c->get_param('jurisdiction_id') });
     }
     $c->detach('bad_request', ['jurisdiction_id']) unless $body;
-    my $user = $body->comment_user;
 
     my $key = $c->get_param('api_key') || '';
     my $token = $c->cobrand->feature('open311_token') || '';
@@ -45,37 +43,36 @@ sub receive : Regex('^open311/v2/servicerequestupdates.(xml|json)$') : Args(0) {
         $request->{$_} = $c->get_param($_) || $c->detach('bad_request', [ $_ ]);
     }
 
-    my %open311_conf = (
-        endpoint => $body->endpoint,
-        api_key => $body->api_key,
-        jurisdiction => $body->jurisdiction,
-        extended_statuses => $body->send_extended_statuses,
-    );
+    $c->forward('process_update', [ $body, $request ]);
+}
 
-    my $cobrand = $body->get_cobrand_handler;
-    $cobrand->call_hook(open311_config_updates => \%open311_conf)
-        if $cobrand;
+sub process_update : Private {
+    my ($self, $c, $body, $request) = @_;
 
-    my $open311 = Open311->new(%open311_conf);
     my $updates = Open311::GetServiceRequestUpdates->new(
-        system_user => $user,
-        current_open311 => $open311,
+        system_user => $body->comment_user,
         current_body => $body,
     );
 
     my $p = $updates->find_problem($request);
     $c->detach('bad_request', [ 'not found' ]) unless $p;
 
-    my $comment = $p->comments->search( { external_id => $request->{update_id} } )->first;
-    $c->detach('bad_request', [ 'already exists' ]) if $comment;
+    $c->forward('check_existing', [ $p, $request, $updates ]);
 
-    $comment = $updates->process_update($request, $p);
+    my $comment = $updates->process_update($request, $p);
 
     my $data = { service_request_updates => { update_id => $comment->id } };
 
     $c->forward('/open311/format_output', [ $data ]);
 }
 
+sub check_existing : Private {
+    my ($self, $c, $p, $request, $updates) = @_;
+
+    my $comment = $p->comments->search( { external_id => $request->{update_id} } )->first;
+    $c->detach('bad_request', [ 'already exists' ]) if $comment;
+}
+
 sub bad_request : Private {
     my ($self, $c, $comment) = @_;
     $c->response->status(400);
diff --git a/perllib/FixMyStreet/App/Controller/Questionnaire.pm b/perllib/FixMyStreet/App/Controller/Questionnaire.pm
index ab6117ae4..ef6152c30 100755
--- a/perllib/FixMyStreet/App/Controller/Questionnaire.pm
+++ b/perllib/FixMyStreet/App/Controller/Questionnaire.pm
@@ -206,16 +206,12 @@ sub submit_standard : Private {
         $update = $c->model('DB::Comment')->new(
             {
                 problem      => $problem,
-                name         => $problem->name,
                 user         => $problem->user,
                 text         => $update,
-                state        => 'confirmed',
                 mark_fixed   => $c->stash->{new_state} eq 'fixed - user' ? 1 : 0,
                 mark_open    => $c->stash->{new_state} eq 'confirmed' ? 1 : 0,
                 lang         => $c->stash->{lang_code},
                 cobrand      => $c->cobrand->moniker,
-                cobrand_data => '',
-                confirmed    => \'current_timestamp',
                 anonymous    => $problem->anonymous,
             }
         );
diff --git a/perllib/FixMyStreet/App/Controller/Report.pm b/perllib/FixMyStreet/App/Controller/Report.pm
index 72f96013a..1e5751588 100644
--- a/perllib/FixMyStreet/App/Controller/Report.pm
+++ b/perllib/FixMyStreet/App/Controller/Report.pm
@@ -85,14 +85,32 @@ sub display :PathPart('') :Chained('id') :Args(0) {
     $c->forward( 'load_updates' );
     $c->forward( 'format_problem_for_display' );
 
-    my $permissions = $c->stash->{_permissions} ||= $c->forward( 'check_has_permission_to',
-        [ qw/report_inspect report_edit_category report_edit_priority report_mark_private triage/ ] );
-    if (any { $_ } values %$permissions) {
+    my $permissions = $c->stash->{permissions} ||= $c->forward('fetch_permissions');
+
+    my $staff_user = $c->user_exists && ($c->user->is_superuser || $c->user->belongs_to_body($c->stash->{problem}->bodies_str));
+
+    if ($staff_user) {
+        # Check assigned categories feature
+        my $okay = 1;
+        my $contact = $c->stash->{problem}->contact;
+        if ($contact && ($c->user->get_extra_metadata('assigned_categories_only') || $contact->get_extra_metadata('assigned_users_only'))) {
+            my $user_cats = $c->user->get_extra_metadata('categories') || [];
+            $okay = any { $contact->id eq $_ } @$user_cats;
+        }
+        if ($okay) {
+            $c->stash->{relevant_staff_user} = 1;
+        } else {
+            # Remove all staff permissions
+            $permissions = $c->stash->{permissions} = {};
+        }
+    }
+
+    if (grep { $permissions->{$_} } qw/report_inspect report_edit_category report_edit_priority report_mark_private triage/) {
         $c->stash->{template} = 'report/inspect.html';
         $c->forward('inspect');
     }
 
-    if ($c->user_exists && $c->user->has_permission_to(contribute_as_another_user => $c->stash->{problem}->bodies_str_ids)) {
+    if ($permissions->{contribute_as_another_user}) {
         $c->stash->{email} = $c->user->email;
     }
 }
@@ -133,11 +151,13 @@ sub support :Chained('id') :Args(0) {
 sub load_problem_or_display_error : Private {
     my ( $self, $c, $id ) = @_;
 
+    my $attrs = { prefetch => 'contact' };
+
     # try to load a report if the id is a number
     my $problem
       = ( !$id || $id =~ m{\D} ) # is id non-numeric?
       ? undef                    # ...don't even search
-      : $c->cobrand->problems->find( { id => $id } )
+      : $c->cobrand->problems->find( { id => $id }, $attrs )
           or $c->detach( '/page_error_404_not_found', [ _('Unknown problem ID') ] );
 
     # check that the problem is suitable to show.
@@ -158,8 +178,7 @@ sub load_problem_or_display_error : Private {
     } elsif ( $problem->non_public ) {
         # Creator, and inspection users can see non_public reports
         $c->stash->{problem} = $problem;
-        my $permissions = $c->stash->{_permissions} = $c->forward( 'check_has_permission_to',
-            [ qw/report_inspect report_edit_category report_edit_priority report_mark_private / ] );
+        my $permissions = $c->stash->{permissions} = $c->forward('fetch_permissions');
 
         # If someone has clicked a unique token link in an email to them
         my $from_email = $c->sessionid && $c->flash->{alert_to_reporter} && $c->flash->{alert_to_reporter} == $problem->id;
@@ -178,6 +197,7 @@ sub load_problem_or_display_error : Private {
         }
     }
 
+    $c->cobrand->call_hook(munge_problem_list => $problem);
     $c->stash->{problem} = $problem;
     if ( $c->user_exists && $c->user->can_moderate($problem) ) {
         $c->stash->{problem_original} = $problem->find_or_new_related(
@@ -232,6 +252,7 @@ sub load_updates : Private {
     my @combined;
     my %questionnaires_with_updates;
     while (my $update = $updates->next) {
+        $c->cobrand->call_hook(munge_update_list => $update);
         push @combined, [ $update->confirmed, $update ];
         if (my $qid = $update->get_extra_metadata('questionnaire_id')) {
             $questionnaires_with_updates{$qid} = $update;
@@ -304,7 +325,7 @@ sub format_problem_for_display : Private {
         $c->res->content_type('application/json; charset=utf-8');
 
         # encode_json doesn't like DateTime objects, so strip them out
-        my $report_hashref = $c->cobrand->problem_as_hashref( $problem, $c );
+        my $report_hashref = $c->cobrand->problem_as_hashref( $problem );
         delete $report_hashref->{created};
         delete $report_hashref->{confirmed};
 
@@ -312,7 +333,7 @@ sub format_problem_for_display : Private {
         my $content = $json->encode(
             {
                 report => $report_hashref,
-                updates => $c->cobrand->updates_as_hashref( $problem, $c ),
+                updates => $c->cobrand->updates_as_hashref( $problem ),
             }
         );
         $c->res->body( $content );
@@ -333,7 +354,7 @@ sub generate_map_tags : Private {
         latitude  => $problem->latitude,
         longitude => $problem->longitude,
         pins      => $problem->used_map
-            ? [ $problem->pin_data($c, 'report', type => 'big', draggable => 1) ]
+            ? [ $problem->pin_data('report', type => 'big', draggable => 1) ]
             : [],
     );
 
@@ -384,7 +405,7 @@ sub delete :Chained('id') :Args(0) {
 sub inspect : Private {
     my ( $self, $c ) = @_;
     my $problem = $c->stash->{problem};
-    my $permissions = $c->stash->{_permissions};
+    my $permissions = $c->stash->{permissions};
 
     $c->forward('/admin/reports/categories_for_point');
     $c->stash->{report_meta} = { map { 'x' . $_->{name} => $_ } @{ $c->stash->{problem}->get_extra_fields() } };
@@ -392,7 +413,8 @@ sub inspect : Private {
     if ($c->cobrand->can('body')) {
         my $priorities_by_category = FixMyStreet::App->model('DB::ResponsePriority')->by_categories(
             $c->stash->{contacts},
-            body_id => $c->cobrand->body->id
+            body_id => $c->cobrand->body->id,
+            problem => $problem,
         );
         $c->stash->{priorities_by_category} = $priorities_by_category;
         my $templates_by_category = FixMyStreet::App->model('DB::ResponseTemplate')->by_categories(
@@ -445,7 +467,7 @@ sub inspect : Private {
                 }
             }
 
-            if ( $c->get_param('include_update') ) {
+            if ( $c->get_param('include_update') or $c->get_param('raise_defect') ) {
                 $update_text = Utils::cleanup_text( $c->get_param('public_update'), { allow_multiline => 1 } );
                 if (!$update_text) {
                     $valid = 0;
@@ -490,6 +512,14 @@ sub inspect : Private {
                 };
                 $c->user->create_alert($problem->id, $options);
             }
+
+            # If the state has been changed to action scheduled and they've said
+            # they want to raise a defect, consider the report to be inspected.
+            if ($problem->state eq 'action scheduled' && $c->get_param('raise_defect') && !$problem->get_extra_metadata('inspected')) {
+                $update_params{extra} = { 'defect_raised' => 1 };
+                $problem->set_extra_metadata( inspected => 1 );
+                $c->forward( '/admin/log_edit', [ $problem->id, 'problem', 'inspected' ] );
+            }
         }
 
         $problem->non_public($c->get_param('non_public') ? 1 : 0);
@@ -533,6 +563,12 @@ sub inspect : Private {
 
         $c->cobrand->call_hook(report_inspect_update_extra => $problem);
 
+        $c->forward('/photo/process_photo');
+        if ( my $photo_error  = delete $c->stash->{photo_error} ) {
+            $valid = 0;
+            push @{ $c->stash->{errors} }, $photo_error;
+        }
+
         if ($valid) {
             $problem->lastupdate( \'current_timestamp' );
             $problem->update;
@@ -548,16 +584,12 @@ sub inspect : Private {
                         epoch => $saved_at
                     );
                 }
-                my $name = $c->user->from_body ? $c->user->from_body->name : $c->user->name;
                 $problem->add_to_comments( {
                     text => $update_text,
                     created => $timestamp,
                     confirmed => $timestamp,
-                    user_id => $c->user->id,
-                    name => $name,
-                    state => 'confirmed',
-                    mark_fixed => 0,
-                    anonymous => 0,
+                    user => $c->user->obj,
+                    photo => $c->stash->{upload_fileid} || undef,
                     %update_params,
                 } );
             }
@@ -647,7 +679,7 @@ sub _nearby_json :Private {
     # Want to treat these as if they were on map
     $nearby = [ map { $_->problem } @$nearby ];
     my @pins = map {
-        my $p = $_->pin_data($c, 'around');
+        my $p = $_->pin_data('around');
         [ $p->{latitude}, $p->{longitude}, $p->{colour},
           $p->{id}, $p->{title}, $pin_size, JSON->false
         ]
@@ -666,30 +698,26 @@ sub _nearby_json :Private {
 }
 
 
-=head2 check_has_permission_to
+=head2 fetch_permissions
 
-Ensure the currently logged-in user has any of the provided permissions applied
-to the current Problem in $c->stash->{problem}. Shows the 403 page if not.
+Returns a hash of the user's permissions, applied to the problem
+in $c->stash->{problem}.
 
 =cut
 
-sub check_has_permission_to : Private {
-    my ( $self, $c, @permissions ) = @_;
+sub fetch_permissions : Private {
+    my ( $self, $c ) = @_;
     return {} unless $c->user_exists;
-    my $bodies = $c->stash->{problem}->bodies_str_ids;
-    my %permissions = map { $_ => $c->user->has_permission_to($_, $bodies) } @permissions;
-    return \%permissions;
+    return $c->user->permissions($c->stash->{problem});
 };
 
-
 sub stash_category_groups : Private {
     my ( $self, $c, $contacts, $combine_multiple ) = @_;
 
     my %category_groups = ();
     for my $category (@$contacts) {
-        my $group = $category->{group} // $category->get_extra_metadata('group') // [''];
-        # this could be an array ref or a string
-        my @groups = ref $group eq 'ARRAY' ? @$group : ($group);
+        my $group = $category->{group} // $category->groups;
+        my @groups = @$group;
         if (scalar @groups > 1 && $combine_multiple) {
             @groups = sort @groups;
             $category->{group} = \@groups;
@@ -708,6 +736,19 @@ sub stash_category_groups : Private {
     $c->stash->{category_groups}  = \@category_groups;
 }
 
+sub assigned_users_only : Private {
+    my ($self, $c, $categories) = @_;
+
+    # Assigned only category checking
+    if ($c->user_exists && $c->user->from_body) {
+        my @assigned_users_only = grep { $_->get_extra_metadata('assigned_users_only') } @$categories;
+        $c->stash->{assigned_users_only} = { map { $_->category => 1 } @assigned_users_only };
+        $c->stash->{assigned_categories_only} = $c->user->get_extra_metadata('assigned_categories_only');
+
+        $c->stash->{user_categories} = { map { $_ => 1 } @{$c->user->categories} };
+    }
+}
+
 __PACKAGE__->meta->make_immutable;
 
 1;
diff --git a/perllib/FixMyStreet/App/Controller/Report/New.pm b/perllib/FixMyStreet/App/Controller/Report/New.pm
index fc1a78cd5..f64a109e8 100644
--- a/perllib/FixMyStreet/App/Controller/Report/New.pm
+++ b/perllib/FixMyStreet/App/Controller/Report/New.pm
@@ -6,8 +6,7 @@ BEGIN { extends 'Catalyst::Controller'; }
 
 use utf8;
 use Encode;
-use List::MoreUtils qw(uniq);
-use List::Util 'first';
+use List::Util qw(uniq);
 use HTML::Entities;
 use Path::Class;
 use Utils;
@@ -30,8 +29,8 @@ Create a new report, or complete a partial one.
 
 submit_problem: true if a problem has been submitted, at all.
 submit_sign_in: true if the sign in button has been clicked by logged out user.
-submit_register: true if the register/confirm by email button has been clicked
-by logged out user.
+submit_register(_mobile): true if the register/confirm by email button has been clicked
+by logged out user, or submit button clicked by logged in user.
 
 =head2 location (required)
 
@@ -127,8 +126,10 @@ sub report_new_ajax : Path('mobile') : Args(0) {
 
     # Apps are sending email as username
     # Prepare for when they upgrade
-    if (!$c->get_param('username')) {
-        $c->set_param('username', $c->get_param('email'));
+    my $username_field = ( $c->get_param('submit_sign_in') || $c->get_param('password_sign_in') )
+        ? 'username': 'username_register';
+    if (!$c->get_param($username_field)) {
+        $c->set_param($username_field, $c->get_param('email'));
     }
 
     # create the report - loading a partial if available
@@ -333,12 +334,14 @@ sub disable_form_message : Private {
             my %category;
             foreach my $opt (@{$_->{values}}) {
                 if ($opt->{disable}) {
-                    $category{message} = $opt->{disable_message} || $_->{datatype_description};
-                    $category{code} = $_->{code};
-                    push @{$category{answers}}, $opt->{key};
+                    my $message = $opt->{disable_message} || $_->{datatype_description};
+                    $category{$message} ||= {};
+                    $category{$message}->{message} = $message;
+                    $category{$message}->{code} = $_->{code};
+                    push @{$category{$message}->{answers}}, $opt->{key};
                 }
             }
-            push @{$out{questions}}, \%category if %category;
+            push @{$out{questions}}, $_ for values %category;
         }
     }
 
@@ -759,14 +762,20 @@ sub setup_categories_and_bodies : Private {
             if !$c->stash->{unresponsive}{ALL} &&
                 ($contact->email =~ /^REFUSED$/i || $body_send_method eq 'Refused');
 
-        push @category_options, $contact unless $seen{$contact->category};
-        $seen{$contact->category} = $contact;
+        if (my $cat = $seen{$contact->category}) {
+            # Make sure the category is listed in all its groups, not just the first set
+            my @groups = uniq @{$cat->groups}, @{$contact->groups};
+            $cat->set_extra_metadata(group => \@groups);
+        } else {
+            push @category_options, $contact;
+            $seen{$contact->category} = $contact;
+        }
     }
 
     if (@category_options) {
         # If there's an Other category present, put it at the bottom
         @category_options = (
-            { category => _('-- Pick a category --'), category_display => _('-- Pick a category --'), group => '' },
+            { category => _('-- Pick a category --'), category_display => _('-- Pick a category --'), group => [''] },
             grep { $_->category ne _('Other') } @category_options );
         push @category_options, $seen{_('Other')} if $seen{_('Other')};
     }
@@ -837,10 +846,16 @@ sub process_user : Private {
 
     # Extract all the params to a hash to make them easier to work with
     my %params = map { $_ => $c->get_param($_) }
-      ( 'email', 'name', 'phone', 'password_register', 'fms_extra_title' );
+      qw( email name phone password_register fms_extra_title update_method );
 
-    # Report form includes two username fields: #form_username_register and #form_username_sign_in
-    $params{username} = (first { $_ } $c->get_param_list('username')) || '';
+    if ($c->user_exists) {
+        $params{username} = $c->get_param('username');
+    } elsif ($c->get_param('submit_sign_in') || $c->get_param('password_sign_in')) {
+        $params{username} = $c->get_param('username');
+    } else {
+        $params{username} = $c->get_param('username_register');
+    }
+    $params{username} ||= '';
 
     my $anon_button = $c->cobrand->allow_anonymous_reports eq 'button' && $c->get_param('report_anonymously');
     my $anon_fallback = $c->cobrand->allow_anonymous_reports eq '1' && !$c->user_exists && !$params{username};
@@ -891,11 +906,24 @@ sub process_user : Private {
         $params{username} = $params{phone};
     }
 
+    # Code to deal with SMS being switched on and so the user being asked to
+    # pick a method and no username field
+    if (!$params{username} && !$params{update_method}) {
+        $c->stash->{field_errors}->{update_method} = _('Please pick your update preference');
+    }
+    if (!$params{username} && $params{update_method}) {
+        if ($params{update_method} eq 'phone') {
+            $params{username} = $params{phone};
+        } else {
+            $params{username} = $params{email};
+        }
+        $c->stash->{update_method} = $params{update_method};
+    }
+
     my $parsed = FixMyStreet::SMS->parse_username($params{username});
     my $type = $parsed->{type} || 'email';
     $type = 'email' unless FixMyStreet->config('SMS_AUTHENTICATION') || $c->stash->{contributing_as_another_user};
-    $report->user( $c->model('DB::User')->find_or_new( { $type => $parsed->{username} } ) )
-        unless $report->user;
+    $report->user( $c->model('DB::User')->find_or_new( { $type => $parsed->{username} } ) );
 
     $c->stash->{phone_may_be_mobile} = $type eq 'phone' && $parsed->{may_be_mobile};
 
@@ -1018,7 +1046,13 @@ sub process_report : Private {
     $report->detail( $detail );
 
     # mobile device type
-    $report->service( $params{service} ) if $params{service};
+    if ($params{service}) {
+        $report->service($params{service});
+    } elsif ($c->get_param('submit_register_mobile')) {
+        $report->service('mobile');
+    } elsif ($c->get_param('submit_register')) {
+        $report->service('desktop');
+    }
 
     # set these straight from the params
     $report->category( _ $params{category} ) if $params{category};
@@ -1106,7 +1140,7 @@ sub process_report : Private {
 
     # save the cobrand and language related information
     $report->cobrand( $c->cobrand->moniker );
-    $report->cobrand_data( '' );
+    $report->cobrand_data( $c->stash->{cobrand_data} || '' );
     $report->lang( $c->stash->{lang_code} );
 
     return 1;
@@ -1228,6 +1262,7 @@ sub check_for_errors : Private {
     # if using social login then we don't care about other errors
     $c->stash->{is_social_user} = $c->get_param('social_sign_in') ? 1 : 0;
     if ( $c->stash->{is_social_user} ) {
+        delete $field_errors{update_method};
         delete $field_errors{name};
         delete $field_errors{username};
     }
@@ -1243,6 +1278,16 @@ sub check_for_errors : Private {
         $field_errors{photo} = $photo_error;
     }
 
+    # Now assign the username error according to where it came from
+    if ($field_errors{username}) {
+        if ($c->get_param('submit_sign_in') || $c->get_param('password_sign_in')) {
+            $field_errors{username_sign_in} = $field_errors{username};
+        } else {
+            $field_errors{username_register} = $field_errors{username};
+        }
+        delete $field_errors{username};
+    }
+
     # all good if no errors
     return 1 unless scalar keys %field_errors || $c->stash->{login_success};
 
@@ -1301,6 +1346,11 @@ sub send_problem_confirm_text : Private {
 
     $data->{id} = $report->id;
     $c->forward('/auth/phone/send_token', [ $data, 'problem', $report->user->phone ]);
+    my $error = $c->render_fragment( 'auth/_username_error.html', { default => 'phone' });
+    if ($error) {
+        $c->stash->{field_errors}{phone} = $error;
+        $c->forward('generate_map');
+    }
     $c->stash->{submit_url} = '/report/new/text';
 }
 
@@ -1626,7 +1676,7 @@ sub redirect_or_confirm_creation : Private {
         $c->forward( 'create_related_things' );
         if ($c->stash->{contributing_as_another_user} && $report->user->email
             && $report->user->id != $c->user->id
-            && !$c->cobrand->report_sent_confirmation_email) {
+            && !$c->cobrand->report_sent_confirmation_email($report)) {
                 $c->send_email( 'other-reported.txt', {
                     to => [ [ $report->user->email, $report->name ] ],
                 } );
@@ -1674,21 +1724,8 @@ sub create_related_things : Private {
     foreach my $body (values %{$problem->bodies}) {
         my $user = $body->comment_user or next;
 
-        my %open311_conf = (
-            endpoint => $body->endpoint || '',
-            api_key => $body->api_key || '',
-            jurisdiction => $body->jurisdiction || '',
-            extended_statuses => $body->send_extended_statuses,
-        );
-
-        my $cobrand = $body->get_cobrand_handler;
-        $cobrand->call_hook(open311_config_updates => \%open311_conf)
-            if $cobrand;
-
-        my $open311 = Open311->new(%open311_conf);
         my $updates = Open311::GetServiceRequestUpdates->new(
             system_user => $user,
-            current_open311 => $open311,
             current_body => $body,
             blank_updates_permitted => 1,
         );
@@ -1699,7 +1736,10 @@ sub create_related_things : Private {
         my $request = {
             service_request_id => $problem->id,
             update_id => 'auto-internal',
-            comment_time => DateTime->now,
+            # Add a second so it is definitely later than problem confirmed timestamp,
+            # which uses current_timestamp (and thus microseconds) whilst this update
+            # is rounded down to the nearest second
+            comment_time => DateTime->now->add( seconds => 1 ),
             status => 'open',
             description => $description,
         };
@@ -1793,6 +1833,24 @@ sub generate_category_extra_json : Private {
     return \@fields;
 }
 
+sub non_map_creation : Private {
+    my ($self, $c, $extras) = @_;
+
+    $c->forward('initialize_report');
+    $c->forward('check_for_category');
+    $c->forward('/auth/check_csrf_token');
+    $c->forward('process_report');
+    $c->forward('process_user');
+    if ($extras) {
+        $c->forward($_) for @$extras;
+    }
+    $c->forward('/photo/process_photo');
+    return 0 unless $c->forward('check_for_errors');
+    $c->forward('save_user_and_report');
+    return 1;
+
+}
+
 __PACKAGE__->meta->make_immutable;
 
 1;
diff --git a/perllib/FixMyStreet/App/Controller/Report/Update.pm b/perllib/FixMyStreet/App/Controller/Report/Update.pm
index 41c42b8a1..2acafc654 100644
--- a/perllib/FixMyStreet/App/Controller/Report/Update.pm
+++ b/perllib/FixMyStreet/App/Controller/Report/Update.pm
@@ -6,7 +6,6 @@ BEGIN { extends 'Catalyst::Controller'; }
 
 use utf8;
 use Path::Class;
-use List::Util 'first';
 use Utils;
 
 =head1 NAME
@@ -103,8 +102,14 @@ sub process_user : Private {
     my %params = map { $_ => $c->get_param($_) }
       ( 'name', 'password_register', 'fms_extra_title' );
 
-    # Update form includes two username fields: #form_username_register and #form_username_sign_in
-    $params{username} = (first { $_ } $c->get_param_list('username')) || '';
+    if ($c->user_exists) {
+        $params{username} = $c->get_param('username');
+    } elsif ($c->get_param('submit_sign_in') || $c->get_param('password_sign_in')) {
+        $params{username} = $c->get_param('username');
+    } else {
+        $params{username} = $c->get_param('username_register');
+    }
+    $params{username} ||= '';
 
     my $anon_button = $c->cobrand->allow_anonymous_reports eq 'button' && $c->get_param('report_anonymously');
     if ($anon_button) {
@@ -121,7 +126,7 @@ sub process_user : Private {
     if ( $c->user_exists ) { {
         my $user = $c->user->obj;
 
-        if ($c->stash->{contributing_as_another_user} = $user->contributing_as('another_user', $c, $update->problem->bodies_str_ids)) {
+        if ($c->stash->{contributing_as_another_user} = $user->contributing_as('another_user', $c, $c->stash->{problem}->bodies_str_ids)) {
             # Act as if not logged in (and it will be auto-confirmed later on)
             last;
         }
@@ -145,8 +150,7 @@ sub process_user : Private {
     my $parsed = FixMyStreet::SMS->parse_username($params{username});
     my $type = $parsed->{type} || 'email';
     $type = 'email' unless FixMyStreet->config('SMS_AUTHENTICATION') || $c->stash->{contributing_as_another_user};
-    $update->user( $c->model('DB::User')->find_or_new( { $type => $parsed->{username} } ) )
-        unless $update->user;
+    $update->user( $c->model('DB::User')->find_or_new( { $type => $parsed->{username} } ) );
 
     $c->stash->{phone_may_be_mobile} = $type eq 'phone' && $parsed->{may_be_mobile};
 
@@ -248,7 +252,7 @@ sub load_problem : Private {
     # Problem ID could come from existing update in token, or from query parameter
     my $problem_id = $update->problem_id || $c->get_param('id');
     $c->forward( '/report/load_problem_or_display_error', [ $problem_id ] );
-    $update->problem($c->stash->{problem});
+    $update->problem_id($c->stash->{problem}->id);
 }
 
 =head2 check_form_submitted
@@ -282,7 +286,8 @@ sub process_update : Private {
 
     my $name = Utils::trim_text( $params{name} );
 
-    $params{reopen} = 0 unless $c->user && $c->user->id == $c->stash->{problem}->user->id;
+    my $problem = $c->stash->{problem};
+    $params{reopen} = 0 unless $c->user && $c->user->id == $problem->user->id;
 
     my $update = $c->stash->{update};
     $update->text($params{update});
@@ -290,11 +295,11 @@ sub process_update : Private {
     $update->mark_fixed($params{fixed} ? 1 : 0);
     $update->mark_open($params{reopen} ? 1 : 0);
 
-    $c->stash->{contributing_as_body} = $c->user_exists && $c->user->contributing_as('body', $c, $update->problem->bodies_str_ids);
-    $c->stash->{contributing_as_anonymous_user} = $c->user_exists && $c->user->contributing_as('anonymous_user', $c, $update->problem->bodies_str_ids);
+    $c->stash->{contributing_as_body} = $c->user_exists && $c->user->contributing_as('body', $c, $problem->bodies_str_ids);
+    $c->stash->{contributing_as_anonymous_user} = $c->user_exists && $c->user->contributing_as('anonymous_user', $c, $problem->bodies_str_ids);
 
     # This is also done in process_user, but is needed here for anonymous() just below
-    my $anon_button = $c->cobrand->allow_anonymous_reports($update->problem->category) eq 'button' && $c->get_param('report_anonymously');
+    my $anon_button = $c->cobrand->allow_anonymous_reports($problem->category) eq 'button' && $c->get_param('report_anonymously');
     if ($anon_button) {
         $c->stash->{contributing_as_anonymous_user} = 1;
         $c->stash->{contributing_as_body} = undef;
@@ -323,7 +328,6 @@ sub process_update : Private {
         # then we are not changing the state of the problem so can use the current
         # problem state
         } else {
-            my $problem = $c->stash->{problem} || $update->problem;
             $update->problem_state( $problem->state );
         }
     }
@@ -332,7 +336,7 @@ sub process_update : Private {
     my @extra; # Next function fills this, but we don't need it here.
     # This is just so that the error checking for these extra fields runs.
     # TODO Use extra here as it is used on reports.
-    my $body = (values %{$update->problem->bodies})[0];
+    my $body = (values %{$problem->bodies})[0];
     $c->cobrand->process_open311_extras( $c, $body, \@extra );
 
     if ( $c->get_param('fms_extra_title') ) {
@@ -404,6 +408,16 @@ sub check_for_errors : Private {
         $field_errors{photo} = $photo_error;
     }
 
+    # Now assign the username error according to where it came from
+    if ($field_errors{username}) {
+        if ($c->get_param('submit_sign_in') || $c->get_param('password_sign_in')) {
+            $field_errors{username_sign_in} = $field_errors{username};
+        } else {
+            $field_errors{username_register} = $field_errors{username};
+        }
+        delete $field_errors{username};
+    }
+
     # all good if no errors
     return 1
       unless ( scalar keys %field_errors
@@ -484,6 +498,13 @@ sub save_update : Private {
         $update->confirm();
     } elsif ($c->stash->{contributing_as_anonymous_user}) {
         $update->set_extra_metadata( contributed_as => 'anonymous_user' );
+        if ( $c->user_exists && $c->user->from_body ) {
+            # If a staff user has clicked the 'report anonymously' button then
+            # there would be no record of who that staff member was as we've
+            # used the cobrand's anonymous_account for the report. In this case
+            # record the staff user ID in the report metadata.
+            $update->set_extra_metadata( contributed_by => $c->user->id );
+        }
         $update->confirm();
     } elsif ( !$update->user->in_storage ) {
         # User does not exist.
@@ -571,6 +592,11 @@ sub send_confirmation_text : Private {
     my ( $self, $c ) = @_;
     my $update = $c->stash->{update};
     $c->forward('/auth/phone/send_token', [ $c->stash->{token_data}, 'comment', $update->user->phone ]);
+    my $error = $c->render_fragment( 'auth/_username_error.html', { default => 'phone' });
+    if ($error) {
+        $c->stash->{field_errors}{username_register} = $error;
+        $c->go( '/report/display', [ $c->stash->{problem}->id ], [] );
+    }
     $c->stash->{submit_url} = '/report/update/text';
 }
 
diff --git a/perllib/FixMyStreet/App/Controller/Reports.pm b/perllib/FixMyStreet/App/Controller/Reports.pm
index 97976ebe3..e65810b91 100644
--- a/perllib/FixMyStreet/App/Controller/Reports.pm
+++ b/perllib/FixMyStreet/App/Controller/Reports.pm
@@ -154,6 +154,8 @@ sub ward : Path : Args(2) {
     $c->forward('stash_report_sort', [ $c->cobrand->reports_ordering ]);
     $c->forward( 'load_and_group_problems' );
 
+    $c->forward('setup_categories');
+
     if ($c->get_param('ajax')) {
         my $ajax_template = $c->stash->{ajax_template} || 'reports/_problem-list.html';
         $c->detach('ajax', [ $ajax_template ]);
@@ -165,7 +167,7 @@ sub ward : Path : Args(2) {
 
     $c->stash->{stats} = $c->cobrand->get_report_stats();
 
-    $c->forward('setup_categories_and_map');
+    $c->forward('setup_map');
 
     # List of wards
     if ( !$c->stash->{wards} && $c->stash->{body}->id && $c->stash->{body}->body_areas->first ) {
@@ -181,7 +183,7 @@ sub ward : Path : Args(2) {
     }
 }
 
-sub setup_categories_and_map :Private {
+sub setup_categories :Private {
     my ($self, $c) = @_;
 
     my @categories = $c->stash->{body}->contacts->not_deleted->search( undef, {
@@ -191,9 +193,15 @@ sub setup_categories_and_map :Private {
 
     $c->cobrand->call_hook('munge_reports_category_list', \@categories);
 
+    $c->forward('/report/assigned_users_only', [ \@categories ]);
+
     $c->stash->{filter_categories} = \@categories;
     $c->stash->{filter_category} = { map { $_ => 1 } $c->get_param_list('filter_category', 1) };
     $c->forward('/report/stash_category_groups', [ \@categories ]) if $c->cobrand->enable_category_groups;
+}
+
+sub setup_map :Private {
+    my ($self, $c) = @_;
 
     my $pins = $c->stash->{pins} || [];
 
@@ -473,10 +481,10 @@ sub summary : Private {
     $c->forward('/admin/fetch_contacts');
     $c->stash->{contacts} = [ $c->stash->{contacts}->all ];
 
-    $c->forward('/dashboard/construct_rs_filter', []);
+    my $reporting = $c->forward('/dashboard/construct_rs_filter', []);
 
     if ( $c->get_param('csv') ) {
-        $c->detach('export_summary_csv');
+        $c->detach('export_summary_csv', [ $reporting ]);
     }
 
     $c->forward('/dashboard/generate_grouped_data');
@@ -486,38 +494,26 @@ sub summary : Private {
 }
 
 sub export_summary_csv : Private {
-    my ( $self, $c ) = @_;
+    my ( $self, $c, $reporting ) = @_;
 
-    $c->stash->{csv} = {
-        objects => $c->stash->{objects_rs}->search_rs({}, {
-            rows => 100,
-            order_by => { '-desc' => 'me.confirmed' },
-        }),
-        headers => [
-            'Report ID',
-            'Title',
-            'Category',
-            'Created',
-            'Confirmed',
-            'Status',
-            'Latitude', 'Longitude',
-            'Query',
-            'Report URL',
-        ],
-        columns => [
-            'id',
-            'title',
-            'category',
-            'created',
-            'confirmed',
-            'state',
-            'latitude', 'longitude',
-            'postcode',
-            'url',
-        ],
-        filename => 'fixmystreet-data',
-    };
-    $c->forward('/dashboard/generate_csv');
+    $reporting->objects_attrs({
+        rows => 100,
+        order_by => { '-desc' => 'me.confirmed' },
+    });
+    $reporting->add_csv_columns(
+        id => 'Report ID',
+        title => 'Title',
+        category => 'Category',
+        created => 'Created',
+        confirmed => 'Confirmed',
+        state => 'Status',
+        latitude => 'Latitude',
+        longitude => 'Longitude',
+        postcode => 'Query',
+        url => 'Report URL',
+    );
+    $reporting->filename('fixmystreet-data');
+    $reporting->generate_csv_http($c);
 }
 
 =head2 check_canonical_url
@@ -620,6 +616,9 @@ sub load_problems_parameters : Private {
     };
     if ($c->user_exists && $body) {
         my $prefetch = [];
+        if ($c->user->from_body || $c->user->is_superuser) {
+            push @$prefetch, 'contact';
+        }
         if ($c->user->has_permission_to('planned_reports', $body->id)) {
             push @$prefetch, 'user_planned_reports';
         }
@@ -646,7 +645,7 @@ sub load_problems_parameters : Private {
     }
 
     if (@$category) {
-        $where->{category} = $category;
+        $where->{'me.category'} = $category;
     }
 
     if ($c->stash->{wards}) {
@@ -687,12 +686,12 @@ sub check_non_public_reports_permission : Private {
         }
 
         if ( $user_has_permission ) {
-            $where->{non_public} = 1 if $c->stash->{only_non_public};
+            $where->{'me.non_public'} = 1 if $c->stash->{only_non_public};
         } else {
-            $where->{non_public} = 0;
+            $where->{'me.non_public'} = 0;
         }
     } else {
-        $where->{non_public} = 0;
+        $where->{'me.non_public'} = 0;
     }
 }
 
@@ -815,7 +814,7 @@ sub stash_report_sort : Private {
 sub add_row {
     my ( $c, $problem, $body, $problems, $pins ) = @_;
     push @{$problems->{$body}}, $problem;
-    push @$pins, $problem->pin_data($c, 'reports');
+    push @$pins, $problem->pin_data('reports');
 }
 
 sub ajax : Private {
diff --git a/perllib/FixMyStreet/App/Controller/Test.pm b/perllib/FixMyStreet/App/Controller/Test.pm
index 5ec4bebf3..ce54f004f 100644
--- a/perllib/FixMyStreet/App/Controller/Test.pm
+++ b/perllib/FixMyStreet/App/Controller/Test.pm
@@ -42,6 +42,28 @@ sub setup : Path('/_test/setup') : Args(1) {
         my $problem = FixMyStreet::DB->resultset("Problem")->find(1);
         $problem->update({ category => 'Skips' });
         $c->response->body("OK");
+    } elsif ( $test eq 'regression-duplicate-stopper') {
+        my $problem = FixMyStreet::DB->resultset("Problem")->find(1);
+        $problem->update({ category => 'Flytipping' });
+        my $category = FixMyStreet::DB->resultset('Contact')->search({
+            category => 'Flytipping',
+        })->first;
+        $category->push_extra_fields({
+            code => 'hazardous',
+            datatype => 'singlevaluelist',
+            description => 'Hazardous material',
+            order => 0,
+            variable => 'true',
+            values => [
+                { key => 'yes', name => 'Yes', disable => 1, disable_message => 'Please phone' },
+                { key => 'no', name => 'No' },
+            ],
+        });
+        $category->update;
+        $c->response->body("OK");
+    } elsif ($test eq 'simple-service-check') {
+        my $problem = FixMyStreet::DB->resultset("Problem")->search(undef, { order_by => { -desc => 'id' } })->first;
+        $c->response->body($problem->service);
     }
 }
 
@@ -51,6 +73,15 @@ sub teardown : Path('/_test/teardown') : Args(1) {
         my $problem = FixMyStreet::DB->resultset("Problem")->find(1);
         $problem->update({ category => 'Potholes' });
         $c->response->body("OK");
+    } elsif ( $test eq 'regression-duplicate-stopper') {
+        my $problem = FixMyStreet::DB->resultset("Problem")->find(1);
+        $problem->update({ category => 'Potholes' });
+        my $category = FixMyStreet::DB->resultset('Contact')->search({
+            category => 'Flytipping',
+        })->first;
+        $category->remove_extra_field('hazardous');
+        $category->update;
+        $c->response->body("OK");
     }
 }
 
diff --git a/perllib/FixMyStreet/App/Controller/Waste.pm b/perllib/FixMyStreet/App/Controller/Waste.pm
new file mode 100644
index 000000000..fe177e9fe
--- /dev/null
+++ b/perllib/FixMyStreet/App/Controller/Waste.pm
@@ -0,0 +1,569 @@
+package FixMyStreet::App::Controller::Waste;
+use Moose;
+use namespace::autoclean;
+
+BEGIN { extends 'Catalyst::Controller' }
+
+use utf8;
+use Lingua::EN::Inflect qw( NUMWORDS );
+use FixMyStreet::App::Form::Waste::UPRN;
+use FixMyStreet::App::Form::Waste::AboutYou;
+use FixMyStreet::App::Form::Waste::Request;
+use FixMyStreet::App::Form::Waste::Report;
+use FixMyStreet::App::Form::Waste::Enquiry;
+use Open311::GetServiceRequestUpdates;
+
+sub auto : Private {
+    my ( $self, $c ) = @_;
+    my $cobrand_check = $c->cobrand->feature('waste');
+    $c->detach( '/page_error_404_not_found' ) if !$cobrand_check;
+    return 1;
+}
+
+sub index : Path : Args(0) {
+    my ( $self, $c ) = @_;
+
+    if (my $id = $c->get_param('address')) {
+        $c->detach('redirect_to_id', [ $id ]);
+    }
+
+    $c->stash->{title} = 'What is your address?';
+    my $form = FixMyStreet::App::Form::Waste::UPRN->new( cobrand => $c->cobrand );
+    $form->process( params => $c->req->body_params );
+    if ($form->validated) {
+        my $addresses = $form->value->{postcode};
+        $form = address_list_form($addresses);
+    }
+    $c->stash->{form} = $form;
+}
+
+sub address_list_form {
+    my $addresses = shift;
+    HTML::FormHandler->new(
+        field_list => [
+            address => {
+                required => 1,
+                type => 'Select',
+                widget => 'RadioGroup',
+                label => 'Select an address',
+                tags => { last_differs => 1, small => 1 },
+                options => $addresses,
+            },
+            go => {
+                type => 'Submit',
+                value => 'Continue',
+                element_attr => { class => 'govuk-button' },
+            },
+        ],
+    );
+}
+
+sub redirect_to_id : Private {
+    my ($self, $c, $id) = @_;
+    my $uri = '/waste/' . $id;
+    my $type = $c->get_param('type') || '';
+    $uri .= '/request' if $type eq 'request';
+    $uri .= '/report' if $type eq 'report';
+    $c->res->redirect($uri);
+    $c->detach;
+}
+
+sub property : Chained('/') : PathPart('waste') : CaptureArgs(1) {
+    my ($self, $c, $id) = @_;
+
+    if ($id eq 'missing') {
+        $c->stash->{template} = 'waste/missing.html';
+        $c->detach;
+    }
+
+    $c->forward('/auth/get_csrf_token');
+
+    my $property = $c->stash->{property} = $c->cobrand->call_hook(look_up_property => $id);
+    $c->detach( '/page_error_404_not_found', [] ) unless $property;
+
+    $c->stash->{latitude} = $property->{latitude};
+    $c->stash->{longitude} = $property->{longitude};
+
+    $c->stash->{service_data} = $c->cobrand->call_hook(bin_services_for_address => $property) || [];
+    $c->stash->{services} = { map { $_->{service_id} => $_ } @{$c->stash->{service_data}} };
+}
+
+sub bin_days : Chained('property') : PathPart('') : Args(0) {
+    my ($self, $c) = @_;
+}
+
+sub calendar : Chained('property') : PathPart('calendar.ics') : Args(0) {
+    my ($self, $c) = @_;
+    $c->res->header(Content_Type => 'text/calendar');
+    require Data::ICal::RFC7986;
+    require Data::ICal::Entry::Event;
+    my $calendar = Data::ICal::RFC7986->new(
+        calname => 'Bin calendar',
+        rfc_strict => 1,
+        auto_uid => 1,
+    );
+    $calendar->add_properties(
+        prodid => '//FixMyStreet//Bin Collection Calendars//EN',
+        method => 'PUBLISH',
+        'refresh-interval' => [ 'P1D', { value => 'DURATION' } ],
+        'x-published-ttl' => 'P1D',
+        calscale => 'GREGORIAN',
+        'x-wr-timezone' => 'Europe/London',
+        source => [ $c->uri_for_action($c->action, [ $c->stash->{property}{id} ]), { value => 'URI' } ],
+        url => $c->uri_for_action('waste/bin_days', [ $c->stash->{property}{id} ]),
+    );
+
+    my $events = $c->cobrand->bin_future_collections;
+    my $stamp = DateTime->now->strftime('%Y%m%dT%H%M%SZ');
+    foreach (@$events) {
+        my $event = Data::ICal::Entry::Event->new;
+        $event->add_properties(
+            summary => $_->{summary},
+            description => $_->{desc},
+            dtstamp => $stamp,
+            dtstart => [ $_->{date}->ymd(''), { value => 'DATE' } ],
+            dtend => [ $_->{date}->add(days=>1)->ymd(''), { value => 'DATE' } ],
+        );
+        $calendar->add_entry($event);
+    }
+
+    $c->res->body($calendar->as_string);
+}
+
+sub construct_bin_request_form {
+    my $c = shift;
+
+    my $field_list = [];
+
+    foreach (@{$c->stash->{service_data}}) {
+        next unless $_->{next} && !$_->{request_open};
+        my $name = $_->{service_name};
+        my $containers = $_->{request_containers};
+        my $max = $_->{request_max};
+        foreach my $id (@$containers) {
+            push @$field_list, "container-$id" => {
+                type => 'Checkbox',
+                apply => [
+                    {
+                        when => { "quantity-$id" => sub { $_[0] > 0 } },
+                        check => qr/^1$/,
+                        message => 'Please tick the box',
+                    },
+                ],
+                label => $name,
+                option_label => $c->stash->{containers}->{$id},
+                tags => { toggle => "form-quantity-$id-row" },
+            };
+            $name = ''; # Only on first container
+            push @$field_list, "quantity-$id" => {
+                type => 'Select',
+                label => 'Quantity',
+                tags => {
+                    hint => "You can request a maximum of " . NUMWORDS($max) . " containers",
+                    initial_hidden => 1,
+                },
+                options => [
+                    { value => "", label => '-' },
+                    map { { value => $_, label => $_ } } (1..$max),
+                ],
+                required_when => { "container-$id" => 1 },
+            };
+        }
+    }
+
+    return $field_list;
+}
+
+sub request : Chained('property') : Args(0) {
+    my ($self, $c) = @_;
+
+    my $field_list = construct_bin_request_form($c);
+
+    $c->stash->{first_page} = 'request';
+    $c->stash->{form_class} = 'FixMyStreet::App::Form::Waste::Request';
+    $c->stash->{page_list} = [
+        request => {
+            fields => [ grep { ! ref $_ } @$field_list, 'submit' ],
+            title => 'Which containers do you need?',
+            next => 'about_you',
+        },
+    ];
+    $c->stash->{field_list} = $field_list;
+    $c->forward('form');
+}
+
+sub process_request_data : Private {
+    my ($self, $c, $form) = @_;
+    my $data = $form->saved_data;
+    my $address = $c->stash->{property}->{address};
+    my @services = grep { /^container-/ && $data->{$_} } keys %$data;
+    foreach (@services) {
+        my ($id) = /container-(.*)/;
+        my $container = $c->stash->{containers}{$id};
+        my $quantity = $data->{"quantity-$id"};
+        $data->{title} = "Request new $container";
+        $data->{detail} = "Quantity: $quantity\n\n$address";
+        $c->set_param('Container_Type', $id);
+        $c->set_param('Quantity', $quantity);
+        $c->forward('add_report', [ $data ]) or return;
+        push @{$c->stash->{report_ids}}, $c->stash->{report}->id;
+    }
+    return 1;
+}
+
+sub construct_bin_report_form {
+    my $c = shift;
+
+    my $field_list = [];
+
+    foreach (@{$c->stash->{service_data}}) {
+        next unless $_->{last} && $_->{report_allowed} && !$_->{report_open};
+        my $id = $_->{service_id};
+        my $name = $_->{service_name};
+        push @$field_list, "service-$id" => {
+            type => 'Checkbox',
+            label => $name,
+            option_label => $name,
+        };
+    }
+
+    return $field_list;
+}
+
+sub report : Chained('property') : Args(0) {
+    my ($self, $c) = @_;
+
+    my $field_list = construct_bin_report_form($c);
+
+    $c->stash->{first_page} = 'report';
+    $c->stash->{form_class} = 'FixMyStreet::App::Form::Waste::Report';
+    $c->stash->{page_list} = [
+        report => {
+            fields => [ grep { ! ref $_ } @$field_list, 'submit' ],
+            title => 'Select your missed collection',
+            next => 'about_you',
+        },
+    ];
+    $c->stash->{field_list} = $field_list;
+    $c->forward('form');
+}
+
+sub process_report_data : Private {
+    my ($self, $c, $form) = @_;
+    my $data = $form->saved_data;
+    my $address = $c->stash->{property}->{address};
+    my @services = grep { /^service-/ && $data->{$_} } keys %$data;
+    foreach (@services) {
+        my ($id) = /service-(.*)/;
+        my $service = $c->stash->{services}{$id}{service_name};
+        $data->{title} = "Report missed $service";
+        $data->{detail} = "$data->{title}\n\n$address";
+        $c->set_param('service_id', $id);
+        $c->forward('add_report', [ $data ]) or return;
+        push @{$c->stash->{report_ids}}, $c->stash->{report}->id;
+    }
+    return 1;
+}
+
+sub enquiry : Chained('property') : Args(0) {
+    my ($self, $c) = @_;
+
+    if (my $template = $c->get_param('template')) {
+        $c->stash->{template} = "waste/enquiry-$template.html";
+        $c->detach;
+    }
+
+    $c->forward('setup_categories_and_bodies');
+
+    my $category = $c->get_param('category');
+    my $service = $c->get_param('service_id');
+    if (!$category || !$service || !$c->stash->{services}{$service}) {
+        $c->res->redirect('/waste/' . $c->stash->{property}{id});
+        $c->detach;
+    }
+    my ($contact) = grep { $_->category eq $category } @{$c->stash->{contacts}};
+    if (!$contact) {
+        $c->res->redirect('/waste/' . $c->stash->{property}{id});
+        $c->detach;
+    }
+
+    my $field_list = [];
+    foreach (@{$contact->get_metadata_for_input}) {
+        next if $_->{code} eq 'service_id' || $_->{code} eq 'uprn' || $_->{code} eq 'property_id';
+        my $type = 'Text';
+        $type = 'TextArea' if 'text' eq ($_->{datatype} || '');
+        my $required = $_->{required} eq 'true' ? 1 : 0;
+        push @$field_list, "extra_$_->{code}" => {
+            type => $type, label => $_->{description}, required => $required
+        };
+    }
+
+    $c->stash->{first_page} = 'enquiry';
+    $c->stash->{form_class} = 'FixMyStreet::App::Form::Waste::Enquiry';
+    $c->stash->{page_list} = [
+        enquiry => {
+            fields => [ 'category', 'service_id', grep { ! ref $_ } @$field_list, 'continue' ],
+            title => $category,
+            next => 'about_you',
+            update_field_list => sub {
+                my $form = shift;
+                my $c = $form->c;
+                return {
+                    category => { default => $c->get_param('category') },
+                    service_id => { default => $c->get_param('service_id') },
+                }
+            }
+        },
+    ];
+    $c->stash->{field_list} = $field_list;
+    $c->forward('form');
+}
+
+sub process_enquiry_data : Private {
+    my ($self, $c, $form) = @_;
+    my $data = $form->saved_data;
+    my $address = $c->stash->{property}->{address};
+    $data->{title} = $data->{category};
+    $data->{detail} = "$data->{category}\n\n$address";
+    # Read extra details in loop
+    foreach (grep { /^extra_/ } keys %$data) {
+        my ($id) = /^extra_(.*)/;
+        $c->set_param($id, $data->{$_});
+    }
+    $c->set_param('service_id', $data->{service_id});
+    $c->forward('add_report', [ $data ]) or return;
+    push @{$c->stash->{report_ids}}, $c->stash->{report}->id;
+    return 1;
+}
+
+sub load_form {
+    my ($c, $previous_form) = @_;
+
+    my $page;
+    if ($previous_form) {
+        $page = $previous_form->next;
+    } else {
+        $page = $c->forward('get_page');
+    }
+
+    my $form = $c->stash->{form_class}->new(
+        page_list => $c->stash->{page_list},
+        $c->stash->{field_list} ? (field_list => $c->stash->{field_list}) : (),
+        page_name => $page,
+        csrf_token => $c->stash->{csrf_token},
+        c => $c,
+        previous_form => $previous_form,
+        saved_data_encoded => $c->get_param('saved_data'),
+        no_preload => 1,
+    );
+
+    if (!$form->has_current_page) {
+        $c->detach('/page_error_400_bad_request', [ 'Bad request' ]);
+    }
+
+    return $form;
+}
+
+sub form : Private {
+    my ($self, $c) = @_;
+
+    my $form = load_form($c);
+    if ($c->get_param('process')) {
+        $c->forward('/auth/check_csrf_token');
+        $form->process(params => $c->req->body_params);
+        if ($form->validated) {
+            $form = load_form($c, $form);
+        }
+    }
+
+    $form->process unless $form->processed;
+
+    $c->stash->{template} = $form->template || 'waste/index.html';
+    $c->stash->{form} = $form;
+}
+
+sub get_page : Private {
+    my ($self, $c) = @_;
+
+    my $goto = $c->get_param('goto') || '';
+    my $process = $c->get_param('process') || '';
+    $goto = $c->stash->{first_page} unless $goto || $process;
+    if ($goto && $process) {
+        $c->detach('/page_error_400_bad_request', [ 'Bad request' ]);
+    }
+
+    return $goto || $process;
+}
+
+sub add_report : Private {
+    my ( $self, $c, $data ) = @_;
+
+    $c->stash->{cobrand_data} = 'waste';
+
+    # XXX Is this best way to do this?
+    if ($c->user_exists && $c->user->from_body && $c->user->email ne $data->{email}) {
+        $c->set_param('form_as', 'another_user');
+        $c->set_param('username', $data->{email} || $data->{phone});
+    } else {
+        $c->set_param('username_register', $data->{email} || $data->{phone});
+    }
+
+    # Set the data as if a new report form has been submitted
+
+    $c->set_param('submit_problem', 1);
+    $c->set_param('pc', '');
+    $c->set_param('non_public', 1);
+
+    $c->set_param('name', $data->{name});
+    $c->set_param('phone', $data->{phone});
+
+    $c->set_param('category', $data->{category});
+    $c->set_param('title', $data->{title});
+    $c->set_param('detail', $data->{detail});
+    $c->set_param('uprn', $c->stash->{property}{uprn});
+    $c->set_param('property_id', $c->stash->{property}{id});
+
+    $c->forward('setup_categories_and_bodies') unless $c->stash->{contacts};
+    $c->forward('/report/new/non_map_creation', [['/waste/remove_name_errors']]) or return;
+    my $report = $c->stash->{report};
+    $report->confirm;
+    $report->update;
+
+    $c->model('DB::Alert')->find_or_create({
+        user => $report->user,
+        alert_type => 'new_updates',
+        parameter => $report->id,
+        cobrand => $report->cobrand,
+        lang => $report->lang,
+    })->confirm;
+
+    return 1;
+}
+
+sub remove_name_errors : Private {
+    my ($self, $c) = @_;
+    # We do not mind about missing title/split name here
+    my $field_errors = $c->stash->{field_errors};
+    delete $field_errors->{fms_extra_title};
+    delete $field_errors->{first_name};
+    delete $field_errors->{last_name};
+}
+
+sub setup_categories_and_bodies : Private {
+    my ($self, $c) = @_;
+
+    $c->stash->{all_areas} = $c->stash->{all_areas_mapit} = { $c->cobrand->council_area_id => { id => $c->cobrand->council_area_id } };
+    $c->forward('/report/new/setup_categories_and_bodies');
+    my $contacts = $c->stash->{contacts};
+    @$contacts = grep { grep { $_ eq 'Waste' } @{$_->groups} } @$contacts;
+}
+
+sub receive_echo_event_notification : Path('/waste/echo') : Args(0) {
+    my ($self, $c) = @_;
+    $c->stash->{format} = 'xml';
+    $c->response->header(Content_Type => 'application/soap+xml');
+
+    require SOAP::Lite;
+
+    $c->detach('soap_error', [ 'Invalid method', 405 ]) unless $c->req->method eq 'POST';
+
+    my $echo = $c->cobrand->feature('echo');
+    $c->detach('soap_error', [ 'Missing config', 500 ]) unless $echo;
+
+    # Make sure we log entire request for debugging
+    $c->detach('soap_error', [ 'Missing body' ]) unless $c->req->body;
+    my $soap = join('', $c->req->body->getlines);
+    $c->log->info($soap);
+
+    my $body = $c->cobrand->body;
+    $c->detach('soap_error', [ 'Bad jurisdiction' ]) unless $body;
+
+    my $env = SOAP::Deserializer->deserialize($soap);
+
+    my $header = $env->header;
+    $c->detach('soap_error', [ 'Missing SOAP header' ]) unless $header;
+    my $action = $header->{Action};
+    $c->detach('soap_error', [ 'Incorrect Action' ]) unless $action && $action eq $echo->{receive_action};
+    $header = $header->{Security};
+    $c->detach('soap_error', [ 'Missing Security header' ]) unless $header;
+    my $token = $header->{UsernameToken};
+    $c->detach('soap_error', [ 'Authentication failed' ])
+        unless $token && $token->{Username} eq $echo->{receive_username} && $token->{Password} eq $echo->{receive_password};
+
+    my $event = $env->result;
+
+    my $cfg = { echo => Integrations::Echo->new(%$echo) };
+    my $request = $c->cobrand->construct_waste_open311_update($cfg, $event);
+    $request->{updated_datetime} = DateTime::Format::W3CDTF->format_datetime(DateTime->now);
+    $request->{service_request_id} = $event->{Guid};
+
+    my $updates = Open311::GetServiceRequestUpdates->new(
+        system_user => $body->comment_user,
+        current_body => $body,
+    );
+
+    my $p = $updates->find_problem($request);
+    if ($p) {
+        $c->forward('check_existing_update', [ $p, $request, $updates ]);
+        my $comment = $updates->process_update($request, $p);
+    }
+    # Still want to say it is okay, even if we did nothing with it
+    $c->forward('soap_ok');
+}
+
+sub soap_error : Private {
+    my ($self, $c, $comment, $code) = @_;
+    $code ||= 400;
+    $c->response->status($code);
+    my $type = $code == 500 ? 'Server' : 'Client';
+    $c->response->body(SOAP::Serializer->fault($type, "Bad request: $comment", soap_header()));
+}
+
+sub soap_ok : Private {
+    my ($self, $c) = @_;
+    $c->response->status(200);
+    my $method = SOAP::Data->name("NotifyEventUpdatedResponse")->attr({
+        xmlns => "http://www.twistedfish.com/xmlns/echo/api/v1"
+    });
+    $c->response->body(SOAP::Serializer->envelope(method => $method, soap_header()));
+}
+
+sub soap_header {
+    my $attr = "http://www.twistedfish.com/xmlns/echo/api/v1";
+    my $action = "NotifyEventUpdatedResponse";
+    my $header = SOAP::Header->name("Action")->attr({
+        xmlns => 'http://www.w3.org/2005/08/addressing',
+        'soap:mustUnderstand' => 1,
+    })->value("$attr/ReceiverService/$action");
+
+    my $dt = DateTime->now();
+    my $dt2 = $dt->clone->add(minutes => 5);
+    my $w3c = DateTime::Format::W3CDTF->new;
+    my $header2 = SOAP::Header->name("Security")->attr({
+        'soap:mustUnderstand' => 'true',
+        'xmlns' => 'http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd'
+    })->value(
+        \SOAP::Header->name(
+            "Timestamp" => \SOAP::Header->value(
+                SOAP::Header->name('Created', $w3c->format_datetime($dt)),
+                SOAP::Header->name('Expires', $w3c->format_datetime($dt2)),
+            )
+        )->attr({
+            xmlns => "http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd",
+        })
+    );
+    return ($header, $header2);
+}
+
+sub check_existing_update : Private {
+    my ($self, $c, $p, $request, $updates) = @_;
+
+    my $cfg = { updates => $updates };
+    $c->detach('soap_ok')
+        unless $c->cobrand->waste_check_last_update(
+            $cfg, $p, $request->{status}, $request->{external_status_code});
+}
+
+__PACKAGE__->meta->make_immutable;
+
+1;