19 files changed, 574 insertions, 104 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index bcaf66717..1ab1c2a1c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,8 @@
         - Move summary failures to a separate script.
         - Add script to export/import body data.
         - Add fetch script that does combined job of fetch-comments and fetch-reports.
+        - Show error page when submitting with web param to /import.
+        - Add a daemon option for sending reports and updates.
     - Open311 improvements:
         - match response templates on external status code over state
         - Add flag to protect category/group names from Open311 overwrite.
diff --git a/bin/send-daemon b/bin/send-daemon
new file mode 100755
index 000000000..dee9e949f
--- /dev/null
+++ b/bin/send-daemon
@@ -0,0 +1,147 @@
+#!/usr/bin/env perl
+#
+# send-daemon
+# FixMyStreet daemon for sending reports and updates.
+
+use strict;
+use warnings;
+use v5.14;
+
+BEGIN {
+    use File::Basename qw(dirname);
+    use File::Spec;
+    my $d = dirname(File::Spec->rel2abs($0));
+    require "$d/../setenv.pl";
+}
+
+use Getopt::Long::Descriptive;
+use Parallel::ForkManager;
+use CronFns;
+use FixMyStreet;
+use FixMyStreet::DB;
+use FixMyStreet::Script::Reports;
+use FixMyStreet::Queue::Item::Report;
+use Open311::PostServiceRequestUpdates;
+
+my ($opts, $usage) = describe_options(
+    '%c %o',
+    ['verbose|v+', 'more verbose output'],
+    ['nomail', 'do not send any email, print instead'],
+    ['debug', 'always try and send reports (no back-off skipping)'],
+    ['help|h', "print usage message and exit" ],
+    [],
+    ['Send a USR1 signal to the parent to cycle through verbose levels.'],
+);
+$usage->die if $opts->help;
+my $verbose = $opts->verbose || 0;
+
+my $site = CronFns::site(FixMyStreet->config('BASE_URL'));
+my $states = [ FixMyStreet::DB::Result::Problem::open_states() ];
+$states = [ 'submitted', 'confirmed', 'in progress', 'feedback pending', 'external', 'wish' ] if $site eq 'zurich';
+
+my $db = FixMyStreet::DB->schema->storage;
+
+my %children;
+
+my $exit = 0;
+$SIG{TERM} = $SIG{INT} = sub { $exit = 1; };
+
+my $changeverboselevel = 0;
+$SIG{USR1} = sub {
+    kill 'USR1', keys %children;
+    ++$changeverboselevel;
+};
+
+my $procs = FixMyStreet->config('QUEUE_DAEMON_PROCESSES') || 4;
+my $pm = Parallel::ForkManager->new($procs);
+
+$pm->run_on_start(sub {
+    my $pid = shift;
+    $children{$pid} = time();
+});
+$pm->run_on_finish(sub {
+    my $pid = shift;
+    if ($children{$pid} > time() - 10) {
+        # It didn't live very long, let's wait a bit
+        sleep(5);
+    }
+    delete $children{$pid};
+});
+
+# The parent loop
+while (!$exit) {
+    while (keys %children < $procs) {
+        $pm->start and next;
+        srand;
+        $SIG{USR1} = sub { ++$changeverboselevel; };
+        while (!$exit) {
+            $0 = "fmsd (running queue)";
+            $db->txn_do(\&look_for_report);
+            $db->txn_do(\&look_for_update);
+            $0 = "fmsd";
+            sleep(5 + rand(10));
+        }
+        $pm->finish;
+    }
+
+    if (!keys %children) { # Very high load, something wrong
+        sleep(10);
+        next;
+    }
+
+    $pm->wait_for_available_procs;
+}
+
+sub look_for_report {
+    my $params = FixMyStreet::Script::Reports::construct_query($opts->debug);
+    my $unsent = FixMyStreet::DB->resultset('Problem')->search($params, {
+        for => \'UPDATE SKIP LOCKED',
+        rows => 1,
+    } )->single or return;
+
+    print_log('debug', "Trying to send report " . $unsent->id);
+    my $item = FixMyStreet::Queue::Item::Report->new(
+        report => $unsent,
+        verbose => $verbose,
+        nomail => $opts->nomail,
+    );
+    $item->process;
+}
+
+sub look_for_update {
+    my $updates = Open311::PostServiceRequestUpdates->new(
+        verbose => $verbose,
+    );
+
+    my $bodies = $updates->fetch_bodies;
+    my $params = $updates->construct_query($opts->debug);
+    my $comment = FixMyStreet::DB->resultset('Comment')
+        ->to_body([ keys %$bodies ])
+        ->search($params, { for => \'UPDATE SKIP LOCKED', rows => 1 })
+        ->single or return;
+
+    print_log('debug', "Trying to send update " . $comment->id);
+
+    my ($body) = grep { $bodies->{$_} } @{$comment->problem->bodies_str_ids};
+    $body = $bodies->{$body};
+
+    $updates->construct_open311($body);
+    $updates->process_update($body, $comment);
+}
+
+sub print_log {
+    my $prio = shift;
+
+    if ($changeverboselevel) {
+        $verbose = ($verbose + $changeverboselevel) % 3;
+        STDERR->print("fmsd: info: verbose level now $verbose\n");
+        $changeverboselevel = 0;
+    }
+
+    if ($verbose < 2) {
+        return if ($prio eq 'noise');
+        return if ($verbose < 1 && $prio eq 'debug');
+        return if ($verbose < 0 && $prio eq 'info');
+    }
+    STDERR->print("[fmsd] [$prio] ", join("", @_), "\n");
+}
diff --git a/conf/general.yml-example b/conf/general.yml-example
index 243d077f0..91507b03d 100644
--- a/conf/general.yml-example
+++ b/conf/general.yml-example
@@ -268,3 +268,7 @@ SIGNUPS_DISABLED: 0
 FETCH_COMMENTS_PROCESSES_MIN: 0
 FETCH_COMMENTS_PROCESSES_MAX: 0
 FETCH_COMMENTS_PROCESS_TIMEOUT: 0
+
+# If you use the daemon for sending reports, rather than the cron script,
+# this is how many children it will have.
+QUEUE_DAEMON_PROCESSES: 4
diff --git a/conf/send-daemon.service.example b/conf/send-daemon.service.example
new file mode 100644
index 000000000..1314ada3c
--- /dev/null
+++ b/conf/send-daemon.service.example
@@ -0,0 +1,17 @@
+#
+# systemd service unit for FixMyStreet Report Sending Daemon
+#
+[Unit]
+Description=FixMyStreet Report Sending Daemon
+After=syslog.target network.target
+
+[Service]
+Type=simple
+ExecStart=/var/www/www.fixmystreet.com/fixmystreet/bin/send-daemon
+User=fms
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=fms-send-daemon
+
+[Install]
+WantedBy=multi-user.target
diff --git a/perllib/FixMyStreet/App/Controller/Report/New.pm b/perllib/FixMyStreet/App/Controller/Report/New.pm
index 516752b89..69d20171a 100644
--- a/perllib/FixMyStreet/App/Controller/Report/New.pm
+++ b/perllib/FixMyStreet/App/Controller/Report/New.pm
@@ -358,8 +358,7 @@ sub report_import : Path('/import') {
     # If this is not a POST then just print out instructions for using page
     return unless $c->req->method eq 'POST';
 
-    # anything else we return is plain text
-    $c->res->content_type('text/plain; charset=utf-8');
+    my $format = $c->get_param('web') ? 'web' : 'text';
 
     my %input =
       map { $_ => $c->get_param($_) || '' } (
@@ -412,8 +411,14 @@ sub report_import : Path('/import') {
 
     # if we have errors then we should bail out
     if (@errors) {
-        my $body = join '', map { "ERROR:$_\n" } @errors;
-        $c->res->body($body);
+        if ($format eq 'web') {
+            $c->stash->{input} = \%input;
+            $c->stash->{errors} = \@errors;
+        } else {
+            my $body = join '', map { "ERROR:$_\n" } @errors;
+            $c->res->content_type('text/plain; charset=utf-8');
+            $c->res->body($body);
+        }
         return;
     }
 
@@ -469,13 +474,13 @@ sub report_import : Path('/import') {
 
     $c->send_email( 'partial.txt', { to => $report->user->email, } );
 
-    if ( $c->get_param('web') ) {
-        $c->res->content_type('text/html; charset=utf-8');
+    if ($format eq 'web') {
         $c->stash->{template}   = 'email_sent.html';
         $c->stash->{email_type} = 'problem';
-        return 1;
+    } else {
+        $c->res->content_type('text/plain; charset=utf-8');
+        $c->res->body('SUCCESS');
     }
-    $c->res->body('SUCCESS');
     return 1;
 }
 
diff --git a/perllib/FixMyStreet/Cobrand/EastSussex.pm b/perllib/FixMyStreet/Cobrand/EastSussex.pm
new file mode 100644
index 000000000..c113a0986
--- /dev/null
+++ b/perllib/FixMyStreet/Cobrand/EastSussex.pm
@@ -0,0 +1,41 @@
+package FixMyStreet::Cobrand::EastSussex;
+use parent 'FixMyStreet::Cobrand::UK';
+
+use strict;
+use warnings;
+
+sub council_area_id { return 2224; }
+
+sub open311_pre_send {
+    my ($self, $row, $open311) = @_;
+
+    my $contact = $row->category_row;
+    my $fields = $contact->get_extra_fields;
+    for my $field ( @$fields ) {
+        if ($field->{variable} && !$field->{automated}) {
+            my $text = $row->detail;
+            my $q = $row->get_extra_field_value( $field->{code} ) || '';
+            $text .= "\n\n" . $field->{description} . "\n" . $q;
+            $row->detail($text);
+        }
+    }
+}
+
+sub open311_post_send {
+    my ($self, $row, $h, $contact) = @_;
+
+    my $fields = $contact->get_extra_fields;
+    my $text = $row->detail;
+    my $added = '';
+    for my $field ( @$fields ) {
+        if ($field->{variable} && !$field->{automated}) {
+            my $q = $row->get_extra_field_value( $field->{code} ) || '';
+            $added .= "\n\n" . $field->{description} . "\n" . $q;
+        }
+    }
+
+    $text =~ s/\Q$added\E//;
+    $row->detail($text);
+}
+
+1;
diff --git a/perllib/FixMyStreet/Cobrand/Oxfordshire.pm b/perllib/FixMyStreet/Cobrand/Oxfordshire.pm
index 12714185d..b110731e6 100644
--- a/perllib/FixMyStreet/Cobrand/Oxfordshire.pm
+++ b/perllib/FixMyStreet/Cobrand/Oxfordshire.pm
@@ -142,8 +142,13 @@ sub should_skip_sending_update {
     my ($self, $update ) = @_;
 
     # Oxfordshire stores the external id of the problem as a customer reference
-    # in metadata
-    return 1 if !$update->problem->get_extra_metadata('customer_reference');
+    # in metadata, it arrives in a fetched update (but give up if it never does,
+    # or the update is for an old pre-ref report)
+    my $customer_ref = $update->problem->get_extra_metadata('customer_reference');
+    my $diff = time() - $update->confirmed->epoch;
+    return 1 if !$customer_ref && $diff > 60*60*24;
+    return 'WAIT' if !$customer_ref;
+    return 0;
 }
 
 sub on_map_default_status { return 'open'; }
@@ -197,4 +202,19 @@ sub available_permissions {
     return $perms;
 }
 
+sub dashboard_export_problems_add_columns {
+    my $self = shift;
+    my $c = $self->{c};
+
+    push @{$c->stash->{csv}->{headers}}, "HIAMS Ref";
+    push @{$c->stash->{csv}->{columns}}, "customer_reference";
+
+    $c->stash->{csv}->{extra_data} = sub {
+        my $ref = shift->get_extra_metadata('customer_reference') || '';
+        return {
+            customer_reference => $ref,
+        };
+    };
+}
+
 1;
diff --git a/perllib/FixMyStreet/Cobrand/Peterborough.pm b/perllib/FixMyStreet/Cobrand/Peterborough.pm
index 882bef7eb..0ddaeacb6 100644
--- a/perllib/FixMyStreet/Cobrand/Peterborough.pm
+++ b/perllib/FixMyStreet/Cobrand/Peterborough.pm
@@ -40,6 +40,19 @@ sub geocoder_munge_results {
 
 sub admin_user_domain { "peterborough.gov.uk" }
 
+around open311_extra_data => sub {
+    my ($orig, $self, $row, $h, $extra) = @_;
+
+    my $open311_only = $self->$orig($row, $h, $extra);
+    foreach (@$open311_only) {
+        if ($_->{name} eq 'description') {
+            my ($ref) = grep { $_->{name} =~ /pcc-Skanska-csc-ref/i } @{$row->get_extra_fields};
+            $_->{value} .= "\n\nSkanska CSC ref: $ref->{value}" if $ref;
+        }
+    }
+    return $open311_only;
+};
+
 # remove categories which are informational only
 sub open311_pre_send {
     my ($self, $row, $open311) = @_;
diff --git a/perllib/FixMyStreet/DB/Result/Problem.pm b/perllib/FixMyStreet/DB/Result/Problem.pm
index 489c43090..37563d327 100644
--- a/perllib/FixMyStreet/DB/Result/Problem.pm
+++ b/perllib/FixMyStreet/DB/Result/Problem.pm
@@ -800,7 +800,7 @@ sub defect_types {
 #     Note:   this only makes sense when called on a problem that has been sent!
 sub can_display_external_id {
     my $self = shift;
-    if ($self->external_id && $self->to_body_named('Oxfordshire|Lincolnshire|Isle of Wight')) {
+    if ($self->external_id && $self->to_body_named('Oxfordshire|Lincolnshire|Isle of Wight|East Sussex')) {
         return 1;
     }
     return 0;
diff --git a/perllib/FixMyStreet/DB/ResultSet/Problem.pm b/perllib/FixMyStreet/DB/ResultSet/Problem.pm
index 3e48170d8..359d5224a 100644
--- a/perllib/FixMyStreet/DB/ResultSet/Problem.pm
+++ b/perllib/FixMyStreet/DB/ResultSet/Problem.pm
@@ -159,7 +159,7 @@ sub _recent {
             $probs = [ $rs->search({
                 id => [ map { $_->id } @$probs ],
                 %$query,
-            })->all ];
+            }, $attrs)->all ];
         } else {
             $probs = [ $rs->search( $query, $attrs )->all ];
             Memcached::set($key, $probs, _cache_timeout());
diff --git a/perllib/FixMyStreet/Queue/Item/Report.pm b/perllib/FixMyStreet/Queue/Item/Report.pm
index 4d0d62752..e38987838 100644
--- a/perllib/FixMyStreet/Queue/Item/Report.pm
+++ b/perllib/FixMyStreet/Queue/Item/Report.pm
@@ -181,7 +181,7 @@ sub _create_reporters {
         }
         $reporters{ $sender } ||= $sender->new();
 
-        $self->log("OK, adding recipient body " . $body->id . ":" . $body->name . ", " . $sender_info->{method});
+        $self->log("Adding recipient body " . $body->id . ":" . $body->name . ", " . $sender_info->{method});
         push @dear, $body->name;
         $reporters{ $sender }->add_body( $body, $sender_info->{config} );
     }
@@ -218,21 +218,23 @@ sub _send {
     my $result = -1;
 
     for my $sender ( keys %{$self->reporters} ) {
-        $self->log("sending using " . $sender);
+        $self->log("Sending using " . $sender);
         $sender = $self->reporters->{$sender};
         my $res = $sender->send( $self->report, $self->h );
         $result *= $res;
         $self->report->add_send_method($sender) if !$res;
-        if ( $sender->unconfirmed_data) {
-            foreach my $e (keys %{ $sender->unconfirmed_data } ) {
-                foreach my $c (keys %{ $sender->unconfirmed_data->{$e} }) {
-                    $self->manager->unconfirmed_data->{$e}{$c}{count} += $sender->unconfirmed_data->{$e}{$c}{count};
-                    $self->manager->unconfirmed_data->{$e}{$c}{note} = $sender->unconfirmed_data->{$e}{$c}{note};
+        if ( $self->manager ) {
+            if ($sender->unconfirmed_data) {
+                foreach my $e (keys %{ $sender->unconfirmed_data } ) {
+                    foreach my $c (keys %{ $sender->unconfirmed_data->{$e} }) {
+                        $self->manager->unconfirmed_data->{$e}{$c}{count} += $sender->unconfirmed_data->{$e}{$c}{count};
+                        $self->manager->unconfirmed_data->{$e}{$c}{note} = $sender->unconfirmed_data->{$e}{$c}{note};
+                    }
                 }
             }
+            $self->manager->test_data->{test_req_used} = $sender->open311_test_req_used
+                if FixMyStreet->test_mode && $sender->can('open311_test_req_used');
         }
-        $self->manager->test_data->{test_req_used} = $sender->open311_test_req_used
-            if FixMyStreet->test_mode && $sender->can('open311_test_req_used');
     }
 
     return $result;
@@ -251,7 +253,7 @@ sub _post_send {
             $self->h->{sent_confirm_id_ref} = $self->report->$send_confirmation_email;
             $self->_send_report_sent_email;
         }
-        $self->log("send successful: OK");
+        $self->log("Send successful");
     } else {
         my @errors;
         for my $sender ( keys %{$self->reporters} ) {
@@ -260,7 +262,7 @@ sub _post_send {
             }
         }
         $self->report->update_send_failed( join( '|', @errors ) );
-        $self->log("send FAILED: " . join( '|', @errors ));
+        $self->log("Send failed");
     }
 }
 
diff --git a/perllib/FixMyStreet/Script/Reports.pm b/perllib/FixMyStreet/Script/Reports.pm
index 3e9b2d693..3d5afe216 100644
--- a/perllib/FixMyStreet/Script/Reports.pm
+++ b/perllib/FixMyStreet/Script/Reports.pm
@@ -20,6 +20,36 @@ sub send {
         verbose => $verbose,
     );
 
+    my $params = construct_query($debug);
+    my $db = FixMyStreet::DB->schema->storage;
+
+    $db->txn_do(sub {
+        my $unsent = FixMyStreet::DB->resultset('Problem')->search($params, {
+            for => \'UPDATE SKIP LOCKED',
+        });
+
+        $manager->log("starting to loop through unsent problem reports...");
+        my $unsent_count = 0;
+        while (my $row = $unsent->next) {
+            $unsent_count++;
+            my $item = FixMyStreet::Queue::Item::Report->new(
+                report => $row,
+                manager => $manager,
+                verbose => $verbose,
+                nomail => $nomail,
+            );
+            $item->process;
+        }
+
+        $manager->end_line($unsent_count);
+        $manager->end_summary_unconfirmed;
+    });
+
+    return $manager->test_data;
+}
+
+sub construct_query {
+    my ($debug) = @_;
     my $site = CronFns::site(FixMyStreet->config('BASE_URL'));
     my $states = [ FixMyStreet::DB::Result::Problem::open_states() ];
     $states = [ 'submitted', 'confirmed', 'in progress', 'feedback pending', 'external', 'wish' ] if $site eq 'zurich';
@@ -55,25 +85,7 @@ sub send {
         ];
     }
 
-    my $unsent = FixMyStreet::DB->resultset('Problem')->search($params);
-
-    $manager->log("starting to loop through unsent problem reports...");
-    my $unsent_count = 0;
-    while (my $row = $unsent->next) {
-        $unsent_count++;
-        my $item = FixMyStreet::Queue::Item::Report->new(
-            report => $row,
-            manager => $manager,
-            verbose => $verbose,
-            nomail => $nomail,
-        );
-        $item->process;
-    }
-
-    $manager->end_line($unsent_count);
-    $manager->end_summary_unconfirmed;
-
-    return $manager->test_data;
+    return $params;
 }
 
 sub end_line {
diff --git a/perllib/Open311/PostServiceRequestUpdates.pm b/perllib/Open311/PostServiceRequestUpdates.pm
index 14bebfcb7..fadd063da 100755
--- a/perllib/Open311/PostServiceRequestUpdates.pm
+++ b/perllib/Open311/PostServiceRequestUpdates.pm
@@ -19,17 +19,31 @@ has current_open311 => ( is => 'rw' );
 sub send {
     my $self = shift;
 
+    my $bodies = $self->fetch_bodies;
+    foreach my $body (values %$bodies) {
+        $self->construct_open311($body);
+        $self->process_body($body);
+    }
+}
+
+sub fetch_bodies {
     my $bodies = FixMyStreet::DB->resultset('Body')->search( {
         send_method => SEND_METHOD_OPEN311,
         send_comments => 1,
-    } );
-
+    }, { prefetch => 'body_areas' } );
+    my %bodies;
     while ( my $body = $bodies->next ) {
         my $cobrand = $body->get_cobrand_handler;
         next if $cobrand && $cobrand->call_hook('open311_post_update_skip');
-        $self->current_open311(Open311->new($self->open311_params($body)));
-        $self->process_body($body);
+        $bodies{$body->id} = $body;
     }
+    return \%bodies;
+}
+
+sub construct_open311 {
+    my ($self, $body) = @_;
+    my $o = Open311->new($self->open311_params($body));
+    $self->current_open311($o);
 }
 
 sub open311_params {
@@ -53,41 +67,57 @@ sub open311_params {
 sub process_body {
     my ($self, $body) = @_;
 
-    my $comments = FixMyStreet::DB->resultset('Comment')->to_body($body)->search( {
-            'me.whensent' => undef,
-            'me.external_id' => undef,
-            'me.state' => 'confirmed',
-            'me.confirmed' => { '!=' => undef },
-            'problem.whensent' => { '!=' => undef },
-            'problem.external_id' => { '!=' => undef },
-            'problem.send_method_used' => { -like => '%Open311%' },
-        },
-        {
+    my $params = $self->construct_query($self->verbose);
+
+    my $db = FixMyStreet::DB->schema->storage;
+    $db->txn_do(sub {
+        my $comments = FixMyStreet::DB->resultset('Comment')->to_body($body)->search($params, {
+            for => \'UPDATE SKIP LOCKED',
             order_by => [ 'confirmed', 'id' ],
-        }
-    );
+        });
 
-    while ( my $comment = $comments->next ) {
-        my $cobrand = $body->get_cobrand_handler || $comment->get_cobrand_logged;
-
-        # Some cobrands (e.g. Buckinghamshire) don't want to receive updates
-        # from anyone except the original problem reporter.
-        if ($cobrand->call_hook(should_skip_sending_update => $comment)) {
-            unless (defined $comment->get_extra_metadata('cobrand_skipped_sending')) {
-                $comment->set_extra_metadata(cobrand_skipped_sending => 1);
-                $comment->update;
-            }
-            next;
+        while ( my $comment = $comments->next ) {
+            $self->process_update($body, $comment);
         }
+    });
+}
 
-        next if !$self->verbose && $comment->send_fail_count && retry_timeout($comment);
-
-        $self->process_update($body, $comment, $cobrand);
+sub construct_query {
+    my ($self, $debug) = @_;
+    my $params = {
+        'me.whensent' => undef,
+        'me.external_id' => undef,
+        'me.state' => 'confirmed',
+        'me.confirmed' => { '!=' => undef },
+        'me.extra' => [ undef, { -not_like => '%cobrand_skipped_sending%' } ],
+        'problem.whensent' => { '!=' => undef },
+        'problem.external_id' => { '!=' => undef },
+        'problem.send_method_used' => { -like => '%Open311%' },
+    };
+    if (!$debug) {
+        $params->{'-or'} = [
+            'me.send_fail_count' => 0,
+            'me.send_fail_timestamp' => { '<', \"current_timestamp - '30 minutes'::interval" },
+        ];
     }
+    return $params;
 }
 
 sub process_update {
-    my ($self, $body, $comment, $cobrand) = @_;
+    my ($self, $body, $comment) = @_;
+
+    my $cobrand = $body->get_cobrand_handler || $comment->get_cobrand_logged;
+
+    # Some cobrands (e.g. Buckinghamshire) don't want to receive updates
+    # from anyone except the original problem reporter.
+    if (my $skip = $cobrand->call_hook(should_skip_sending_update => $comment)) {
+        if ($skip ne 'WAIT' && !defined $comment->get_extra_metadata('cobrand_skipped_sending')) {
+            $comment->set_extra_metadata(cobrand_skipped_sending => 1);
+            $comment->update;
+        }
+        $self->log($comment, 'Skipping');
+        return;
+    }
 
     my $o = $self->current_open311;
 
@@ -100,30 +130,21 @@ sub process_update {
             external_id => $id,
             whensent => \'current_timestamp',
         } );
+        $self->log($comment, 'Send successful');
     } else {
         $comment->update( {
             send_fail_count => $comment->send_fail_count + 1,
             send_fail_timestamp => \'current_timestamp',
             send_fail_reason => "Failed to post over Open311\n\n" . $o->error,
         } );
-
-        if ( $self->verbose && $o->error ) {
-            warn $o->error;
-        }
+        $self->log($comment, 'Send failed');
     }
 }
 
-sub retry_timeout {
-    my $row = shift;
-
-    my $tz = FixMyStreet->local_time_zone;
-    my $now = DateTime->now( time_zone => $tz );
-    my $diff = $now - $row->send_fail_timestamp;
-    if ( $diff->in_units( 'minutes' ) < 30 ) {
-        return 1;
-    }
-
-    return 0;
+sub log {
+    my ($self, $comment, $msg) = @_;
+    return unless $self->verbose;
+    STDERR->print("[fmsd] [" . $comment->id . "] $msg\n");
 }
 
 1;
diff --git a/t/cobrand/northamptonshire.t b/t/cobrand/northamptonshire.t
index 70df10340..57fe319a9 100644
--- a/t/cobrand/northamptonshire.t
+++ b/t/cobrand/northamptonshire.t
@@ -83,6 +83,7 @@ subtest 'Check updates not sent for defects' => sub {
 };
 
 $report->update({ user => $user });
+$comment->update({ extra => undef });
 subtest 'check updates sent for non defects' => sub {
     FixMyStreet::override_config {
         ALLOWED_COBRANDS => [ { northamptonshire => '.' } ],
diff --git a/t/cobrand/oxfordshire.t b/t/cobrand/oxfordshire.t
index dd5eedc8d..907c66c19 100644
--- a/t/cobrand/oxfordshire.t
+++ b/t/cobrand/oxfordshire.t
@@ -5,6 +5,7 @@ use FixMyStreet::Script::Alerts;
 my $mech = FixMyStreet::TestMech->new;
 
 my $oxon = $mech->create_body_ok(2237, 'Oxfordshire County Council');
+my $counciluser = $mech->create_user_ok('counciluser@example.com', name => 'Council User', from_body => $oxon);
 
 subtest 'check /around?ajax defaults to open reports only' => sub {
     my $categories = [ 'Bridges', 'Fences', 'Manhole' ];
@@ -105,6 +106,34 @@ subtest 'check unable to fix label' => sub {
     };
 };
 
+subtest 'extra CSV columns are present' => sub {
+    FixMyStreet::override_config {
+        ALLOWED_COBRANDS => [ 'oxfordshire' ],
+        MAPIT_URL => 'http://mapit.uk/',
+    }, sub {
+
+        $mech->log_in_ok( $counciluser->email );
+
+        $mech->get_ok('/dashboard?export=1');
+
+        my @rows = $mech->content_as_csv;
+        is scalar @rows, 7, '1 (header) + 6 (reports) = 7 lines';
+        is scalar @{$rows[0]}, 21, '21 columns present';
+
+        is_deeply $rows[0],
+            [
+                'Report ID', 'Title', 'Detail', 'User Name', 'Category',
+                'Created', 'Confirmed', 'Acknowledged', 'Fixed', 'Closed',
+                'Status', 'Latitude', 'Longitude', 'Query', 'Ward',
+                'Easting', 'Northing', 'Report URL', 'Site Used',
+                'Reported As', 'HIAMS Ref',
+            ],
+            'Column headers look correct';
+
+        is $rows[1]->[20], 'ENQ12456', 'HIAMS reference included in row';
+    };
+};
+
 END {
     done_testing();
 }
diff --git a/t/cobrand/peterborough.t b/t/cobrand/peterborough.t
index d61b3de75..5d07acb9f 100644
--- a/t/cobrand/peterborough.t
+++ b/t/cobrand/peterborough.t
@@ -25,6 +25,7 @@ subtest 'open311 request handling', sub {
         $p->push_extra_fields({ name => 'emergency', value => 'no'});
         $p->push_extra_fields({ name => 'private_land', value => 'no'});
         $p->push_extra_fields({ name => 'PCC-light', value => 'whatever'});
+        $p->push_extra_fields({ name => 'PCC-skanska-csc-ref', value => '1234'});
         $p->push_extra_fields({ name => 'tree_code', value => 'tree-42'});
         $p->update;
 
@@ -38,6 +39,7 @@ subtest 'open311 request handling', sub {
 
         my $req = $test_data->{test_req_used};
         my $c = CGI::Simple->new($req->content);
+        is $c->param('attribute[description]'), "Title Test 1 for " . $peterborough->id . " Detail\r\n\r\nSkanska CSC ref: 1234", 'Ref added to description';
         is $c->param('attribute[emergency]'), undef, 'no emergency param sent';
         is $c->param('attribute[private_land]'), undef, 'no private_land param sent';
         is $c->param('attribute[PCC-light]'), undef, 'no pcc- param sent';
diff --git a/templates/web/base/report/new/report_import.html b/templates/web/base/report/new/report_import.html
index f2ead081b..f68e21301 100644
--- a/templates/web/base/report/new/report_import.html
+++ b/templates/web/base/report/new/report_import.html
@@ -2,6 +2,8 @@
 
 <h1>External import</h1>
 
+[% INCLUDE 'errors.html' %]
+
 <p>You may inject problem reports into FixMyStreet programatically using this
 simple interface. Upon receipt, an email will be sent to the address given,
 with a link the user must click in order to check the details of their report,
@@ -14,12 +16,14 @@ line each starting with <samp>ERROR:</samp>.
 <p>You may submit the following information by POST to this URL
 (i.e. <samp>[% c.uri_for('/import') %]</samp> ):</p>
 
+[% IF NOT errors AND NOT c.req.params.web %]
 <style type="text/css" media="screen">
     input {
         /* Hide the form elements - they are just here for simpler testing */
-        display: none;
+        display: none !important;
     }
 </style>
+[% END %]
 
 <form method="POST" action="/import" enctype="multipart/form-data">
 
@@ -28,66 +32,76 @@ line each starting with <samp>ERROR:</samp>.
     <dd>
         <em>Required</em>.
         Name of application/service using this interface.
-        <input type="text" name="service" />
+        <input type="text" name="service" value="[% input.service %]">
     </dd>
 
     <dt>id</dt>
     <dd>
         Unique ID of a user/device, for possible future use.<br>
         <small>(e.g. used by Flickr import to know which accounts to look at)</small>
-        <input type="text" name="id" />
+        <input type="text" name="id" value="[% input.id %]">
     </dd>
 
     <dt>subject</dt>
     <dd>
         <em>Required</em>. Subject of problem report.
-        <input type="text" name="subject" />
+        <input type="text" name="subject" value="[% input.subject %]">
     </dd>
 
     <dt>detail</dt>
     <dd>
         Main body and details of problem report.
-        <input type="text" name="detail" />
+        <input type="text" name="detail" value="[% input.detail %]">
     </dd>
 
     <dt>name</dt>
     <dd>
         <em>Required</em>. Name of problem reporter.
-        <input type="text" name="name" />
+        <input type="text" name="name" value="[% input.name %]">
     </dd>
 
     <dt>email</dt>
     <dd>
         <em>Required</em>. Email address of problem reporter.
-        <input type="text" name="email" />
+        <input type="text" name="email" value="[% input.email %]">
     </dd>
 
     <dt>phone</dt>
     <dd>
         Telephone number of problem reporter.
-        <input type="text" name="phone" />
+        <input type="text" name="phone" value="[% input.phone %]">
     </dd>
 
     <dt>easting / northing</dt>
     <dt>lat / lon</dt>
     <dd>
         Location of problem report. You can either supply eastings/northings, or WGS84 latitude/longitude.
-        <input type="text" name="easting" />
-        <input type="text" name="northing" />
-        <input type="text" name="lat" />
-        <input type="text" name="lon" />
+        <input type="text" name="easting" value="[% input.easting %]">
+        <input type="text" name="northing" value="[% input.northing %]">
+        <input type="text" name="lat" value="[% input.lat %]">
+        <input type="text" name="lon" value="[% input.lon %]">
     </dd>
 
+  [% IF c.cobrand.allow_photo_upload %]
+    <input type="hidden" name="upload_fileid" value="[% upload_fileid %]">
+      [% IF upload_fileid %]
+        [% FOREACH id IN upload_fileid.split(',') %]
+        <img align="right" src="/photo/temp.[% id %]" alt="">
+        [% END %]
+      [% END %]
+
     <dt>photo</dt>
     <dd>
-        Photo of problem (JPEG only).
+        Photo of problem.
         <input type="file" name="photo" />
     </dd>
+  [% END %]
+
 </dl>
 
-<input type="hidden" name="web" value="0">
-<input type="submit" />
+<input type="hidden" name="web" value="[% c.req.params.web ? 1 : 0 %]">
+<input type="submit" value="[% loc('Submit') %]">
 
 </form>
 
-[% INCLUDE 'footer.html' %]
\ No newline at end of file
+[% INCLUDE 'footer.html' %]
diff --git a/templates/web/fixmystreet.com/footer_extra_js.html b/templates/web/fixmystreet.com/footer_extra_js.html
index d2fe1e588..289e230e7 100644
--- a/templates/web/fixmystreet.com/footer_extra_js.html
+++ b/templates/web/fixmystreet.com/footer_extra_js.html
@@ -11,6 +11,7 @@ IF bodyclass.match('mappage');
     scripts.push( version('/cobrands/bromley/assets.js') );
     scripts.push( version('/cobrands/buckinghamshire/assets.js') );
     scripts.push( version('/cobrands/cheshireeast/assets.js') );
+    scripts.push( version('/cobrands/eastsussex/assets.js') );
     scripts.push( version('/cobrands/isleofwight/assets.js') );
     scripts.push( version('/cobrands/lincolnshire/assets.js') );
     scripts.push( version('/cobrands/northamptonshire/assets.js') );
diff --git a/web/cobrands/eastsussex/assets.js b/web/cobrands/eastsussex/assets.js
new file mode 100644
index 000000000..296c8ede4
--- /dev/null
+++ b/web/cobrands/eastsussex/assets.js
@@ -0,0 +1,139 @@
+(function(){
+
+if (!fixmystreet.maps) {
+    return;
+}
+
+
+OpenLayers.Format.EastSussex = OpenLayers.Class(OpenLayers.Format.JSON, {
+    read: function(json, type, filter) {
+        var obj = json;
+        if (typeof json == "string") {
+            obj = OpenLayers.Format.JSON.prototype.read.apply(this,
+                                                              [json, filter]);
+        }
+
+        var results = [];
+        for (var i=0, len=obj.length; i<len; i++) {
+            var item = obj[i];
+            var geom = new OpenLayers.Geometry.Point(item.Mid_Location__c.longitude, item.Mid_Location__c.latitude);
+            var vec = new OpenLayers.Feature.Vector(geom, item);
+            results.push(vec);
+        }
+
+        return results;
+    },
+    CLASS_NAME: "OpenLayers.Format.EastSussex"
+});
+
+OpenLayers.Protocol.EastSussex = OpenLayers.Class(OpenLayers.Protocol.HTTP, {
+    read: function(options) {
+        OpenLayers.Protocol.prototype.read.apply(this, arguments);
+        options = options || {};
+        options.params = OpenLayers.Util.applyDefaults(
+            options.params, this.options.params);
+        options = OpenLayers.Util.applyDefaults(options, this.options);
+        var types = options.types.join('&types=');
+        var coords = fixmystreet.map.getCenterWGS84();
+        options.url = options.url + '?longitude=' + coords.lat + '&latitude=' + coords.lon + '&types=' + types;
+        var resp = new OpenLayers.Protocol.Response({requestType: "read"});
+        resp.priv = OpenLayers.Request.GET({
+            url: options.url,
+            callback: this.createCallback(this.handleRead, resp, options),
+            params: options.params,
+            headers: options.headers
+        });
+    },
+    CLASS_NAME: "OpenLayers.Protocol.EastSussex"
+});
+
+var defaults = {
+    http_options: {
+      url: fixmystreet.staging ? "https://tilma.staging.mysociety.org/proxy/escc/" : "https://tilma.mysociety.org/proxy/escc/"
+    },
+    max_resolution: 1.194328566789627,
+    geometryName: 'msGeometry',
+    srsName: "EPSG:4326",
+    body: "East Sussex County Council",
+    format_class: OpenLayers.Format.EastSussex,
+    protocol_class: OpenLayers.Protocol.EastSussex,
+    asset_id_field: 'asset_id',
+    attributes: {
+      asset_id: 'id'
+    }
+};
+
+fixmystreet.assets.add(defaults, {
+    http_options: {
+      types: [
+        "Bollard", "Central Refuge Beacon", "External Illuminated Sign", "Floodlight", "Internal Illuminated Sign", "Lighting Column", "Reflect Bollard", "Safety bollards", "Solar Bollard", "Subway Unit", "Zebra X Beacon"
+      ]
+    },
+    asset_item: 'street light',
+    asset_category: ["Burning By Day", "Intermittent", "Lamp Dim", "Lamp Flashing", "Lamp Obscured", "Lamp Out", "Missing Number", "Noisy Column", "Vandalism" ],
+    select_action: true,
+    actions: {
+        asset_found: function(asset) {
+          var id = asset.attributes.Name || '';
+          if (id !== '') {
+              $('.category_meta_message').html('You have selected <b>' + id + '</b>');
+          } else {
+              $('.category_meta_message').html('You can pick a <b class="asset-spot">' + this.fixmystreet.asset_item + '</b> from the map &raquo;');
+          }
+        },
+        asset_not_found: function() {
+           $('.category_meta_message').html('You can pick a <b class="asset-spot">' + this.fixmystreet.asset_item + '</b> from the map &raquo;');
+        }
+    }
+});
+
+fixmystreet.assets.add(defaults, {
+    http_options: {
+      types: [
+        "Grit bin"
+      ]
+    },
+    asset_item: 'grit bin',
+    asset_category: ["Broken Grit Bin", "Request For New Grit Bin", "Request To Refill Grit Bin"]
+});
+
+fixmystreet.assets.add(defaults, {
+    http_options: {
+      types: [
+        "Filter Drain", "Gully and Catchpit"
+      ]
+    },
+    asset_item: 'drain',
+    asset_category: ["Blocked Drain", "Culvert", "Broken Drain Cover", "Smell", "Sunken Drain", "Missing Drain Cover"],
+    select_action: true,
+    actions: {
+        asset_found: function(asset) {
+          var last_clean = asset.attributes.Gully_Last_Clean_Date__c || '';
+          var next_clean = asset.attributes.Gully_Next_Clean_Date__c || '';
+          if (last_clean !== '' || next_clean !== '') {
+              var message = '';
+              if (last_clean) { message += '<b>Last Cleaned</b>: ' + last_clean; }
+              if (next_clean) { message += ' <b>Next Clean</b>: ' + next_clean; }
+              $('.category_meta_message').html(message);
+          } else {
+              $('.category_meta_message').html('You can pick a <b class="asset-spot">' + this.fixmystreet.asset_item + '</b> from the map &raquo;');
+          }
+        },
+        asset_not_found: function() {
+           $('.category_meta_message').html('You can pick a <b class="asset-spot">' + this.fixmystreet.asset_item + '</b> from the map &raquo;');
+        }
+    }
+});
+
+// can have multiple group
+$(function(){
+    $("#problem_form").on("change.category", function() {
+        var group = '';
+        if (OpenLayers.Util.indexOf(fixmystreet.bodies, 'East Sussex County Council') != -1 ) {
+          group = $('#form_category :selected').parent().attr('label');
+        }
+        $('#form_group').val(group);
+    });
+});
+
+})();