aboutsummaryrefslogtreecommitdiffstats
path: root/bin/send-reports
diff options
context:
space:
mode:
Diffstat (limited to 'bin/send-reports')
-rwxr-xr-xbin/send-reports304
1 files changed, 286 insertions, 18 deletions
diff --git a/bin/send-reports b/bin/send-reports
index 22bd12732..5f3a508b7 100755
--- a/bin/send-reports
+++ b/bin/send-reports
@@ -22,6 +22,7 @@ use CronFns;
use FixMyStreet::App;
use EastHantsWSDL;
+use BarnetInterfaces::service::ZLBB_SERVICE_ORDER;
use Utils;
use mySociety::Config;
use mySociety::EmailUtil;
@@ -30,6 +31,22 @@ use mySociety::Web qw(ent);
use Open311;
+# maximum number of webservice attempts to send before not trying any more (XXX may be better in config?)
+use constant SEND_FAIL_RETRIES_CUTOFF => 3;
+
+# specific council numbers
+use constant COUNCIL_ID_BARNET => 2489;
+use constant COUNCIL_ID_EAST_HANTS => 2330;
+
+use constant MAX_LINE_LENGTH => 132;
+
+# send_method config values found in by-area config data, for selecting to appropriate method
+use constant SEND_METHOD_EMAIL => 'email';
+use constant SEND_METHOD_OPEN311 => 'open311';
+use constant SEND_METHOD_BARNET => 'barnet';
+use constant SEND_METHOD_EAST_HANTS => 'easthants';
+use constant SEND_METHOD_LONDON => 'london';
+
# Set up site, language etc.
my ($verbose, $nomail) = CronFns::options();
my $base_url = mySociety::Config::get('BASE_URL');
@@ -40,6 +57,9 @@ my $unsent = FixMyStreet::App->model("DB::Problem")->search( {
whensent => undef,
council => { '!=', undef },
} );
+
+my %sending_skipped_by_method = ();
+
my (%notgot, %note);
while (my $row = $unsent->next) {
@@ -56,8 +76,10 @@ while (my $row = $unsent->next) {
next;
}
- my $send_email = 0;
- my $send_web = 0;
+ # Due to multiple councils, it's possible to want to send both by email *and* another method
+ # NB: might need to revist this if multiple councils have custom send methods
+ my $send_email = 0;
+ my $send_method = 0;
# Template variables for the email
my $email_base_url = $cobrand->base_url_for_emails($row->cobrand_data);
@@ -118,6 +140,7 @@ while (my $row = $unsent->next) {
push @to, [ $council_email, $name ];
@recips = ($council_email);
+ $send_method = 0;
$send_email = 1;
$template = Utils::read_file("$FindBin::Bin/../templates/email/emptyhomes/" . $row->lang . "/submit.txt");
@@ -133,15 +156,37 @@ while (my $row = $unsent->next) {
foreach my $council (@councils) {
my $name = $areas_info->{$council}->{name};
push @dear, $name;
- if ($council == 2330) { # E. Hants have a web service
- $send_web = 'easthants';
+
+ # look in the DB to determine if there is a special handler for this council (e.g., open311, or custom)
+ my $council_config = FixMyStreet::App->model("DB::Open311conf")->search( { area_id => $council} )->first;
+ $send_method = $council_config->send_method if ($council_config and $council_config->send_method);
+ if ($council == COUNCIL_ID_EAST_HANTS) { # E. Hants have a web service
+ $send_method = SEND_METHOD_EAST_HANTS; # TODO: delete? should be in the db
$h{category} = 'Customer Services' if $h{category} eq 'Other';
- } elsif ($areas_info->{$council}->{type} eq 'LBO') { # London
- $send_web = 'london';
- } elsif ( my $endpoint = FixMyStreet::App->model("DB::Open311conf")->search( { area_id => $council, endpoint => { '!=', '' } } )->first ) {
- push @open311_councils, $endpoint;
- $send_web = 'open311';
- } else {
+ }
+
+ # if council lookup provided no explicit send_method, maybe there's some other criterion for setting it:
+ if (! $send_method) {
+ if ($areas_info->{$council}->{type} eq 'LBO') { # London
+ $send_method = SEND_METHOD_LONDON;
+ }
+ }
+ $send_email = 1 unless $send_method; # default to email if nothing explicit was provided
+
+ # currently: open311 or Barnet without an endpoint is useless, so check the endpoint is set
+ if ($send_method eq SEND_METHOD_OPEN311 or $send_method eq SEND_METHOD_BARNET) {
+ if ($council_config->endpoint) {
+ if ($send_method eq SEND_METHOD_OPEN311) {
+ push @open311_councils, $council_config;
+ }
+ } else {
+ print "Warning: no endpoint specified in config data for council=$council (will try email instead)\n";
+ $send_method = 0;
+ $send_email = 1;
+ }
+ }
+
+ if ($send_email) {
my $contact = FixMyStreet::App->model("DB::Contact")->find( {
deleted => 0,
area_id => $council,
@@ -160,7 +205,6 @@ while (my $row = $unsent->next) {
}
push @to, [ $council_email, $name ];
$recips{$council_email} = 1;
- $send_email = 1;
}
}
@recips = keys %recips;
@@ -199,15 +243,19 @@ while (my $row = $unsent->next) {
}
- unless ($send_email || $send_web) {
+ unless ($send_email || $send_method) {
die 'Report not going anywhere for ID ' . $row->id . '!';
}
if (mySociety::Config::get('STAGING_SITE')) {
# on a staging server send emails to ourselves rather than the councils
- @recips = ( mySociety::Config::get('CONTACT_EMAIL') );
- $send_web = 0;
- $send_email = 1;
+ # ...webservice calls will only go through if explictly allowed here:
+ my @testing_councils = (COUNCIL_ID_BARNET);
+ unless (grep {$row->council eq $_} @testing_councils) {
+ @recips = ( mySociety::Config::get('CONTACT_EMAIL') );
+ $send_method = 0;
+ $send_email = 1;
+ }
} elsif ($site eq 'emptyhomes') {
my $council = $row->council;
my $country = $areas_info->{$council}->{country};
@@ -243,17 +291,28 @@ while (my $row = $unsent->next) {
);
}
- if ($send_web eq 'easthants') {
+ if ($send_method eq SEND_METHOD_EAST_HANTS) {
$h{message} = construct_easthants_message(%h);
if (!$nomail) {
$result *= post_easthants_message(%h);
}
- } elsif ($send_web eq 'london') {
+ } elsif ($send_method eq SEND_METHOD_BARNET) {
+ $h{message} = construct_barnet_message(%h);
+ if (!$nomail) {
+ if (my $cutoff_msg = does_exceed_cutoff_limit($row, "barnet")) {
+ print "$cutoff_msg\n" if $verbose;
+ } else {
+ my ($barnet_result, $err_msg) = post_barnet_message( $row, %h );
+ update_send_fail_data($row, $err_msg) if $barnet_result;
+ $result *= $barnet_result;
+ }
+ }
+ } elsif ($send_method eq SEND_METHOD_LONDON) {
$h{message} = construct_london_message(%h);
if (!$nomail) {
$result *= post_london_report( $row, %h );
}
- } elsif ($send_web eq 'open311') {
+ } elsif ($send_method eq SEND_METHOD_OPEN311) {
foreach my $conf ( @open311_councils ) {
print 'posting to end point for ' . $conf->area_id . "\n" if $verbose;
@@ -316,6 +375,16 @@ if ($verbose) {
}
}
+if ($verbose and keys %sending_skipped_by_method) {
+ my $c = 0;
+ print "\nProblem reports that send-reports did not attempt to send because retries >= " . SEND_FAIL_RETRIES_CUTOFF . ":\n";
+ foreach my $send_method (sort keys %sending_skipped_by_method) {
+ printf " %-24s %4d\n", "$send_method:", $sending_skipped_by_method{$send_method};
+ $c+=$sending_skipped_by_method{$send_method};
+ }
+ printf " %-24s %4d\n", "Total:", $c;
+}
+
sub _get_district_for_contact {
my ( $lat, $lon ) = @_;
my $district =
@@ -358,6 +427,8 @@ sub construct_easthants_message {
$message .= <<EOF;
Subject: $h{title}
+Category: $h{category}
+
Details: $h{detail}
$h{fuzzy}, or to provide an update on the problem, please visit the following link:
@@ -390,6 +461,126 @@ sub post_easthants_message {
return $return;
}
+# currently just blind copy of construct_easthants_message
+sub construct_barnet_message {
+ my %h = @_;
+ my $message = <<EOF;
+Subject: $h{title}
+
+Details: $h{detail}
+
+$h{fuzzy}, or to provide an update on the problem, please visit the following link:
+
+$h{url}
+
+$h{closest_address}
+EOF
+}
+
+sub post_barnet_message {
+ my ( $problem, %h ) = @_;
+ my $return = 1;
+ my $err_msg = "";
+
+ my $default_kbid = 14; # This is the default, "Street Scene"
+ my $kbid = sprintf( "%050d", Utils::barnet_categories()->{$h{category}} || $default_kbid);
+
+ my $geo_code = "$h{easting} $h{northing}";
+
+ my $interface = BarnetInterfaces::service::ZLBB_SERVICE_ORDER->new();
+
+ my ($nearest_postcode, $nearest_street) = ('', '');
+ for ($h{closest_address}) {
+ $nearest_postcode = sprintf("%-10s", $1) if /Nearest postcode [^:]+: ((\w{1,4}\s?\w+|\w+))/;
+ # use partial postcode or comma as delimiter, strip leading number (possible letter 221B) off too
+ # "99 Foo Street, London N11 1XX" becomes Foo Street
+ # "99 Foo Street N11 1XX" becomes Foo Street
+ $nearest_street = $1 if /Nearest road [^:]+: (?:\d+\w? )?(.*?)(\b[A-Z]+\d|,|$)/m;
+ }
+ my $postcode = mySociety::PostcodeUtil::is_valid_postcode($h{query})
+ ? $h{query} : $nearest_postcode; # use given postcode if available
+
+ # note: endpoint can be of form 'https://username:password@url'
+ my $council_config = FixMyStreet::App->model("DB::Open311conf")->search( { area_id => COUNCIL_ID_BARNET} )->first;
+ if ($council_config and $council_config->endpoint) {
+ $interface->set_proxy($council_config->endpoint);
+ # Barnet web service doesn't like namespaces in the elements so use a prefix
+ $interface->set_prefix('urn');
+ } else {
+ die "Barnet webservice FAIL: looks like you're missing some config data: no endpoint (URL) found for area_id=" . COUNCIL_ID_BARNET;
+ }
+
+ eval {
+ my $result = $interface->Z_CRM_SERVICE_ORDER_CREATE( {
+ ET_RETURN => { # ignored by server
+ item => {
+ TYPE => "", ID => "", NUMBER => "", MESSAGE => "", LOG_NO => "", LOG_MSG_NO => "",
+ MESSAGE_V1 => "", MESSAGE_V2 => "", MESSAGE_V3 => "", MESSAGE_V4 => "", PARAMETER => "",
+ ROW => "", FIELD => "", SYSTEM => "",
+ },
+ },
+ IT_PROBLEM_DESC => { # MyTypes::TABLE_OF_CRMT_SERVICE_REQUEST_TEXT
+ item => [ # MyTypes::CRMT_SERVICE_REQUEST_TEXT
+ map { { TEXT_LINE => $_ } } split_text_with_entities(ent(encode_utf8($h{message})), 132) # char132
+ ],
+ },
+ IV_CUST_EMAIL => truncate_string_with_entities(ent(encode_utf8($h{email})), 241), # char241
+ IV_CUST_NAME => truncate_string_with_entities(ent(encode_utf8($h{name})), 50), # char50
+ IV_KBID => $kbid, # char50
+ IV_PROBLEM_ID => $h{id}, # char35
+ IV_PROBLEM_LOC => { # MyTypes::BAPI_TTET_ADDRESS_COM
+ COUNTRY2 => 'GB', # char2
+ REGION => "", # char3
+ COUNTY => "", # char30
+ CITY => "", # char30
+ POSTALCODE => $postcode, # char10
+ STREET => truncate_string_with_entities(ent(encode_utf8($nearest_street)), 30), # char30
+ STREETNUMBER => "", # char5
+ GEOCODE => $geo_code, # char32
+ },
+ IV_PROBLEM_SUB => truncate_string_with_entities(ent(encode_utf8($h{title})), 40), # char40
+ },
+ );
+ if ($result) {
+ # currently not using this: get_EV_ORDER_GUID (maybe that's the customer number in the CRM)
+ if (my $barnet_id = $result->get_EV_ORDER_NO()) {
+ $problem->external_id( $barnet_id );
+ $problem->external_body( 'Barnet Borough Council' ); # better to use $problem->body()?
+ $problem->send_method_used('barnet');
+ $return = 0;
+ } else {
+ my @returned_items = split /<item[^>]*>/, $result->get_ET_RETURN;
+ my @messages = ();
+ foreach my $item (@returned_items) {
+ if ($item=~/<MESSAGE [^>]*>\s*(\S.*?)<\/MESSAGE>/) { # if there's a non-null MESSAGE in there, grab it
+ push @messages, $1; # best stab at extracting useful error message back from convoluted response
+ }
+ }
+ push @messages, "service returned no external id" unless @messages;
+ $err_msg = "Failed (problem id $h{id}): " . join(" \n ", @messages);
+ }
+ } else {
+ my %fault = (
+ 'code' => $result->get_faultcode(),
+ 'actor' => $result->get_faultactor(),
+ 'string' => $result->get_faultstring(),
+ 'detail' => $result->get_detail(), # possibly only contains debug info
+ );
+ $fault{$_}=~s/^\s*|\s*$//g foreach keys %fault;
+ $fault{actor}&&=" (actor: $fault{actor})";
+ $fault{'detail'} &&= "\n" . $fault{'detail'};
+ $err_msg = "Failed (problem id $h{id}): Fault $fault{code}$fault{actor}\n$fault{string}$fault{detail}";
+ }
+
+ };
+ print "$err_msg\n" if $err_msg;
+ if ($@) {
+ my $e = shift;
+ print "Caught an error: $@\n";
+ }
+ return ($return, $err_msg);
+}
+
# London
sub construct_london_message {
@@ -489,3 +680,80 @@ sub london_lookup {
return $str;
}
+# for barnet webservice: max-length fields require truncate and split
+
+# truncate_string_with_entities
+# args: text to truncate
+# max number of chars
+# returns: string truncated
+# Note: must not partially truncate an entity (e.g., &amp;)
+sub truncate_string_with_entities {
+ my ($str, $max_len) = @_;
+ my $retVal = "";
+ foreach my $chunk (split /(\&(?:\#\d+|\w+);)/, $str) {
+ if ($chunk=~/^\&(\#\d+|\w+);$/){
+ my $next = $retVal.$chunk;
+ last if length $next > $max_len;
+ $retVal=$next
+ } else {
+ $retVal.=$chunk;
+ if (length $retVal > $max_len) {
+ $retVal = substr($retVal, 0, $max_len);
+ last
+ }
+ }
+ }
+ return $retVal
+}
+
+# split_text_with_entities into lines
+# args: text to be broken into lines
+# max length (option: uses constant MAX_LINE_LENGTH)
+# returns: array of lines
+# Must not to split an entity (e.g., &amp;)
+# Not worrying about hyphenating here, since a word is only ever split if
+# it's longer than the whole line, which is uncommon in genuine problem reports
+sub split_text_with_entities {
+ my ($text, $max_line_length) = @_;
+ $max_line_length ||= MAX_LINE_LENGTH;
+ my @lines;
+ foreach my $line (split "\n", $text) {
+ while (length $line > $max_line_length) {
+ if (! ($line =~ s/^(.{1,$max_line_length})\s// # break on a space
+ or $line =~ s/^(.{1,$max_line_length})(\&(\#\d+|\w+);)/$2/ # break before an entity
+ or $line =~ s/(.{$max_line_length})//)) { # break the word ruthlessly
+ $line =~ s/(.*)//; # otherwise gobble whole line (which is now shorter than max length)
+ }
+ push @lines, $1;
+ }
+ push @lines, $line;
+ }
+ return @lines;
+}
+
+# tests send_fail_count agains cutoff limit
+# args: problem (row from problem db)
+# returns false if there is no cutoff, otherwise error message
+sub does_exceed_cutoff_limit {
+ my ($problem, $council_name) = @_;
+ my $err_msg = "";
+ if ($problem->send_fail_count >= SEND_FAIL_RETRIES_CUTOFF) {
+ $sending_skipped_by_method{$council_name || '?'}++;
+ $council_name &&= " to $council_name";
+ $err_msg = "skipped: problem id=" . $problem->id . " send$council_name has failed "
+ . $problem->send_fail_count . " times, cutoff is " . SEND_FAIL_RETRIES_CUTOFF;
+ }
+ return $err_msg;
+}
+
+# update_send_fail_data records the failure (of a webservice send)
+# args: problem (row from problem db)
+# returns: no return value (updates record)
+sub update_send_fail_data {
+ my ($problem, $err_msg) = @_;
+ $problem->update( {
+ send_fail_count => $problem->send_fail_count + 1,
+ send_fail_timestamp => \'ms_current_timestamp()',
+ send_fail_reason => $err_msg
+ } );
+}
generated by cgit v1.2.3 (git 2.25.1) at 2025-04-26 13:52:41 +0000