aboutsummaryrefslogtreecommitdiffstats
path: root/bin
diff options
context:
space:
mode:
Diffstat (limited to 'bin')
-rwxr-xr-xbin/fetch-comments58
-rwxr-xr-xbin/send-comments92
-rwxr-xr-xbin/send-reports151
3 files changed, 282 insertions, 19 deletions
diff --git a/bin/fetch-comments b/bin/fetch-comments
new file mode 100755
index 000000000..e0e53826e
--- /dev/null
+++ b/bin/fetch-comments
@@ -0,0 +1,58 @@
+#!/usr/bin/env perl
+
+# send-reports:
+# Send new problem reports to councils
+#
+# Copyright (c) 2011 UK Citizens Online Democracy. All rights reserved.
+# Email: matthew@mysociety.org. WWW: http://www.mysociety.org
+
+use strict;
+use warnings;
+require 5.8.0;
+
+use Digest::MD5;
+use Encode;
+use Error qw(:try);
+use CronFns;
+
+use FixMyStreet::App;
+
+use Utils;
+use mySociety::Config;
+use mySociety::EmailUtil;
+
+use Open311;
+use Open311::GetServiceRequestUpdates;
+
+# send_method config values found in by-area config data, for selecting to appropriate method
+use constant SEND_METHOD_EMAIL => 'email';
+use constant SEND_METHOD_OPEN311 => 'open311';
+
+# Set up site, language etc.
+my ( $verbose, $nomail ) = CronFns::options();
+my $base_url = mySociety::Config::get('BASE_URL');
+my $site = CronFns::site($base_url);
+
+my $councils = FixMyStreet::App->model('DB::Open311Conf')->search(
+ {
+ send_method => SEND_METHOD_OPEN311,
+ send_comments => 1,
+ comment_user_id => { '!=', undef },
+ endpoint => { '!=', '' },
+ }
+);
+
+while ( my $council = $councils->next ) {
+
+ my $o = Open311->new(
+ endpoint => $council->endpoint,
+ api_key => $council->api_key,
+ jurisdiction => $council->jurisdiction,
+ );
+
+ my $updates =
+ Open311::GetServiceRequestUpdates->new(
+ system_user => $council->comment_user );
+
+ $updates->update_comments( $o, { areaid => $council->area_id }, );
+}
diff --git a/bin/send-comments b/bin/send-comments
new file mode 100755
index 000000000..df646a682
--- /dev/null
+++ b/bin/send-comments
@@ -0,0 +1,92 @@
+#!/usr/bin/env perl
+
+# send-reports:
+# Send new problem reports to councils
+#
+# Copyright (c) 2011 UK Citizens Online Democracy. All rights reserved.
+# Email: matthew@mysociety.org. WWW: http://www.mysociety.org
+
+use strict;
+use warnings;
+require 5.8.0;
+
+use Digest::MD5;
+use Encode;
+use Error qw(:try);
+use CronFns;
+
+use FixMyStreet::App;
+
+use Utils;
+use mySociety::Config;
+use mySociety::EmailUtil;
+
+use Open311;
+
+# maximum number of webservice attempts to send before not trying any more (XXX may be better in config?)
+use constant SEND_FAIL_RETRIES_CUTOFF => 3;
+
+# send_method config values found in by-area config data, for selecting to appropriate method
+use constant SEND_METHOD_EMAIL => 'email';
+use constant SEND_METHOD_OPEN311 => 'open311';
+
+# Set up site, language etc.
+my ($verbose, $nomail) = CronFns::options();
+my $base_url = mySociety::Config::get('BASE_URL');
+my $site = CronFns::site($base_url);
+
+my $councils = FixMyStreet::App->model('DB::Open311Conf')->search( {
+ send_method => SEND_METHOD_OPEN311,
+ send_comments => 1,
+} );
+
+while ( my $council = $councils->next ) {
+ my $comments = FixMyStreet::App->model('DB::Comment')->search( {
+ 'me.whensent' => undef,
+ 'problem.whensent' => { '!=' => undef },
+ 'problem.external_id' => { '!=' => undef },
+ 'problem.council' => { -like => '%' . $council->area_id .'%' },
+ },
+ {
+ join => 'problem',
+ }
+ );
+
+ my $o = Open311->new(
+ endpoint => $council->endpoint
+ );
+
+ while ( my $comment = $comments->next ) {
+ my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($comment->cobrand)->new();
+
+ if ( $comment->send_fail_count ) {
+ next if bromley_retry_timeout( $comment );
+ }
+
+ my $id = $o->post_service_request_update( $comment );
+
+ if ( $id ) {
+ $comment->external_id( $id );
+ $comment->update();
+ } else {
+ $comment->update( {
+ send_fail_count => $comment->send_fail_count + 1,
+ send_fail_timestamp => \'ms_current_timestamp()',
+ send_fail_reason => 'Failed to post over Open311',
+ } );
+ }
+ }
+}
+
+sub bromley_retry_timeout {
+ my $row = shift;
+
+ my $tz = DateTime::TimeZone->new( name => 'local' );
+ my $now = DateTime->now( time_zone => $tz );
+ my $diff = $now - $row->send_fail_timestamp;
+ if ( $diff->minutes < 30 ) {
+ return 1;
+ }
+
+ return 0;
+}
diff --git a/bin/send-reports b/bin/send-reports
index 22bd12732..1b93f8c5a 100755
--- a/bin/send-reports
+++ b/bin/send-reports
@@ -30,6 +30,20 @@ use mySociety::Web qw(ent);
use Open311;
+# maximum number of webservice attempts to send before not trying any more (XXX may be better in config?)
+use constant SEND_FAIL_RETRIES_CUTOFF => 3;
+
+# specific council numbers
+use constant COUNCIL_ID_EAST_HANTS => 2330;
+
+use constant MAX_LINE_LENGTH => 132;
+
+# send_method config values found in by-area config data, for selecting to appropriate method
+use constant SEND_METHOD_EMAIL => 'email';
+use constant SEND_METHOD_OPEN311 => 'open311';
+use constant SEND_METHOD_EAST_HANTS => 'easthants';
+use constant SEND_METHOD_LONDON => 'london';
+
# Set up site, language etc.
my ($verbose, $nomail) = CronFns::options();
my $base_url = mySociety::Config::get('BASE_URL');
@@ -40,6 +54,9 @@ my $unsent = FixMyStreet::App->model("DB::Problem")->search( {
whensent => undef,
council => { '!=', undef },
} );
+
+my %sending_skipped_by_method = ();
+
my (%notgot, %note);
while (my $row = $unsent->next) {
@@ -56,8 +73,10 @@ while (my $row = $unsent->next) {
next;
}
- my $send_email = 0;
- my $send_web = 0;
+ # Due to multiple councils, it's possible to want to send both by email *and* another method
+ # NB: might need to revist this if multiple councils have custom send methods
+ my $send_email = 0;
+ my $send_method = 0;
# Template variables for the email
my $email_base_url = $cobrand->base_url_for_emails($row->cobrand_data);
@@ -118,6 +137,7 @@ while (my $row = $unsent->next) {
push @to, [ $council_email, $name ];
@recips = ($council_email);
+ $send_method = 0;
$send_email = 1;
$template = Utils::read_file("$FindBin::Bin/../templates/email/emptyhomes/" . $row->lang . "/submit.txt");
@@ -133,15 +153,37 @@ while (my $row = $unsent->next) {
foreach my $council (@councils) {
my $name = $areas_info->{$council}->{name};
push @dear, $name;
- if ($council == 2330) { # E. Hants have a web service
- $send_web = 'easthants';
+
+ # look in the DB to determine if there is a special handler for this council (e.g., open311, or custom)
+ my $council_config = FixMyStreet::App->model("DB::Open311conf")->search( { area_id => $council} )->first;
+ $send_method = $council_config->send_method if $council_config;
+ if ($council == COUNCIL_ID_EAST_HANTS) { # E. Hants have a web service
+ $send_method = SEND_METHOD_EAST_HANTS; # TODO: delete? should be in the db
$h{category} = 'Customer Services' if $h{category} eq 'Other';
- } elsif ($areas_info->{$council}->{type} eq 'LBO') { # London
- $send_web = 'london';
- } elsif ( my $endpoint = FixMyStreet::App->model("DB::Open311conf")->search( { area_id => $council, endpoint => { '!=', '' } } )->first ) {
- push @open311_councils, $endpoint;
- $send_web = 'open311';
- } else {
+ }
+
+ # if council lookup provided no explicit send_method, maybe there's some other criterion for setting it:
+ if (! $send_method) {
+ if ($areas_info->{$council}->{type} eq 'LBO') { # London
+ $send_method = SEND_METHOD_LONDON;
+ }
+ }
+ $send_email = 1 unless $send_method; # default to email if nothing explicit was provided
+
+ # currently: open311 without an endpoint is useless, so check the endpoint is set
+ if ($send_method eq SEND_METHOD_OPEN311) {
+ if ($council_config->endpoint) {
+ if ($send_method eq SEND_METHOD_OPEN311) {
+ push @open311_councils, $council_config;
+ }
+ } else {
+ print "Warning: no endpoint specified in config data for council=$council (will try email instead)\n";
+ $send_method = 0;
+ $send_email = 1;
+ }
+ }
+
+ if ($send_email) {
my $contact = FixMyStreet::App->model("DB::Contact")->find( {
deleted => 0,
area_id => $council,
@@ -160,7 +202,6 @@ while (my $row = $unsent->next) {
}
push @to, [ $council_email, $name ];
$recips{$council_email} = 1;
- $send_email = 1;
}
}
@recips = keys %recips;
@@ -199,15 +240,19 @@ while (my $row = $unsent->next) {
}
- unless ($send_email || $send_web) {
+ unless ($send_method) {
die 'Report not going anywhere for ID ' . $row->id . '!';
}
if (mySociety::Config::get('STAGING_SITE')) {
# on a staging server send emails to ourselves rather than the councils
- @recips = ( mySociety::Config::get('CONTACT_EMAIL') );
- $send_web = 0;
- $send_email = 1;
+ # ...webservice calls will only go through if explictly allowed here:
+ my @testing_councils = ();
+ unless (grep {$row->council eq $_} @testing_councils) {
+ @recips = ( mySociety::Config::get('CONTACT_EMAIL') );
+ $send_method = 0;
+ $send_email = 1;
+ }
} elsif ($site eq 'emptyhomes') {
my $council = $row->council;
my $country = $areas_info->{$council}->{country};
@@ -243,20 +288,36 @@ while (my $row = $unsent->next) {
);
}
- if ($send_web eq 'easthants') {
+ if ($send_method eq SEND_METHOD_EAST_HANTS) {
$h{message} = construct_easthants_message(%h);
if (!$nomail) {
$result *= post_easthants_message(%h);
}
- } elsif ($send_web eq 'london') {
+ } elsif ($send_method eq SEND_METHOD_LONDON) {
$h{message} = construct_london_message(%h);
if (!$nomail) {
$result *= post_london_report( $row, %h );
}
- } elsif ($send_web eq 'open311') {
+ } elsif ($send_method eq SEND_METHOD_OPEN311) {
foreach my $conf ( @open311_councils ) {
print 'posting to end point for ' . $conf->area_id . "\n" if $verbose;
+ # Extra bromley fields
+ if ( $row->council =~ /2482/ ) {
+ if ( $row->send_fail_count > 0 ) {
+ next if bromley_retry_timeout( $row );
+ }
+
+ my $extra = $row->extra;
+ push @$extra, { name => 'northing', value => $h{northing} };
+ push @$extra, { name => 'easting', value => $h{easting} };
+ push @$extra, { name => 'report_url', value => $h{url} };
+ push @$extra, { name => 'service_request_id', value => $row->id };
+ push @$extra, { name => 'report_title', value => $row->title };
+ push @$extra, { name => 'public_anonymity_required', value => $row->anonymous ? 'TRUE' : 'FALSE' };
+ $row->extra( $extra );
+ }
+
my $contact = FixMyStreet::App->model("DB::Contact")->find( {
deleted => 0,
area_id => $conf->area_id,
@@ -282,7 +343,7 @@ while (my $row = $unsent->next) {
my $resp = $open311->send_service_request( $row, \%h, $contact->email );
# make sure we don't save user changes from above
- if ( $row->council =~ /2218/ ) {
+ if ( $row->council =~ /2218/ or $row->council =~ /2482/ ) {
$row->discard_changes();
}
@@ -291,6 +352,7 @@ while (my $row = $unsent->next) {
$result *= 0;
} else {
$result *= 1;
+ update_send_fail_data( $row, 'Open311 failed' );
# temporary fix to resolve some issues with west berks
if ( $row->council =~ /2619/ ) {
$result *= 0;
@@ -316,6 +378,17 @@ if ($verbose) {
}
}
+# not conditional on verbose because these can be considered failures (more relevant than one-off error messages?)
+if (keys %sending_skipped_by_method) {
+ my $c = 0;
+ print "\nProblem reports that send-reports did not attempt to send because retries >= " . SEND_FAIL_RETRIES_CUTOFF . ":\n";
+ foreach my $send_method (sort keys %sending_skipped_by_method) {
+ printf " %-24s %4d\n", "$send_method:", $sending_skipped_by_method{$send_method};
+ $c+=$sending_skipped_by_method{$send_method};
+ }
+ printf " %-24s %4d\n", "Total:", $c;
+}
+
sub _get_district_for_contact {
my ( $lat, $lon ) = @_;
my $district =
@@ -489,3 +562,43 @@ sub london_lookup {
return $str;
}
+
+# tests send_fail_count agains cutoff limit
+# args: problem (row from problem db)
+# returns false if there is no cutoff, otherwise error message
+sub does_exceed_cutoff_limit {
+ my ($problem, $council_name) = @_;
+ my $err_msg = "";
+ if ($problem->send_fail_count >= SEND_FAIL_RETRIES_CUTOFF) {
+ $sending_skipped_by_method{$council_name || '?'}++;
+ $council_name &&= " to $council_name";
+ $err_msg = "skipped: problem id=" . $problem->id . " send$council_name has failed "
+ . $problem->send_fail_count . " times, cutoff is " . SEND_FAIL_RETRIES_CUTOFF;
+ }
+ return $err_msg;
+}
+
+# update_send_fail_data records the failure (of a webservice send)
+# args: problem (row from problem db)
+# returns: no return value (updates record)
+sub update_send_fail_data {
+ my ($problem, $err_msg) = @_;
+ $problem->update( {
+ send_fail_count => $problem->send_fail_count + 1,
+ send_fail_timestamp => \'ms_current_timestamp()',
+ send_fail_reason => $err_msg
+ } );
+}
+
+sub bromley_retry_timeout {
+ my $row = shift;
+
+ my $tz = DateTime::TimeZone->new( name => 'local' );
+ my $now = DateTime->now( time_zone => $tz );
+ my $diff = $now - $row->send_fail_timestamp;
+ if ( $diff->minutes < 30 ) {
+ return 1;
+ }
+
+ return 0;
+}
generated by cgit v1.2.3 (git 2.25.1) at 2025-04-27 01:14:15 +0000