diff options
Diffstat (limited to 'bin')
| -rwxr-xr-x | bin/kasabi | 61 | ||||
| -rwxr-xr-x | bin/make_css | 3 | ||||
| -rwxr-xr-x | bin/populate_bing_cache | 4 | ||||
| -rwxr-xr-x | bin/problem-creation-graph | 2 | ||||
| -rwxr-xr-x | bin/problems-filed-graph | 2 | ||||
| -rwxr-xr-x | bin/send-reports | 304 |
6 files changed, 331 insertions, 45 deletions
diff --git a/bin/kasabi b/bin/kasabi index 843531d51..5b99ba4ff 100755 --- a/bin/kasabi +++ b/bin/kasabi @@ -1,5 +1,6 @@ #!/usr/bin/env python +import sys import datetime import json import os.path @@ -11,6 +12,7 @@ import pytassium import psycopg2 import psycopg2.extras from rdfchangesets import BatchChangeSet +from rdflib.namespace import XSD # Set up data access config = yaml.load(open(os.path.abspath(os.path.join(os.path.dirname(__file__), '../conf/general.yml')))) @@ -25,20 +27,27 @@ cursor = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor) report_cursor = db.cursor(cursor_factory=psycopg2.extras.RealDictCursor) def main(): - # Fetch reports that have changed recently - #response, data = dataset.select('select (max(?lastupdate) as ?max) where { ?report <http://data.kasabi.com/dataset/fixmystreet/def/lastUpdate> ?lastupdate }') - #max_lastUpdate = data[1][0]['max'] - last_hour = datetime.datetime.now().replace(minute=0, second=0, microsecond=0) - datetime.timedelta(hours=1) - cursor.execute(""" + + # Check the status of our dataset + response, status = dataset.status() + if response.status not in range(200, 300) or status['storageMode'] == 'read-only': + # We can't import anything, so let's not bother + sys.exit() + + # Fetch reports that have changed since last update in dataset + response, data = dataset.select('select (max(?lastupdate) as ?max) where { ?report <http://data.kasabi.com/dataset/fixmystreet/def/lastUpdate> ?lastupdate }') + max_lastUpdate = data[1][0]['max'] + query = """ SELECT id, latitude, longitude, used_map, council, category, title, detail, (photo IS NOT NULL) as photo, confirmed, lastupdate, whensent, state FROM problem WHERE state not in ('unconfirmed', 'partial') - AND date_trunc('hour', lastupdate) = %s - """, (last_hour,)) -# AND lastupdate > %s -# """, (max_lastUpdate,)) + """ + if len(sys.argv) > 1 and sys.argv[1].isdigit(): + cursor.execute("%s AND id=%%s" % query, (sys.argv[1],)) + else: + cursor.execute("%s AND lastupdate > %%s ORDER BY lastupdate" % query, (str(max_lastUpdate),)) for report in cursor: changeset = FixMyStreetChangeSet(dataset) @@ -50,8 +59,8 @@ def main(): # Canonicalise some values report['latitude'] = round(report['latitude'], 6) # <10cm report['longitude'] = round(report['longitude'], 6) - report['title'] = report['title'].replace('"', r'\"') # Escape double quotes - report['detail'] = report['detail'].replace('"', r'\"') + report['title'] = tidy_string(report['title']) + report['detail'] = tidy_string(report['detail']) report['confirmed'] = report['confirmed'].replace(microsecond=0).isoformat() # Don't want microseconds report['lastupdate'] = report['lastupdate'].replace(microsecond=0).isoformat() report['council'] = sorted(re.sub('\|.*', '', report['council']).split(',')) # Remove missing councils @@ -77,7 +86,10 @@ def main(): changeset.remove_report(report) changeset.add_report(report, states) changeset.apply() - print '{id} change applied'.format(id=report['id']) + +# Escape double quotes and backslashes, remove carriage returns +def tidy_string(s): + return s.replace('\r', '').replace('\\', '\\\\').replace('"', r'\"') class FixMyStreetChangeSet(object): """Something that hosts either or both of a BatchChangeSet and a Turtle @@ -107,27 +119,30 @@ class FixMyStreetChangeSet(object): def apply(self): if len(self.changeset.changesets): - response, data = self.dataset.apply_changeset(self.changeset) - if response.status in range(200, 300): - print 'Change accepted:', data - else: + #response, data = self.dataset.apply_changeset(self.changeset) + # XXX Do everything the above call does, but additionally escape carriage returns to prevent 409 error + api = self.dataset.get_api('update') + g = self.changeset.getGraph() + data = g.serialize(format='xml') + data = data.replace('\r', ' ') + response, data = api.client.request(api.uri, "POST", body=data, headers={"accept" : "*/*", 'content-type':'application/vnd.talis.changeset+xml', 'X_KASABI_APIKEY':api.apikey}) + if response.status not in range(200, 300): print 'Error:', response.status, response.reason, data if self.data: response, data = self.dataset.store_data(self.data, media_type='text/turtle') - if response.status in range(200, 300): - print 'New data accepted:', data - else: + if response.status not in range(200, 300): print 'Error:', response.status, response.reason, data def remove_report(self, report): uri = 'http://data.kasabi.com/dataset/fixmystreet/report/{id}'.format(**report) response, data = self.dataset.select('select ?p ?o where {{ <{0}> ?p ?o }}'.format(uri)) for row in data[1]: - # XXX This throws an error + # Need to set the datatype correctly for the lastUpdate if str(row['p']) == 'http://data.kasabi.com/dataset/fixmystreet/def/lastUpdate': - continue - if re.match('http://data.kasabi.com/dataset/fixmystreet/report/\d+/status/\d+$', str(row['o'])): - uri2 = str(row['o']) + row['o'].datatype = XSD.dateTime + # Delete the referenced statuses + if re.match('http://data.kasabi.com/dataset/fixmystreet/report/\d+/status/\d+$', unicode(row['o'])): + uri2 = unicode(row['o']) response2, data2 = self.dataset.select('select ?p ?o where {{ <{0}> ?p ?o }}'.format(uri2)) for row2 in data2[1]: self.changeset.remove(uri2, row2['p'], row2['o']) diff --git a/bin/make_css b/bin/make_css index 70625b2f4..27cbec1b5 100755 --- a/bin/make_css +++ b/bin/make_css @@ -15,9 +15,10 @@ DIRECTORY=$(cd `dirname $0`/../web && pwd) # FixMyStreet uses compass compass compile --output-style compressed $DIRECTORY/cobrands/fixmystreet +compass compile --output-style compressed $DIRECTORY/cobrands/bromley # The rest are plain sass -for scss in `find $DIRECTORY -name "*.scss" -exec dirname {} \; | uniq | grep -v cobrands/fixmystreet` +for scss in `find $DIRECTORY -name "*.scss" -exec dirname {} \; | uniq | grep -v "cobrands/\(fixmystreet\|bromley\)"` do sass --scss --update --style compressed $scss done diff --git a/bin/populate_bing_cache b/bin/populate_bing_cache index a3bef6759..17c8911d0 100755 --- a/bin/populate_bing_cache +++ b/bin/populate_bing_cache @@ -9,6 +9,8 @@ use Data::Dumper; use FixMyStreet::App; use FixMyStreet::Geocode::Bing; +my $bing_culture = 'en-GB'; + my $reports = FixMyStreet::App->model('DB::Problem')->search( { geocode => undef, @@ -46,7 +48,7 @@ while ( my $report = $reports->next ) { next if $report->geocode; my $j = FixMyStreet::Geocode::Bing::reverse( $report->latitude, - $report->longitude ); + $report->longitude, $bing_culture ); $report->geocode($j); $report->update; diff --git a/bin/problem-creation-graph b/bin/problem-creation-graph index 4bba1cdb8..6692ae724 100755 --- a/bin/problem-creation-graph +++ b/bin/problem-creation-graph @@ -104,6 +104,6 @@ END #echo "gpscript $GPSCRIPT" export GDFONTPATH=/usr/share/fonts/truetype/ttf-bitstream-vera -gnuplot < $GPSCRIPT > fixmystreet/web/bci-live-creation$EXTENSION +gnuplot < $GPSCRIPT > fixmystreet/web/bci-live-creation$EXTENSION 2>/dev/null diff --git a/bin/problems-filed-graph b/bin/problems-filed-graph index dbac35639..8addacd62 100755 --- a/bin/problems-filed-graph +++ b/bin/problems-filed-graph @@ -57,5 +57,5 @@ END #echo "gpscript $GPSCRIPT" export GDFONTPATH=/usr/share/fonts/truetype/ttf-bitstream-vera -gnuplot < $GPSCRIPT > fixmystreet/web/bci-live-line$EXTENSION +gnuplot < $GPSCRIPT > fixmystreet/web/bci-live-line$EXTENSION 2>/dev/null diff --git a/bin/send-reports b/bin/send-reports index 22bd12732..5f3a508b7 100755 --- a/bin/send-reports +++ b/bin/send-reports @@ -22,6 +22,7 @@ use CronFns; use FixMyStreet::App; use EastHantsWSDL; +use BarnetInterfaces::service::ZLBB_SERVICE_ORDER; use Utils; use mySociety::Config; use mySociety::EmailUtil; @@ -30,6 +31,22 @@ use mySociety::Web qw(ent); use Open311; +# maximum number of webservice attempts to send before not trying any more (XXX may be better in config?) +use constant SEND_FAIL_RETRIES_CUTOFF => 3; + +# specific council numbers +use constant COUNCIL_ID_BARNET => 2489; +use constant COUNCIL_ID_EAST_HANTS => 2330; + +use constant MAX_LINE_LENGTH => 132; + +# send_method config values found in by-area config data, for selecting to appropriate method +use constant SEND_METHOD_EMAIL => 'email'; +use constant SEND_METHOD_OPEN311 => 'open311'; +use constant SEND_METHOD_BARNET => 'barnet'; +use constant SEND_METHOD_EAST_HANTS => 'easthants'; +use constant SEND_METHOD_LONDON => 'london'; + # Set up site, language etc. my ($verbose, $nomail) = CronFns::options(); my $base_url = mySociety::Config::get('BASE_URL'); @@ -40,6 +57,9 @@ my $unsent = FixMyStreet::App->model("DB::Problem")->search( { whensent => undef, council => { '!=', undef }, } ); + +my %sending_skipped_by_method = (); + my (%notgot, %note); while (my $row = $unsent->next) { @@ -56,8 +76,10 @@ while (my $row = $unsent->next) { next; } - my $send_email = 0; - my $send_web = 0; + # Due to multiple councils, it's possible to want to send both by email *and* another method + # NB: might need to revist this if multiple councils have custom send methods + my $send_email = 0; + my $send_method = 0; # Template variables for the email my $email_base_url = $cobrand->base_url_for_emails($row->cobrand_data); @@ -118,6 +140,7 @@ while (my $row = $unsent->next) { push @to, [ $council_email, $name ]; @recips = ($council_email); + $send_method = 0; $send_email = 1; $template = Utils::read_file("$FindBin::Bin/../templates/email/emptyhomes/" . $row->lang . "/submit.txt"); @@ -133,15 +156,37 @@ while (my $row = $unsent->next) { foreach my $council (@councils) { my $name = $areas_info->{$council}->{name}; push @dear, $name; - if ($council == 2330) { # E. Hants have a web service - $send_web = 'easthants'; + + # look in the DB to determine if there is a special handler for this council (e.g., open311, or custom) + my $council_config = FixMyStreet::App->model("DB::Open311conf")->search( { area_id => $council} )->first; + $send_method = $council_config->send_method if ($council_config and $council_config->send_method); + if ($council == COUNCIL_ID_EAST_HANTS) { # E. Hants have a web service + $send_method = SEND_METHOD_EAST_HANTS; # TODO: delete? should be in the db $h{category} = 'Customer Services' if $h{category} eq 'Other'; - } elsif ($areas_info->{$council}->{type} eq 'LBO') { # London - $send_web = 'london'; - } elsif ( my $endpoint = FixMyStreet::App->model("DB::Open311conf")->search( { area_id => $council, endpoint => { '!=', '' } } )->first ) { - push @open311_councils, $endpoint; - $send_web = 'open311'; - } else { + } + + # if council lookup provided no explicit send_method, maybe there's some other criterion for setting it: + if (! $send_method) { + if ($areas_info->{$council}->{type} eq 'LBO') { # London + $send_method = SEND_METHOD_LONDON; + } + } + $send_email = 1 unless $send_method; # default to email if nothing explicit was provided + + # currently: open311 or Barnet without an endpoint is useless, so check the endpoint is set + if ($send_method eq SEND_METHOD_OPEN311 or $send_method eq SEND_METHOD_BARNET) { + if ($council_config->endpoint) { + if ($send_method eq SEND_METHOD_OPEN311) { + push @open311_councils, $council_config; + } + } else { + print "Warning: no endpoint specified in config data for council=$council (will try email instead)\n"; + $send_method = 0; + $send_email = 1; + } + } + + if ($send_email) { my $contact = FixMyStreet::App->model("DB::Contact")->find( { deleted => 0, area_id => $council, @@ -160,7 +205,6 @@ while (my $row = $unsent->next) { } push @to, [ $council_email, $name ]; $recips{$council_email} = 1; - $send_email = 1; } } @recips = keys %recips; @@ -199,15 +243,19 @@ while (my $row = $unsent->next) { } - unless ($send_email || $send_web) { + unless ($send_email || $send_method) { die 'Report not going anywhere for ID ' . $row->id . '!'; } if (mySociety::Config::get('STAGING_SITE')) { # on a staging server send emails to ourselves rather than the councils - @recips = ( mySociety::Config::get('CONTACT_EMAIL') ); - $send_web = 0; - $send_email = 1; + # ...webservice calls will only go through if explictly allowed here: + my @testing_councils = (COUNCIL_ID_BARNET); + unless (grep {$row->council eq $_} @testing_councils) { + @recips = ( mySociety::Config::get('CONTACT_EMAIL') ); + $send_method = 0; + $send_email = 1; + } } elsif ($site eq 'emptyhomes') { my $council = $row->council; my $country = $areas_info->{$council}->{country}; @@ -243,17 +291,28 @@ while (my $row = $unsent->next) { ); } - if ($send_web eq 'easthants') { + if ($send_method eq SEND_METHOD_EAST_HANTS) { $h{message} = construct_easthants_message(%h); if (!$nomail) { $result *= post_easthants_message(%h); } - } elsif ($send_web eq 'london') { + } elsif ($send_method eq SEND_METHOD_BARNET) { + $h{message} = construct_barnet_message(%h); + if (!$nomail) { + if (my $cutoff_msg = does_exceed_cutoff_limit($row, "barnet")) { + print "$cutoff_msg\n" if $verbose; + } else { + my ($barnet_result, $err_msg) = post_barnet_message( $row, %h ); + update_send_fail_data($row, $err_msg) if $barnet_result; + $result *= $barnet_result; + } + } + } elsif ($send_method eq SEND_METHOD_LONDON) { $h{message} = construct_london_message(%h); if (!$nomail) { $result *= post_london_report( $row, %h ); } - } elsif ($send_web eq 'open311') { + } elsif ($send_method eq SEND_METHOD_OPEN311) { foreach my $conf ( @open311_councils ) { print 'posting to end point for ' . $conf->area_id . "\n" if $verbose; @@ -316,6 +375,16 @@ if ($verbose) { } } +if ($verbose and keys %sending_skipped_by_method) { + my $c = 0; + print "\nProblem reports that send-reports did not attempt to send because retries >= " . SEND_FAIL_RETRIES_CUTOFF . ":\n"; + foreach my $send_method (sort keys %sending_skipped_by_method) { + printf " %-24s %4d\n", "$send_method:", $sending_skipped_by_method{$send_method}; + $c+=$sending_skipped_by_method{$send_method}; + } + printf " %-24s %4d\n", "Total:", $c; +} + sub _get_district_for_contact { my ( $lat, $lon ) = @_; my $district = @@ -358,6 +427,8 @@ sub construct_easthants_message { $message .= <<EOF; Subject: $h{title} +Category: $h{category} + Details: $h{detail} $h{fuzzy}, or to provide an update on the problem, please visit the following link: @@ -390,6 +461,126 @@ sub post_easthants_message { return $return; } +# currently just blind copy of construct_easthants_message +sub construct_barnet_message { + my %h = @_; + my $message = <<EOF; +Subject: $h{title} + +Details: $h{detail} + +$h{fuzzy}, or to provide an update on the problem, please visit the following link: + +$h{url} + +$h{closest_address} +EOF +} + +sub post_barnet_message { + my ( $problem, %h ) = @_; + my $return = 1; + my $err_msg = ""; + + my $default_kbid = 14; # This is the default, "Street Scene" + my $kbid = sprintf( "%050d", Utils::barnet_categories()->{$h{category}} || $default_kbid); + + my $geo_code = "$h{easting} $h{northing}"; + + my $interface = BarnetInterfaces::service::ZLBB_SERVICE_ORDER->new(); + + my ($nearest_postcode, $nearest_street) = ('', ''); + for ($h{closest_address}) { + $nearest_postcode = sprintf("%-10s", $1) if /Nearest postcode [^:]+: ((\w{1,4}\s?\w+|\w+))/; + # use partial postcode or comma as delimiter, strip leading number (possible letter 221B) off too + # "99 Foo Street, London N11 1XX" becomes Foo Street + # "99 Foo Street N11 1XX" becomes Foo Street + $nearest_street = $1 if /Nearest road [^:]+: (?:\d+\w? )?(.*?)(\b[A-Z]+\d|,|$)/m; + } + my $postcode = mySociety::PostcodeUtil::is_valid_postcode($h{query}) + ? $h{query} : $nearest_postcode; # use given postcode if available + + # note: endpoint can be of form 'https://username:password@url' + my $council_config = FixMyStreet::App->model("DB::Open311conf")->search( { area_id => COUNCIL_ID_BARNET} )->first; + if ($council_config and $council_config->endpoint) { + $interface->set_proxy($council_config->endpoint); + # Barnet web service doesn't like namespaces in the elements so use a prefix + $interface->set_prefix('urn'); + } else { + die "Barnet webservice FAIL: looks like you're missing some config data: no endpoint (URL) found for area_id=" . COUNCIL_ID_BARNET; + } + + eval { + my $result = $interface->Z_CRM_SERVICE_ORDER_CREATE( { + ET_RETURN => { # ignored by server + item => { + TYPE => "", ID => "", NUMBER => "", MESSAGE => "", LOG_NO => "", LOG_MSG_NO => "", + MESSAGE_V1 => "", MESSAGE_V2 => "", MESSAGE_V3 => "", MESSAGE_V4 => "", PARAMETER => "", + ROW => "", FIELD => "", SYSTEM => "", + }, + }, + IT_PROBLEM_DESC => { # MyTypes::TABLE_OF_CRMT_SERVICE_REQUEST_TEXT + item => [ # MyTypes::CRMT_SERVICE_REQUEST_TEXT + map { { TEXT_LINE => $_ } } split_text_with_entities(ent(encode_utf8($h{message})), 132) # char132 + ], + }, + IV_CUST_EMAIL => truncate_string_with_entities(ent(encode_utf8($h{email})), 241), # char241 + IV_CUST_NAME => truncate_string_with_entities(ent(encode_utf8($h{name})), 50), # char50 + IV_KBID => $kbid, # char50 + IV_PROBLEM_ID => $h{id}, # char35 + IV_PROBLEM_LOC => { # MyTypes::BAPI_TTET_ADDRESS_COM + COUNTRY2 => 'GB', # char2 + REGION => "", # char3 + COUNTY => "", # char30 + CITY => "", # char30 + POSTALCODE => $postcode, # char10 + STREET => truncate_string_with_entities(ent(encode_utf8($nearest_street)), 30), # char30 + STREETNUMBER => "", # char5 + GEOCODE => $geo_code, # char32 + }, + IV_PROBLEM_SUB => truncate_string_with_entities(ent(encode_utf8($h{title})), 40), # char40 + }, + ); + if ($result) { + # currently not using this: get_EV_ORDER_GUID (maybe that's the customer number in the CRM) + if (my $barnet_id = $result->get_EV_ORDER_NO()) { + $problem->external_id( $barnet_id ); + $problem->external_body( 'Barnet Borough Council' ); # better to use $problem->body()? + $problem->send_method_used('barnet'); + $return = 0; + } else { + my @returned_items = split /<item[^>]*>/, $result->get_ET_RETURN; + my @messages = (); + foreach my $item (@returned_items) { + if ($item=~/<MESSAGE [^>]*>\s*(\S.*?)<\/MESSAGE>/) { # if there's a non-null MESSAGE in there, grab it + push @messages, $1; # best stab at extracting useful error message back from convoluted response + } + } + push @messages, "service returned no external id" unless @messages; + $err_msg = "Failed (problem id $h{id}): " . join(" \n ", @messages); + } + } else { + my %fault = ( + 'code' => $result->get_faultcode(), + 'actor' => $result->get_faultactor(), + 'string' => $result->get_faultstring(), + 'detail' => $result->get_detail(), # possibly only contains debug info + ); + $fault{$_}=~s/^\s*|\s*$//g foreach keys %fault; + $fault{actor}&&=" (actor: $fault{actor})"; + $fault{'detail'} &&= "\n" . $fault{'detail'}; + $err_msg = "Failed (problem id $h{id}): Fault $fault{code}$fault{actor}\n$fault{string}$fault{detail}"; + } + + }; + print "$err_msg\n" if $err_msg; + if ($@) { + my $e = shift; + print "Caught an error: $@\n"; + } + return ($return, $err_msg); +} + # London sub construct_london_message { @@ -489,3 +680,80 @@ sub london_lookup { return $str; } +# for barnet webservice: max-length fields require truncate and split + +# truncate_string_with_entities +# args: text to truncate +# max number of chars +# returns: string truncated +# Note: must not partially truncate an entity (e.g., &) +sub truncate_string_with_entities { + my ($str, $max_len) = @_; + my $retVal = ""; + foreach my $chunk (split /(\&(?:\#\d+|\w+);)/, $str) { + if ($chunk=~/^\&(\#\d+|\w+);$/){ + my $next = $retVal.$chunk; + last if length $next > $max_len; + $retVal=$next + } else { + $retVal.=$chunk; + if (length $retVal > $max_len) { + $retVal = substr($retVal, 0, $max_len); + last + } + } + } + return $retVal +} + +# split_text_with_entities into lines +# args: text to be broken into lines +# max length (option: uses constant MAX_LINE_LENGTH) +# returns: array of lines +# Must not to split an entity (e.g., &) +# Not worrying about hyphenating here, since a word is only ever split if +# it's longer than the whole line, which is uncommon in genuine problem reports +sub split_text_with_entities { + my ($text, $max_line_length) = @_; + $max_line_length ||= MAX_LINE_LENGTH; + my @lines; + foreach my $line (split "\n", $text) { + while (length $line > $max_line_length) { + if (! ($line =~ s/^(.{1,$max_line_length})\s// # break on a space + or $line =~ s/^(.{1,$max_line_length})(\&(\#\d+|\w+);)/$2/ # break before an entity + or $line =~ s/(.{$max_line_length})//)) { # break the word ruthlessly + $line =~ s/(.*)//; # otherwise gobble whole line (which is now shorter than max length) + } + push @lines, $1; + } + push @lines, $line; + } + return @lines; +} + +# tests send_fail_count agains cutoff limit +# args: problem (row from problem db) +# returns false if there is no cutoff, otherwise error message +sub does_exceed_cutoff_limit { + my ($problem, $council_name) = @_; + my $err_msg = ""; + if ($problem->send_fail_count >= SEND_FAIL_RETRIES_CUTOFF) { + $sending_skipped_by_method{$council_name || '?'}++; + $council_name &&= " to $council_name"; + $err_msg = "skipped: problem id=" . $problem->id . " send$council_name has failed " + . $problem->send_fail_count . " times, cutoff is " . SEND_FAIL_RETRIES_CUTOFF; + } + return $err_msg; +} + +# update_send_fail_data records the failure (of a webservice send) +# args: problem (row from problem db) +# returns: no return value (updates record) +sub update_send_fail_data { + my ($problem, $err_msg) = @_; + $problem->update( { + send_fail_count => $problem->send_fail_count + 1, + send_fail_timestamp => \'ms_current_timestamp()', + send_fail_reason => $err_msg + } ); +} |