path: root/bin/send-reports

#!/usr/bin/env perl

# send-reports:
# Send new problem reports to councils
#
# Copyright (c) 2011 UK Citizens Online Democracy. All rights reserved.
# Email: matthew@mysociety.org. WWW: http://www.mysociety.org

use strict;
use warnings;
require 5.8.0;

use Digest::MD5;
use Encode;
use Error qw(:try);
use JSON;
use LWP::UserAgent;
use LWP::Simple;
use CGI; # Trying awkward kludge
use CronFns;

use FixMyStreet::App;

use EastHantsWSDL;
use BarnetInterfaces::service::ZLBB_SERVICE_ORDER;
use Utils;
use mySociety::Config;
use mySociety::EmailUtil;
use mySociety::MaPit;
use mySociety::Web qw(ent);

use Open311;

# maximum number of webservice attempts to send before not trying any more (XXX may be better in config?)
use constant SEND_FAIL_RETRIES_CUTOFF => 3;

# specific council numbers
use constant COUNCIL_ID_BARNET     => 2489;
use constant COUNCIL_ID_EAST_HANTS => 2330;

# FMB aka FixMyBarangay numbers
use constant FMB_ID_LUZ     => 1;   # barangay Luz
use constant FMB_ID_BSN     => 2;   # barangay Basak San Nicolas

use constant MAX_LINE_LENGTH       => 132;

# send_method config values found in by-area config data, for selecting to appropriate method
use constant SEND_METHOD_EMAIL      => 'email';
use constant SEND_METHOD_OPEN311    => 'open311';
use constant SEND_METHOD_BARNET     => 'barnet';
use constant SEND_METHOD_EAST_HANTS => 'easthants';
use constant SEND_METHOD_LONDON     => 'london';

# Set up site, language etc.
my ($verbose, $nomail) = CronFns::options();
my $base_url = mySociety::Config::get('BASE_URL');
my $site = CronFns::site($base_url);

my $unsent = FixMyStreet::App->model("DB::Problem")->search( {
    state => [ 'confirmed', 'fixed' ],
    whensent => undef,
    council => { '!=', undef },
} );

my %sending_skipped_by_method = ();

my (%notgot, %note);
while (my $row = $unsent->next) {

    my $cobrand = FixMyStreet::Cobrand->get_class_for_moniker($row->cobrand)->new();

    # Cobranded and non-cobranded messages can share a database. In this case, the conf file 
    # should specify a vhost to send the reports for each cobrand, so that they don't get sent 
    # more than once if there are multiple vhosts running off the same database. The email_host
    # call checks if this is the host that sends mail for this cobrand.
    next unless $cobrand->email_host();
    $cobrand->set_lang_and_domain($row->lang, 1);
    if ( $row->is_from_abuser ) {
        $row->update( { state => 'hidden' } );
        next;
    }

    # Due to multiple councils, it's possible to want to send both by email *and* another method
    # NB: might need to revist this if multiple councils have custom send methods
    my $send_email = 0; 
    my $send_method = 0;

    # Template variables for the email
    my $email_base_url = $cobrand->base_url_for_emails($row->cobrand_data);
    my %h = map { $_ => $row->$_ } qw/id title detail name category latitude longitude used_map/;
    map { $h{$_} = $row->user->$_ } qw/email phone/;
    $h{confirmed} = DateTime::Format::Pg->format_datetime( $row->confirmed->truncate (to => 'second' ) );

    $h{query} = $row->postcode;
    $h{url} = $email_base_url . '/report/' . $row->id;
    $h{phone_line} = $h{phone} ? _('Phone:') . " $h{phone}\n\n" : '';
    if ($row->photo) {
        $h{has_photo} = _("This web page also contains a photo of the problem, provided by the user.") . "\n\n";
        $h{image_url} = $email_base_url . '/photo/' . $row->id . '.full.jpeg';
    } else {
        $h{has_photo} = '';
        $h{image_url} = '';
    }
    $h{fuzzy} = $row->used_map ? _('To view a map of the precise location of this issue')
        : _('The user could not locate the problem on a map, but to see the area around the location they entered');
    $h{closest_address} = '';

    # If we are in the UK include eastings and northings, and nearest stuff
    $h{easting_northing} = '';
    if ( $cobrand->country eq 'GB' ) {

        ( $h{easting}, $h{northing} ) = Utils::convert_latlon_to_en( $h{latitude}, $h{longitude} );

        # email templates don't have conditionals so we need to farmat this here
        $h{easting_northing}                             #
          = "Easting: $h{easting}\n\n"                   #
          . "Northing: $h{northing}\n\n";

    }

    if ( $row->used_map ) {
        $h{closest_address} = $cobrand->find_closest( $h{latitude}, $h{longitude}, $row );
    }

    my (@to, @recips, $template, $areas_info, @open311_councils);
    if ($site eq 'emptyhomes') {

        my $council = $row->council;
        $areas_info = mySociety::MaPit::call('areas', $council);
        my $name = $areas_info->{$council}->{name};
        my $contact = FixMyStreet::App->model("DB::Contact")->find( {
            deleted => 0,
            area_id => $council,
            category => 'Empty property',
        } );
        my ($council_email, $confirmed, $note) = ( $contact->email, $contact->confirmed, $contact->note );
        unless ($confirmed) {
            $note = 'Council ' . $council . ' deleted' unless $note;
            $council_email = 'N/A' unless $council_email;
            $notgot{$council_email}{$row->category}++;
            $note{$council_email}{$row->category} = $note;
            next;
        }

        push @to, [ $council_email, $name ];
        @recips = ($council_email);
        $send_method = 0;
        $send_email = 1;
        $template = Utils::read_file("$FindBin::Bin/../templates/email/emptyhomes/" . $row->lang . "/submit.txt");

    } else {

        # XXX Needs locks!
        my @all_councils = split /,|\|/, $row->council;
        my ($councils, $missing) = $row->council =~ /^([\d,]+)(?:\|([\d,]+))?/;
        my @councils = split(/,/, $councils);
        $areas_info = mySociety::MaPit::call('areas', \@all_councils);
        my (@dear, %recips);
        my $all_confirmed = 1;
        foreach my $council (@councils) {
            my $name = $areas_info->{$council}->{name};
            push @dear, $name;
            
            # look in the DB to determine if there is a special handler for this council (e.g., open311, or custom)
            my $council_config = FixMyStreet::App->model("DB::Open311conf")->search( { area_id => $council} )->first;
            $send_method = $council_config->send_method if ($council_config and $council_config->send_method);
            if ($council == COUNCIL_ID_EAST_HANTS) { # E. Hants have a web service
                $send_method = SEND_METHOD_EAST_HANTS; # TODO: delete? should be in the db
                $h{category} = 'Customer Services' if $h{category} eq 'Other';
            } 

            # if council lookup provided no explicit send_method, maybe there's some other criterion for setting it:
            if (! $send_method) {
                if ($areas_info->{$council}->{type} eq 'LBO') { # London
                    $send_method = SEND_METHOD_LONDON;
                }
            }
            $send_email = 1 unless $send_method; # default to email if nothing explicit was provided
            
            # currently: open311 or Barnet without an endpoint is useless, so check the endpoint is set
            if ($send_method eq SEND_METHOD_OPEN311 or $send_method eq SEND_METHOD_BARNET) { 
                if ($council_config->endpoint) {
                    if ($send_method eq SEND_METHOD_OPEN311) {
                        push @open311_councils, $council_config;                        
                    }
                } else {
                    print "Warning: no endpoint specified in config data for council=$council (will try email instead)\n";
                    $send_method = 0;
                    $send_email = 1;
                }
            }
            
            if ($send_email) {
                my $contact = FixMyStreet::App->model("DB::Contact")->find( {
                    deleted => 0,
                    area_id => $council,
                    category => $row->category
                } );
                my ($council_email, $confirmed, $note) = ( $contact->email, $contact->confirmed, $contact->note );
                $council_email = essex_contact($row->latitude, $row->longitude) if $council == 2225;
                $council_email = oxfordshire_contact($row->latitude, $row->longitude) if $council == 2237 && $council_email eq 'SPECIAL';
                unless ($confirmed) {
                    $all_confirmed = 0;
                    $note = 'Council ' . $row->council . ' deleted'
                        unless $note;
                    $council_email = 'N/A' unless $council_email;
                    $notgot{$council_email}{$row->category}++;
                    $note{$council_email}{$row->category} = $note;
                }
                push @to, [ $council_email, $name ];
                $recips{$council_email} = 1;
            }
        }
        @recips = keys %recips;
        next unless $all_confirmed;

        $template = 'submit.txt';
        $template = 'submit-brent.txt' if $row->council eq 2488 || $row->council eq 2237;
        my $template_path = FixMyStreet->path_to( "templates", "email", $cobrand->moniker, $template )->stringify;
        $template_path = FixMyStreet->path_to( "templates", "email", "default", $template )->stringify
            unless -e $template_path;
        $template = Utils::read_file( $template_path );

        if ($h{category} eq _('Other')) {
            $h{category_footer} = _('this type of local problem');
            $h{category_line} = '';
        } else {
            $h{category_footer} = "'" . $h{category} . "'";
            $h{category_line} = sprintf(_("Category: %s"), $h{category}) . "\n\n";
        }

        $h{councils_name} = join(_(' and '), @dear);
        if ($h{category} eq _('Other')) {
            $h{multiple} = @dear>1 ? "[ " . _("This email has been sent to both councils covering the location of the problem, as the user did not categorise it; please ignore it if you're not the correct council to deal with the issue, or let us know what category of problem this is so we can add it to our system.") . " ]\n\n"
                : '';
        } else {
            $h{multiple} = @dear>1 ? "[ " . _("This email has been sent to several councils covering the location of the problem, as the category selected is provided for all of them; please ignore it if you're not the correct council to deal with the issue.") . " ]\n\n"
                : '';
        }
        $h{missing} = ''; 
        if ($missing) {
            my $name = $areas_info->{$missing}->{name};
            $h{missing} = '[ '
              . sprintf(_('We realise this problem might be the responsibility of %s; however, we don\'t currently have any contact details for them. If you know of an appropriate contact address, please do get in touch.'), $name)
              . " ]\n\n";
        }

    }

    unless ($send_email || $send_method) {
        die 'Report not going anywhere for ID ' . $row->id . '!';
    }

    if (mySociety::Config::get('STAGING_SITE')) {
        # on a staging server send emails to ourselves rather than the councils
        # ...webservice calls will only go through if explictly allowed here:
        my @testing_councils = (COUNCIL_ID_BARNET, FMB_ID_LUZ, FMB_ID_BSN);
        unless (grep {$row->council eq $_} @testing_councils) {
            @recips = ( mySociety::Config::get('CONTACT_EMAIL') );
            $send_method = 0;
            $send_email = 1;
        }
    } elsif ($site eq 'emptyhomes') {
        my $council = $row->council;
        my $country = $areas_info->{$council}->{country};
        if ($country eq 'W') {
            push @recips, 'shelter@' . mySociety::Config::get('EMAIL_DOMAIN');
        } else {
            push @recips, 'eha@' . mySociety::Config::get('EMAIL_DOMAIN');
        }
    }

    # Special case for this parish council
    # if ($address && $address =~ /Sprowston/ && $row->council == 2233 && $row->category eq 'Street lighting') {
    #     $h{councils_name} = 'Sprowston Parish Council';
    #     my $e = 'parishclerk' . '@' . 'sprowston-pc.gov.uk';
    #     @to = ( [ $e, $h{councils_name} ] );
    #     @recips = ($e);
    # }

    # Multiply results together, so one success counts as a success.
    my $result = -1;

    if ($send_email) {
        $result *= FixMyStreet::App->send_email_cron(
            {
                _template_ => $template,
                _parameters_ => \%h,
                To => \@to,
                From => [ $row->user->email, $row->name ],
            },
            mySociety::Config::get('CONTACT_EMAIL'),
            \@recips,
            $nomail
        );
    }

    if ($send_method eq SEND_METHOD_EAST_HANTS) {
        $h{message} = construct_easthants_message(%h);
        if (!$nomail) {
            $result *= post_easthants_message(%h);
        }
    } elsif ($send_method eq SEND_METHOD_BARNET) {
        $h{message} = construct_barnet_message(%h);
        if (!$nomail) {
            if (my $cutoff_msg = does_exceed_cutoff_limit($row, "barnet")) {
                print "$cutoff_msg\n" if $verbose;
            } else {
                my ($barnet_result, $err_msg) = post_barnet_message( $row, %h );
                update_send_fail_data($row, $err_msg) if $barnet_result;
                $result *= $barnet_result;
            } 
        }
    } elsif ($send_method eq SEND_METHOD_LONDON) {
        $h{message} = construct_london_message(%h);
        if (!$nomail) {
            $result *= post_london_report( $row, %h );
        }
    } elsif ($send_method eq SEND_METHOD_OPEN311) {
        foreach my $conf ( @open311_councils ) {
            print 'posting to end point for ' . $conf->area_id . "\n" if $verbose;

            my $contact = FixMyStreet::App->model("DB::Contact")->find( {
                deleted => 0,
                area_id => $conf->area_id,
                category => $row->category
            } );

            my $open311 = Open311->new(
                jurisdiction => $conf->jurisdiction,
                endpoint     => $conf->endpoint,
                api_key      => $conf->api_key,
            );

            # non standard west berks end points
            if ( $row->council =~ /2619/ ) {
                $open311->endpoints( { services => 'Services', requests => 'Requests' } );
            }

            # required to get round issues with CRM constraints
            if ( $row->council =~ /2218/ ) {
                $row->user->name( $row->user->id . ' ' . $row->user->name );
            }

            if ($cobrand->moniker eq 'fixmybarangay') {
                # FixMyBarangay endpoints expect external_id as an attribute
                # TODO: nice if this was indicated in the open311conf entry?
                $row->extra( [ { 'name' => 'external_id', 'value' => $row->id  } ]  ); 
            }
            
            my $resp = $open311->send_service_request( $row, \%h, $contact->email );

            # make sure we don't save user changes from above
            if ( $row->council =~ /2218/ ) {
                $row->discard_changes();
            }

            if ($cobrand->moniker eq 'fixmybarangay') {
                $row->discard_changes();              
            }

            if ( $resp ) {
                $row->external_id( $resp );
                $result *= 0;
            } else {
                $result *= 1;
                # temporary fix to resolve some issues with west berks
                if ( $row->council =~ /2619/ ) {
                    $result *= 0;
                }
            }
        }
    }

    if ($result == mySociety::EmailUtil::EMAIL_SUCCESS) {
        $row->update( {
            whensent => \'ms_current_timestamp()',
            lastupdate => \'ms_current_timestamp()',
        } );
    }
}

if ($verbose) {
    print "Council email addresses that need checking:\n" if keys %notgot;
    foreach my $e (keys %notgot) {
        foreach my $c (keys %{$notgot{$e}}) {
            print $notgot{$e}{$c} . " problem, to $e category $c (" . $note{$e}{$c}. ")\n";
        }
    }
}

if ($verbose and keys %sending_skipped_by_method) {
    my $c = 0;
    print "\nProblem reports that send-reports did not attempt to send because retries >= " . SEND_FAIL_RETRIES_CUTOFF . ":\n";
    foreach my $send_method (sort keys %sending_skipped_by_method) {
        printf "    %-24s %4d\n", "$send_method:", $sending_skipped_by_method{$send_method};
        $c+=$sending_skipped_by_method{$send_method};
    }
    printf "    %-24s %4d\n", "Total:", $c;
}

sub _get_district_for_contact {
    my ( $lat, $lon ) = @_;
    my $district =
      mySociety::MaPit::call( 'point', "4326/$lon,$lat", type => 'DIS' );
    ($district) = keys %$district;
    return $district;
}

# Essex has different contact addresses depending upon the district
# Might be easier if we start storing in the db all areas covered by a point
# Will do for now :)
sub essex_contact {
    my $district = _get_district_for_contact(@_);
    my $email;
    $email = 'eastarea' if $district == 2315 || $district == 2312;
    $email = 'midarea' if $district == 2317 || $district == 2314 || $district == 2316;
    $email = 'southarea' if $district == 2319 || $district == 2320 || $district == 2310;
    $email = 'westarea' if $district == 2309 || $district == 2311 || $district == 2318 || $district == 2313;
    die "Returned district $district which is not in Essex!" unless $email;
    return "highways.$email\@essexcc.gov.uk";
}

# Oxfordshire has different contact addresses depending upon the district
sub oxfordshire_contact {
    my $district = _get_district_for_contact(@_);
    my $email;
    $email = 'northernarea' if $district == 2419 || $district == 2420 || $district == 2421;
    $email = 'southernarea' if $district == 2417 || $district == 2418;
    die "Returned district $district which is not in Oxfordshire!" unless $email;
    return "$email\@oxfordshire.gov.uk";
}

# East Hampshire

sub construct_easthants_message {
    my %h = @_;
    my $message = '';
    $message .= "[ This report was also sent to the district council covering the location of the problem, as the user did not categorise it; please ignore if you're not the correct council to deal with the issue. ]\n\n"
        if $h{multiple};
    $message .= <<EOF;
Subject: $h{title}

Category: $h{category}

Details: $h{detail}

$h{fuzzy}, or to provide an update on the problem, please visit the following link:

$h{url}

$h{closest_address}
EOF
    return $message;
}

my $eh_service;
sub post_easthants_message {
    my %h = @_;
    my $return = 1;
    $eh_service ||= EastHantsWSDL->on_fault(sub { my($soap, $res) = @_; die ref $res ? $res->faultstring : $soap->transport->status, "\n"; });
    try {
        # ServiceName, RemoteCreatedBy, Salutation, FirstName, Name, Email, Telephone, HouseNoName, Street, Town, County, Country, Postcode, Comments, FurtherInfo, ImageURL
        my $message = ent(encode_utf8($h{message}));
        my $name = ent(encode_utf8($h{name}));
        my $result = $eh_service->INPUTFEEDBACK(
            $h{category}, 'FixMyStreet', '', '', $name, $h{email}, $h{phone},
            '', '', '', '', '', '', $message, 'Yes', $h{image_url}
        );
        $return = 0 if $result eq 'Report received';
    } otherwise {
        my $e = shift;
        print "Caught an error: $e\n";
    };
    return $return;
}

# currently just blind copy of construct_easthants_message
sub construct_barnet_message {
    my %h = @_;
    my $message = <<EOF;
Subject: $h{title}

Details: $h{detail}

$h{fuzzy}, or to provide an update on the problem, please visit the following link:

$h{url}

$h{closest_address}
EOF
}

sub post_barnet_message {
    my ( $problem, %h ) = @_;
    my $return = 1;
    my $err_msg = "";

    my $default_kbid = 14; # This is the default, "Street Scene"
    my $kbid = sprintf( "%050d",  Utils::barnet_categories()->{$h{category}} || $default_kbid);

    my $geo_code = "$h{easting} $h{northing}"; 

    my $interface = BarnetInterfaces::service::ZLBB_SERVICE_ORDER->new();
    
    my ($nearest_postcode, $nearest_street);
    for ($h{closest_address}) {
        $nearest_postcode = sprintf("%-10s", $1) if /Nearest postcode [^:]+: ((\w{1,4}\s?\w+|\w+))/;
        # use partial postcode or comma as delimiter, strip leading number (possible letter 221B) off too
        #    "99 Foo Street, London N11 1XX" becomes Foo Street
        #    "99 Foo Street N11 1XX" becomes Foo Street
        $nearest_street = sprintf("%-30s", $1) if /Nearest road [^:]+: (?:\d+\w? )?(.*?)(\b[A-Z]+\d|,|$)/m;         
    }
    my $postcode = $h{postcode} || $nearest_postcode; # use given postcode if available
    
    # note: endpoint can be of form 'https://username:password@:url'
    my $council_config = FixMyStreet::App->model("DB::Open311conf")->search( { area_id => COUNCIL_ID_BARNET} )->first;
    if ($council_config and $council_config->endpoint) {
        $interface->set_proxy($council_config->endpoint);
        # Barnet web service doesn't like namespaces in the elements so use a prefix
        $interface->set_prefix('urn');
    } else {
        die "Barnet webservice FAIL: looks like you're missing some config data: no endpoint (URL) found for area_id=" . COUNCIL_ID_BARNET;
    }
    
    eval {    
        my $result = $interface->Z_CRM_SERVICE_ORDER_CREATE( {
            ET_RETURN =>  { # ignored by server
              item =>  {
                  TYPE => "", ID => "", NUMBER => "", MESSAGE => "", LOG_NO => "", LOG_MSG_NO => "", 
                  MESSAGE_V1 => "", MESSAGE_V2 => "", MESSAGE_V3 => "", MESSAGE_V4 => "", PARAMETER => "", 
                  ROW =>  "", FIELD => "", SYSTEM => "",
                },
            },
            IT_PROBLEM_DESC =>  { # MyTypes::TABLE_OF_CRMT_SERVICE_REQUEST_TEXT
              item =>  [ # MyTypes::CRMT_SERVICE_REQUEST_TEXT
                map { { TEXT_LINE => $_ } } split_text_with_entities(ent(encode_utf8($h{message})), 132) # char132
              ],
            },
            IV_CUST_EMAIL => truncate_string_with_entities(ent(encode_utf8($h{email})), 241), # char241
            IV_CUST_NAME  => truncate_string_with_entities(ent(encode_utf8($h{name})),   50), # char50
            IV_KBID => $kbid,        # char50
            IV_PROBLEM_ID => $h{id}, # char35
            IV_PROBLEM_LOC =>  {     # MyTypes::BAPI_TTET_ADDRESS_COM
              COUNTRY2 => 'GB',      # char2
              REGION => "",          # char3
              COUNTY => "",          # char30
              CITY => "",            # char30
              POSTALCODE => $postcode,   # char10
              STREET => $nearest_street, # char30
              STREETNUMBER => "",    # char5
              GEOCODE => $geo_code,  # char32
            },
            IV_PROBLEM_SUB => truncate_string_with_entities(ent(encode_utf8($h{title})), 40), # char40
          },
        );
        if ($result) {
            # currently not using this: get_EV_ORDER_GUID (maybe that's the customer number in the CRM)
            if (my $barnet_id = $result->get_EV_ORDER_NO()) {
                $problem->external_id( $barnet_id );
                $problem->external_body( 'Barnet Borough Council' ); # better to use $problem->body()?
                $problem->send_method_used('barnet');
                $return = 0;
            } else { 
                $err_msg = "Failed (problem id $h{id}): service returned no external id";
            }
        } else {
            my %fault = (
                    'code' =>  $result->get_faultcode(),
                    'actor' =>  $result->get_faultactor(),
                    'string' =>  $result->get_faultstring(),
                    # 'detail' =>  $result->get_detail(), # possibly only contains debug info
                );
            $fault{$_}=~s/^\s*|\s*$//g foreach keys %fault;
            $fault{actor}&&=" (actor: $fault{actor})";
            $err_msg = "Failed (problem id $h{id}): Fault $fault{code}$fault{actor}\n$fault{string}";
        }

    };
    print "$err_msg\n" if $err_msg;
    if ($@) {
        my $e = shift;
        print "Caught an error: $@\n"; 
    }
    return ($return, $err_msg);
}

# London

sub construct_london_message {
    my %h = @_;
    return <<EOF,
A user of FixMyStreet has submitted the following report of a local
problem that they believe might require your attention.

Subject: $h{title}

Details: $h{detail}

$h{fuzzy}, or to provide an update on the problem, please visit the
following link:

$h{url}

$h{closest_address}
Yours,
The FixMyStreet team
EOF
}

sub post_london_report {
    my ( $problem, %h ) = @_;
    my $phone = $h{phone};
    my $mobile = '';
    if ($phone && $phone =~ /^\s*07/) {
        $mobile = $phone;
        $phone = '';
    }
    my ($first, $last) = $h{name} =~ /^(\S*)(?: (.*))?$/;
    my %params = (
        Key => mySociety::Config::get('LONDON_REPORTIT_KEY'),
        Signature => Digest::MD5::md5_hex( $h{confirmed} . mySociety::Config::get('LONDON_REPORTIT_SECRET') ),
        Type => Utils::london_categories()->{$h{category}},
        RequestDate => $h{confirmed},
        RequestMethod => 'Web',
        ExternalId => $h{url},
        'Customer.Title' => '',
        'Customer.FirstName' => $first,
        'Customer.Surname' => $last,
        'Customer.Email' => $h{email},
        'Customer.Phone' => $phone,
        'Customer.Mobile' => $mobile,
        'ProblemDescription' => $h{message},
    );
    if ($h{used_map}) {
        $params{'Location.Latitude'} = $h{latitude};
        $params{'Location.Longitude'} = $h{longitude};
    } elsif (mySociety::PostcodeUtil::is_valid_postcode($h{query})) {
        # Didn't use map, and entered postcode, so use that.
        $params{'Location.Postcode'} = $h{query};
    } else {
        # Otherwise, lat/lon is all we have, even if it's wrong.
        $params{'Location.Latitude'} = $h{latitude};
        $params{'Location.Longitude'} = $h{longitude};
    }
    if ($h{has_photo}) {
        $params{'Document1.Name'} = 'Photograph';
        $params{'Document1.MimeType'} = 'image/jpeg';
        $params{'Document1.URL'} = $h{image_url};
        $params{'Document1.URLPublic'} = 'true';
    }
    my $browser = LWP::UserAgent->new;
    my $response = $browser->post( mySociety::Config::get('LONDON_REPORTIT_URL'), \%params );
    my $out = $response->content;
    if ($response->code ne 200) {
        print "Failed to post $h{id} to London API, response was " . $response->code . " $out\n";
        return 1;
    }
    my ($id) = $out =~ /<caseid>(.*?)<\/caseid>/;
    my ($org) = $out =~ /<organisation>(.*?)<\/organisation>/;
    my ($team) = $out =~ /<team>(.*?)<\/team>/;

    $org = london_lookup($org);
    $problem->external_id( $id );
    $problem->external_body( $org );
    $problem->external_team( $team );
    return 0;
}

# Nearest things

sub london_lookup {
    my $org = shift || '';
    my $str = "Unknown ($org)";
    open(FP, "$FindBin::Bin/../data/dft.csv");
    while (<FP>) {
        /^(.*?),(.*)/;
        if ($org eq $1) {
            $str = $2;
            last;
        }
    }
    close FP;
    return $str;
}

# for barnet webservice: max-length fields require truncate and split

# truncate_string_with_entities
# args:    text to truncate
#          max number of chars
# returns: string truncated
# Note: must not partially truncate an entity (e.g., &amp;)
sub truncate_string_with_entities {
    my ($str, $max_len) = @_;
    my $retVal = "";
    foreach my $chunk  (split /(\&(?:\#\d+|\w+);)/, $str) {
        if ($chunk=~/^\&(\#\d+|\w+);$/){
            my $next = $retVal.$chunk;
            last if length $next > $max_len;
            $retVal=$next
        } else {
            $retVal.=$chunk;
            if (length $retVal > $max_len) {
                $retVal = substr($retVal, 0, $max_len);
                last
            }
        } 
    }
    return $retVal
}

# split_text_with_entities into lines
# args:    text to be broken into lines
#          max length (option: uses constant MAX_LINE_LENGTH)
# returns: array of lines
# Must not to split an entity (e.g., &amp;)
# Not worrying about hyphenating here, since a word is only ever split if 
# it's longer than the whole line, which is uncommon in genuine problem reports
sub split_text_with_entities {
    my ($text, $max_line_length) = @_;
    $max_line_length ||= MAX_LINE_LENGTH;
    my @lines;
    foreach my $line (split "\n", $text) {
        while (length $line > $max_line_length) {
            if (! ($line =~ s/^(.{1,$max_line_length})\s//                 # break on a space
                or $line =~ s/^(.{1,$max_line_length})(\&(\#\d+|\w+);)/$2/ # break before an entity
                or $line =~ s/(.{$max_line_length})//)) {                  # break the word ruthlessly
                $line =~ s/(.*)//; # otherwise gobble whole line (which is now shorter than max length)
            }
            push @lines, $1;
        }
        push @lines, $line;
    }
    return @lines;
}

# tests send_fail_count agains cutoff limit
#   args: problem (row from problem db)
#   returns false if there is no cutoff, otherwise error message
sub does_exceed_cutoff_limit {
    my ($problem, $council_name) = @_;
    my $err_msg = "";
    if ($problem->send_fail_count >= SEND_FAIL_RETRIES_CUTOFF) {
        $sending_skipped_by_method{$council_name || '?'}++;
        $council_name &&= " to $council_name";
        $err_msg = "skipped: problem id=" . $problem->id . " send$council_name has failed " 
                . $problem->send_fail_count . " times, cutoff is " . SEND_FAIL_RETRIES_CUTOFF;
    }
    return $err_msg;
}

# update_send_fail_data records the failure (of a webservice send)
#  args: problem (row from problem db)
#  returns: no return value (updates record)
sub update_send_fail_data {
    my ($problem, $err_msg) = @_;
    $problem->update( {
        send_fail_count => $problem->send_fail_count + 1,
        send_fail_timestamp => \'ms_current_timestamp()',
        send_fail_reason => $err_msg
    } ); 
}