fetch new problems over open311

This enables the display of existing reports from the back end on FMS
if the body is configured to do this.

Reports will not be created if they are missing an id, a lat or a long,
if the lat/long is outside the area covered by the body, if there is
already a report with a matching id, or if we can't parse out the
request time.

14 files changed, 572 insertions, 35 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fdfad3359..22989fe31 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,8 @@
 ## Releases
 
 * Unreleased
+    - New features
+        - Fetch problems over Open311 #1986
     - Front end improvements:
         - Improve questionnaire process. #1939 #1998
         - Increase size of "sub map links" (hide pins, permalink, etc) #2003
diff --git a/bin/fetch-reports b/bin/fetch-reports
new file mode 100755
index 000000000..665b4aff0
--- /dev/null
+++ b/bin/fetch-reports
@@ -0,0 +1,25 @@
+#!/usr/bin/env perl
+#
+# This script utilises the Open311 extension explained at
+# https://github.com/mysociety/FixMyStreet/wiki/Open311-FMS---Proposed-differences-to-Open311
+# to fetch updates on service requests.
+
+use strict;
+use warnings;
+require 5.8.0;
+
+BEGIN {
+    use File::Basename qw(dirname);
+    use File::Spec;
+    my $d = dirname(File::Spec->rel2abs($0));
+    require "$d/../setenv.pl";
+}
+
+use CronFns;
+my ($verbose, $nomail) = CronFns::options();
+
+use Open311::GetServiceRequests;
+
+my $reports = Open311::GetServiceRequests->new( verbose => $verbose );
+
+$reports->fetch;
diff --git a/bin/update-schema b/bin/update-schema
index fea316bd6..d9322d80b 100755
--- a/bin/update-schema
+++ b/bin/update-schema
@@ -212,6 +212,7 @@ else {
 # (assuming schema change files are never half-applied, which should be the case)
 sub get_db_version {
     return 'EMPTY' if ! table_exists('problem');
+    return '0057' if column_exists('body', 'fetch_problems');
     return '0056' if column_exists('users', 'email_verified');
     return '0055' if column_exists('response_priorities', 'is_default');
     return '0054' if table_exists('state');
diff --git a/db/downgrade_0057---0056.sql b/db/downgrade_0057---0056.sql
new file mode 100644
index 000000000..a87488e41
--- /dev/null
+++ b/db/downgrade_0057---0056.sql
@@ -0,0 +1,5 @@
+BEGIN;
+
+ALTER TABLE body DROP fetch_problems;
+
+COMMIT;
diff --git a/db/schema.sql b/db/schema.sql
index f2197dc52..9d212e1da 100644
--- a/db/schema.sql
+++ b/db/schema.sql
@@ -54,6 +54,7 @@ create table body (
     suppress_alerts boolean not null default 'f',
     can_be_devolved boolean not null default 'f',
     send_extended_statuses boolean not null default 'f',
+    fetch_problems boolean not null default 'f',
     deleted boolean not null default 'f'
 );
 
diff --git a/db/schema_0057-fetch-problems.sql b/db/schema_0057-fetch-problems.sql
new file mode 100644
index 000000000..7419eb032
--- /dev/null
+++ b/db/schema_0057-fetch-problems.sql
@@ -0,0 +1,5 @@
+BEGIN;
+
+ALTER TABLE body ADD fetch_problems boolean default 'f' not null;
+
+COMMIT;
diff --git a/perllib/FixMyStreet/App/Controller/Admin.pm b/perllib/FixMyStreet/App/Controller/Admin.pm
index 5e69d5bf3..27792cd9d 100644
--- a/perllib/FixMyStreet/App/Controller/Admin.pm
+++ b/perllib/FixMyStreet/App/Controller/Admin.pm
@@ -445,6 +445,7 @@ sub body_params : Private {
     my %defaults = map { $_ => '' } @fields;
     %defaults = ( %defaults,
         send_comments => 0,
+        fetch_problems => 0,
         suppress_alerts => 0,
         comment_user_id => undef,
         send_extended_statuses => 0,
diff --git a/perllib/FixMyStreet/DB/Result/Body.pm b/perllib/FixMyStreet/DB/Result/Body.pm
index 07bea276c..95c176084 100644
--- a/perllib/FixMyStreet/DB/Result/Body.pm
+++ b/perllib/FixMyStreet/DB/Result/Body.pm
@@ -44,6 +44,8 @@ __PACKAGE__->add_columns(
   { data_type => "boolean", default_value => \"false", is_nullable => 0 },
   "external_url",
   { data_type => "text", is_nullable => 1 },
+  "fetch_problems",
+  { data_type => "boolean", default_value => \"false", is_nullable => 0 },
 );
 __PACKAGE__->set_primary_key("id");
 __PACKAGE__->has_many(
@@ -118,8 +120,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07035 @ 2017-02-13 15:11:11
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:BOJANVwg3kR/1VjDq0LykA
+# Created by DBIx::Class::Schema::Loader v0.07035 @ 2017-12-20 14:32:39
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:4RaXIyQmPdyDRW+0pmk0cg
 
 use Moo;
 use namespace::clean;
diff --git a/perllib/Open311.pm b/perllib/Open311.pm
index 44af0b134..22293ee9f 100644
--- a/perllib/Open311.pm
+++ b/perllib/Open311.pm
@@ -213,14 +213,20 @@ sub _generate_service_request_description {
 
 sub get_service_requests {
     my $self = shift;
-    my $report_ids = shift;
+    my $args = shift;
 
     my $params = {};
 
-    if ( $report_ids ) {
-        $params->{service_request_id} = join ',', @$report_ids;
+    if ( $args->{report_ids} ) {
+        $params->{service_request_id} = join ',', @{$args->{report_ids}};
+        delete $args->{report_ids};
     }
 
+    $params = {
+      %$params,
+      %$args
+    };
+
     my $service_request_xml = $self->_get( $self->endpoints->{requests}, $params || undef );
     return $self->_get_xml_object( $service_request_xml );
 }
@@ -291,6 +297,37 @@ sub post_service_request_update {
     return 0;
 }
 
+sub add_media {
+    my ($self, $url, $object) = @_;
+
+    my $ua = LWP::UserAgent->new;
+    my $res = $ua->get($url);
+    if ( $res->is_success && $res->content_type eq 'image/jpeg' ) {
+        my $photoset = FixMyStreet::App::Model::PhotoSet->new({
+            data_items => [ $res->decoded_content ],
+        });
+        $object->photo($photoset->data);
+    }
+}
+
+sub map_state {
+    my $self           = shift;
+    my $incoming_state = shift;
+
+    $incoming_state = lc($incoming_state);
+    $incoming_state =~ s/_/ /g;
+
+    my %state_map = (
+        fixed                         => 'fixed - council',
+        'not councils responsibility' => 'not responsible',
+        'no further action'           => 'unable to fix',
+        open                          => 'confirmed',
+        closed                        => 'fixed - council',
+    );
+
+    return $state_map{$incoming_state} || $incoming_state;
+}
+
 sub _populate_service_request_update_params {
     my $self = shift;
     my $comment = shift;
diff --git a/perllib/Open311/GetServiceRequestUpdates.pm b/perllib/Open311/GetServiceRequestUpdates.pm
index 2620b176a..8e83d08b9 100644
--- a/perllib/Open311/GetServiceRequestUpdates.pm
+++ b/perllib/Open311/GetServiceRequestUpdates.pm
@@ -107,7 +107,7 @@ sub update_comments {
             my $c = $p->comments->search( { external_id => $request->{update_id} } );
 
             if ( !$c->first ) {
-                my $state = $self->map_state( $request->{status} );
+                my $state = $open311->map_state( $request->{status} );
                 my $comment = $self->schema->resultset('Comment')->new(
                     {
                         problem => $p,
@@ -124,16 +124,8 @@ sub update_comments {
                     }
                 );
 
-                if ($request->{media_url}) {
-                    my $ua = LWP::UserAgent->new;
-                    my $res = $ua->get($request->{media_url});
-                    if ( $res->is_success && $res->content_type eq 'image/jpeg' ) {
-                        my $photoset = FixMyStreet::App::Model::PhotoSet->new({
-                            data_items => [ $res->decoded_content ],
-                        });
-                        $comment->photo($photoset->data);
-                    }
-                }
+                $open311->add_media($request->{media_url}, $comment)
+                    if $request->{media_url};
 
                 # if the comment is older than the last update
                 # do not change the status of the problem as it's
@@ -195,22 +187,4 @@ sub comment_text_for_request {
     return "";
 }
 
-sub map_state {
-    my $self           = shift;
-    my $incoming_state = shift;
-
-    $incoming_state = lc($incoming_state);
-    $incoming_state =~ s/_/ /g;
-
-    my %state_map = (
-        fixed                         => 'fixed - council',
-        'not councils responsibility' => 'not responsible',
-        'no further action'           => 'unable to fix',
-        open                          => 'confirmed',
-        closed                        => 'fixed - council'
-    );
-
-    return $state_map{$incoming_state} || $incoming_state;
-}
-
 1;
diff --git a/perllib/Open311/GetServiceRequests.pm b/perllib/Open311/GetServiceRequests.pm
new file mode 100644
index 000000000..2a82c64a1
--- /dev/null
+++ b/perllib/Open311/GetServiceRequests.pm
@@ -0,0 +1,168 @@
+package Open311::GetServiceRequests;
+
+use Moo;
+use Open311;
+use FixMyStreet::DB;
+use FixMyStreet::App::Model::PhotoSet;
+use DateTime::Format::W3CDTF;
+
+has system_user => ( is => 'rw' );
+has start_date => ( is => 'ro', default => sub { undef } );
+has end_date => ( is => 'ro', default => sub { undef } );
+has verbose => ( is => 'ro', default => 0 );
+has schema => ( is =>'ro', lazy => 1, default => sub { FixMyStreet::DB->schema->connect } );
+
+sub fetch {
+    my $self = shift;
+
+    my $bodies = $self->schema->resultset('Body')->search(
+        {
+            send_method     => 'Open311',
+            fetch_problems  => 1,
+            comment_user_id => { '!=', undef },
+            endpoint        => { '!=', '' },
+        }
+    );
+
+    while ( my $body = $bodies->next ) {
+
+        my $o = Open311->new(
+            endpoint     => $body->endpoint,
+            api_key      => $body->api_key,
+            jurisdiction => $body->jurisdiction,
+        );
+
+        $self->system_user( $body->comment_user );
+        $self->create_problems( $o, $body );
+    }
+}
+
+sub create_problems {
+    my ( $self, $open311, $body ) = @_;
+
+    my $args = {};
+
+    if ( $self->start_date || $self->end_date ) {
+        return 0 unless $self->start_date && $self->end_date;
+
+
+        $args->{start_date} = DateTime::Format::W3CDTF->format_datetime( $self->start_date );
+        $args->{end_date} = DateTime::Format::W3CDTF->format_datetime( $self->end_date );
+    }
+
+    my $requests = $open311->get_service_requests( $args );
+
+    unless ( $open311->success ) {
+        warn "Failed to fetch ServiceRequest Updates for " . $body->name . ":\n" . $open311->error
+            if $self->verbose;
+        return 0;
+    }
+
+    my $contacts = $self->schema->resultset('Contact')
+        ->active
+        ->search( { body_id => $body->id } );
+
+    for my $request (@{$requests->{request}}) {
+        # no point importing if we can't put it on the map
+        unless ($request->{service_request_id} && $request->{lat} && $request->{long}) {
+            warn "Not creating request '$request->{description}' for @{[$body->name]} as missing one of id, lat or long"
+                if $self->verbose;
+            next;
+        }
+        my $request_id = $request->{service_request_id};
+
+        my %params;
+        $params{generation} = mySociety::Config::get('MAPIT_GENERATION')
+            if mySociety::Config::get('MAPIT_GENERATION');
+
+        my ($latitude, $longitude) = ( $request->{lat}, $request->{long} );
+        my $all_areas =
+          mySociety::MaPit::call( 'point',
+            "4326/$longitude,$latitude", %params );
+
+        # skip if it doesn't look like it's for this body
+        my @areas = grep { $all_areas->{$_->area_id} } $body->body_areas;
+        unless (@areas) {
+            warn "Not creating request id $request_id for @{[$body->name]} as outside body area"
+                if $self->verbose;
+            next;
+        }
+
+        my $updated_time = eval {
+            DateTime::Format::W3CDTF->parse_datetime(
+                $request->{updated_datetime} || ""
+            )->set_time_zone(
+                FixMyStreet->time_zone || FixMyStreet->local_time_zone
+            );
+        };
+        if ($@) {
+            warn "Not creating problem $request_id for @{[$body->name]}, bad update time"
+                if $self->verbose;
+            next;
+        }
+
+        my $updated = DateTime::Format::W3CDTF->format_datetime(
+            $updated_time->clone->set_time_zone('UTC')
+        );
+        if ($args->{start_date} && $args->{end_date} && ($updated lt $args->{start_date} || $updated gt $args->{end_date}) ) {
+            warn "Problem id $request_id for @{[$body->name]} has an invalid time, not creating"
+                if $self->verbose;
+            next;
+        }
+
+        my $created_time = eval {
+            DateTime::Format::W3CDTF->parse_datetime(
+                $request->{requested_datetime} || ""
+            )->set_time_zone(
+                FixMyStreet->time_zone || FixMyStreet->local_time_zone
+            );
+        };
+        $created_time = $updated_time if $@;
+
+        my $problems;
+        my $criteria = {
+            external_id => $request_id,
+        };
+        $problems = $self->schema->resultset('Problem')->to_body($body)->search( $criteria );
+
+        my @contacts = grep { $request->{service_code} eq $_->category } $contacts->all;
+        my $contact = $contacts[0] ? $contacts[0]->category : 'Other';
+
+        my $state = $open311->map_state($request->{status});
+
+        unless (my $p = $problems->first) {
+            my $problem = $self->schema->resultset('Problem')->new(
+                {
+                    user => $self->system_user,
+                    external_id => $request_id,
+                    detail => $request->{description},
+                    title => $request->{title} || $request->{service_name} . ' problem',
+                    anonymous => 0,
+                    name => $self->system_user->name,
+                    confirmed => $created_time,
+                    created => $created_time,
+                    lastupdate => $updated_time,
+                    whensent => $created_time,
+                    state => $state,
+                    postcode => '',
+                    used_map => 1,
+                    latitude => $request->{lat},
+                    longitude => $request->{long},
+                    areas => ',' . $body->id . ',',
+                    bodies_str => $body->id,
+                    send_method_used => 'Open311',
+                    category => $contact,
+                }
+            );
+
+            $open311->add_media($request->{media_url}, $problem)
+                if $request->{media_url};
+
+            $problem->insert();
+        }
+    }
+
+    return 1;
+}
+
+1;
diff --git a/perllib/Open311/GetUpdates.pm b/perllib/Open311/GetUpdates.pm
index 1b1e339e3..f62acf4a8 100644
--- a/perllib/Open311/GetUpdates.pm
+++ b/perllib/Open311/GetUpdates.pm
@@ -41,7 +41,7 @@ sub get_updates {
 sub update_reports {
     my ( $self, $report_ids, $open311, $body ) = @_;
 
-    my $service_requests = $open311->get_service_requests( $report_ids );
+    my $service_requests = $open311->get_service_requests( { report_ids => $report_ids } );
     my $requests = $service_requests->{request};
 
     for my $request (@$requests) {
diff --git a/t/open311/getservicerequests.t b/t/open311/getservicerequests.t
new file mode 100644
index 000000000..878c178ef
--- /dev/null
+++ b/t/open311/getservicerequests.t
@@ -0,0 +1,301 @@
+#!/usr/bin/env perl
+
+use FixMyStreet::TestMech;
+
+use_ok( 'Open311' );
+use_ok( 'Open311::GetServiceRequests' );
+use DateTime;
+use DateTime::Format::W3CDTF;
+
+my $mech = FixMyStreet::TestMech->new;
+
+my $user = $mech->create_user_ok('system_user@example.com', name => 'test users');
+my $body = $mech->create_body_ok(2482, 'Bromley');
+my $contact = $mech->create_contact_ok( body_id => $body->id, category => 'sidewalks', email => 'sidewalks@example.com' );
+
+my $dtf = DateTime::Format::W3CDTF->new;
+
+my $requests_xml = qq{<?xml version="1.0" encoding="utf-8"?>
+<service_requests>
+<request>
+<service_request_id>638344</service_request_id>
+<status>open</status>
+<status_notes>This is a note.</status_notes>
+<service_name>Sidewalk and Curb Issues</service_name>
+<service_code>sidewalks</service_code>
+<description>This is a sidewalk problem</description>
+<agency_responsible></agency_responsible>
+<service_notice></service_notice>
+<requested_datetime>2010-04-14T06:37:38-08:00</requested_datetime>
+<updated_datetime>2010-04-14T06:37:38-08:00</updated_datetime>
+<expected_datetime>2010-04-15T06:37:38-08:00</expected_datetime>
+<lat>51.4021</lat>
+<long>0.01578</long>
+</request>
+<request>
+<service_request_id>638345</service_request_id>
+<status>investigating</status>
+<status_notes>This is a for a different issue.</status_notes>
+<service_name>Not Sidewalk and Curb Issues</service_name>
+<service_code>not_sidewalks</service_code>
+<description>This is a problem</description>
+<agency_responsible></agency_responsible>
+<service_notice></service_notice>
+<requested_datetime>2010-04-15T06:37:38-08:00</requested_datetime>
+<updated_datetime>2010-04-15T06:37:38-08:00</updated_datetime>
+<expected_datetime>2010-04-15T06:37:38-08:00</expected_datetime>
+<lat>51.4021</lat>
+<long>0.01578</long>
+</request>
+</service_requests>
+};
+
+my $o = Open311->new(
+    jurisdiction => 'mysociety',
+    endpoint => 'http://example.com',
+    test_mode => 1,
+    test_get_returns => { 'requests.xml' => $requests_xml }
+);
+
+subtest 'basic parsing checks' => sub {
+    my $update = Open311::GetServiceRequests->new( system_user => $user );
+    FixMyStreet::override_config {
+        MAPIT_URL => 'http://mapit.uk/',
+    }, sub {
+        $update->create_problems( $o, $body );
+    };
+
+    my $p1_date = $dtf->parse_datetime('2010-04-14T06:37:38-08:00')
+                    ->set_time_zone(
+                        FixMyStreet->time_zone || FixMyStreet->local_time_zone
+                  );
+
+    my $p = FixMyStreet::DB->resultset('Problem')->search(
+                { external_id => 638344 }
+            )->first;
+
+    ok $p, 'Found problem';
+    is $p->detail, 'This is a sidewalk problem', 'correct problem description';
+    is $p->created, $p1_date, 'Problem has correct creation date';
+    is $p->confirmed, $p1_date, 'Problem has correct confirmed date';
+    is $p->whensent, $p1_date, 'Problem has whensent set';
+    is $p->state, 'confirmed', 'correct problem state';
+    is $p->user->id, $user->id, 'user set to system user';
+    is $p->category, 'sidewalks', 'correct problem category';
+
+    my $p2 = FixMyStreet::DB->resultset('Problem')->search( { external_id => 638345 } )->first;
+    ok $p2, 'second problem found';
+    ok $p2->whensent, 'second problem marked sent';
+    is $p2->state, 'investigating', 'second problem correct state';
+    is $p2->category, 'Other', 'category falls back to Other';
+};
+
+subtest 'check problems not re-created' => sub {
+    my $update = Open311::GetServiceRequests->new( system_user => $user );
+    FixMyStreet::override_config {
+        MAPIT_URL => 'http://mapit.uk/',
+    }, sub {
+        $update->create_problems( $o, $body );
+    };
+
+    my $count = FixMyStreet::DB->resultset('Problem')->count;
+
+    FixMyStreet::override_config {
+        MAPIT_URL => 'http://mapit.uk/',
+    }, sub {
+        $update->create_problems( $o, $body );
+    };
+
+    my $after_count = FixMyStreet::DB->resultset('Problem')->count;
+
+    is $count, $after_count, "problems not re-created";
+};
+
+for my $test (
+  {
+      desc => 'problem with no id is not created',
+      detail => 'This is a problem with no service_code',
+      subs => { id => '', desc => 'This is a problem with service code' },
+  },
+  {
+      desc => 'problem with no lat is not created',
+      detail => 'This is a problem with no lat',
+      subs => { lat => '', desc => 'This is a problem with no lat' },
+  },
+  {
+      desc => 'problem with no long is not created',
+      detail => 'This is a problem with no long',
+      subs => { long => '', desc => 'This is a problem with no long' },
+  },
+  {
+      desc => 'problem with bad lat/long is not created',
+      detail => 'This is a problem with bad lat/long',
+      subs => { lat => '51', long => 0.1, desc => 'This is a problem with bad lat/long' },
+  },
+) {
+    subtest $test->{desc} => sub {
+        my $xml = prepare_xml( $test->{subs} );
+        my $o = Open311->new(
+            jurisdiction => 'mysociety',
+            endpoint => 'http://example.com',
+            test_mode => 1,
+            test_get_returns => { 'requests.xml' => $xml}
+        );
+
+        my $count = FixMyStreet::DB->resultset('Problem')->count;
+        my $update = Open311::GetServiceRequests->new( system_user => $user );
+        FixMyStreet::override_config {
+            MAPIT_URL => 'http://mapit.uk/',
+        }, sub {
+            $update->create_problems( $o, $body );
+        };
+        my $after_count = FixMyStreet::DB->resultset('Problem')->count;
+
+        warn $count;
+        is $count, $after_count, "problems not created";
+
+        my $with_text = FixMyStreet::DB->resultset('Problem')->search( {
+              detail => $test->{detail}
+        } )->count;
+
+        is $with_text, 0, 'no matching problem created';
+    };
+}
+
+my $date = DateTime->new(
+    year => 2010,
+    month => 4,
+    day => 14,
+    hour => 6,
+    minute => 37
+);
+
+for my $test (
+  {
+      start_date => '1',
+      end_date => '',
+      desc => 'do not process if only a start_date',
+      subs => {},
+  },
+  {
+      start_date => '',
+      end_date => '1',
+      desc => 'do not process if only an end_date',
+      subs => {},
+  },
+) {
+    subtest $test->{desc} => sub {
+        my $xml = prepare_xml( $test->{subs} );
+        my $o = Open311->new(
+            jurisdiction => 'mysociety',
+            endpoint => 'http://example.com',
+            test_mode => 1,
+            test_get_returns => { 'requests.xml' => $xml}
+        );
+
+        my $update = Open311::GetServiceRequests->new(
+            start_date => $test->{start_date},
+            end_date => $test->{end_date},
+            system_user => $user,
+        );
+        my $ret = $update->create_problems( $o, $body );
+
+        is $ret, 0, 'failed correctly'
+    };
+}
+
+$date = DateTime->new(
+    year => 2010,
+    month => 4,
+    day => 14,
+    hour => 6,
+    minute => 37
+);
+
+for my $test (
+  {
+      start_date => $date->clone->add(hours => -2),
+      end_date => $date->clone->add(hours => -1),
+      desc => 'do not process if update time after end_date',
+      subs => {},
+  },
+  {
+      start_date => $date->clone->add(hours => 2),
+      end_date => $date->clone->add(hours => 4),
+      desc => 'do not process if update time before start_date',
+      subs => {},
+  },
+  {
+      start_date => $date->clone->add(hours => -2),
+      end_date => $date->clone->add(hours => 4),
+      desc => 'do not process if update time is bad',
+      subs => { update_time => '2010/12/12' },
+  },
+) {
+    subtest $test->{desc} => sub {
+        my $xml = prepare_xml( $test->{subs} );
+        my $o = Open311->new(
+            jurisdiction => 'mysociety',
+            endpoint => 'http://example.com',
+            test_mode => 1,
+            test_get_returns => { 'requests.xml' => $xml}
+        );
+
+        my $update = Open311::GetServiceRequests->new(
+            start_date => $test->{start_date},
+            end_date => $test->{end_date},
+            system_user => $user,
+        );
+        my $count = FixMyStreet::DB->resultset('Problem')->count;
+        FixMyStreet::override_config {
+            MAPIT_URL => 'http://mapit.uk/',
+        }, sub {
+            $update->create_problems( $o, $body );
+        };
+        my $after = FixMyStreet::DB->resultset('Problem')->count;
+
+        is $count, $after, 'problem not added';
+    };
+}
+
+sub prepare_xml {
+    my $replacements = shift;
+
+    my %defaults = (
+        desc => 'this is a problem',
+        lat => 51.4021,
+        long => 0.01578,
+        id => 123456,
+        update_time => '2010-04-14T06:37:38-08:00',
+        %$replacements
+    );
+
+    my $xml = qq[<?xml version="1.0" encoding="utf-8"?>
+<service_requests>
+<request>
+<service_request_id>XXX_ID</service_request_id>
+<status>open</status>
+<status_notes></status_notes>
+<service_name>Sidewalk and Curb Issues</service_name>
+<service_code>sidewalks</service_code>
+<description>XXX_DESC</description>
+<agency_responsible></agency_responsible>
+<service_notice></service_notice>
+<requested_datetime>2010-04-14T06:37:38-08:00</requested_datetime>
+<updated_datetime>XXX_UPDATE_TIME</updated_datetime>
+<expected_datetime>2010-04-15T06:37:38-08:00</expected_datetime>
+<lat>XXX_LAT</lat>
+<long>XXX_LONG</long>
+</request>
+</service_requests>
+];
+
+    for my $key (keys %defaults) {
+        my $string = 'XXX_' . uc $key;
+        $xml =~ s/$string/$defaults{$key}/;
+    }
+
+    return $xml;
+}
+
+done_testing();
diff --git a/templates/web/base/admin/open311-form-fields.html b/templates/web/base/admin/open311-form-fields.html
index d1067205c..393693264 100644
--- a/templates/web/base/admin/open311-form-fields.html
+++ b/templates/web/base/admin/open311-form-fields.html
@@ -68,6 +68,21 @@
     <div class="admin-hint">
       <p>
         [% loc(
+            "Enable <strong>Open311 problem-fetching</strong> if you want to display reports created at
+          the endpoint to FixMyStreet. If you're not sure, you probably do not, so leave this unchecked.
+          For more information, see 
+          <a href='https://www.mysociety.org/2013/02/20/open311-extended/' class='admin-offsite-link'>this article</a>."
+        ) %]
+      </p>
+    </div>
+    <p>
+        <input type="checkbox" id="fetch_problems" name="fetch_problems"[% ' checked' IF object.fetch_problems %]>
+        <label for="fetch_problems" class="inline">[% loc('Use Open311 problem fetching') %]</label>
+    </p>
+
+    <div class="admin-hint">
+      <p>
+        [% loc(
           "If you've enabled Open311 update-sending above, you must identify which 
           FixMyStreet <strong>user</strong> will be attributed as the creator of those updates
           when they are shown on the site. Enter the ID (number) of that user."