14 files changed, 572 insertions, 35 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index fdfad3359..22989fe31 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,8 @@
 ## Releases
 
 * Unreleased
+    - New features
+        - Fetch problems over Open311 #1986
     - Front end improvements:
         - Improve questionnaire process. #1939 #1998
         - Increase size of "sub map links" (hide pins, permalink, etc) #2003
diff --git a/bin/fetch-reports b/bin/fetch-reports
new file mode 100755
index 000000000..665b4aff0
--- /dev/null
+++ b/bin/fetch-reports
@@ -0,0 +1,25 @@
+#!/usr/bin/env perl
+#
+# This script utilises the Open311 extension explained at
+# https://github.com/mysociety/FixMyStreet/wiki/Open311-FMS---Proposed-differences-to-Open311
+# to fetch updates on service requests.
+
+use strict;
+use warnings;
+require 5.8.0;
+
+BEGIN {
+    use File::Basename qw(dirname);
+    use File::Spec;
+    my $d = dirname(File::Spec->rel2abs($0));
+    require "$d/../setenv.pl";
+}
+
+use CronFns;
+my ($verbose, $nomail) = CronFns::options();
+
+use Open311::GetServiceRequests;
+
+my $reports = Open311::GetServiceRequests->new( verbose => $verbose );
+
+$reports->fetch;
diff --git a/bin/update-schema b/bin/update-schema
index fea316bd6..d9322d80b 100755
--- a/bin/update-schema
+++ b/bin/update-schema
@@ -212,6 +212,7 @@ else {
 # (assuming schema change files are never half-applied, which should be the case)
 sub get_db_version {
     return 'EMPTY' if ! table_exists('problem');
+    return '0057' if column_exists('body', 'fetch_problems');
     return '0056' if column_exists('users', 'email_verified');
     return '0055' if column_exists('response_priorities', 'is_default');
     return '0054' if table_exists('state');
diff --git a/db/downgrade_0057---0056.sql b/db/downgrade_0057---0056.sql
new file mode 100644
index 000000000..a87488e41
--- /dev/null
+++ b/db/downgrade_0057---0056.sql
@@ -0,0 +1,5 @@
+BEGIN;
+
+ALTER TABLE body DROP fetch_problems;
+
+COMMIT;
diff --git a/db/schema.sql b/db/schema.sql
index f2197dc52..9d212e1da 100644
--- a/db/schema.sql
+++ b/db/schema.sql
@@ -54,6 +54,7 @@ create table body (
     suppress_alerts boolean not null default 'f',
     can_be_devolved boolean not null default 'f',
     send_extended_statuses boolean not null default 'f',
+    fetch_problems boolean not null default 'f',
     deleted boolean not null default 'f'
 );
 
diff --git a/db/schema_0057-fetch-problems.sql b/db/schema_0057-fetch-problems.sql
new file mode 100644
index 000000000..7419eb032
--- /dev/null
+++ b/db/schema_0057-fetch-problems.sql
@@ -0,0 +1,5 @@
+BEGIN;
+
+ALTER TABLE body ADD fetch_problems boolean default 'f' not null;
+
+COMMIT;
diff --git a/perllib/FixMyStreet/App/Controller/Admin.pm b/perllib/FixMyStreet/App/Controller/Admin.pm
index 5e69d5bf3..27792cd9d 100644
--- a/perllib/FixMyStreet/App/Controller/Admin.pm
+++ b/perllib/FixMyStreet/App/Controller/Admin.pm
@@ -445,6 +445,7 @@ sub body_params : Private {
     my %defaults = map { $_ => '' } @fields;
     %defaults = ( %defaults,
         send_comments => 0,
+        fetch_problems => 0,
         suppress_alerts => 0,
         comment_user_id => undef,
         send_extended_statuses => 0,
diff --git a/perllib/FixMyStreet/DB/Result/Body.pm b/perllib/FixMyStreet/DB/Result/Body.pm
index 07bea276c..95c176084 100644
--- a/perllib/FixMyStreet/DB/Result/Body.pm
+++ b/perllib/FixMyStreet/DB/Result/Body.pm
@@ -44,6 +44,8 @@ __PACKAGE__->add_columns(
   { data_type => "boolean", default_value => \"false", is_nullable => 0 },
   "external_url",
   { data_type => "text", is_nullable => 1 },
+  "fetch_problems",
+  { data_type => "boolean", default_value => \"false", is_nullable => 0 },
 );
 __PACKAGE__->set_primary_key("id");
 __PACKAGE__->has_many(
@@ -118,8 +120,8 @@ __PACKAGE__->has_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07035 @ 2017-02-13 15:11:11
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:BOJANVwg3kR/1VjDq0LykA
+# Created by DBIx::Class::Schema::Loader v0.07035 @ 2017-12-20 14:32:39
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:4RaXIyQmPdyDRW+0pmk0cg
 
 use Moo;
 use namespace::clean;
diff --git a/perllib/Open311.pm b/perllib/Open311.pm
index 44af0b134..22293ee9f 100644
--- a/perllib/Open311.pm
+++ b/perllib/Open311.pm
@@ -213,14 +213,20 @@ sub _generate_service_request_description {
 
 sub get_service_requests {
     my $self = shift;
-    my $report_ids = shift;
+    my $args = shift;
 
     my $params = {};
 
-    if ( $report_ids ) {
-        $params->{service_request_id} = join ',', @$report_ids;
+    if ( $args->{report_ids} ) {
+        $params->{service_request_id} = join ',', @{$args->{report_ids}};
+        delete $args->{report_ids};
     }
 
+    $params = {
+      %$params,
+      %$args
+    };
+
     my $service_request_xml = $self->_get( $self->endpoints->{requests}, $params || undef );
     return $self->_get_xml_object( $service_request_xml );
 }
@@ -291,6 +297,37 @@ sub post_service_request_update {
     return 0;
 }
 
+sub add_media {
+    my ($self, $url, $object) = @_;
+
+    my $ua = LWP::UserAgent->new;
+    my $res = $ua->get($url);
+    if ( $res->is_success && $res->content_type eq 'image/jpeg' ) {
+        my $photoset = FixMyStreet::App::Model::PhotoSet->new({
+            data_items => [ $res->decoded_content ],
+        });
+        $object->photo($photoset->data);
+    }
+}
+
+sub map_state {
+    my $self           = shift;
+    my $incoming_state = shift;
+
+    $incoming_state = lc($incoming_state);
+    $incoming_state =~ s/_/ /g;
+
+    my %state_map = (
+        fixed                         => 'fixed - council',
+        'not councils responsibility' => 'not responsible',
+        'no further action'           => 'unable to fix',
+        open                          => 'confirmed',
+        closed                        => 'fixed - council',
+    );
+
+    return $state_map{$incoming_state} || $incoming_state;
+}
+
 sub _populate_service_request_update_params {
     my $self = shift;
     my $comment = shift;
diff --git a/perllib/Open311/GetServiceRequestUpdates.pm b/perllib/Open311/GetServiceRequestUpdates.pm
index 2620b176a..8e83d08b9 100644
--- a/perllib/Open311/GetServiceRequestUpdates.pm
+++ b/perllib/Open311/GetServiceRequestUpdates.pm
@@ -107,7 +107,7 @@ sub update_comments {
             my $c = $p->comments->search( { external_id => $request->{update_id} } );
 
             if ( !$c->first ) {
-                my $state = $self->map_state( $request->{status} );
+                my $state = $open311->map_state( $request->{status} );
                 my $comment = $self->schema->resultset('Comment')->new(
                     {
                         problem => $p,
@@ -124,16 +124,8 @@ sub update_comments {
                     }
                 );
 
-                if ($request->{media_url}) {
-                    my $ua = LWP::UserAgent->new;
-                    my $res = $ua->get($request->{media_url});
-                    if ( $res->is_success && $res->content_type eq 'image/jpeg' ) {
-                        my $photoset = FixMyStreet::App::Model::PhotoSet->new({
-                            data_items => [ $res->decoded_content ],
-                        });
-                        $comment->photo($photoset->data);
-                    }
-                }
+                $open311->add_media($request->{media_url}, $comment)
+                    if $request->{media_url};
 
                 # if the comment is older than the last update
                 # do not change the status of the problem as it's
@@ -195,22 +187,4 @@ sub comment_text_for_request {
     return "";
 }
 
-sub map_state {
-    my $self           = shift;
-    my $incoming_state = shift;
-
-    $incoming_state = lc($incoming_state);
-    $incoming_state =~ s/_/ /g;
-
-    my %state_map = (
-        fixed                         => 'fixed - council',
-        'not councils responsibility' => 'not responsible',
-        'no further action'           => 'unable to fix',
-        open                          => 'confirmed',
-        closed                        => 'fixed - council'
-    );
-
-    return $state_map{$incoming_state} || $incoming_state;
-}
-
 1;
diff --git a/perllib/Open311/GetServiceRequests.pm b/perllib/Open311/GetServiceRequests.pm
new file mode 100644
index 000000000..2a82c64a1
--- /dev/null
+++ b/perllib/Open311/GetServiceRequests.pm
@@ -0,0 +1,168 @@
+package Open311::GetServiceRequests;
+
+use Moo;
+use Open311;
+use FixMyStreet::DB;
+use FixMyStreet::App::Model::PhotoSet;
+use DateTime::Format::W3CDTF;
+
+has system_user => ( is => 'rw' );
+has start_date => ( is => 'ro', default => sub { undef } );
+has end_date => ( is => 'ro', default => sub { undef } );
+has verbose => ( is => 'ro', default => 0 );
+has schema => ( is =>'ro', lazy => 1, default => sub { FixMyStreet::DB->schema->connect } );
+
+sub fetch {
+    my $self = shift;
+
+    my $bodies = $self->schema->resultset('Body')->search(
+        {
+            send_method     => 'Open311',
+            fetch_problems  => 1,
+            comment_user_id => { '!=', undef },
+            endpoint        => { '!=', '' },
+        }
+    );
+
+    while ( my $body = $bodies->next ) {
+
+        my $o = Open311->new(
+            endpoint     => $body->endpoint,
+            api_key      => $body->api_key,
+            jurisdiction => $body->jurisdiction,
+        );
+
+        $self->system_user( $body->comment_user );
+        $self->create_problems( $o, $body );
+    }
+}
+
+sub create_problems {
+    my ( $self, $open311, $body ) = @_;
+
+    my $args = {};
+
+    if ( $self->start_date || $self->end_date ) {
+        return 0 unless $self->start_date && $self->end_date;
+
+
+        $args->{start_date} = DateTime::Format::W3CDTF->format_datetime( $self->start_date );
+        $args->{end_date} = DateTime::Format::W3CDTF->format_datetime( $self->end_date );
+    }
+
+    my $requests = $open311->get_service_requests( $args );
+
+    unless ( $open311->success ) {
+        warn "Failed to fetch ServiceRequest Updates for " . $body->name . ":\n" . $open311->error
+            if $self->verbose;
+        return 0;
+    }
+
+    my $contacts = $self->schema->resultset('Contact')
+        ->active
+        ->search( { body_id => $body->id } );
+
+    for my $request (@{$requests->{request}}) {
+        # no point importing if we can't put it on the map
+        unless ($request->{service_request_id} && $request->{lat} && $request->{long}) {
+            warn "Not creating request '$request->{description}' for @{[$body->name]} as missing one of id, lat or long"
+                if $self->verbose;
+            next;
+        }
+        my $request_id = $request->{service_request_id};
+
+        my %params;
+        $params{generation} = mySociety::Config::get('MAPIT_GENERATION')
+            if mySociety::Config::get('MAPIT_GENERATION');
+
+        my ($latitude, $longitude) = ( $request->{lat}, $request->{long} );
+        my $all_areas =
+          mySociety::MaPit::call( 'point',
+            "4326/$longitude,$latitude", %params );
+
+        # skip if it doesn't look like it's for this body
+        my @areas = grep { $all_areas->{$_->area_id} } $body->body_areas;
+        unless (@areas) {
+            warn "Not creating request id $request_id for @{[$body->name]} as outside body area"
+                if $self->verbose;
+            next;
+        }
+
+        my $updated_time = eval {
+            DateTime::Format::W3CDTF->parse_datetime(
+                $request->{updated_datetime} || ""
+            )->set_time_zone(
+                FixMyStreet->time_zone || FixMyStreet->local_time_zone
+            );
+        };
+        if ($@) {
+            warn "Not creating problem $request_id for @{[$body->name]}, bad update time"
+                if $self->verbose;
+            next;
+        }
+
+        my $updated = DateTime::Format::W3CDTF->format_datetime(
+            $updated_time->clone->set_time_zone('UTC')
+        );
+        if ($args->{start_date} && $args->{end_date} && ($updated lt $args->{start_date} || $updated gt $args->{end_date}) ) {
+            warn "Problem id $request_id for @{[$body->name]} has an invalid time, not creating"
+                if $self->verbose;
+            next;
+        }
+
+        my $created_time = eval {
+            DateTime::Format::W3CDTF->parse_datetime(
+                $request->{requested_datetime} || ""
+            )->set_time_zone(
+                FixMyStreet->time_zone || FixMyStreet->local_time_zone
+            );
+        };
+        $created_time = $updated_time if $@;
+
+        my $problems;
+        my $criteria = {
+            external_id => $request_id,
+        };
+        $problems = $self->schema->resultset('Problem')->to_body($body)->search( $criteria );
+
+        my @contacts = grep { $request->{service_code} eq $_->category } $contacts->all;
+        my $contact = $contacts[0] ? $contacts[0]->category : 'Other';
+
+        my $state = $open311->map_state($request->{status});
+
+        unless (my $p = $problems->first) {
+            my $problem = $self->schema->resultset('Problem')->new(
+                {
+                    user => $self->system_user,
+                    external_id => $request_id,
+                    detail => $request->{description},
+                    title => $request->{title} || $request->{service_name} . ' problem',
+                    anonymous => 0,
+                    name => $self->system_user->name,
+                    confirmed => $created_time,
+                    created => $created_time,
+                    lastupdate => $updated_time,
+                    whensent => $created_time,
+                    state => $state,
+                    postcode => '',
+                    used_map => 1,
+                    latitude => $request->{lat},
+                    longitude => $request->{long},
+                    areas => ',' . $body->id . ',',
+                    bodies_str => $body->id,
+                    send_method_used => 'Open311',
+                    category => $contact,
+                }
+            );
+
+            $open311->add_media($request->{media_url}, $problem)
+                if $request->{media_url};
+
+            $problem->insert();
+        }
+    }
+
+    return 1;
+}
+
+1;
diff --git a/perllib/Open311/GetUpdates.pm b/perllib/Open311/GetUpdates.pm
index 1b1e339e3..f62acf4a8 100644
--- a/perllib/Open311/GetUpdates.pm
+++ b/perllib/Open311/GetUpdates.pm
@@ -41,7 +41,7 @@ sub get_updates {
 sub update_reports {
     my ( $self, $report_ids, $open311, $body ) = @_;
 
-    my $service_requests = $open311->get_service_requests( $report_ids );
+    my $service_requests = $open311->get_service_requests( { report_ids => $report_ids } );
     my $requests = $service_requests->{request};
 
     for my $request (@$requests) {
diff --git a/t/open311/getservicerequests.t b/t/open311/getservicerequests.t
new file mode 100644
index 000000000..878c178ef
--- /dev/null
+++ b/t/open311/getservicerequests.t
@@ -0,0 +1,301 @@
+#!/usr/bin/env perl
+
+use FixMyStreet::TestMech;
+
+use_ok( 'Open311' );
+use_ok( 'Open311::GetServiceRequests' );
+use DateTime;
+use DateTime::Format::W3CDTF;
+
+my $mech = FixMyStreet::TestMech->new;
+
+my $user = $mech->create_user_ok('system_user@example.com', name => 'test users');
+my $body = $mech->create_body_ok(2482, 'Bromley');
+my $contact = $mech->create_contact_ok( body_id => $body->id, category => 'sidewalks', email => 'sidewalks@example.com' );
+
+my $dtf = DateTime::Format::W3CDTF->new;
+
+my $requests_xml = qq{<?xml version="1.0" encoding="utf-8"?>
+<service_requests>
+<request>
+<service_request_id>638344</service_request_id>
+<status>open</status>
+<status_notes>This is a note.</status_notes>
+<service_name>Sidewalk and Curb Issues</service_name>
+<service_code>sidewalks</service_code>
+<description>This is a sidewalk problem</description>
+<agency_responsible></agency_responsible>
+<service_notice></service_notice>
+<requested_datetime>2010-04-14T06:37:38-08:00</requested_datetime>
+<updated_datetime>2010-04-14T06:37:38-08:00</updated_datetime>
+<expected_datetime>2010-04-15T06:37:38-08:00</expected_datetime>
+<lat>51.4021</lat>
+<long>0.01578</long>
+</request>
+<request>
+<service_request_id>638345</service_request_id>
+<status>investigating</status>
+<status_notes>This is a for a different issue.</status_notes>
+<service_name>Not Sidewalk and Curb Issues</service_name>
+<service_code>not_sidewalks</service_code>
+<description>This is a problem</description>
+<agency_responsible></agency_responsible>
+<service_notice></service_notice>
+<requested_datetime>2010-04-15T06:37:38-08:00</requested_datetime>
+<updated_datetime>2010-04-15T06:37:38-08:00</updated_datetime>
+<expected_datetime>2010-04-15T06:37:38-08:00</expected_datetime>
+<lat>51.4021</lat>
+<long>0.01578</long>
+</request>
+</service_requests>
+};
+
+my $o = Open311->new(
+    jurisdiction => 'mysociety',
+    endpoint => 'http://example.com',
+    test_mode => 1,
+    test_get_returns => { 'requests.xml' => $requests_xml }
+);
+
+subtest 'basic parsing checks' => sub {
+    my $update = Open311::GetServiceRequests->new( system_user => $user );
+    FixMyStreet::override_config {
+        MAPIT_URL => 'http://mapit.uk/',
+    }, sub {
+        $update->create_problems( $o, $body );
+    };
+
+    my $p1_date = $dtf->parse_datetime('2010-04-14T06:37:38-08:00')
+                    ->set_time_zone(
+                        FixMyStreet->time_zone || FixMyStreet->local_time_zone
+                  );
+
+    my $p = FixMyStreet::DB->resultset('Problem')->search(
+                { external_id => 638344 }
+            )->first;
+
+    ok $p, 'Found problem';
+    is $p->detail, 'This is a sidewalk problem', 'correct problem description';
+    is $p->created, $p1_date, 'Problem has correct creation date';
+    is $p->confirmed, $p1_date, 'Problem has correct confirmed date';
+    is $p->whensent, $p1_date, 'Problem has whensent set';
+    is $p->state, 'confirmed', 'correct problem state';
+    is $p->user->id, $user->id, 'user set to system user';
+    is $p->category, 'sidewalks', 'correct problem category';
+
+    my $p2 = FixMyStreet::DB->resultset('Problem')->search( { external_id => 638345 } )->first;
+    ok $p2, 'second problem found';
+    ok $p2->whensent, 'second problem marked sent';
+    is $p2->state, 'investigating', 'second problem correct state';
+    is $p2->category, 'Other', 'category falls back to Other';
+};
+
+subtest 'check problems not re-created' => sub {
+    my $update = Open311::GetServiceRequests->new( system_user => $user );
+    FixMyStreet::override_config {
+        MAPIT_URL => 'http://mapit.uk/',
+    }, sub {
+        $update->create_problems( $o, $body );
+    };
+
+    my $count = FixMyStreet::DB->resultset('Problem')->count;
+
+    FixMyStreet::override_config {
+        MAPIT_URL => 'http://mapit.uk/',
+    }, sub {
+        $update->create_problems( $o, $body );
+    };
+
+    my $after_count = FixMyStreet::DB->resultset('Problem')->count;
+
+    is $count, $after_count, "problems not re-created";
+};
+
+for my $test (
+  {
+      desc => 'problem with no id is not created',
+      detail => 'This is a problem with no service_code',
+      subs => { id => '', desc => 'This is a problem with service code' },
+  },
+  {
+      desc => 'problem with no lat is not created',
+      detail => 'This is a problem with no lat',
+      subs => { lat => '', desc => 'This is a problem with no lat' },
+  },
+  {
+      desc => 'problem with no long is not created',
+      detail => 'This is a problem with no long',
+      subs => { long => '', desc => 'This is a problem with no long' },
+  },
+  {
+      desc => 'problem with bad lat/long is not created',
+      detail => 'This is a problem with bad lat/long',
+      subs => { lat => '51', long => 0.1, desc => 'This is a problem with bad lat/long' },
+  },
+) {
+    subtest $test->{desc} => sub {
+        my $xml = prepare_xml( $test->{subs} );
+        my $o = Open311->new(
+            jurisdiction => 'mysociety',
+            endpoint => 'http://example.com',
+            test_mode => 1,
+            test_get_returns => { 'requests.xml' => $xml}
+        );
+
+        my $count = FixMyStreet::DB->resultset('Problem')->count;
+        my $update = Open311::GetServiceRequests->new( system_user => $user );
+        FixMyStreet::override_config {
+            MAPIT_URL => 'http://mapit.uk/',
+        }, sub {
+            $update->create_problems( $o, $body );
+        };
+        my $after_count = FixMyStreet::DB->resultset('Problem')->count;
+
+        warn $count;
+        is $count, $after_count, "problems not created";
+
+        my $with_text = FixMyStreet::DB->resultset('Problem')->search( {
+              detail => $test->{detail}
+        } )->count;
+
+        is $with_text, 0, 'no matching problem created';
+    };
+}
+
+my $date = DateTime->new(
+    year => 2010,
+    month => 4,
+    day => 14,
+    hour => 6,
+    minute => 37
+);
+
+for my $test (
+  {
+      start_date => '1',
+      end_date => '',
+      desc => 'do not process if only a start_date',
+      subs => {},
+  },
+  {
+      start_date => '',
+      end_date => '1',
+      desc => 'do not process if only an end_date',
+      subs => {},
+  },
+) {
+    subtest $test->{desc} => sub {
+        my $xml = prepare_xml( $test->{subs} );
+        my $o = Open311->new(
+            jurisdiction => 'mysociety',
+            endpoint => 'http://example.com',
+            test_mode => 1,
+            test_get_returns => { 'requests.xml' => $xml}
+        );
+
+        my $update = Open311::GetServiceRequests->new(
+            start_date => $test->{start_date},
+            end_date => $test->{end_date},
+            system_user => $user,
+        );
+        my $ret = $update->create_problems( $o, $body );
+
+        is $ret, 0, 'failed correctly'
+    };
+}
+
+$date = DateTime->new(
+    year => 2010,
+    month => 4,
+    day => 14,
+    hour => 6,
+    minute => 37
+);
+
+for my $test (
+  {
+      start_date => $date->clone->add(hours => -2),
+      end_date => $date->clone->add(hours => -1),
+      desc => 'do not process if update time after end_date',
+      subs => {},
+  },
+  {
+      start_date => $date->clone->add(hours => 2),
+      end_date => $date->clone->add(hours => 4),
+      desc => 'do not process if update time before start_date',
+      subs => {},
+  },
+  {
+      start_date => $date->clone->add(hours => -2),
+      end_date => $date->clone->add(hours => 4),
+      desc => 'do not process if update time is bad',
+      subs => { update_time => '2010/12/12' },
+  },
+) {
+    subtest $test->{desc} => sub {
+        my $xml = prepare_xml( $test->{subs} );
+        my $o = Open311->new(
+            jurisdiction => 'mysociety',
+            endpoint => 'http://example.com',
+            test_mode => 1,
+            test_get_returns => { 'requests.xml' => $xml}
+        );
+
+        my $update = Open311::GetServiceRequests->new(
+            start_date => $test->{start_date},
+            end_date => $test->{end_date},
+            system_user => $user,
+        );
+        my $count = FixMyStreet::DB->resultset('Problem')->count;
+        FixMyStreet::override_config {
+            MAPIT_URL => 'http://mapit.uk/',
+        }, sub {
+            $update->create_problems( $o, $body );
+        };
+        my $after = FixMyStreet::DB->resultset('Problem')->count;
+
+        is $count, $after, 'problem not added';
+    };
+}
+
+sub prepare_xml {
+    my $replacements = shift;
+
+    my %defaults = (
+        desc => 'this is a problem',
+        lat => 51.4021,
+        long => 0.01578,
+        id => 123456,
+        update_time => '2010-04-14T06:37:38-08:00',
+        %$replacements
+    );
+
+    my $xml = qq[<?xml version="1.0" encoding="utf-8"?>
+<service_requests>
+<request>
+<service_request_id>XXX_ID</service_request_id>
+<status>open</status>
+<status_notes></status_notes>
+<service_name>Sidewalk and Curb Issues</service_name>
+<service_code>sidewalks</service_code>
+<description>XXX_DESC</description>
+<agency_responsible></agency_responsible>
+<service_notice></service_notice>
+<requested_datetime>2010-04-14T06:37:38-08:00</requested_datetime>
+<updated_datetime>XXX_UPDATE_TIME</updated_datetime>
+<expected_datetime>2010-04-15T06:37:38-08:00</expected_datetime>
+<lat>XXX_LAT</lat>
+<long>XXX_LONG</long>
+</request>
+</service_requests>
+];
+
+    for my $key (keys %defaults) {
+        my $string = 'XXX_' . uc $key;
+        $xml =~ s/$string/$defaults{$key}/;
+    }
+
+    return $xml;
+}
+
+done_testing();
diff --git a/templates/web/base/admin/open311-form-fields.html b/templates/web/base/admin/open311-form-fields.html
index d1067205c..393693264 100644
--- a/templates/web/base/admin/open311-form-fields.html
+++ b/templates/web/base/admin/open311-form-fields.html
@@ -68,6 +68,21 @@
     <div class="admin-hint">
       <p>
         [% loc(
+            "Enable <strong>Open311 problem-fetching</strong> if you want to display reports created at
+          the endpoint to FixMyStreet. If you're not sure, you probably do not, so leave this unchecked.
+          For more information, see 
+          <a href='https://www.mysociety.org/2013/02/20/open311-extended/' class='admin-offsite-link'>this article</a>."
+        ) %]
+      </p>
+    </div>
+    <p>
+        <input type="checkbox" id="fetch_problems" name="fetch_problems"[% ' checked' IF object.fetch_problems %]>
+        <label for="fetch_problems" class="inline">[% loc('Use Open311 problem fetching') %]</label>
+    </p>
+
+    <div class="admin-hint">
+      <p>
+        [% loc(
           "If you've enabled Open311 update-sending above, you must identify which 
           FixMyStreet <strong>user</strong> will be attributed as the creator of those updates
           when they are shown on the site. Enter the ID (number) of that user."