[Open311] fetch_all option for open311 problem fetching

If a body has a `fetch_all_reports` setting in the extra metadata then
all reports are fetched over Open311 and processed regardless of age.

This is useful for bodies where the API endpoint always returns all the
reports as it suppresses the error messages you would otherwise get
about reports with invalid dates.

3 files changed, 59 insertions, 8 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 835f428ae..7b8c484f9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,7 +2,7 @@
 
 * Unreleased
     - New features
-        - Fetch problems over Open311 #1986
+        - Fetch problems over Open311 #1986 #2067
         - Option to send multiple photos over Open311 #1986
         - Allow Open311 service definitions to include automated
           attributes #1986
diff --git a/perllib/Open311/GetServiceRequests.pm b/perllib/Open311/GetServiceRequests.pm
index 78e9647fa..48e35acab 100644
--- a/perllib/Open311/GetServiceRequests.pm
+++ b/perllib/Open311/GetServiceRequests.pm
@@ -9,6 +9,7 @@ use DateTime::Format::W3CDTF;
 has system_user => ( is => 'rw' );
 has start_date => ( is => 'ro', default => sub { undef } );
 has end_date => ( is => 'ro', default => sub { undef } );
+has fetch_all => ( is => 'rw', default => 0 );
 has verbose => ( is => 'ro', default => 0 );
 has schema => ( is =>'ro', lazy => 1, default => sub { FixMyStreet::DB->schema->connect } );
 has convert_latlong => ( is => 'rw', default => 0 );
@@ -26,19 +27,28 @@ sub fetch {
     );
 
     while ( my $body = $bodies->next ) {
-
-        my $o = Open311->new(
-            endpoint     => $body->endpoint,
-            api_key      => $body->api_key,
-            jurisdiction => $body->jurisdiction,
-        );
+        my $o = $self->create_open311_object( $body );
 
         $self->system_user( $body->comment_user );
         $self->convert_latlong( $body->convert_latlong );
+        $self->fetch_all( $body->get_extra_metadata('fetch_all_problems') );
         $self->create_problems( $o, $body );
     }
 }
 
+# this is so we can test
+sub create_open311_object {
+    my ($self, $body) = @_;
+
+    my $o = Open311->new(
+        endpoint     => $body->endpoint,
+        api_key      => $body->api_key,
+        jurisdiction => $body->jurisdiction,
+    );
+
+    return $o;
+}
+
 sub create_problems {
     my ( $self, $open311, $body ) = @_;
 
@@ -49,7 +59,7 @@ sub create_problems {
 
         $args->{start_date} = DateTime::Format::W3CDTF->format_datetime( $self->start_date );
         $args->{end_date} = DateTime::Format::W3CDTF->format_datetime( $self->end_date );
-    } else {
+    } elsif ( !$self->fetch_all ) {
         my $end_dt = DateTime->now();
         my $start_dt = $end_dt->clone;
         $start_dt->add( hours => -1 );
diff --git a/t/open311/getservicerequests.t b/t/open311/getservicerequests.t
index c59e5cc41..57f112e2f 100644
--- a/t/open311/getservicerequests.t
+++ b/t/open311/getservicerequests.t
@@ -273,6 +273,47 @@ for my $test (
     };
 }
 
+subtest 'check fetch_all body setting ignores date errors' => sub {
+    my $xml = prepare_xml({ id => '12345678' });
+
+    $body->update( {
+        send_method => 'Open311',
+        fetch_problems => 1,
+        comment_user_id => $user->id,
+        endpoint => 'http://open311.localhost/',
+        api_key => 'KEY',
+        jurisdiction => 'test',
+    } );
+    $body->set_extra_metadata( fetch_all_problems => 1 );
+    $body->update();
+
+    my $update = Open311::GetServiceRequests->new(
+        verbose => 1,
+        system_user => $user,
+    );
+    $update = Test::MockObject::Extends->new($update);
+
+    $update->mock('create_open311_object', sub {
+        return Open311->new(
+            jurisdiction => 'mysociety',
+            endpoint => 'http://example.com',
+            test_mode => 1,
+            test_get_returns => { 'requests.xml' => $xml}
+        );
+    } );
+
+    my $count = FixMyStreet::DB->resultset('Problem')->count;
+    FixMyStreet::override_config {
+        MAPIT_URL => 'http://mapit.uk/',
+    }, sub {
+        $update->fetch;
+    };
+
+    my $after = FixMyStreet::DB->resultset('Problem')->count;
+
+    is $after, $count + 1, 'problem created';
+};
+
 for my $test (
   {
       desc => 'convert easting/northing to lat/long',