Expire cached geolocations after a week.

And refactor geoloction caching code into one function
(except for Zurich which has its own SOAP based system).

6 files changed, 44 insertions, 117 deletions

diff --git a/perllib/FixMyStreet/Geocode.pm b/perllib/FixMyStreet/Geocode.pm
index b5be152a8..616df87b0 100644
--- a/perllib/FixMyStreet/Geocode.pm
+++ b/perllib/FixMyStreet/Geocode.pm
@@ -7,6 +7,11 @@
 package FixMyStreet::Geocode;
 
 use strict;
+use Digest::MD5 qw(md5_hex);
+use Encode;
+use File::Slurp;
+use File::Path ();
+use LWP::Simple qw($ua);
 use URI::Escape;
 use FixMyStreet::Geocode::Bing;
 use FixMyStreet::Geocode::Google;
@@ -55,4 +60,25 @@ sub escape {
     return $s;
 }
 
+sub cache {
+    my ($type, $url, $args, $re) = @_;
+    my $cache_dir = FixMyStreet->config('GEO_CACHE') . $type . '/';
+    my $cache_file = $cache_dir . md5_hex($url);
+    my $js;
+    if (-s $cache_file && -M $cache_file <= 7) {
+        $js = File::Slurp::read_file($cache_file);
+    } else {
+        $url .= '&' . $args if $args;
+        $ua->timeout(15);
+        $js = LWP::Simple::get($url);
+        $js = encode_utf8($js) if utf8::is_utf8($js);
+        File::Path::mkpath($cache_dir);
+        if ($js && (!$re || $js !~ $re)) {
+            File::Slurp::write_file($cache_file, $js);
+        }
+    }
+    $js = JSON->new->utf8->allow_nonref->decode($js) if $js;
+    return $js;
+}
+
 1;
diff --git a/perllib/FixMyStreet/Geocode/Bing.pm b/perllib/FixMyStreet/Geocode/Bing.pm
index 702e19814..c62a87767 100644
--- a/perllib/FixMyStreet/Geocode/Bing.pm
+++ b/perllib/FixMyStreet/Geocode/Bing.pm
@@ -7,11 +7,6 @@
 package FixMyStreet::Geocode::Bing;
 
 use strict;
-use Encode;
-use File::Slurp;
-use File::Path ();
-use LWP::Simple;
-use Digest::MD5 qw(md5_hex);
 
 use mySociety::Locale;
 
@@ -34,24 +29,10 @@ sub string {
     $url .= '&userLocation=' . $params->{centre} if $params->{centre};
     $url .= '&c=' . $params->{bing_culture} if $params->{bing_culture};
 
-    my $cache_dir = FixMyStreet->config('GEO_CACHE') . 'bing/';
-    my $cache_file = $cache_dir . md5_hex($url);
-    my $js;
-    if (-s $cache_file) {
-        $js = File::Slurp::read_file($cache_file);
-    } else {
-        $url .= '&key=' . FixMyStreet->config('BING_MAPS_API_KEY');
-        $js = LWP::Simple::get($url);
-        $js = encode_utf8($js) if utf8::is_utf8($js);
-        File::Path::mkpath($cache_dir);
-        File::Slurp::write_file($cache_file, $js) if $js;
-    }
-
+    my $js = FixMyStreet::Geocode::cache('bing', $url, 'key=' . FixMyStreet->config('BING_MAPS_API_KEY'));
     if (!$js) {
         return { error => _('Sorry, we could not parse that location. Please try again.') };
     }
-
-    $js = JSON->new->utf8->allow_nonref->decode($js);
     if ($js->{statusCode} ne '200') {
         return { error => _('Sorry, we could not find that location.') };
     }
@@ -88,33 +69,15 @@ sub string {
 }
 
 sub reverse {
-    my ( $latitude, $longitude, $bing_culture, $cache ) = @_;
+    my ( $latitude, $longitude, $bing_culture ) = @_;
 
     # Get nearest road-type thing from Bing
     my $key = mySociety::Config::get('BING_MAPS_API_KEY', '');
     if ($key) {
         my $url = "http://dev.virtualearth.net/REST/v1/Locations/$latitude,$longitude?key=$key";
         $url .= '&c=' . $bing_culture if $bing_culture;
-        my $j;
-        if ( $cache ) {
-            my $cache_dir = FixMyStreet->config('GEO_CACHE') . 'bing/';
-            my $cache_file = $cache_dir . md5_hex($url);
-
-            if (-s $cache_file) {
-                $j = File::Slurp::read_file($cache_file);
-            } else {
-                $j = LWP::Simple::get($url);
-                File::Path::mkpath($cache_dir);
-                File::Slurp::write_file($cache_file, $j) if $j;
-            }
-        } else {
-            $j = LWP::Simple::get($url);
-        }
-
-        if ($j) {
-            $j = JSON->new->utf8->allow_nonref->decode($j);
-            return $j;
-        }
+        my $j = FixMyStreet::Geocode::cache('bing', $url);
+        return $j if $j;
     }
 
     return undef;
diff --git a/perllib/FixMyStreet/Geocode/FixaMinGata.pm b/perllib/FixMyStreet/Geocode/FixaMinGata.pm
index 2ea92c422..68e5bc8a5 100644
--- a/perllib/FixMyStreet/Geocode/FixaMinGata.pm
+++ b/perllib/FixMyStreet/Geocode/FixaMinGata.pm
@@ -14,13 +14,8 @@ package FixMyStreet::Geocode::FixaMinGata;
 
 use warnings;
 use strict;
-use Data::Dumper;
 
-use Digest::MD5 qw(md5_hex);
-use Encode;
-use File::Slurp;
-use File::Path ();
-use LWP::Simple qw($ua);
+use LWP::Simple;
 use Memcached;
 use XML::Simple;
 use mySociety::Locale;
@@ -56,25 +51,11 @@ sub string {
         if $params->{country};
     $url .= join('&', map { "$_=$query_params{$_}" } keys %query_params);
 
-    my $cache_dir = FixMyStreet->config('GEO_CACHE') . 'osm/';
-    my $cache_file = $cache_dir . md5_hex($url);
-    my $js;
-    if (-s $cache_file) {
-        $js = File::Slurp::read_file($cache_file);
-    } else {
-        $ua->timeout(15);
-        $js = LWP::Simple::get($url);
-        $js = encode_utf8($js) if utf8::is_utf8($js);
-        File::Path::mkpath($cache_dir);
-        File::Slurp::write_file($cache_file, $js) if $js;
-    }
-
+    my $js = FixMyStreet::Geocode::cache('osm', $url);
     if (!$js) {
         return { error => _('Sorry, we could not find that location.') };
     }
 
-    $js = JSON->new->utf8->allow_nonref->decode($js);
-
     my ( %locations, $error, @valid_locations, $latitude, $longitude );
     foreach (@$js) {
         # These co-ordinates are output as query parameters in a URL, make sure they have a "."
diff --git a/perllib/FixMyStreet/Geocode/Google.pm b/perllib/FixMyStreet/Geocode/Google.pm
index 11ff8ef80..8d3280ae7 100644
--- a/perllib/FixMyStreet/Geocode/Google.pm
+++ b/perllib/FixMyStreet/Geocode/Google.pm
@@ -7,11 +7,6 @@
 package FixMyStreet::Geocode::Google;
 
 use strict;
-use Encode;
-use File::Slurp;
-use File::Path ();
-use LWP::Simple;
-use Digest::MD5 qw(md5_hex);
 use mySociety::Locale;
 
 # string STRING CONTEXT
@@ -24,6 +19,13 @@ sub string {
 
     my $params = $c->cobrand->disambiguate_location($s);
 
+    # For some reason adding gl=uk is no longer sufficient to make google
+    # think we are in the UK for some locations so we explictly add UK to
+    # the address.
+    if ($c->cobrand->country eq 'GB' && $s !~ /, *UK/ && $s !~ /united *kingdom$/) {
+        $s .= ', UK';
+    }
+
     $s = FixMyStreet::Geocode::escape($s);
 
     my $url = 'http://maps.google.com/maps/geo?q=' . $s;
@@ -36,38 +38,11 @@ sub string {
     }
     $url .=  '&hl=' . $params->{lang}    if $params->{lang};
 
-    my $cache_dir = FixMyStreet->config('GEO_CACHE') . 'google/';
-    my $cache_file = $cache_dir . md5_hex($url);
-    my $js;
-    if (-s $cache_file) {
-        $js = File::Slurp::read_file($cache_file);
-    } else {
-        # For some reason adding gl=uk is no longer sufficient to make google
-        # think we are in the UK for some locations so we explictly add UK to
-        # the address. We do it here so as not to invalidate existing cache
-        # entries
-        if (   $c->cobrand->country eq 'GB'
-            && $url !~ /,\+UK/
-            && $url !~ /united\++kingdom$/ )
-        {
-            if ( $url =~ /&/ ) {
-                $url =~ s/&/,+UK&/;
-            } else {
-                $url .= ',+UK';
-            }
-        }
-        $url .= '&sensor=false&key=' . FixMyStreet->config('GOOGLE_MAPS_API_KEY');
-        $js = LWP::Simple::get($url);
-        $js = encode_utf8($js) if utf8::is_utf8($js);
-        File::Path::mkpath($cache_dir);
-        File::Slurp::write_file($cache_file, $js) if $js && $js !~ /"code":6[12]0/;
-    }
-
+    my $args = 'sensor=false&key=' . FixMyStreet->config('GOOGLE_MAPS_API_KEY');
+    my $js = FixMyStreet::Geocode::cache('google', $url, $args, qr/"code":6[12]0/);
     if (!$js) {
         return { error => _('Sorry, we could not parse that location. Please try again.') };
     }
-
-    $js = JSON->new->utf8->allow_nonref->decode($js);
     if ($js->{Status}->{code} ne '200') {
         return { error => _('Sorry, we could not find that location.') };
     }
diff --git a/perllib/FixMyStreet/Geocode/OSM.pm b/perllib/FixMyStreet/Geocode/OSM.pm
index 919940f78..9a27b1576 100644
--- a/perllib/FixMyStreet/Geocode/OSM.pm
+++ b/perllib/FixMyStreet/Geocode/OSM.pm
@@ -9,11 +9,7 @@ package FixMyStreet::Geocode::OSM;
 use warnings;
 use strict;
 
-use Digest::MD5 qw(md5_hex);
-use Encode;
-use File::Slurp;
-use File::Path ();
-use LWP::Simple qw($ua);
+use LWP::Simple;
 use Memcached;
 use XML::Simple;
 use mySociety::Locale;
@@ -47,25 +43,11 @@ sub string {
         if $params->{country};
     $url .= join('&', map { "$_=$query_params{$_}" } keys %query_params);
 
-    my $cache_dir = FixMyStreet->config('GEO_CACHE') . 'osm/';
-    my $cache_file = $cache_dir . md5_hex($url);
-    my $js;
-    if (-s $cache_file) {
-        $js = File::Slurp::read_file($cache_file);
-    } else {
-        $ua->timeout(15);
-        $js = LWP::Simple::get($url);
-        $js = encode_utf8($js) if utf8::is_utf8($js);
-        File::Path::mkpath($cache_dir);
-        File::Slurp::write_file($cache_file, $js) if $js;
-    }
-
+    my $js = FixMyStreet::Geocode::cache('osm', $url);
     if (!$js) {
         return { error => _('Sorry, we could not find that location.') };
     }
 
-    $js = JSON->new->utf8->allow_nonref->decode($js);
-
     my ( $error, @valid_locations, $latitude, $longitude );
     foreach (@$js) {
         # These co-ordinates are output as query parameters in a URL, make sure they have a "."
diff --git a/perllib/FixMyStreet/Geocode/Zurich.pm b/perllib/FixMyStreet/Geocode/Zurich.pm
index 1f0b4fc16..d0355e1e5 100644
--- a/perllib/FixMyStreet/Geocode/Zurich.pm
+++ b/perllib/FixMyStreet/Geocode/Zurich.pm
@@ -66,7 +66,7 @@ sub string {
     my $cache_dir = FixMyStreet->config('GEO_CACHE') . 'zurich/';
     my $cache_file = $cache_dir . md5_hex($s);
     my $result;
-    if (-s $cache_file) {
+    if (-s $cache_file && -M $cache_file <= 7) {
         $result = retrieve($cache_file);
     } else {
         my $search = SOAP::Data->name('search' => $s)->type('');