diff options
| author | Matthew Somerville <matthew-github@dracos.co.uk> | 2017-11-14 20:42:48 +0000 |
|---|---|---|
| committer | Matthew Somerville <matthew-github@dracos.co.uk> | 2017-11-15 11:50:27 +0000 |
| commit | c9dd01a66866dbd7ff867608edcd5c82a0b5e1db (patch) | |
| tree | b7930ae97d714616d1c7fb2b21f8bcc2e9d74cdc | |
| parent | a5bde7c6cd2bca5a8f1dfebdfb0a3040ead1566b (diff) | |
Make sure returned geocoder data is UTF-8 decoded.
It turns out that HTTP::Message (as used by LWP::Simple::get) only
decodes text/* and 'XML', not application/json, as used by all the
geocoders. Work around this, hopefully in a future-proof way.
| -rw-r--r-- | perllib/FixMyStreet/Geocode.pm | 6 | ||||
| -rw-r--r-- | t/Mock/Nominatim.pm | 4 | ||||
| -rw-r--r-- | t/cobrand/fixamingata.t | 2 |
3 files changed, 8 insertions, 4 deletions
diff --git a/perllib/FixMyStreet/Geocode.pm b/perllib/FixMyStreet/Geocode.pm index dd1adfe71..aeac0ab6d 100644 --- a/perllib/FixMyStreet/Geocode.pm +++ b/perllib/FixMyStreet/Geocode.pm @@ -79,8 +79,12 @@ sub cache { $url .= '&' . $args if $args; $ua->timeout(15); $js = LWP::Simple::get($url); - $cache_dir->mkpath; + # The returned data is not correctly decoded if the content type is + # e.g. application/json. Which all of our geocoders return. + # uncoverable branch false + $js = decode_utf8($js) if !utf8::is_utf8($js); if ($js && (!$re || $js !~ $re) && !FixMyStreet->config('STAGING_SITE')) { + $cache_dir->mkpath; # uncoverable statement # uncoverable statement $cache_file->spew_utf8($js); } diff --git a/t/Mock/Nominatim.pm b/t/Mock/Nominatim.pm index 5edbc9b30..6a61b423e 100644 --- a/t/Mock/Nominatim.pm +++ b/t/Mock/Nominatim.pm @@ -6,7 +6,7 @@ use Web::Simple; has json => ( is => 'lazy', default => sub { - JSON->new->pretty->allow_blessed->convert_blessed; + JSON->new->utf8->pretty->allow_blessed->convert_blessed; }, ); @@ -30,7 +30,7 @@ sub query { my ($self, $q) = @_; if ($q eq 'high street') { return [ - {"osm_type"=>"way","osm_id"=>"4684282","lat"=>"55.9504009","lon"=>"-3.1858425","display_name"=>"High Street, Old Town, City of Edinburgh, Scotland, EH1 1SP, United Kingdom","class"=>"highway","type"=>"tertiary","importance"=>0.55892577838734}, + {"osm_type"=>"way","osm_id"=>"4684282","lat"=>"55.9504009","lon"=>"-3.1858425","display_name"=>"High Street, Old Town, City of Ed\x{ed}nburgh, Scotland, EH1 1SP, United Kingdom","class"=>"highway","type"=>"tertiary","importance"=>0.55892577838734}, {"osm_type"=>"node","osm_id"=>"27424410","lat"=>"55.8596449","lon"=>"-4.240377","display_name"=>"High Street, Collegelands, Merchant City, Glasgow, Glasgow City, Scotland, G, United Kingdom","class"=>"railway","type"=>"station","importance"=>0.53074299592768} ]; } diff --git a/t/cobrand/fixamingata.t b/t/cobrand/fixamingata.t index 06b5830e3..133a8c950 100644 --- a/t/cobrand/fixamingata.t +++ b/t/cobrand/fixamingata.t @@ -102,7 +102,7 @@ subtest "Test ajax decimal points" => sub { $mech->content_contains('51.5'); $mech->get_ok('/ajax/lookup_location?term=high+street'); - $mech->content_contains('Edinburgh'); + $mech->content_contains("Ed\xc3\xadnburgh"); }; }; |