Various encoding fixing.

12 files changed, 61 insertions, 47 deletions

diff --git a/bin/send-questionnaires b/bin/send-questionnaires
index eb67cd5d1..0df4c45a8 100755
--- a/bin/send-questionnaires
+++ b/bin/send-questionnaires
@@ -75,7 +75,7 @@ foreach my $row (@$unsent) {
 
     my %h = map { $_ => $row->{$_} } qw/name title detail category/;
     $h{created} = Page::prettify_duration($row->{created}, 'week');
-    $h{councils} = join(' and ', map { $areas_info->{$_}->{name} } @councils);
+    $h{councils} = join(' and ', map { encode_utf8($areas_info->{$_}->{name}) } @councils);
 
     my $id = dbh()->selectrow_array("select nextval('questionnaire_id_seq');");
     dbh()->do('insert into questionnaire (id, problem_id, whensent)
diff --git a/bin/send-reports b/bin/send-reports
index ef3da8c46..7f9454685 100755
--- a/bin/send-reports
+++ b/bin/send-reports
@@ -115,7 +115,7 @@ foreach my $row (@$unsent) {
 
         my $council = $row->{council};
         $areas_info = mySociety::MaPit::call('areas', $council);
-        my $name = $areas_info->{$council}->{name};
+        my $name = encode_utf8($areas_info->{$council}->{name});
         my ($council_email, $confirmed, $note) = dbh()->selectrow_array(
             "SELECT email,confirmed,note FROM contacts WHERE deleted='f'
                 and area_id=? AND category=?", {}, $council, 'Empty property');
@@ -142,7 +142,7 @@ foreach my $row (@$unsent) {
         my (@dear, %recips);
         my $all_confirmed = 1;
         foreach my $council (@councils) {
-            my $name = $areas_info->{$council}->{name};
+            my $name = encode_utf8($areas_info->{$council}->{name});
             push @dear, $name;
             if ($council == 2330) { # E. Hants have a web service
                 $send_web = 1;
@@ -186,7 +186,7 @@ foreach my $row (@$unsent) {
             : '';
         $h{missing} = ''; 
         if ($missing) {
-            my $name = $areas_info->{$missing}->{name};
+            my $name = encode_utf8($areas_info->{$missing}->{name});
             $h{missing} = '[ We realise this problem might be the responsibility of ' . $name
                 . "; however, we don't currently have any contact details for them.
 If you know of an appropriate contact address, please do get in touch. ]\n\n";
diff --git a/perllib/Cobrand.pm b/perllib/Cobrand.pm
index 3d604984d..fa2dd9e88 100644
--- a/perllib/Cobrand.pm
+++ b/perllib/Cobrand.pm
@@ -55,7 +55,7 @@ my %fns = (
     'front_stats' => { default => '\&Problems::front_stats' },
     # Given a STRING ($_[1]) representing a location and a QUERY, return a string that
     # includes any disambiguating information available
-    'disambiguate_location' => { default => '$_[1]' },
+    'disambiguate_location' => { default => '"$_[1]&gl=uk"' },
     # Parameter is EPOCHTIME
     'prettify_epoch' => { default => '0' },
     # Parameters are FORM_NAME, QUERY. Return HTML for any extra needed elements for FORM_NAME
diff --git a/perllib/Cobrands/Barnet/Util.pm b/perllib/Cobrands/Barnet/Util.pm
index 8ce296aaf..e0821e4ba 100644
--- a/perllib/Cobrands/Barnet/Util.pm
+++ b/perllib/Cobrands/Barnet/Util.pm
@@ -79,7 +79,7 @@ sub council_check {
     }
     my $url = 'http://www.fixmystreet.com/';
     $url .= 'alert' if $context eq 'alert';
-    $url .= '?pc=' . URI::Escape::uri_escape_utf8($q->param('pc')) if $q->param('pc');
+    $url .= '?pc=' . URI::Escape::uri_escape($q->param('pc') if $q->param('pc');
     my $error_msg = "That location is not covered by Barnet.
 Please visit <a href=\"$url\">the main FixMyStreet site</a>.";
     return (0, $error_msg);
diff --git a/perllib/FixMyStreet/Geocode.pm b/perllib/FixMyStreet/Geocode.pm
index 892077d6e..475b2f9bc 100644
--- a/perllib/FixMyStreet/Geocode.pm
+++ b/perllib/FixMyStreet/Geocode.pm
@@ -9,6 +9,7 @@
 package FixMyStreet::Geocode;
 
 use strict;
+use Encode;
 use Error qw(:try);
 use File::Slurp;
 use File::Path ();
@@ -87,8 +88,9 @@ sub geocoded_string_coordinates {
 # of the site. 
 sub string {
     my ($s, $q) = @_;
+    $s = decode_utf8($s);
     $s = lc($s);
-    $s =~ s/[^-&0-9a-z ']/ /g;
+    $s =~ s/[^-&\w ']/ /g;
     $s =~ s/\s+/ /g;
     $s = URI::Escape::uri_escape_utf8($s);
     $s = Cobrand::disambiguate_location(Page::get_cobrand($q), "q=$s", $q);
@@ -102,7 +104,7 @@ sub string {
     } else {
         $url .= ',+UK' unless $url =~ /united\++kingdom$/ || $url =~ /uk$/i
             || mySociety::Config::get('COUNTRY') ne 'GB';
-        $url .= '&sensor=false&gl=uk&key=' . mySociety::Config::get('GOOGLE_MAPS_API_KEY');
+        $url .= '&sensor=false&key=' . mySociety::Config::get('GOOGLE_MAPS_API_KEY');
         $js = LWP::Simple::get($url);
         File::Path::mkpath($cache_dir);
         File::Slurp::write_file($cache_file, $js) if $js && $js !~ /"code":6[12]0/;
diff --git a/perllib/Page.pm b/perllib/Page.pm
index 4281e584b..308d2a5c6 100644
--- a/perllib/Page.pm
+++ b/perllib/Page.pm
@@ -51,7 +51,7 @@ my $lastmodified;
 sub do_fastcgi {
     my ($func, $lm) = @_;
 
-    binmode(STDOUT, ":utf8");
+    #binmode(STDOUT, ":utf8");
 
     try {
         my $W = new mySociety::WatchUpdate();
@@ -113,7 +113,7 @@ sub microsite {
     my $lang;
     $lang = 'cy' if $host =~ /cy/;
     $lang = 'en-gb' if $host =~ /^en\./;
-    Cobrand::set_lang_and_domain(get_cobrand($q), $lang, 1);
+    Cobrand::set_lang_and_domain(get_cobrand($q), $lang); # , 1);
 
     Problems::set_site_restriction($q);
     Memcached::set_namespace(mySociety::Config::get('BCI_DB_NAME') . ":");
@@ -304,7 +304,7 @@ sub footer {
     my ($q, %params) = @_;
 
     my $pc = $q->param('pc') || '';
-    $pc = "?pc=" . ent($pc) if $pc;
+    $pc = '?pc=' . URI::Escape::uri_escape($pc) if $pc;
 
     my $creditline = _('Built by <a href="http://www.mysociety.org/">mySociety</a>, using some <a href="http://github.com/mysociety/fixmystreet">clever</a>&nbsp;<a href="https://secure.mysociety.org/cvstrac/dir?d=mysociety/services/TilMa">code</a>.');
     if (mySociety::Config::get('COUNTRY') eq 'NO') {
@@ -561,7 +561,7 @@ sub display_problem_meta_line($$) {
             $problem->{council} =~ s/\|.*//g;
             my @councils = split /,/, $problem->{council};
             my $areas_info = mySociety::MaPit::call('areas', \@councils);
-            my $council = join(' and ', map { $areas_info->{$_}->{name} } @councils);
+            my $council = join(' and ', map { encode_utf8($areas_info->{$_}->{name}) } @councils);
             $out .= '<small class="council_sent_info">';
             $out .= $q->br() . sprintf(_('Sent to %s %s later'), $council, prettify_duration($problem->{whensent}, 'minute'));
             $out .= '</small>';
@@ -684,7 +684,7 @@ sub short_name {
     return 'Durham+City' if $area->{name} eq 'Durham City Council';
     if ($area->{name} =~ /^(Os|Nes|V\xe5ler|Sande|B\xf8|Her\xf8y)$/) {
         my $parent = $info->{$area->{parent_area}}->{name};
-        return "$area->{name},+$parent";
+        return URI::Escape::uri_escape_utf8("$area->{name}, $parent");
     }
     my $name = $area->{name};
     $name =~ s/ (Borough|City|District|County) Council$//;
diff --git a/perllib/Problems.pm b/perllib/Problems.pm
index 8c6eeccad..ff222f874 100644
--- a/perllib/Problems.pm
+++ b/perllib/Problems.pm
@@ -93,12 +93,14 @@ sub recent_photos {
         my $key = "recent_photos:$site_key:$num:$lat:$lon:$dist";
         $probs = Memcached::get($key);
         unless ($probs) {
-            $probs = select_all("select id, title
+            $probs = mySociety::Locale::in_gb_locale {
+                select_all("select id, title
                 from problem_find_nearby(?, ?, ?) as nearby, problem
                 where nearby.problem_id = problem.id
                 and state in ('confirmed', 'fixed') and photo is not null
                 $site_restriction
                 order by confirmed desc limit $num", $lat, $lon, $dist);
+            };
             Memcached::set($key, $probs, 3600);
         }
     } else {
@@ -259,7 +261,7 @@ sub problems_matching_criteria {
     my $areas_info = mySociety::MaPit::call('areas', \@councils);
     foreach my $problem (@$problems){
         if ($problem->{council}) {
-             my @council_names = map { $areas_info->{$_}->{name}} @{$problem->{council}} ;
+             my @council_names = map { encode_utf8($areas_info->{$_}->{name}) } @{$problem->{council}} ;
              $problem->{council} = join(' and ', @council_names);
         }
     }
diff --git a/web-admin/index.cgi b/web-admin/index.cgi
index be37a479b..b2322cda3 100755
--- a/web-admin/index.cgi
+++ b/web-admin/index.cgi
@@ -18,6 +18,7 @@ use strict;
 use FindBin;
 use lib "$FindBin::Bin/../perllib";
 use lib "$FindBin::Bin/../commonlib/perllib";
+use Encode;
 use POSIX qw(strftime);
 use Digest::MD5 qw(md5_hex);
 
@@ -215,7 +216,7 @@ sub admin_councils_list ($) {
                 if $areas->{$_}->{parent_area};
 
             push @li, $q->li($q->a({ href => NewURL($q, area_id => $_, page => 'councilcontacts') }, 
-                  $areas->{$_}->{name}) . $parent .
+                  encode_utf8($areas->{$_}->{name})) . encode_utf8($parent) .
                     ($bci_info->{$_} && $q->{site} ne 'emptyhomes' ?
                         $bci_info->{$_}->{c} . ' addresses'
                     : ''));
@@ -309,7 +310,7 @@ sub admin_council_contacts ($$) {
 
     # Title
     my $mapit_data = mySociety::MaPit::call('area', $area_id);
-    my $title = 'Council contacts for ' . $mapit_data->{name};
+    my $title = sprintf(_('Council contacts for %s'), encode_utf8($mapit_data->{name}));
     print html_head($q, $title);
     print $q->h1($title);
     print $updated;
@@ -392,7 +393,7 @@ sub admin_council_edit ($$$) {
     my $mapit_data = mySociety::MaPit::call('area', $area_id);
     
     # Title
-    my $title = 'Council contacts for ' . $mapit_data->{name};
+    my $title = sprintf(_('Council contacts for %s'), encode_utf8($mapit_data->{name}));
     print html_head($q, $title);
     print $q->h1($title);
 
diff --git a/web/alert.cgi b/web/alert.cgi
index 896ca9ccc..313d55c7d 100755
--- a/web/alert.cgi
+++ b/web/alert.cgi
@@ -11,6 +11,7 @@
 use strict;
 use Standard;
 use Digest::SHA1 qw(sha1_hex);
+use Encode;
 use Error qw(:try);
 use CrossSell;
 use FixMyStreet::Alert;
@@ -110,7 +111,7 @@ sub alert_list {
     my $areas = mySociety::MaPit::call('point', "4326/$lon,$lat", type => \@types);
     my $cobrand = Page::get_cobrand($q);
     my ($success, $error_msg) = Cobrand::council_check($cobrand, { all_councils => $areas }, $q, 'alert');    
-    if (!$success){
+    if (!$success) {
         return alert_front_page($q, $error_msg);
     }
 
@@ -128,10 +129,12 @@ sub alert_list {
                 $ward = $_;
             }
         }
-        push @options, [ 'council', $council->{id}, Page::short_name($council->{name}),
-            sprintf(_("Problems within %s"), $council->{name}) ];
-        push @options, [ 'ward', $council->{id}.':'.$ward->{id}, Page::short_name($council->{name}) . '/'
-            . Page::short_name($ward->{name}), sprintf(_("Problems within %s ward"), $ward->{name}) ];
+        my $council_name = encode_utf8($council->{name});
+        my $ward_name = encode_utf8($ward->{name});
+        push @options, [ 'council', $council->{id}, Page::short_name($council),
+            sprintf(_("Problems within %s"), $council_name) ];
+        push @options, [ 'ward', $council->{id}.':'.$ward->{id}, Page::short_name($council) . '/'
+            . Page::short_name($ward), sprintf(_("Problems within %s ward"), $ward_name) ];
         
         $options_start = "<div><ul id='rss_feed'>";
         $options = alert_list_options($q, @options);
@@ -144,8 +147,9 @@ sub alert_list {
         foreach (values %$areas) {
             $council = $_;
         }
-        push @options, [ 'council', $council->{id}, Page::short_name($council->{name}),
-            sprintf(_("Problems within %s"), $council->{name}) ];
+        my $council_name = encode_utf8($council->{name});
+        push @options, [ 'council', $council->{id}, Page::short_name($council),
+            sprintf(_("Problems within %s"), $council_name) ];
         
         $options_start = "<div><ul id='rss_feed'>"; 
         $options = alert_list_options($q, @options);
@@ -166,26 +170,30 @@ sub alert_list {
                 $d_ward = $_;
             }
         }
+        my $district_name = encode_utf8($district->{name});
+        my $d_ward_name = encode_utf8($d_ward->{name});
+        my $county_name = encode_utf8($county->{name});
+        my $c_ward_name = encode_utf8($c_ward->{name});
         push @options,
-            [ 'area', $district->{id}, Page::short_name($district->{name}), $district->{name} ],
-            [ 'area', $district->{id}.':'.$d_ward->{id}, Page::short_name($district->{name}) . '/'
-              . Page::short_name($d_ward->{name}), "$d_ward->{name} ward, $district->{name}" ],
-            [ 'area', $county->{id}, Page::short_name($county->{name}), $county->{name} ],
-            [ 'area', $county->{id}.':'.$c_ward->{id}, Page::short_name($county->{name}) . '/'
-              . Page::short_name($c_ward->{name}), "$c_ward->{name} ward, $county->{name}" ];
+            [ 'area', $district->{id}, Page::short_name($district), $district_name ],
+            [ 'area', $district->{id}.':'.$d_ward->{id}, Page::short_name($district) . '/'
+              . Page::short_name($d_ward), "$d_ward_name ward, $district_name" ],
+            [ 'area', $county->{id}, Page::short_name($county), $county_name ],
+            [ 'area', $county->{id}.':'.$c_ward->{id}, Page::short_name($county) . '/'
+              . Page::short_name($c_ward), "$c_ward_name ward, $county_name" ];
         $options_start = '<div id="rss_list">';
         $options = $q->p($q->strong(_('Problems within the boundary of:'))) .
             $q->ul(alert_list_options($q, @options));
         @options = ();
         push @options,
-            [ 'council', $district->{id}, Page::short_name($district->{name}), $district->{name} ],
-            [ 'ward', $district->{id}.':'.$d_ward->{id}, Page::short_name($district->{name}) . '/' . Page::short_name($d_ward->{name}),
-              "$district->{name}, within $d_ward->{name} ward" ];
+            [ 'council', $district->{id}, Page::short_name($district), $district_name ],
+            [ 'ward', $district->{id}.':'.$d_ward->{id}, Page::short_name($district) . '/' . Page::short_name($d_ward),
+              "$district_name, within $d_ward_name ward" ];
         if ($q->{site} ne 'emptyhomes') {
             push @options,
-                [ 'council', $county->{id}, Page::short_name($county->{name}), $county->{name} ],
-                [ 'ward', $county->{id}.':'.$c_ward->{id}, Page::short_name($county->{name}) . '/'
-                  . Page::short_name($c_ward->{name}), "$county->{name}, within $c_ward->{name} ward" ];
+                [ 'council', $county->{id}, Page::short_name($county), $county_name ],
+                [ 'ward', $county->{id}.':'.$c_ward->{id}, Page::short_name($county) . '/'
+                  . Page::short_name($c_ward), "$county_name, within $c_ward_name ward" ];
             $options .= $q->p($q->strong(_('Or problems reported to:'))) .
                 $q->ul(alert_list_options($q, @options));
             $options_end = $q->p($q->small(_('FixMyStreet sends different categories of problem
diff --git a/web/index.cgi b/web/index.cgi
index 758ca8c50..189231682 100755
--- a/web/index.cgi
+++ b/web/index.cgi
@@ -9,6 +9,7 @@
 use strict;
 use Standard;
 use Utils;
+use Encode;
 use Error qw(:try);
 use File::Slurp;
 use LWP::Simple;
@@ -629,7 +630,7 @@ If this is not the correct location, simply click on the map again. '));
     $vars{page_heading} = $q->h1(_('Reporting a problem'));
 
     if ($details eq 'all') {
-        my $council_list = join('</strong> or <strong>', map { $_->{name} } values %$all_councils);
+        my $council_list = join('</strong> or <strong>', map { encode_utf8($_->{name}) } values %$all_councils);
         if ($q->{site} eq 'emptyhomes'){
             $vars{text_help} = '<p>' . sprintf(_('All the information you provide here will be sent to <strong>%s</strong>.
 On the site, we will show the subject and details of the problem, plus your
@@ -648,9 +649,9 @@ name if you give us permission.'), $council_list);
             push @missing, $_ unless $councils{$_};
         }
         my $n = @missing;
-        my $list = join(' or ', map { $all_councils->{$_}->{name} } @missing);
+        my $list = join(' or ', map { encode_utf8($all_councils->{$_}->{name}) } @missing);
         $vars{text_help} = '<p>All the information you provide here will be sent to <strong>'
-            . join('</strong> or <strong>', map { $all_councils->{$_}->{name} } @councils)
+            . join('</strong> or <strong>', map { encode_utf8($all_councils->{$_}->{name}) } @councils)
             . '</strong>. The subject and details of the problem will be public, plus your
 name if you give us permission.';
         $vars{text_help} .= ' We do <strong>not</strong> yet have details for the other council';
@@ -661,7 +662,7 @@ problems for $list and emailing it to us at <a href='mailto:$e'>$e</a>.";
             . '|' . join(',', @missing) . '">';
     } else {
         my $e = Cobrand::contact_email($cobrand);
-        my $list = join(' or ', map { $_->{name} } values %$all_councils);
+        my $list = join(' or ', map { encode_utf8($_->{name}) } values %$all_councils);
         my $n = scalar keys %$all_councils;
         if ($q->{site} ne 'emptyhomes') {
             $vars{text_help} = '<p>We do not yet have details for the council';
diff --git a/web/reports.cgi b/web/reports.cgi
index 3ac22b0ce..12e3a3014 100755
--- a/web/reports.cgi
+++ b/web/reports.cgi
@@ -130,7 +130,7 @@ sub main {
         }
         my $type = 'council_problems'; # Problems sent to a council
         my (@params, %title_params);
-        $title_params{COUNCIL} = $area_name;
+        $title_params{COUNCIL} = encode_utf8($area_name);
         push @params, $one_council->{id} if $rss eq 'reports';
         push @params, $ward ? $ward->{id} : $one_council->{id};
         if ($ward && $rss eq 'reports') {
@@ -208,9 +208,9 @@ sub main {
             my $url = Page::short_name($areas_info->{$_}, $areas_info);
             my $cobrand_url = Cobrand::url($cobrand, "/reports/$url", $q);
             print '><td align="left"><a href="' . $cobrand_url . '">' .
-                $areas_info->{$_}->{name};
-            if ($areas_info->{$_}->{parent_area} && $url =~ /,/) {
-                print ', ' . $areas_info->{$areas_info->{$_}->{parent_area}}->{name};
+                encode_utf8($areas_info->{$_}->{name});
+            if ($areas_info->{$_}->{parent_area} && $url =~ /,|%2C/) {
+                print ', ' . encode_utf8($areas_info->{$areas_info->{$_}->{parent_area}}->{name});
             }
             print '</a></td>';
             summary_cell(\@{$open{$_}{new}});
@@ -229,7 +229,7 @@ sub main {
         }
         print '</table>';
     } else {
-        my $name = $one_council->{name};
+        my $name = encode_utf8($one_council->{name});
         if (!$name) {
             print Page::header($q, title=>_("Summary reports"));
             print "Council with identifier " . ent($one_council->{id}). " not found. ";
diff --git a/web/rss.cgi b/web/rss.cgi
index 6ba23abe2..5c2d3701e 100755
--- a/web/rss.cgi
+++ b/web/rss.cgi
@@ -46,7 +46,7 @@ sub main {
         my $id = $q->param('id');
         my $va_info = mySociety::MaPit::call('area', $id);
         my $qs = '/'.$id;
-        $out = FixMyStreet::Alert::generate_rss($type, $xsl, $qs, [$id], { NAME => $va_info->{name} }, $cobrand, $q);
+        $out = FixMyStreet::Alert::generate_rss($type, $xsl, $qs, [$id], { NAME => encode_utf8($va_info->{name}) }, $cobrand, $q);
     } elsif ($type eq 'all_problems') {
         $out = FixMyStreet::Alert::generate_rss($type, $xsl, '', undef, undef, $cobrand, $q);
     } else {