diff options
Diffstat (limited to 'bin')
| -rwxr-xr-x | bin/gettext-extract | 4 | ||||
| -rwxr-xr-x | bin/gettext-merge | 45 | ||||
| -rwxr-xr-x | bin/import-flickr | 27 | ||||
| -rwxr-xr-x | bin/make_css | 18 | ||||
| -rwxr-xr-x | bin/send-questionnaires | 3 | ||||
| -rwxr-xr-x | bin/send-reports | 90 | ||||
| -rwxr-xr-x | bin/test-run | 72 | ||||
| -rwxr-xr-x | bin/update-areas | 4 |
8 files changed, 205 insertions, 58 deletions
diff --git a/bin/gettext-extract b/bin/gettext-extract index b354c11f2..24defd014 100755 --- a/bin/gettext-extract +++ b/bin/gettext-extract @@ -40,7 +40,7 @@ PO=locale/FixMyStreet.po rm -f $PO # Extract from Perl -xgettext --add-comments=TRANS --language=Perl --keyword=_ --keyword=nget:1,2 --from-code=utf-8 -o $PO perllib/*.pm perllib/FixMyStreet/*.pm web/*.cgi bin/send-* db/alert_types_eha.pl +xgettext --add-comments=TRANS --language=Perl --keyword=_ --keyword=nget:1,2 --from-code=utf-8 -o $PO `find perllib -name "*.pm"` web/*.cgi bin/send-* db/*.pl web-admin/*.cgi # Fix headers TEMP=`tempfile` @@ -76,5 +76,5 @@ do fi done -fixmystreet/bin/make_emptyhomes_po +bin/make_emptyhomes_po diff --git a/bin/gettext-merge b/bin/gettext-merge new file mode 100755 index 000000000..84aa4fdb7 --- /dev/null +++ b/bin/gettext-merge @@ -0,0 +1,45 @@ +#!/bin/bash +# +# bin/gettext-merge +# Update all .po files from new .pot +# +# Copyright (c) 2011 UK Citizens Online Democracy. All rights reserved. +# Email: matthew@mysociety.org; WWW: http://www.mysociety.org/ + +# Yuck +if [ -e ../../locale ] +then + cd ../../ +else if [ -e ../locale ] +then + cd ../ +else if [ -e locale ] +then + cd . +else + echo "Please run with current directory fixmystreet" + exit 1 +fi +fi +fi + +for X in locale/*.UTF-8 +do + if [ -d $X ] + then + cd $X/LC_MESSAGES + echo $X + if [ -e EmptyHomes.po ] + then + msgmerge -o New.po EmptyHomes.po ../../FixMyStreet-EmptyHomes.po + mv New.po EmptyHomes.po + fi + if [ -e FixMyStreet.po ] + then + msgmerge -o New.po FixMyStreet.po ../../FixMyStreet.po + mv New.po FixMyStreet.po + fi + cd - >/dev/null + fi +done + diff --git a/bin/import-flickr b/bin/import-flickr index 992a6ac8e..f4a838547 100755 --- a/bin/import-flickr +++ b/bin/import-flickr @@ -24,7 +24,6 @@ use mySociety::Config; use mySociety::DBHandle qw(dbh select_all); use mySociety::EmailUtil; use mySociety::Email; -use mySociety::GeoUtil; BEGIN { mySociety::Config::set_file("$FindBin::Bin/../conf/general"); @@ -55,36 +54,36 @@ foreach (@$st) { # XXX: Hmm... Use format=perl now Cal has added it for me! :) while ($result =~ /<photo id="([^"]*)" owner="([^"]*)" secret="([^"]*)" server="([^"]*)" farm="([^"]*)" title="([^"]*)".*?latitude="([^"]*)" longitude="([^"]*)".*?machine_tags="([^"]*)"/g) { - my ($id, $owner, $secret, $server, $farm, $title, $lat, $lon, $machine) = ($1, $2, $3, $4, $5, $6, $7, $8, $9); + my ($id, $owner, $secret, $server, $farm, $title, $latitude, $longitude, $machine) = ($1, $2, $3, $4, $5, $6, $7, $8, $9); next if $ids{$id}; - if ($machine =~ /geo:/ && !$lat && !$lon) { + if ($machine =~ /geo:/ && !$latitude && !$longitude) { # Have to fetch raw tags, as otherwise can't tell if it's negative, or how many decimal places my $url = 'http://api.flickr.com/services/rest/?method=flickr.tags.getListPhoto&api_key=' . $key . '&photo_id=' . $id; my $tags = get($url); - ($lon) = $tags =~ /raw="geo:lon=([^"]*)"/i; - ($lat) = $tags =~ /raw="geo:lat=([^"]*)"/i; + ($longitude) = $tags =~ /raw="geo:lon=([^"]*)"/i; + ($latitude) = $tags =~ /raw="geo:lat=([^"]*)"/i; } my $url = "http://farm$farm.static.flickr.com/$server/".$id.'_'.$secret.'_m.jpg'; my $image = get($url); - problem_create($id, $owner, $title, $lat, $lon, $image); + problem_create($id, $owner, $title, $latitude, $longitude, $image); } sub problem_create { - my ($photo_id, $owner, $title, $lat, $lon, $image) = @_; + my ($photo_id, $owner, $title, $latitude, $longitude, $image) = @_; my ($name, $email) = dbh()->selectrow_array("select name, email from partial_user where service='flickr' and nsid=?", {}, $owner); - my ($easting, $northing) = (0,0); + + # set some defaults $name ||= ''; - if ($lat && $lon) { - # XXX This appears to be going wrong :( - ($easting, $northing) = mySociety::GeoUtil::wgs84_to_national_grid($lat, $lon, 'G'); - } + $latitude ||= 0; + $longitude ||= 0; + my $id = dbh()->selectrow_array("select nextval('problem_id_seq')"); Utils::workaround_pg_bytea("insert into problem - (id, postcode, easting, northing, title, detail, name, + (id, postcode, latitude, longitude, title, detail, name, email, phone, photo, state, used_map, anonymous, category, areas) values (?, '', ?, ?, ?, '', ?, ?, '', ?, 'partial', 't', 'f', '', '')", 7, - $id, $easting, $northing, $title, $name, $email, $image + $id, $latitude, $longitude, $title, $name, $email, $image ); dbh()->do('insert into flickr_imported (id, problem_id) values (?, ?)', {}, $photo_id, $id); diff --git a/bin/make_css b/bin/make_css new file mode 100755 index 000000000..7e01da267 --- /dev/null +++ b/bin/make_css @@ -0,0 +1,18 @@ +#!/bin/bash +# +# make_css: +# Generate CSS files from SCSS files. +# Curerntly the CSS files are also in version control, though I guess +# in future they don't need to be, and compressed style could then be used. +# +# Copyright (c) 2011 UK Citizens Online Democracy. All rights reserved. +# Email: matthew@mysociety.org. WWW: http://www.mysociety.org +# +# $Id: send-reports,v 1.79 2010-01-06 16:50:26 louise Exp $ + +DIRECTORY=$(cd `dirname $0` && pwd) + +for file in `find $DIRECTORY/../ -name "*.scss" ! -name "_*"`; do + sass --style expanded $file ${file/scss/css} +done + diff --git a/bin/send-questionnaires b/bin/send-questionnaires index eb67cd5d1..d3945bf12 100755 --- a/bin/send-questionnaires +++ b/bin/send-questionnaires @@ -17,6 +17,7 @@ use lib "$FindBin::Bin/../perllib"; use lib "$FindBin::Bin/../commonlib/perllib"; use File::Slurp; use CGI; # XXX Awkward kludge +use Encode; use CronFns; use Page; @@ -62,7 +63,7 @@ foreach my $row (@$unsent) { my @all_councils = split /,|\|/, $row->{council}; my $cobrand = $row->{cobrand}; my $lang = $row->{lang}; - Cobrand::set_lang_and_domain($cobrand, $lang); + Cobrand::set_lang_and_domain($cobrand, $lang, 1); # Cobranded and non-cobranded messages can share a database. In this case, the conf file # should specify a vhost to send the reports for each cobrand, so that they don't get sent # more than once if there are multiple vhosts running off the same database. The email_host diff --git a/bin/send-reports b/bin/send-reports index d61269d96..96f259d5c 100755 --- a/bin/send-reports +++ b/bin/send-reports @@ -18,11 +18,14 @@ use lib "$FindBin::Bin/../commonlib/perllib"; use Encode; use Error qw(:try); use File::Slurp; +use JSON; +use LWP::Simple; use CGI; # Trying awkward kludge use CronFns; use EastHantsWSDL; use Cobrand; +use Utils; use mySociety::Config; use mySociety::DBHandle qw(dbh); use mySociety::Email; @@ -49,10 +52,12 @@ my $base_url = mySociety::Config::get('BASE_URL'); my $site = CronFns::site($base_url); my $query = "SELECT id, council, category, title, detail, name, email, phone, - used_map, easting, northing, (photo is not null) as has_photo, lang, + used_map, latitude, longitude, (photo is not null) as has_photo, lang, cobrand, cobrand_data - FROM problem WHERE state in ('confirmed','fixed') AND whensent IS NULL - AND council IS NOT NULL"; + FROM problem + WHERE state in ('confirmed','fixed') + AND whensent IS NULL + AND council IS NOT NULL"; my $unsent = dbh()->selectall_arrayref($query, { Slice => {} }); my (%notgot, %note); @@ -67,7 +72,7 @@ foreach my $row (@$unsent) { # more than once if there are multiple vhosts running off the same database. The email_host # call checks if this is the host that sends mail for this cobrand. next unless (Cobrand::email_host($cobrand)); - Cobrand::set_lang_and_domain($cobrand, $row->{lang}); + Cobrand::set_lang_and_domain($cobrand, $row->{lang}, 1); if (dbh()->selectrow_array('select email from abuse where lower(email)=?', {}, lc($row->{email}))) { dbh()->do("update problem set state='hidden' where id=?", {}, $row->{id}); dbh()->commit(); @@ -81,7 +86,7 @@ foreach my $row (@$unsent) { # Template variables for the email my $email_base_url = Cobrand::base_url_for_emails($cobrand, $cobrand_data); - my %h = map { $_ => $row->{$_} } qw/title detail name email phone category easting northing/; + my %h = map { $_ => $row->{$_} } qw/title detail name email phone category latitude longitude/; $h{url} = $email_base_url . '/report/' . $row->{id}; $h{phone_line} = $h{phone} ? _('Phone:') . " $h{phone}\n\n" : ''; if ($row->{has_photo}) { @@ -94,7 +99,21 @@ foreach my $row (@$unsent) { $h{fuzzy} = $row->{used_map} ? _('To view a map of the precise location of this issue') : _('The user could not locate the problem on a map, but to see the area around the location they entered'); $h{closest_address} = ''; - $h{closest_address_machine} = ''; + + # If we are in the UK include eastings and northings, and nearest stuff + $h{easting_northing} = ''; + if ( mySociety::Config::get('COUNTRY') eq 'GB' ) { + + ( $h{easting}, $h{northing} ) = Utils::convert_latlon_to_en( $h{latitude}, $h{longitude} ); + + # email templates don't have conditionals so we need to farmat this here + $h{easting_northing} # + = "Easting: $h{easting}\n\n" # + . "Northing: $h{northing}\n\n"; + + $h{closest_address} = find_closest($row, $h{latitude}, $h{longitude}); + } + $h{closest_address_machine} = $h{closest_address}; my (@to, @recips, $template, $areas_info); if ($site eq 'emptyhomes') { @@ -137,8 +156,8 @@ foreach my $row (@$unsent) { my ($council_email, $confirmed, $note) = dbh()->selectrow_array( "SELECT email,confirmed,note FROM contacts WHERE deleted='f' and area_id=? AND category=?", {}, $council, $row->{category}); - $council_email = essex_contact($row->{easting}, $row->{northing}) if $council == 2225; - $council_email = oxfordshire_contact($row->{easting}, $row->{northing}) if $council == 2237 && $council_email eq 'SPECIAL'; + $council_email = essex_contact($row->{latitude}, $row->{longitude}) if $council == 2225; + $council_email = oxfordshire_contact($row->{latitude}, $row->{longitude}) if $council == 2237 && $council_email eq 'SPECIAL'; unless ($confirmed) { $all_confirmed = 0; $note = 'Council ' . $row->{council} . ' deleted' @@ -159,23 +178,23 @@ foreach my $row (@$unsent) { $template = 'submit-brent' if $row->{council} eq 2488 || $row->{council} eq 2237; $template = File::Slurp::read_file("$FindBin::Bin/../templates/emails/$template"); - if ($h{category} eq 'Other') { - $h{category_footer} = 'this type of local problem'; + if ($h{category} eq _('Other')) { + $h{category_footer} = _('this type of local problem'); $h{category_line} = ''; } else { $h{category_footer} = "'" . $h{category} . "'"; - $h{category_line} = "Category: $h{category}\n\n"; + $h{category_line} = sprintf(_("Category: %s"), $h{category}) . "\n\n"; } - $h{councils_name} = join(' and ', @dear); - $h{multiple} = @dear>1 ? "[ This email has been sent to both councils covering the location of the problem, as the user did not categorise it; please ignore it if you're not the correct council to deal with the issue, or let us know what category of problem this is so we can add it to our system. ]\n\n" + $h{councils_name} = join(_(' and '), @dear); + $h{multiple} = @dear>1 ? "[ " . _("This email has been sent to both councils covering the location of the problem, as the user did not categorise it; please ignore it if you're not the correct council to deal with the issue, or let us know what category of problem this is so we can add it to our system.") . " ]\n\n" : ''; $h{missing} = ''; if ($missing) { my $name = $areas_info->{$missing}->{name}; - $h{missing} = '[ We realise this problem might be the responsibility of ' . $name - . "; however, we don't currently have any contact details for them. -If you know of an appropriate contact address, please do get in touch. ]\n\n"; + $h{missing} = '[ ' + . sprintf(_('We realise this problem might be the responsibility of %s; however, we don\'t currently have any contact details for them. If you know of an appropriate contact address, please do get in touch.'), $name) + . " ]\n\n"; } } @@ -190,6 +209,7 @@ If you know of an appropriate contact address, please do get in touch. ]\n\n"; $send_web = 0; $send_email = 1; } elsif (mySociety::Config::get('STAGING_SITE')) { + # on a staging server send emails to ourselves rather than the councils @recips = ( mySociety::Config::get('CONTACT_EMAIL') ); } elsif ($site eq 'emptyhomes') { my $council = $row->{council}; @@ -254,13 +274,19 @@ if ($verbose) { } } +sub _get_district_for_contact { + my ( $lat, $lon ) = @_; + my $district = + mySociety::MaPit::call( 'point', "4326/$lon,$lat", type => 'DIS' ); + ($district) = keys %$district; + return $district; +} + # Essex has different contact addresses depending upon the district # Might be easier if we start storing in the db all areas covered by a point # Will do for now :) sub essex_contact { - my ($E, $N) = @_; - my $district = mySociety::MaPit::call('point', "27700/$E,$N", type => 'DIS'); - ($district) = keys %$district; + my $district = _get_district_for_contact(@_); my $email; $email = 'eastarea' if $district == 2315 || $district == 2312; $email = 'midarea' if $district == 2317 || $district == 2314 || $district == 2316; @@ -272,9 +298,7 @@ sub essex_contact { # Oxfordshire has different contact addresses depending upon the district sub oxfordshire_contact { - my ($E, $N) = @_; - my $district = mySociety::MaPit::call('point', "27700/$E,$N", type => 'DIS'); - ($district) = keys %$district; + my $district = _get_district_for_contact(@_); my $email; $email = 'northernarea' if $district == 2419 || $district == 2420 || $district == 2421; $email = 'southernarea' if $district == 2417 || $district == 2418; @@ -322,3 +346,25 @@ sub post_easthants_message { return $return; } +sub find_closest { + my ($row, $latitude, $longitude) = @_; + my $str = ''; + + return '' unless $row->{used_map}; + + # Get nearest road-type thing from Bing + my $url = "http://dev.virtualearth.net/REST/v1/Locations/$latitude,$longitude?c=en-GB&key=" . mySociety::Config::get('BING_MAPS_API_KEY'); + my $j = LWP::Simple::get($url); + $j = JSON->new->utf8->allow_nonref->decode($j); + $str .= "Nearest road to the pin placed on the map (automatically generated by Bing Maps): $j->{resourceSets}[0]{resources}[0]{name}\n\n"; + + # Get nearest postcode from Matthew's random gazetteer (put in MaPit? Or elsewhere?) + $url = "http://gazetteer.dracos.vm.bytemark.co.uk/point/$latitude,$longitude.json"; + $j = LWP::Simple::get($url); + $j = JSON->new->utf8->allow_nonref->decode($j); + if ($j->{postcode}) { + $str .= "Nearest postcode to the pin placed on the map (automatically generated): $j->{postcode}[0] ($j->{postcode}[1]m away)\n\n"; + } + return $str; +} + diff --git a/bin/test-run b/bin/test-run index 15f9c3ed2..9c3142207 100755 --- a/bin/test-run +++ b/bin/test-run @@ -32,6 +32,8 @@ use Test::Harness; use File::Find; use lib "$FindBin::Bin/../perllib"; use Cobrand; +use FixMyStreet::Geocode; +use Utils; my @actions = ('report', 'update', 'questionnaire', 'alert', 'static', 'cobrand', 'unit', 'eha_alert', 'import', 'rss'); my %actions_desc = ( @@ -213,6 +215,11 @@ sub submit_postcode{ sub submit_report { my ($postcode, $x, $y, $easting, $northing, $user_num, $council, $texts, $cobrand ) = @_; my @messages = @{$texts}; + + # convert easting, northing to lat lon + my ( $latitude, $longitude ) = + Utils::convert_en_to_latlon( $easting, $northing ); + submit_postcode($cobrand, $postcode, $messages[0]); { # Writing values to hidden fields, so switching @@ -235,8 +242,8 @@ sub submit_report { y => $y, pc => $postcode, council => -1, - easting => $easting, - northing => $northing, + latitude => $latitude, + longitude => $longitude, title => 'My test problem', detail => 'Detail of my test problem', anonymous => 1, @@ -377,13 +384,20 @@ sub do_alert { # sign up for alerts in an area my $postcode = 'EH1 2NG'; - my $x = 2015; my $e = 325066; - my $y = 4175; my $n = 673533; + + my $x = 2015; + my $y = 4175; + + my $e = 325066; + my $n = 673533; + + my ( $lat, $lon ) = (55.948967, -3.201478); + my $messages = english_fms_messages(); submit_postcode('', $postcode, 'Problems in this area'); $wth->browser_follow_link(text => 'Email me new local problems'); $wth->browser_submit_form(form_name => 'alerts', - fields => {feed => "local:" . $e . ":" . $n, + fields => {feed => "local:$lat:$lon", rznvy => email_n(2)} ); $wth->browser_check_contents('Nearly Done!'); my $confirmation_email = $wth->email_get_containing( @@ -396,7 +410,7 @@ sub do_alert { $wth->browser_check_contents('successfully confirmed your alert'); # create and confirm a new problem in the area - submit_report($postcode, $x, $y, 325000, 673387.096774193, 3, undef, $messages, ''); + submit_report($postcode, $x, $y, $e, $n, 3, undef, $messages, ''); # run the alert script call_send_emails(); @@ -419,6 +433,9 @@ sub do_eha_alert { my $postcode = 'EH1 2NG'; my $x = 2015; my $e = 325066; my $y = 4175; my $n = 673533; + + my ( $lat, $lon ) = (55.948967, -3.201478); + my @texts = ('Eiddo gwag yn yr ardal hon', 'Adrodd am eiddo gwag', 'Nawr, gwiriwch eich e-bost', @@ -428,7 +445,7 @@ sub do_eha_alert { submit_postcode('cy.emptyhomes.', $postcode, $texts[0]); $wth->browser_follow_link(text => 'Anfonwch fanylion eiddo gwag lleol newydd ataf i drwy\'r e-bost'); $wth->browser_submit_form(form_name => 'alerts', - fields => {feed => "local:" . $e . ":" . $n, + fields => {feed => "local:$lat:$lon", rznvy => email_n(4)} ); $wth->browser_check_contents($texts[2]); my $confirmation_email = $wth->email_get_containing( @@ -441,7 +458,7 @@ sub do_eha_alert { $wth->browser_check_contents('Rydych wedi cadarnhau\'ch hysbysiad yn llwyddiannus.'); # create and confirm a new problem in the area - submit_report($postcode, $x, $y, 325000, 673387.096774193, 3, undef, \@texts, 'cy.emptyhomes.'); + submit_report($postcode, $x, $y, $e, $n, 3, undef, \@texts, 'cy.emptyhomes.'); # run the alert script call_send_emails(); @@ -513,16 +530,37 @@ sub do_static { # Check RSS feeds redirect to the right places and so on. # Just checks header, doesn't check any contents. sub do_rss { - $wth->browser_get($base_url . '/rss/l/52.5/-1.9'); - die "Lat/lon redirect failed" unless $wth->browser_uri() =~ m{/rss/n/406886,289126$}; - $wth->browser_get($base_url . '/rss/2524/1779'); - die "Tile redirect failed" unless $wth->browser_uri() =~ m{/rss/n/407097,286935$}; - $wth->browser_get($base_url . '/rss/n/407097,286935'); + + my %redirects = ( + + # should always go to lat lon, except postcode (actually, query string) + '/rss/n/406886,289126' => '/rss/l/52.499994,-1.899993', + '/rss/2524/1779' => '/rss/l/52.480294,-1.896931', + '/rss/pc/SW1A1AA' => '/rss/pc/SW1A1AA', + '/rss/l/52.5/-1.9' => '/rss/l/52.5/-1.9', + + # go to reports + '/rss/area/Birmingham' => '/rss/reports/Birmingham', + '/rss/area/Birmingham/Lozells' => '/rss/reports/Birmingham/Lozells+and+East+Handsworth', + ); + + my $error_count = 0; + foreach my $from ( sort keys %redirects ) { + my $to = $redirects{$from}; + $wth->browser_get( $base_url . $from ); + my ($got) = $wth->browser_uri() =~ m{(/rss/.*)$}; + + next if $got eq $to; + + warn "RSS redirect from '$from' to '$to' failed - got '$got'"; + $error_count++; + } + + die "Found errors doing redirect - aborting" if $error_count; + + $wth->browser_get($base_url . '/rss/l/52.5/-1.94'); $wth->browser_check_contents('New local problems on FixMyStreet'); - $wth->browser_get($base_url . '/rss/pc/SW1A1AA'); - die "Postcode redirect failed" unless $wth->browser_uri() =~ m{/rss/n/529090,179645$}; - $wth->browser_get($base_url . '/rss/area/Birmingham'); - die "One-tier redirect failed" unless $wth->browser_uri() =~ m{/rss/reports/Birmingham$}; + $wth->browser_get($base_url . '/rss/reports/Birmingham'); $wth->browser_check_contents('New problems to Birmingham City Council on FixMyStreet'); $wth->browser_get($base_url . '/rss/reports/Birmingham/Lozells'); diff --git a/bin/update-areas b/bin/update-areas index ed463708a..bbc933faf 100755 --- a/bin/update-areas +++ b/bin/update-areas @@ -33,11 +33,11 @@ BEGIN { } print "Fetching problems...\n"; -my $ids = select_all("select id, easting, northing from problem where areas=''"); +my $ids = select_all("select id, latitude, longitude from problem where areas=''"); print "Updating areas...\n"; my $c = 0; foreach (@$ids) { - my $areas = mySociety::MaPit::get_voting_areas_by_location({easting=>$_->{easting}, northing=>$_->{northing}}, 'polygon'); + my $areas = mySociety::MaPit::get_voting_areas_by_location({latitude=>$_->{latitude}, longitude=>$_->{longitude}}, 'polygon'); $areas = ',' . join(',', sort keys %$areas) . ','; dbh()->do('update problem set areas=? where id=?', {}, $areas, $_->{id}); dbh()->commit(); |