diff options
| -rwxr-xr-x | bin/canonicalise-csv | 110 | ||||
| -rwxr-xr-x | bin/councils-no-contact | 106 | ||||
| -rwxr-xr-x | bin/load-contacts | 48 | ||||
| -rw-r--r-- | perllib/Page.pm | 7 | ||||
| -rwxr-xr-x | web/index.cgi | 8 |
5 files changed, 167 insertions, 112 deletions
diff --git a/bin/canonicalise-csv b/bin/canonicalise-csv new file mode 100755 index 000000000..72502e2d8 --- /dev/null +++ b/bin/canonicalise-csv @@ -0,0 +1,110 @@ +#!/usr/bin/perl -w + +# canonicalise-csv: +# Convert provided CSV file into one with standard names for MaPit +# +# Copyright (c) 2006 UK Citizens Online Democracy. All rights reserved. +# Email: matthew@mysociety.org. WWW: http://www.mysociety.org +# +# $Id: canonicalise-csv,v 1.1 2006-10-13 15:37:48 matthew Exp $ + +use strict; +require 5.8.0; + +# Horrible boilerplate to set up appropriate library paths. +use FindBin; +use lib "$FindBin::Bin/../perllib"; +use lib "$FindBin::Bin/../../perllib"; + +use mySociety::Config; +use mySociety::MaPit; +use mySociety::VotingArea; + +BEGIN { + mySociety::Config::set_file("$FindBin::Bin/../conf/general"); + mySociety::MaPit::configure(); +} + +my %councils; +open(FP, "$FindBin::Bin/../data/councils.csv"); +while (<FP>) { + s/\r?\n//g; + my ($name, $email) = split /,/; + $email ||= ''; + + # Canonicalise + next if $name eq 'Londonderry'; # Dupe of Derry + next if $name eq 'Kingston upon Hull'; # Dupe of Hull + next if $name eq 'London' || $name eq 'Greater London'; # Untrustworthy + + # Different + $name = 'Corporation of London' if $name eq 'City of London'; + $name = "Renfrewsh'r" if $name eq 'Renfrewshire'; + # Shorter + $name = 'Blackburn' if $name eq 'Blackburn with Darwen'; + $name = 'Dungannon' if $name eq 'Dungannon & South Tyrone'; + $name = 'Staffordshire' if $name eq 'Staffordshire County'; + $name = 'Armagh' if $name eq 'Armagh City'; + # Longer + $name = 'Kingston upon Hull' if $name eq 'Hull'; + $name = "City of $name" if $name =~ /^(Edinburgh|Glasgow|York)$/; + $name .= ' Islands' if $name eq 'Shetland'; + $name .= ' & Chelsea' if $name eq 'Kensington'; + # Wrong + $name =~ s/King's Lynn/Kings Lynn/; + $name = 'Surrey Heath' if $name eq 'Surrey Health'; + $name = 'Barking & Dagenham' if $name eq 'Barking-Dagenham'; + $name = 'Newtownabbey' if $name eq 'Newtonabbey'; + $name = 'Isles of Scilly' if $name eq 'Isle of Scilly'; + # Compass + $name =~ s/North East /N. E. /; + $name =~ s/^North West /N. W. /; + $name =~ s/^North (?!Lincolnshire|Norfolk|Somerset)/N. /; + $name =~ s/^South (?!Shropshire|Staffordshire|Somerset)/S. /; + $name =~ s/^East (?!Staffordshire)/E. /; + $name =~ s/^West(ern)? (?!Berkshire|Wiltshire)/W. /; + $name =~ s/ W / W. /; + # Various + $name =~ s/^Great /Gt. /; + $name =~ s/^St /St. /; + $name =~ s/ and / & /; + $name =~ s/ ?Royal$//; + $name =~ s/ Borough$//; + $name =~ s/-(upon|on|le)-/ $1 /; + $councils{$name} = $email; +} +close(FP); + +my $types = $mySociety::VotingArea::council_parent_types; +my (%out, @missing); +foreach my $type (@$types) { + my $areas = mySociety::MaPit::get_areas_by_type($type); + my $areas_info = mySociety::MaPit::get_voting_areas_info($areas); + foreach my $id (keys %$areas_info) { + my $area_info = $areas_info->{$id}; + my $name = $area_info->{name}; + if ($name eq 'Durham City Council') { + $out{$id} = $councils{'Durham City'}; + next; + } elsif ($name eq 'Durham County Council') { + $out{$id} = $councils{'Durham County'}; + next; + } + $name =~ s/( (Borough|City|District|County))* Council//; + if ($councils{$name} && $councils{$name} =~ /@/) { + $out{$id} = $councils{$name}; + } elsif ($councils{$name} || exists($councils{$name})) { + push @missing, $id; + } + } +} + +# Output missing IDs to STDOUT +print join(',', @missing) . "\n"; + +# Output emails to canonical CSV +open(FP, ">$FindBin::Bin/../data/councils_canonical.csv"); +foreach (sort keys %out) { + print FP "$_," . $out{$_} . "\n"; +} +close FP; diff --git a/bin/councils-no-contact b/bin/councils-no-contact deleted file mode 100755 index 22855aaf0..000000000 --- a/bin/councils-no-contact +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/perl -w - -# councils-no-contact: -# Given list of councils without email, generate list of MaPit area IDs. -# -# Copyright (c) 2006 UK Citizens Online Democracy. All rights reserved. -# Email: matthew@mysociety.org. WWW: http://www.mysociety.org -# -# $Id: councils-no-contact,v 1.2 2006-10-07 21:06:31 matthew Exp $ - -use strict; -require 5.8.0; - -# Horrible boilerplate to set up appropriate library paths. -use FindBin; -use lib "$FindBin::Bin/../perllib"; -use lib "$FindBin::Bin/../../perllib"; - -use mySociety::Config; -use mySociety::MaPit; -use mySociety::VotingArea; - -BEGIN { - mySociety::Config::set_file("$FindBin::Bin/../conf/general"); - mySociety::MaPit::configure(); -} - -my @councils_no_email = ( -'Aberdeenshire', -'Antrim', -'Ashford', -'Blaenau Gwent', -'Bradford', -'Breckland', -'Castlereagh', -'Charnwood', -'Cherwell', -'Chesterfield', -'Corporation of London', -'Congleton', -'Derwentside', -'Dorset', -'Durham County', -'E. Ayrshire', -'E. Dorset', -'E. Hampshire', -'E. Lothian', -'E. Sussex', -'Enfield', -'Essex', -'Halton', -'Harborough', -'Inverclyde', -'Kingston upon Hull', -'Magherafelt', -'Melton', -'Middlesbrough', -'Mole Valley', -'Newham', -'N. Devon', -'North Norfolk', -'N. Tyneside', -'N. Yorkshire', -'Northumberland', -'Nottinghamshire', -'Oadby & Wigston', -'Oldham', -'Purbeck', -'Reading', -'Rhondda Cynon Taff', -'Scottish Borders', -'S. Bedfordshire', -'S. Ribble', -'South Somerset', -'Southend on Sea', -'Surrey', -'Tendring', -'Thurrock', -'Tonbridge & Malling', -'Wakefield', -'Wansbeck', -'W. Devon', -'W. Sussex', -'Westminster', -'Weymouth & Portland', -'Worcestershire' -); - -my $types = $mySociety::VotingArea::council_parent_types; -my @out; -foreach my $type (@$types) { - my $areas = mySociety::MaPit::get_areas_by_type($type); - my $areas_info = mySociety::MaPit::get_voting_areas_info($areas); - foreach my $id (keys %$areas_info) { - my $area_info = $areas_info->{$id}; - my $name = $area_info->{name}; - foreach my $council (@councils_no_email) { - if ($name =~ /^$council( ((Borough|City|District|County) )*Council)?$/) { - push(@out, $id); - } - } - } -} - -print join(',', @out); - diff --git a/bin/load-contacts b/bin/load-contacts new file mode 100755 index 000000000..bac8497d3 --- /dev/null +++ b/bin/load-contacts @@ -0,0 +1,48 @@ +#!/usr/bin/perl -w + +# canonicalise-csv: +# Convert provided CSV file into one with standard names for MaPit +# +# Copyright (c) 2006 UK Citizens Online Democracy. All rights reserved. +# Email: matthew@mysociety.org. WWW: http://www.mysociety.org +# +# $Id: load-contacts,v 1.1 2006-10-13 15:37:48 matthew Exp $ + +use strict; +require 5.8.0; + +# Horrible boilerplate to set up appropriate library paths. +use FindBin; +use lib "$FindBin::Bin/../perllib"; +use lib "$FindBin::Bin/../../perllib"; + +use mySociety::Config; +use mySociety::DBHandle qw(dbh select_all); + +BEGIN { + mySociety::Config::set_file("$FindBin::Bin/../conf/general"); + mySociety::DBHandle::configure( + Name => mySociety::Config::get('BCI_DB_NAME'), + User => mySociety::Config::get('BCI_DB_USER'), + Password => mySociety::Config::get('BCI_DB_PASS'), + Host => mySociety::Config::get('BCI_DB_HOST', undef), + Port => mySociety::Config::get('BCI_DB_PORT', undef) + ); + + if (!dbh()->selectrow_array('select secret from secret for update of secret')) { + local dbh()->{HandleError}; + dbh()->do('insert into secret (secret) values (?)', {}, unpack('h*', mySociety::Util::random_bytes(32))); + } + dbh()->commit(); +} + +open(FP, "$FindBin::Bin/../data/councils_canonical.csv"); +while (<FP>) { + s/\r?\n//g; + my ($id, $email) = split /,/; + dbh()->do("INSERT INTO contacts (area_id, email, editor, whenedited, note) + VALUES (?, ?, 'import', ms_current_timestamp(), 'Initial import')", + {}, $id, $email); +} +dbh()->commit(); +close(FP); diff --git a/perllib/Page.pm b/perllib/Page.pm index 5938b907d..52b9439e1 100644 --- a/perllib/Page.pm +++ b/perllib/Page.pm @@ -6,7 +6,7 @@ # Copyright (c) 2006 UK Citizens Online Democracy. All rights reserved. # Email: matthew@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: Page.pm,v 1.28 2006-10-10 15:53:05 matthew Exp $ +# $Id: Page.pm,v 1.29 2006-10-13 15:37:49 matthew Exp $ # package Page; @@ -18,6 +18,9 @@ use Error qw(:try); use mySociety::Config; use mySociety::WatchUpdate; use mySociety::Web qw(ent NewURL); +BEGIN { + mySociety::Config::set_file("$FindBin::Bin/../conf/general"); +} sub do_fastcgi { my $func = shift; @@ -92,7 +95,7 @@ EOF $html .= $home ? '</h1>' : '</a></div>'; $html .= '<div id="wrapper"><div id="content">'; if (mySociety::Config::get('STAGING_SITE')) { - $html .= '<p id="error">This is a developer site, things might break at any time.</p>'; + $html .= '<p id="error">This is a developer site; things might break at any time, and councils are not sent emails (they\'d get annoyed!).</p>'; } return $html; } diff --git a/web/index.cgi b/web/index.cgi index 7b0eb8e10..0483b30ba 100755 --- a/web/index.cgi +++ b/web/index.cgi @@ -6,7 +6,7 @@ # Copyright (c) 2006 UK Citizens Online Democracy. All rights reserved. # Email: matthew@mysociety.org. WWW: http://www.mysociety.org # -# $Id: index.cgi,v 1.47 2006-10-10 22:02:51 matthew Exp $ +# $Id: index.cgi,v 1.48 2006-10-13 15:37:49 matthew Exp $ # TODO # Nothing is done about the update checkboxes - not stored anywhere on anything! @@ -304,8 +304,8 @@ drag_x = $px - 254; drag_y = 254 - $py; EOF } $out .= '<h1>Reporting a problem</h1>'; - $out .= '<p>You have located the problem at the location marked with a yellow pin on the map, which is within ' - . $council . '. If this is not the correct location, simply click on the map again.</p> + $out .= '<p>You have located the problem at the location marked with a purple pin on the map. If this is not the correct location, simply click on the map again.</p> +<p>This problem will be reported to <strong>' . $council . '</strong>.</p> <p>Please fill in details of the problem below. Your council won\'t be able to help unless you leave as much detail as you can, so please describe the exact location of the problem (ie. on a wall or the floor), and so on.</p>'; @@ -702,7 +702,7 @@ sub postcode_check { my @councils_allowed = (2510, 2492, 2507); my @councils_no_email = (2288,2402,2390,2252,2351,2430,2375,2285,2377,2374,2330,2454,2284,2378,2294,2312,2419,2386,2363,2353,2296,2300,2291,2268,2512,2504,2495,# 2510 - 2530,2516,2531,2545,2586,2554,2574,2580,2615,2596,2599,2601,2648,2563,2652,2607,2582,14287,14317,14328,2223,2225,2242,2222,2248,2246,2235,2224,2244,2236); + 2530,2516,2531,2545,2586,2554,2574,2580,2615,2596,2599,2601,2648,2652,2607,2582,14287,14317,14328,2225,2242,2222,2248,2246,2235,2224,2244,2236); my ($valid_councils, $invalid_councils); grep (vec($valid_councils, $_, 1) = 1, @councils_allowed); grep (vec($invalid_councils, $_, 1) = 1, @councils_no_email); |