diff options
-rwxr-xr-x | bin/open311-populate-service-list | 108 | ||||
-rwxr-xr-x | bin/send-reports | 29 | ||||
-rw-r--r-- | db/schema.sql | 13 | ||||
-rw-r--r-- | db/schema_0009-add_extra_to_problem.sql | 6 | ||||
-rw-r--r-- | db/schema_0010-add_open311_conf.sql | 11 | ||||
-rw-r--r-- | db/schema_0011-add_extra_to_contacts.sql | 6 | ||||
-rw-r--r-- | perllib/FixMyStreet/App/Controller/Report/New.pm | 31 | ||||
-rw-r--r-- | perllib/FixMyStreet/DB/Result/Contact.pm | 25 | ||||
-rw-r--r-- | perllib/FixMyStreet/DB/Result/Open311conf.pm | 39 | ||||
-rw-r--r-- | perllib/FixMyStreet/DB/Result/Problem.pm | 26 | ||||
-rw-r--r-- | perllib/Open311.pm | 179 | ||||
-rw-r--r-- | t/open311.t | 24 | ||||
-rw-r--r-- | templates/web/default/common_header_tags.html | 6 | ||||
-rw-r--r-- | templates/web/default/report/new/fill_in_details.html | 19 | ||||
-rw-r--r-- | web/js/fixmystreet.js | 17 |
15 files changed, 533 insertions, 6 deletions
diff --git a/bin/open311-populate-service-list b/bin/open311-populate-service-list new file mode 100755 index 000000000..f9f183f26 --- /dev/null +++ b/bin/open311-populate-service-list @@ -0,0 +1,108 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use LWP::Simple; +use XML::Simple; +use FixMyStreet::App; +use Open311; + +use Data::Dumper; + +my $council_list = FixMyStreet::App->model('DB::Open311conf'); + +while ( my $council = $council_list->next ) { + + my $open311 = Open311->new( + endpoint => $council->endpoint, + jurisdiction => $council->jurisdiction, + api_key => $council->api_key + ); + + my $list = $open311->get_service_list; + + my @found_contacts; + + # print Dumper $list; + + foreach my $service ( @{ $list->{service} } ) { + print $service->{service_code} . ': ' . $service->{service_name} . "\n"; + my $contacts = FixMyStreet::App->model( 'DB::Contact')->search( + { + area_id => $council->area_id, + -OR => [ + email => $service->{service_code}, + category => $service->{service_name} + ] + } + ); + + my $contact = $contacts->first; + + # FIXME - handle change of service name or service code + if ( $contact ) { + + print $council->area_id . " already has a contact for service code " . $service->{service_code} . "\n"; + push @found_contacts, $service->{service_code}; + + if ( $contact->deleted ) { + $contact->update( + { + category => $service->{service_name}, + email => $service->{service_code}, + confirmed => 1, + deleted => 0, + editor => $0, + whenedited => \'ms_current_timestamp()', + note => 'automatically undeleted by script', + } + ); + } + } else { + my $contact = FixMyStreet::App->model( 'DB::Contact')->create( + { + email => $service->{service_code}, + area_id => $council->area_id, + category => $service->{service_name}, + confirmed => 1, + deleted => 0, + editor => $0, + whenedited => \'ms_current_timestamp()', + note => 'created automatically by script', + } + ); + + if ( lc( $service->{metadata} ) eq 'true' ) { + print "Fetching meta data for $service->{service_code}\n"; + my $meta_data = $open311->get_service_meta_info( $service->{service_code} ); + + # turn the data into something a bit more friendly to use + my %meta = (); + foreach my $attribute ( @{ $meta_data->{attribute} } ) { + $meta{ $attribute->{code} } = $attribute; + } + + $contact->extra( \%meta ); + $contact->update; + } + print "created contact for service code " . $service->{service_code} . " for council @{[$council->area_id]}\n"; + } + } + + my $found_contacts = FixMyStreet::App->model( 'DB::Contact')->search( + { + email => { -not_in => \@found_contacts }, + area_id => $council->area_id, + deleted => 0, + } + ); + + $found_contacts->update( + { + deleted => 1, + editor => $0, + whenedited => \'ms_current_timestamp()', + note => 'automatically marked as deleted by script' + } + ); +} diff --git a/bin/send-reports b/bin/send-reports index 1af3ba1ea..298eb458d 100755 --- a/bin/send-reports +++ b/bin/send-reports @@ -28,6 +28,8 @@ use mySociety::EmailUtil; use mySociety::MaPit; use mySociety::Web qw(ent); +use Open311; + # Set up site, language etc. my ($verbose, $nomail) = CronFns::options(); my $base_url = mySociety::Config::get('BASE_URL'); @@ -136,6 +138,8 @@ while (my $row = $unsent->next) { $h{category} = 'Customer Services' if $h{category} eq 'Other'; } elsif ($areas_info->{$council}->{type} eq 'LBO') { # London $send_web = 'london'; + } elsif ( FixMyStreet::App->model("DB::Open311conf")->find( { area_id => $council } ) ) { + $send_web = 'open311'; } else { my $contact = FixMyStreet::App->model("DB::Contact")->find( { deleted => 0, @@ -248,6 +252,31 @@ while (my $row = $unsent->next) { if (!$nomail) { $result *= post_london_report( $row, %h ); } + } elsif ($send_web eq 'open311') { + # FIXME - looking this up twice :( + my $conf = FixMyStreet::App->model('DB::Open311conf')->find( { area_id => $row->council } ); + + # FIXME - doesn't deal with multiple recipients + my $contact = FixMyStreet::App->model("DB::Contact")->find( { + deleted => 0, + area_id => $row->council, + category => $row->category + } ); + + my $open311 = Open311->new( + jurisdiction => $conf->jurisdiction, + endpoint => $conf->endpoint, + api_key => $conf->api_key, + ); + + my $resp = $open311->send_service_request( $row, \%h, $contact->email ); + + if ( $resp ) { + $row->external_id( $resp ); + $result = 0; + } else { + $result = 1; + } } if ($result == mySociety::EmailUtil::EMAIL_SUCCESS) { diff --git a/db/schema.sql b/db/schema.sql index 9c5b3d8fd..3ec3c6756 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -79,6 +79,9 @@ create table contacts ( whenedited timestamp not null, -- what the last change was for: author's notes note text not null + + -- extra fields required for open311 + extra text ); create unique index contacts_area_id_category_idx on contacts(area_id, category); @@ -176,6 +179,7 @@ create table problem ( lastupdate timestamp not null default ms_current_timestamp(), whensent timestamp, send_questionnaire boolean not null default 't' + extra text -- extra fields required for open311 ); create index problem_state_latitude_longitude_idx on problem(state, latitude, longitude); create index problem_user_id_idx on problem ( user_id ); @@ -387,3 +391,12 @@ create table admin_log ( whenedited timestamp not null default ms_current_timestamp() ); +-- Record open 311 configuration details + +create table open311conf ( + id serial primary key, + area_id integer not null unique, + endpoint text not null, + jurisdiction text, + api_key text +); diff --git a/db/schema_0009-add_extra_to_problem.sql b/db/schema_0009-add_extra_to_problem.sql new file mode 100644 index 000000000..bac5806c7 --- /dev/null +++ b/db/schema_0009-add_extra_to_problem.sql @@ -0,0 +1,6 @@ +begin; + +ALTER TABLE problem + ADD COLUMN extra TEXT; + +commit; diff --git a/db/schema_0010-add_open311_conf.sql b/db/schema_0010-add_open311_conf.sql new file mode 100644 index 000000000..920272c05 --- /dev/null +++ b/db/schema_0010-add_open311_conf.sql @@ -0,0 +1,11 @@ +begin; + +CREATE TABLE open311conf ( + id SERIAL PRIMARY KEY, + area_id INTEGER NOT NULL unique, + endpoint TEXT NOT NULL, + jurisdiction TEXT, + api_key TEXT +); + +commit; diff --git a/db/schema_0011-add_extra_to_contacts.sql b/db/schema_0011-add_extra_to_contacts.sql new file mode 100644 index 000000000..fd6eae807 --- /dev/null +++ b/db/schema_0011-add_extra_to_contacts.sql @@ -0,0 +1,6 @@ +begin; + +ALTER TABLE contacts + ADD COLUMN extra TEXT; + +commit; diff --git a/perllib/FixMyStreet/App/Controller/Report/New.pm b/perllib/FixMyStreet/App/Controller/Report/New.pm index 346dfb377..2311b4aff 100644 --- a/perllib/FixMyStreet/App/Controller/Report/New.pm +++ b/perllib/FixMyStreet/App/Controller/Report/New.pm @@ -15,6 +15,7 @@ use Path::Class; use Utils; use mySociety::EmailUtil; use mySociety::TempFiles; +use JSON; =head1 NAME @@ -449,6 +450,7 @@ sub setup_categories_and_councils : Private { my %area_ids_to_list = (); # Areas with categories assigned my @category_options = (); # categories to show my $category_label = undef; # what to call them + my %category_extras = (); # extra fields to fill in for open311 # FIXME - implement in cobrand if ( $c->cobrand->moniker eq 'emptyhomes' ) { @@ -495,8 +497,12 @@ sub setup_categories_and_councils : Private { next if $contact->category eq _('Other'); - push @category_options, $contact->category - unless $seen{$contact->category}; + unless ( $seen{$contact->category} ) { + push @category_options, $contact->category; + + $category_extras{ $contact->category } = $contact->extra + if $contact->extra; + } $seen{$contact->category} = 1; } @@ -511,6 +517,7 @@ sub setup_categories_and_councils : Private { $c->stash->{area_ids_to_list} = [ keys %area_ids_to_list ]; $c->stash->{category_label} = $category_label; $c->stash->{category_options} = \@category_options; + $c->stash->{category_extras} = encode_json \%category_extras; my @missing_details_councils = grep { !$area_ids_to_list{$_} } # @@ -689,6 +696,26 @@ sub process_report : Private { if $council_string && @{ $c->stash->{missing_details_councils} }; $report->council($council_string); + my @extra = (); + my $metas = $contacts[0]->extra; + + foreach my $field ( sort { $a->{order} <=> $b->{order} } values %$metas ) { + if ( lc( $field->{required} ) eq 'true' ) { + unless ( $c->request->param( $field->{code} ) ) { + $c->stash->{field_errors}->{ $field->{code} } = _('This information is required'); + } + } + push @extra, { + name => $field->{code}, + description => $field->{description}, + value => $c->request->param( $field->{code} ), + }; + } + + if ( @extra ) { + $c->stash->{report_meta} = \@extra; + $report->extra( \@extra ); + } } elsif ( @{ $c->stash->{area_ids_to_list} } ) { # There was an area with categories, but we've not been given one. Bail. diff --git a/perllib/FixMyStreet/DB/Result/Contact.pm b/perllib/FixMyStreet/DB/Result/Contact.pm index 001fb4ac6..779ca9bc2 100644 --- a/perllib/FixMyStreet/DB/Result/Contact.pm +++ b/perllib/FixMyStreet/DB/Result/Contact.pm @@ -34,12 +34,33 @@ __PACKAGE__->add_columns( { data_type => "timestamp", is_nullable => 0 }, "note", { data_type => "text", is_nullable => 0 }, + "extra", + { data_type => "text", is_nullable => 1 }, ); __PACKAGE__->set_primary_key("id"); __PACKAGE__->add_unique_constraint("contacts_area_id_category_idx", ["area_id", "category"]); +__PACKAGE__->filter_column( + extra => { + filter_from_storage => sub { + my $self = shift; + my $ser = shift; + return undef unless defined $ser; + my $h = new IO::String($ser); + return RABX::wire_rd($h); + }, + filter_to_storage => sub { + my $self = shift; + my $data = shift; + my $ser = ''; + my $h = new IO::String($ser); + RABX::wire_wr( $data, $h ); + return $ser; + }, + } +); -# Created by DBIx::Class::Schema::Loader v0.07010 @ 2011-06-23 15:49:48 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:BXGd4uk1ybC5RTKlInTr0w +# Created by DBIx::Class::Schema::Loader v0.07010 @ 2011-08-01 10:07:59 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:4y6yRz4rMN66pBpkzfJJhg 1; diff --git a/perllib/FixMyStreet/DB/Result/Open311conf.pm b/perllib/FixMyStreet/DB/Result/Open311conf.pm new file mode 100644 index 000000000..0a5784560 --- /dev/null +++ b/perllib/FixMyStreet/DB/Result/Open311conf.pm @@ -0,0 +1,39 @@ +package FixMyStreet::DB::Result::Open311conf; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +__PACKAGE__->load_components("FilterColumn", "InflateColumn::DateTime", "EncodedColumn"); +__PACKAGE__->table("open311conf"); +__PACKAGE__->add_columns( + "id", + { + data_type => "integer", + is_auto_increment => 1, + is_nullable => 0, + sequence => "open311conf_id_seq", + }, + "area_id", + { data_type => "integer", is_nullable => 0 }, + "endpoint", + { data_type => "text", is_nullable => 0 }, + "jurisdiction", + { data_type => "text", is_nullable => 1 }, + "api_key", + { data_type => "text", is_nullable => 1 }, +); +__PACKAGE__->set_primary_key("id"); +__PACKAGE__->add_unique_constraint("open311conf_area_id_key", ["area_id"]); + + +# Created by DBIx::Class::Schema::Loader v0.07010 @ 2011-07-29 18:09:25 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ryqCpvwjNtQrZm4I3s0hxg + + +# You can replace this text with custom code or comments, and it will be preserved on regeneration +1; diff --git a/perllib/FixMyStreet/DB/Result/Problem.pm b/perllib/FixMyStreet/DB/Result/Problem.pm index ff730958a..09d1e058a 100644 --- a/perllib/FixMyStreet/DB/Result/Problem.pm +++ b/perllib/FixMyStreet/DB/Result/Problem.pm @@ -78,6 +78,8 @@ __PACKAGE__->add_columns( { data_type => "timestamp", is_nullable => 1 }, "send_questionnaire", { data_type => "boolean", default_value => \"true", is_nullable => 0 }, + "extra", + { data_type => "text", is_nullable => 1 }, ); __PACKAGE__->set_primary_key("id"); __PACKAGE__->has_many( @@ -100,8 +102,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.07010 @ 2011-06-23 15:49:48 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:3sw/1dqxlTvcWEI/eJTm4w +# Created by DBIx::Class::Schema::Loader v0.07010 @ 2011-07-29 16:26:23 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ifvx9FOlbui66hPyzNIAPA # Add fake relationship to stored procedure table __PACKAGE__->has_many( @@ -111,11 +113,31 @@ __PACKAGE__->has_many( { cascade_copy => 0, cascade_delete => 0 }, ); +__PACKAGE__->filter_column( + extra => { + filter_from_storage => sub { + my $self = shift; + my $ser = shift; + return undef unless defined $ser; + my $h = new IO::String($ser); + return RABX::wire_rd($h); + }, + filter_to_storage => sub { + my $self = shift; + my $data = shift; + my $ser = ''; + my $h = new IO::String($ser); + RABX::wire_wr( $data, $h ); + return $ser; + }, + } +); use DateTime::TimeZone; use Image::Size; use Moose; use namespace::clean -except => [ 'meta' ]; use Utils; +use RABX; with 'FixMyStreet::Roles::Abuser'; diff --git a/perllib/Open311.pm b/perllib/Open311.pm new file mode 100644 index 000000000..0c8c3fde6 --- /dev/null +++ b/perllib/Open311.pm @@ -0,0 +1,179 @@ +package Open311; + +use URI; +use Moose; +use XML::Simple; +use LWP::Simple; +use LWP::UserAgent; +use HTTP::Request::Common qw(POST); + +has jurisdiction => ( is => 'ro', isa => 'Str' );; +has api_key => ( is => 'ro', isa => 'Str' ); +has endpoint => ( is => 'ro', isa => 'Str' ); + +sub get_service_list { + my $self = shift; + + my $service_list_xml = $self->_get( 'services.xml' ); + + return $self->_get_xml_object( $service_list_xml ); +} + +sub get_service_meta_info { + my $self = shift; + my $service_id = shift; + + my $service_meta_xml = $self->_get( "services/$service_id.xml" ); + return $self->_get_xml_object( $service_meta_xml ); +} + +sub send_service_request { + my $self = shift; + my $problem = shift; + my $extra = shift; + my $service_code = shift; + + my $description = <<EOT; +title: @{[$problem->title()]} + +detail: @{[$problem->detail()]} + +url: $extra->{url} + +Submitted via FixMyStreet +EOT +; + + my $params = { + lat => $problem->latitude, + long => $problem->longitude, + email => $problem->user->email, + description => $description, + service_code => $service_code, + }; + + if ( $problem->user->phone ) { + $params->{ phone } = $problem->user->phone; + } + + if ( $extra->{image_url} ) { + $params->{media_url} = $extra->{image_url}; + } + + if ( $problem->extra ) { + my $extras = $problem->extra; + + for my $attr ( @$extras ) { + my $name = sprintf( 'attribute[%s]', $attr->{name} ); + $params->{ $name } = $attr->{value}; + } + } + + my $response = $self->_post( 'requests.xml', $params ); + + if ( $response ) { + my $obj = $self->_get_xml_object( $response ); + + if ( $obj ) { + if ( $obj->{ request }->{ service_request_id } ) { + return $obj->{ request }->{ service_request_id }; + } else { + my $token = $obj->{ request }->{ token }; + return $self->get_service_request_id_from_token( $token ); + } + } + } +} + +sub get_service_requests { + my $self = shift; + + my $service_request_xml = $self->_get( 'requests.xml' ); +} + +sub get_service_request_id_from_token { + my $self = shift; + my $token = shift; + + my $service_token_xml = $self->_get( "tokens/$token.xml" ); + + my $obj = $self->_get_xml_object( $service_token_xml ); + + if ( $obj && $obj->{ request }->{ service_request_id } ) { + return $obj->{ request }->{ service_request_id }; + } else { + return 0; + } +} + +sub _get { + my $self = shift; + my $path = shift; + my $params = shift; + + my $uri = URI->new( $self->endpoint ); + $uri->path( $uri->path . $path ); + $uri->query_form( jurisdiction_id => $self->jurisdiction ); + + my $content = get( $uri->as_string ); + + return $content; +} + +sub _post { + my $self = shift; + my $path = shift; + my $params = shift; + + my $uri = URI->new( $self->endpoint ); + $uri->path( $uri->path . $path ); + + my $req = POST $uri->as_string, + [ + jurisdiction_id => $self->jurisdiction, + api_key => $self->api_key, + %{ $params } + ]; + + my $ua = LWP::UserAgent->new(); + my $res = $ua->request( $req ); + + if ( $res->is_success ) { + return $res->decoded_content; + } else { + warn "request failed: " . $res->status_line; + warn $self->_process_error( $res->decoded_content ); + return 0; + } +} + +sub _process_error { + my $self = shift; + my $error = shift; + + my $obj = $self->_get_xml_object( $error ); + + my $msg = ''; + if ( ref $obj && exists $obj->{error} ) { + my $errors = $obj->{error}; + $errors = [ $errors ] if ref $errors ne 'ARRAY'; + $msg .= sprintf( "%s: %s\n", $_->{code}, $_->{description} ) for @{ $errors }; + } + + return $msg || 'unknown error'; +} + +sub _get_xml_object { + my $self = shift; + my $xml= shift; + + my $simple = XML::Simple->new(); + my $obj; + + eval { + $obj = $simple ->XMLin( $xml ); + }; + + return $obj; +} +1; diff --git a/t/open311.t b/t/open311.t new file mode 100644 index 000000000..f7a8cd815 --- /dev/null +++ b/t/open311.t @@ -0,0 +1,24 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use Test::More tests => 4; + +use FindBin; +use lib "$FindBin::Bin/../perllib"; +use lib "$FindBin::Bin/../commonlib/perllib"; + +use_ok( 'Open311' ); + +my $o = Open311->new(); +ok $o, 'created object'; + +my $err_text = <<EOT +<?xml version="1.0" encoding="utf-8"?><errors><error><code>400</code><description>Service Code cannot be null -- can't proceed with the request.</description></error></errors> +EOT +; + +is $o->_process_error( $err_text ), "400: Service Code cannot be null -- can't proceed with the request.\n", 'error text parsing'; +is $o->_process_error( '503 - service unavailable' ), 'unknown error', 'error text parsing of bad error'; + + diff --git a/templates/web/default/common_header_tags.html b/templates/web/default/common_header_tags.html index f9048b067..35218bc1c 100644 --- a/templates/web/default/common_header_tags.html +++ b/templates/web/default/common_header_tags.html @@ -4,6 +4,12 @@ [% map_js %] +[% IF category_extras && category_extras != '{}' %] +<script type="text/javascript"> + category_extras = [% category_extras %]; +</script> +[% END %] + [% IF robots %] <meta name="robots" content="[% robots %]"> [% ELSIF c.config.STAGING_SITE %] diff --git a/templates/web/default/report/new/fill_in_details.html b/templates/web/default/report/new/fill_in_details.html index 8150ba894..6c40697eb 100644 --- a/templates/web/default/report/new/fill_in_details.html +++ b/templates/web/default/report/new/fill_in_details.html @@ -102,6 +102,25 @@ <textarea name="detail" id="form_detail" rows="7" cols="26">[% report.detail | html %]</textarea> </div> +[%- IF category_extras %] +<div id="category_meta"> + [%- IF report_meta %] + [%- FOR meta IN report_meta %] + [%- meta_name = meta.name -%] + +[% IF field_errors.$meta_name %] + <div class='form-error'>[% field_errors.$meta_name %]</div> +[% END -%] + + <div class="form-field"> + <label for="form_[% meta_name %]">[% meta.description _ ':' %]</label> + <input type="text" value="[% meta.value | html %]" name="[% meta_name %]" id="form_[% meta_name %]"> + </div> + [%- END %] + [%- END %] +</div> +[%- END %] + [% IF c.cobrand.allow_photo_upload %] [% IF field_errors.photo %] <div class='form-error'>[% field_errors.photo %]</div> diff --git a/web/js/fixmystreet.js b/web/js/fixmystreet.js index 4b19dc53e..22cd1c64f 100644 --- a/web/js/fixmystreet.js +++ b/web/js/fixmystreet.js @@ -57,4 +57,21 @@ $(function(){ timer = window.setTimeout(email_alert_close, 2000); }); + $('#form_category').change(function() { + if ( category_extras ) { + $('#category_meta').empty(); + if ( category_extras[this.options[ this.selectedIndex ].text] ) { + extras = category_extras[this.options[ this.selectedIndex ].text]; + for ( i in extras ) { + meta = extras[i]; + field = '<div class="form-field">'; + field += '<label for="form_' + meta.code + '">' + meta.description + '</label>'; + field += '<input type="text" value="" name="' + meta.code + '" id="form_' + meta.code + '">'; + field += '</div>'; + $('<p>' + field + '</p>').appendTo('#category_meta'); + } + } + } + }); + }); |