Merge branch 'stevenage'

Conflicts:
	.gitignore
	bin/make_css
	conf/crontab.ugly
	perllib/FixMyStreet/Cobrand/Default.pm
	perllib/Open311.pm
	templates/web/fixmystreet/around/postcode_form.html

4 files changed, 259 insertions, 35 deletions

diff --git a/t/app/controller/report_new_open311.t b/t/app/controller/report_new_open311.t
index 863571ad0..55c5a92e8 100644
--- a/t/app/controller/report_new_open311.t
+++ b/t/app/controller/report_new_open311.t
@@ -29,7 +29,7 @@ my $contact1 = FixMyStreet::App->model('DB::Contact')->find_or_create( {
     email => '100',
     extra => [ { description => 'Lamppost number', code => 'number', required => 'True' },
                { description => 'Lamppost type', code => 'type', required => 'False', values =>
-                   { value => { Yellow => { key => 'modern' }, 'Gas' => { key => 'old' } } }
+                   { value => [ { name => ['Gas'], key => ['old'] }, { name => [ 'Yellow' ], key => [ 'modern' ] } ] }
                } 
              ],
 } );
diff --git a/t/open311.t b/t/open311.t
index ce0a4ed90..2371c53bc 100644
--- a/t/open311.t
+++ b/t/open311.t
@@ -207,6 +207,26 @@ subtest 'basic request update post parameters' => sub {
     is $c->param('description'), 'this is a comment', 'email correct';
     is $c->param('email'), 'test@example.com', 'email correct';
     is $c->param('status'), 'OPEN', 'status correct';
+    is $c->param('service_request_id'), 81, 'request id correct';
+    is $c->param('updated_datetime'), DateTime::Format::W3CDTF->format_datetime($dt), 'correct date';
+    is $c->param('title'), 'Mr', 'correct title';
+    is $c->param('last_name'), 'User', 'correct first name';
+    is $c->param('first_name'), 'Test', 'correct second name';
+    is $c->param('media_url'), undef, 'no media url';
+};
+
+subtest 'extended request update post parameters' => sub {
+    my $results = make_update_req( $comment, '<?xml version="1.0" encoding="utf-8"?><service_request_updates><request_update><update_id>248</update_id></request_update></service_request_updates>', 1 );
+
+    is $results->{ res }, 248, 'got update id';
+
+    my $req = $o->test_req_used;
+
+    my $c = CGI::Simple->new( $results->{ req }->content );
+
+    is $c->param('description'), 'this is a comment', 'email correct';
+    is $c->param('email'), 'test@example.com', 'email correct';
+    is $c->param('status'), 'OPEN', 'status correct';
     is $c->param('service_request_id_ext'), 80, 'external request id correct';
     is $c->param('service_request_id'), 81, 'request id correct';
     is $c->param('public_anonymity_required'), 'FALSE', 'anon status correct';
@@ -237,42 +257,57 @@ foreach my $test (
     {
         desc => 'comment with fixed state sends status of CLOSED',
         state => 'fixed',
-        anon  => 0,
         status => 'CLOSED',
     },
     {
         desc => 'comment with fixed - user state sends status of CLOSED',
         state => 'fixed - user',
-        anon  => 0,
         status => 'CLOSED',
     },
     {
         desc => 'comment with fixed - council state sends status of CLOSED',
         state => 'fixed - council',
-        anon  => 0,
         status => 'CLOSED',
     },
     {
         desc => 'comment with closed state sends status of CLOSED',
         state => 'closed',
-        anon  => 0,
         status => 'CLOSED',
     },
     {
         desc => 'comment with investigating state sends status of OPEN',
         state => 'investigating',
-        anon  => 0,
         status => 'OPEN',
     },
     {
         desc => 'comment with planned state sends status of OPEN',
         state => 'planned',
-        anon  => 0,
         status => 'OPEN',
     },
     {
         desc => 'comment with in progress state sends status of OPEN',
         state => 'in progress',
+        status => 'OPEN',
+    },
+    {
+        state => 'confirmed',
+        status => 'OPEN',
+    },
+) {
+    subtest $test->{desc} => sub {
+        $comment->problem->state( $test->{state} );
+
+        my $results = make_update_req( $comment, '<?xml version="1.0" encoding="utf-8"?><service_request_updates><request_update><update_id>248</update_id></request_update></service_request_updates>' );
+
+        my $c = CGI::Simple->new( $results->{ req }->content );
+        is $c->param('status'), $test->{status}, 'correct status';
+    };
+}
+
+for my $test (
+    {
+        desc => 'public comment sets public_anonymity_required to false',
+        state => 'confirmed',
         anon  => 0,
         status => 'OPEN',
     },
@@ -287,10 +322,9 @@ foreach my $test (
         $comment->problem->state( $test->{state} );
         $comment->anonymous( $test->{anon} );
 
-        my $results = make_update_req( $comment, '<?xml version="1.0" encoding="utf-8"?><service_request_updates><request_update><update_id>248</update_id></request_update></service_request_updates>' );
+        my $results = make_update_req( $comment, '<?xml version="1.0" encoding="utf-8"?><service_request_updates><request_update><update_id>248</update_id></request_update></service_request_updates>', 1 );
 
         my $c = CGI::Simple->new( $results->{ req }->content );
-        is $c->param('status'), $test->{status}, 'correct status';
         is $c->param('public_anonymity_required'), $test->{anon} ? 'TRUE' : 'FALSE', 'correct anonymity';
     };
 }
@@ -505,15 +539,19 @@ done_testing();
 sub make_update_req {
     my $comment = shift;
     my $xml = shift;
+    my $extended = shift;
 
-    return make_req(
-        {
-            object => $comment,
-              xml  => $xml,
-            method => 'post_service_request_update',
-            path   => 'update.xml',
-        }
-    );
+    my $params = {
+        object => $comment,
+          xml  => $xml,
+        method => 'post_service_request_update',
+        path   => 'servicerequestupdates.xml',
+    };
+
+    if ( $extended ) {
+        $params->{ open311_conf } = { use_extended_updates => 1 };
+    }
+    return make_req( $params );
 }
 
 sub make_service_req {
diff --git a/t/open311/getservicerequestupdates.t b/t/open311/getservicerequestupdates.t
index 7ec8d5ae0..aba811e58 100644
--- a/t/open311/getservicerequestupdates.t
+++ b/t/open311/getservicerequestupdates.t
@@ -26,7 +26,6 @@ my $requests_xml = qq{<?xml version="1.0" encoding="utf-8"?>
 <request_update>
 <update_id>638344</update_id>
 <service_request_id>1</service_request_id>
-<service_request_id_ext>1</service_request_id_ext>
 <status>open</status>
 <description>This is a note</description>
 UPDATED_DATETIME
@@ -42,25 +41,25 @@ for my $test (
     {
         desc => 'basic parsing - element missing',
         updated_datetime => '',
-        res => { update_id => 638344, service_request_id => 1, service_request_id_ext => 1, 
+        res => { update_id => 638344, service_request_id => 1,
                 status => 'open', description => 'This is a note' },
     },
     {
         desc => 'basic parsing - empty element',
         updated_datetime => '<updated_datetime />',
-        res =>  { update_id => 638344, service_request_id => 1, service_request_id_ext => 1, 
+        res =>  { update_id => 638344, service_request_id => 1,
                 status => 'open', description => 'This is a note', updated_datetime => {} } ,
     },
     {
         desc => 'basic parsing - element with no content',
         updated_datetime => '<updated_datetime></updated_datetime>',
-        res =>  { update_id => 638344, service_request_id => 1, service_request_id_ext => 1, 
+        res =>  { update_id => 638344, service_request_id => 1,
                 status => 'open', description => 'This is a note', updated_datetime => {} } ,
     },
     {
         desc => 'basic parsing - element with content',
         updated_datetime => sprintf( '<updated_datetime>%s</updated_datetime>', $dt ),
-        res =>  { update_id => 638344, service_request_id => 1, service_request_id_ext => 1, 
+        res =>  { update_id => 638344, service_request_id => 1,
                 status => 'open', description => 'This is a note', updated_datetime => $dt } ,
     },
 ) {
@@ -68,7 +67,7 @@ for my $test (
         my $local_requests_xml = $requests_xml;
         $local_requests_xml =~ s/UPDATED_DATETIME/$test->{updated_datetime}/;
 
-        my $o = Open311->new( jurisdiction => 'mysociety', endpoint => 'http://example.com', test_mode => 1, test_get_returns => { 'update.xml' => $local_requests_xml } );
+        my $o = Open311->new( jurisdiction => 'mysociety', endpoint => 'http://example.com', test_mode => 1, test_get_returns => { 'servicerequestupdates.xml' => $local_requests_xml } );
 
         my $res = $o->get_service_request_updates;
         is_deeply $res->[0], $test->{ res }, 'result looks correct';
@@ -76,6 +75,33 @@ for my $test (
     };
 }
 
+subtest 'check extended request parsed correctly' => sub {
+    my $extended_requests_xml = qq{<?xml version="1.0" encoding="utf-8"?>
+    <service_requests_updates>
+    <request_update>
+    <update_id>638344</update_id>
+    <service_request_id_ext>120384</service_request_id_ext>
+    <service_request_id>1</service_request_id>
+    <status>open</status>
+    <description>This is a note</description>
+    UPDATED_DATETIME
+    </request_update>
+    </service_requests_updates>
+    };
+
+    my $updated_datetime = sprintf( '<updated_datetime>%s</updated_datetime>', $dt );
+    my $expected_res =  { update_id => 638344, service_request_id => 1, service_request_id_ext => 120384,
+            status => 'open', description => 'This is a note', updated_datetime => $dt };
+
+    $extended_requests_xml =~ s/UPDATED_DATETIME/$updated_datetime/;
+
+    my $o = Open311->new( jurisdiction => 'mysociety', endpoint => 'http://example.com', test_mode => 1, test_get_returns => { 'servicerequestupdates.xml' => $extended_requests_xml } );
+
+    my $res = $o->get_service_request_updates;
+    is_deeply $res->[0], $expected_res, 'result looks correct';
+
+};
+
 my $problem_rs = FixMyStreet::App->model('DB::Problem');
 my $problem = $problem_rs->new(
     {
@@ -171,7 +197,7 @@ for my $test (
         $local_requests_xml =~ s#<service_request_id_ext>\d+</service_request_id_ext>#<service_request_id_ext>@{[$problem->id]}</service_request_id_ext>#;
         $local_requests_xml =~ s#<status>\w+</status>#<status>closed</status># if $test->{close_comment};
 
-        my $o = Open311->new( jurisdiction => 'mysociety', endpoint => 'http://example.com', test_mode => 1, test_get_returns => { 'update.xml' => $local_requests_xml } );
+        my $o = Open311->new( jurisdiction => 'mysociety', endpoint => 'http://example.com', test_mode => 1, test_get_returns => { 'servicerequestupdates.xml' => $local_requests_xml } );
 
         $problem->comments->delete;
         $problem->lastupdate( DateTime->now()->subtract( days => 1 ) );
@@ -213,7 +239,7 @@ foreach my $test (
         $local_requests_xml =~ s#<service_request_id>\d+</service_request_id>#<service_request_id>@{[$problem->external_id]}</service_request_id>#;
         $local_requests_xml =~ s#<service_request_id_ext>\d+</service_request_id_ext>#<service_request_id_ext>@{[$problem->id]}</service_request_id_ext>#;
 
-        my $o = Open311->new( jurisdiction => 'mysociety', endpoint => 'http://example.com', test_mode => 1, test_get_returns => { 'update.xml' => $local_requests_xml } );
+        my $o = Open311->new( jurisdiction => 'mysociety', endpoint => 'http://example.com', test_mode => 1, test_get_returns => { 'servicerequestupdates.xml' => $local_requests_xml } );
 
         $problem->comments->delete;
 
@@ -283,7 +309,7 @@ for my $test (
         $local_requests_xml =~ s#<service_request_id>\d+</service_request_id>#<service_request_id>$test->{request_id}</service_request_id>#;
         $local_requests_xml =~ s#<service_request_id_ext>\d+</service_request_id_ext>#<service_request_id_ext>$test->{request_id_ext}</service_request_id_ext>#;
 
-        my $o = Open311->new( jurisdiction => 'mysociety', endpoint => 'http://example.com', test_mode => 1, test_get_returns => { 'update.xml' => $local_requests_xml } );
+        my $o = Open311->new( jurisdiction => 'mysociety', endpoint => 'http://example.com', test_mode => 1, test_get_returns => { 'servicerequestupdates.xml' => $local_requests_xml } );
 
 
         my $council_details = { areaid => $test->{area_id} };
@@ -297,7 +323,7 @@ for my $test (
 
 subtest 'using start and end date' => sub {
     my $local_requests_xml = $requests_xml;
-    my $o = Open311->new( jurisdiction => 'mysociety', endpoint => 'http://example.com', test_mode => 1, test_get_returns => { 'update.xml' => $local_requests_xml } );
+    my $o = Open311->new( jurisdiction => 'mysociety', endpoint => 'http://example.com', test_mode => 1, test_get_returns => { 'servicerequestupdates.xml' => $local_requests_xml } );
 
     my $start_dt = DateTime->now();
     $start_dt->subtract( days => 1 );
@@ -345,7 +371,6 @@ subtest 'check that existing comments are not duplicated' => sub {
     <request_update>
     <update_id>638344</update_id>
     <service_request_id>@{[ $problem->external_id ]}</service_request_id>
-    <service_request_id_ext>@{[ $problem->id ]}</service_request_id_ext>
     <status>open</status>
     <description>This is a note</description>
     <updated_datetime>UPDATED_DATETIME</updated_datetime>
@@ -353,7 +378,6 @@ subtest 'check that existing comments are not duplicated' => sub {
     <request_update>
     <update_id>638354</update_id>
     <service_request_id>@{[ $problem->external_id ]}</service_request_id>
-    <service_request_id_ext>@{[ $problem->id ]}</service_request_id_ext>
     <status>open</status>
     <description>This is a different note</description>
     <updated_datetime>UPDATED_DATETIME2</updated_datetime>
@@ -382,7 +406,7 @@ subtest 'check that existing comments are not duplicated' => sub {
     $requests_xml =~ s/UPDATED_DATETIME2/$dt/;
     $requests_xml =~ s/UPDATED_DATETIME/@{[ $comment->confirmed ]}/;
 
-    my $o = Open311->new( jurisdiction => 'mysociety', endpoint => 'http://example.com', test_mode => 1, test_get_returns => { 'update.xml' => $requests_xml } );
+    my $o = Open311->new( jurisdiction => 'mysociety', endpoint => 'http://example.com', test_mode => 1, test_get_returns => { 'servicerequestupdates.xml' => $requests_xml } );
 
     my $update = Open311::GetServiceRequestUpdates->new(
         system_user => $user,
@@ -421,7 +445,6 @@ foreach my $test ( {
         <request_update>
         <update_id>638344</update_id>
         <service_request_id>@{[ $problem->external_id ]}</service_request_id>
-        <service_request_id_ext>@{[ $problem->id ]}</service_request_id_ext>
         <status>closed</status>
         <description>This is a note</description>
         <updated_datetime>UPDATED_DATETIME</updated_datetime>
@@ -429,7 +452,6 @@ foreach my $test ( {
         <request_update>
         <update_id>638354</update_id>
         <service_request_id>@{[ $problem->external_id ]}</service_request_id>
-        <service_request_id_ext>@{[ $problem->id ]}</service_request_id_ext>
         <status>open</status>
         <description>This is a different note</description>
         <updated_datetime>UPDATED_DATETIME2</updated_datetime>
@@ -445,7 +467,7 @@ foreach my $test ( {
         $requests_xml =~ s/UPDATED_DATETIME/$test->{dt1}/;
         $requests_xml =~ s/UPDATED_DATETIME2/$test->{dt2}/;
 
-        my $o = Open311->new( jurisdiction => 'mysociety', endpoint => 'http://example.com', test_mode => 1, test_get_returns => { 'update.xml' => $requests_xml } );
+        my $o = Open311->new( jurisdiction => 'mysociety', endpoint => 'http://example.com', test_mode => 1, test_get_returns => { 'servicerequestupdates.xml' => $requests_xml } );
 
         my $update = Open311::GetServiceRequestUpdates->new(
             system_user => $user,
@@ -475,7 +497,6 @@ foreach my $test ( {
         <request_update>
         <update_id>638344</update_id>
         <service_request_id>@{[ $problem->external_id ]}</service_request_id>
-        <service_request_id_ext>@{[ $problem->id ]}</service_request_id_ext>
         <status>closed</status>
         <description>This is a note</description>
         <updated_datetime>UPDATED_DATETIME</updated_datetime>
@@ -497,7 +518,7 @@ foreach my $test ( {
 
         $requests_xml =~ s/UPDATED_DATETIME/$dt/;
 
-        my $o = Open311->new( jurisdiction => 'mysociety', endpoint => 'http://example.com', test_mode => 1, test_get_returns => { 'update.xml' => $requests_xml } );
+        my $o = Open311->new( jurisdiction => 'mysociety', endpoint => 'http://example.com', test_mode => 1, test_get_returns => { 'servicerequestupdates.xml' => $requests_xml } );
 
         my $update = Open311::GetServiceRequestUpdates->new(
             system_user => $user,
diff --git a/t/open311/populate-service-list.t b/t/open311/populate-service-list.t
index fbd729f3a..5910a8f13 100644
--- a/t/open311/populate-service-list.t
+++ b/t/open311/populate-service-list.t
@@ -269,6 +269,171 @@ subtest 'check meta data population' => sub {
     is_deeply $contact->extra, $extra, 'meta data saved';
 };
 
+for my $test (
+    {
+        desc => 'check meta data added to existing contact',
+        has_meta => 1,
+        orig_meta => undef,
+        end_meta => [ {
+                variable => 'true',
+                code => 'type',
+                datatype => 'string',
+                required => 'true',
+                datatype_description => 'Type of bin',
+                order => 1,
+                description => 'Type of bin'
+
+        } ],
+        meta_xml => '<?xml version="1.0" encoding="utf-8"?>
+    <service_definition>
+        <service_code>100</service_code>
+        <attributes>
+            <attribute>
+                <variable>true</variable>
+                <code>type</code>
+                <datatype>string</datatype>
+                <required>true</required>
+                <datatype_description>Type of bin</datatype_description>
+                <order>1</order>
+                <description>Type of bin</description>
+            </attribute>
+        </attributes>
+    </service_definition>
+        ',
+    },
+    {
+        desc => 'check meta data updated',
+        has_meta => 1,
+        orig_meta => [ {
+                variable => 'true',
+                code => 'type',
+                datatype => 'string',
+                required => 'true',
+                datatype_description => 'Type of bin',
+                order => 1,
+                description => 'Type of bin'
+
+        } ],
+        end_meta => [ {
+                variable => 'true',
+                code => 'type',
+                datatype => 'string',
+                required => 'true',
+                datatype_description => 'Colour of bin',
+                order => 1,
+                description => 'Colour of bin'
+
+        } ],
+        meta_xml => '<?xml version="1.0" encoding="utf-8"?>
+    <service_definition>
+        <service_code>100</service_code>
+        <attributes>
+            <attribute>
+                <variable>true</variable>
+                <code>type</code>
+                <datatype>string</datatype>
+                <required>true</required>
+                <datatype_description>Colour of bin</datatype_description>
+                <order>1</order>
+                <description>Colour of bin</description>
+            </attribute>
+        </attributes>
+    </service_definition>
+        ',
+    },
+    {
+        desc => 'check meta data removed',
+        has_meta => 0,
+        end_meta => undef,
+        orig_meta => [ {
+                variable => 'true',
+                code => 'type',
+                datatype => 'string',
+                required => 'true',
+                datatype_description => 'Type of bin',
+                order => 1,
+                description => 'Type of bin'
+
+        } ],
+        meta_xml => '<?xml version="1.0" encoding="utf-8"?>
+    <service_definition>
+        <service_code>100</service_code>
+        <attributes>
+            <attribute>
+                <variable>true</variable>
+                <code>type</code>
+                <datatype>string</datatype>
+                <required>true</required>
+                <datatype_description>Type of bin</datatype_description>
+                <order>1</order>
+                <description>Type of bin</description>
+            </attribute>
+        </attributes>
+    </service_definition>
+        ',
+    },
+) {
+    subtest $test->{desc} => sub {
+        my $processor = Open311::PopulateServiceList->new( council_list => [] );
+
+        my $services_xml = '<?xml version="1.0" encoding="utf-8"?>
+    <services>
+      <service>
+        <service_code>100</service_code>
+        <service_name>Cans left out 24x7</service_name>
+        <description>Garbage or recycling cans that have been left out for more than 24 hours after collection. Violators will be cited.</description>
+        <metadata>false</metadata>
+        <type>realtime</type>
+        <keywords>lorem, ipsum, dolor</keywords>
+        <group>sanitation</group>
+      </service>
+    </services>
+        ';
+
+        if ( $test->{has_meta} ) {
+            $services_xml =~ s/metadata>false/metadata>true/ms;
+        }
+
+        my $contact = FixMyStreet::App->model('DB::Contact')->find_or_create(
+            {
+                area_id => 1,
+                email =>   '100',
+                category => 'Cans left out 24x7',
+                confirmed => 1,
+                deleted => 0,
+                editor => $0,
+                whenedited => \'ms_current_timestamp()',
+                note => 'test contact',
+            }
+        );
+
+        $contact->update( { extra => $test->{orig_meta} } );
+
+        my $o = Open311->new(
+            jurisdiction => 'mysociety',
+            endpoint => 'http://example.com',
+            test_mode => 1,
+            test_get_returns => { 'services.xml' => $services_xml, 'services/100.xml' => $test->{meta_xml} }
+        );
+
+        my $service_list = get_xml_simple_object( $services_xml );
+        $service_list = { service => [ $service_list->{ service } ] };
+
+        my $council = FixMyStreet::App->model('DB::Open311conf')->new( {
+            area_id => 1
+        } );
+
+        $processor->_current_open311( $o );
+        $processor->_current_council( $council );
+
+        $processor->process_services( $service_list );
+
+        $contact->discard_changes;
+
+        is_deeply $contact->extra, $test->{end_meta}, 'meta data saved';
+    };
+}
+
 subtest 'check attribute ordering' => sub {
     my $processor = Open311::PopulateServiceList->new( council_list => [] );