From 5ce552af2b66613272dc9cfe2d46532e057f44f7 Mon Sep 17 00:00:00 2001 From: Matthew Somerville Date: Thu, 7 May 2020 12:44:23 +0100 Subject: Fix dashboard report CSV export. The change to join in e16054150 did not include a collapse that the previous prefetch was doing, so multiple rows per result were being returned. --- t/app/controller/dashboard.t | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 't/app/controller/dashboard.t') diff --git a/t/app/controller/dashboard.t b/t/app/controller/dashboard.t index 72fc00128..c62ada89a 100644 --- a/t/app/controller/dashboard.t +++ b/t/app/controller/dashboard.t @@ -70,6 +70,7 @@ foreach my $problem (@fixed_problems) { foreach my $problem (@closed_problems) { $problem->update({ state => 'closed' }); + $mech->create_comment_for_problem($problem, $counciluser, 'Name', 'in progress text', 0, 'confirmed', 'in progress'); $mech->create_comment_for_problem($problem, $counciluser, 'Title', 'text', 0, 'confirmed', 'closed'); } @@ -214,7 +215,7 @@ FixMyStreet::override_config { subtest 'export updates as csv' => sub { $mech->get_ok('/dashboard?updates=1&export=1'); my @rows = $mech->content_as_csv; - is scalar @rows, 15, '1 (header) + 14 (updates) = 15 lines'; + is scalar @rows, 18, '1 (header) + 17 (updates) = 18 lines'; is scalar @{$rows[0]}, 8, '8 columns present'; is_deeply $rows[0], -- cgit v1.2.3 From 51eae76dd663d23c1f4bb1e809e9c258e800cb73 Mon Sep 17 00:00:00 2001 From: Matthew Somerville Date: Wed, 10 Jun 2020 14:29:35 +0100 Subject: Only show access tokens once, and store hashed. --- t/app/controller/dashboard.t | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 't/app/controller/dashboard.t') diff --git a/t/app/controller/dashboard.t b/t/app/controller/dashboard.t index c62ada89a..0f07bcae0 100644 --- a/t/app/controller/dashboard.t +++ b/t/app/controller/dashboard.t @@ -236,14 +236,15 @@ FixMyStreet::override_config { subtest 'export as csv using token' => sub { $mech->log_out_ok; - $counciluser->set_extra_metadata('access_token', '1234567890abcdefgh'); + my $u = FixMyStreet::DB->resultset("User")->new({ password => '1234567890abcdefgh' }); + $counciluser->set_extra_metadata('access_token', $u->password); $counciluser->update(); $mech->get_ok('/dashboard?export=1'); like $mech->res->header('Content-type'), qr'text/html'; $mech->content_lacks('Report ID'); - $mech->add_header('Authorization', 'Bearer 1234567890abcdefgh'); + $mech->add_header('Authorization', 'Bearer ' . $counciluser->id . '-1234567890abcdefgh'); $mech->get_ok('/dashboard?export=1'); like $mech->res->header('Content-type'), qr'text/csv'; $mech->content_contains('Report ID'); -- cgit v1.2.3 From 994093c4675393bca6b84747e87a63e10cb063d9 Mon Sep 17 00:00:00 2001 From: M Somerville Date: Tue, 4 Aug 2020 15:05:55 +0100 Subject: Do not error in export if report has bad category. If category groups are enabled, the CSV export includes a subcategory, but it was only fetching a valid groups value if the row's contact was found. --- t/app/controller/dashboard.t | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 't/app/controller/dashboard.t') diff --git a/t/app/controller/dashboard.t b/t/app/controller/dashboard.t index 0f07bcae0..526b1a4c3 100644 --- a/t/app/controller/dashboard.t +++ b/t/app/controller/dashboard.t @@ -83,6 +83,7 @@ my $categories = scraper { FixMyStreet::override_config { ALLOWED_COBRANDS => 'no2fa', + COBRAND_FEATURES => { category_groups => { no2fa => 1 } }, MAPIT_URL => 'http://mapit.uk/', }, sub { @@ -174,13 +175,14 @@ FixMyStreet::override_config { subtest 'export as csv' => sub { $mech->create_problems_for_body(1, $body->id, 'Title', { detail => "this report\nis split across\nseveral lines", + category => 'Problem one', areas => ",$alt_area_id,2651,", }); $mech->get_ok('/dashboard?export=1'); my @rows = $mech->content_as_csv; is scalar @rows, 19, '1 (header) + 18 (reports) = 19 lines'; - is scalar @{$rows[0]}, 20, '20 columns present'; + is scalar @{$rows[0]}, 21, '21 columns present'; is_deeply $rows[0], [ @@ -189,6 +191,7 @@ FixMyStreet::override_config { 'Detail', 'User Name', 'Category', + 'Subcategory', 'Created', 'Confirmed', 'Acknowledged', @@ -207,9 +210,9 @@ FixMyStreet::override_config { ], 'Column headers look correct'; - is $rows[5]->[14], 'Trowbridge', 'Ward column is name not ID'; - is $rows[5]->[15], '529025', 'Correct Easting conversion'; - is $rows[5]->[16], '179716', 'Correct Northing conversion'; + is $rows[5]->[15], 'Trowbridge', 'Ward column is name not ID'; + is $rows[5]->[16], '529025', 'Correct Easting conversion'; + is $rows[5]->[17], '179716', 'Correct Northing conversion'; }; subtest 'export updates as csv' => sub { -- cgit v1.2.3 From e461de75b26e74c0d8c154a1a17d6019c2be30dd Mon Sep 17 00:00:00 2001 From: M Somerville Date: Fri, 7 Aug 2020 20:02:17 +0100 Subject: Offline process for CSV generation. Include a status page, the option for access token requests to use this system, and a script for manual generation. --- t/app/controller/dashboard.t | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) (limited to 't/app/controller/dashboard.t') diff --git a/t/app/controller/dashboard.t b/t/app/controller/dashboard.t index 526b1a4c3..fd491b540 100644 --- a/t/app/controller/dashboard.t +++ b/t/app/controller/dashboard.t @@ -20,6 +20,8 @@ use strict; use warnings; use FixMyStreet::TestMech; +use File::Temp 'tempdir'; +use Path::Tiny; use Web::Scraper; set_absolute_time('2014-02-01T12:00:00'); @@ -81,10 +83,15 @@ my $categories = scraper { }, }; +my $UPLOAD_DIR = tempdir( CLEANUP => 1 ); + FixMyStreet::override_config { ALLOWED_COBRANDS => 'no2fa', COBRAND_FEATURES => { category_groups => { no2fa => 1 } }, MAPIT_URL => 'http://mapit.uk/', + PHOTO_STORAGE_OPTIONS => { + UPLOAD_DIR => $UPLOAD_DIR, + }, }, sub { subtest 'not logged in, redirected to login' => sub { @@ -252,7 +259,37 @@ FixMyStreet::override_config { like $mech->res->header('Content-type'), qr'text/csv'; $mech->content_contains('Report ID'); $mech->delete_header('Authorization'); + + my $token = 'access_token=' . $counciluser->id . '-1234567890abcdefgh'; + $mech->get_ok("/dashboard?export=2&$token"); + is $mech->res->code, 202; + my $loc = $mech->res->header('Location'); + like $loc, qr{/dashboard/csv/.*\.csv$}; + $mech->get_ok("$loc?$token"); + like $mech->res->header('Content-type'), qr'text/csv'; + $mech->content_contains('Report ID'); }; + + subtest 'view status page' => sub { + # Simulate a partly done file + my $f = Path::Tiny->tempfile(SUFFIX => '.csv-part', DIR => path($UPLOAD_DIR, 'dashboard_csv', $counciluser->id)); + (my $name = $f->basename) =~ s/-part$//;; + + my $token = 'access_token=' . $counciluser->id . '-1234567890abcdefgh'; + $mech->get_ok("/dashboard/csv/$name?$token"); + is $mech->res->code, 202; + + $mech->log_in_ok( $counciluser->email ); + $mech->get_ok('/dashboard/status'); + $mech->content_contains('/dashboard/csv/www.example.org-body-' . $body->id . '-start_date-2014-01-02.csv'); + $mech->content_like(qr/$name\s*
0KB\s*In progress/); + + $f->remove; + $mech->get_ok('/dashboard/status'); + $mech->content_contains('/dashboard/csv/www.example.org-body-' . $body->id . '-start_date-2014-01-02.csv'); + $mech->content_lacks('In progress'); + $mech->content_lacks('setTimeout'); + } }; FixMyStreet::override_config { -- cgit v1.2.3