From 89e7318805b09cf32c4f919f2b09f522830fb9ec Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 12 Aug 2013 11:57:27 +0100 Subject: Add a page with experimental statistics on public bodies The statistics on the status of the requests to a particular public body are too slow to calculate on-the-fly, so this commit adds: * Extra columns on public_bodies to store counts of the successful, not held, and overdue request counts for each public body. * A rake task which should be run periodically to update the overdue request count column. If Javascript is not available, the summary statistics are shown as tables. If Javascript is available, graphs are drawn with Flot. --- lib/tasks/stats.rake | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) (limited to 'lib/tasks/stats.rake') diff --git a/lib/tasks/stats.rake b/lib/tasks/stats.rake index 9d7d70540..f7a3b07a5 100644 --- a/lib/tasks/stats.rake +++ b/lib/tasks/stats.rake @@ -91,4 +91,40 @@ namespace :stats do end end + desc 'Update statistics in the public_bodies table' + task :update_public_bodies_stats => :environment do + PublicBody.all.each do |public_body| + puts "Finding statistics for #{public_body.name}" + [["info_requests_count=", nil], + ["info_requests_successful_count=", ['successful', 'partially_successful']], + ["info_requests_not_held_count=", ['not_held']]].each do |column, states| + puts " Aggregating data for column #{column}" + where_clause = 'public_body_id = :pb' + parameters = {:pb => public_body.id} + if states + where_clause += " AND described_state in (:states)" + parameters[:states] = states + end + public_body.send(column, + InfoRequest.where(where_clause, + parameters).count.to_s) + end + # Now looking for values of 'waiting_response_overdue' and + # 'waiting_response_very_overdue' which aren't directly in the + # described_state column, and instead need to + puts " Counting overdue requests" + overdue_count = 0 + very_overdue_count = 0 + InfoRequest.find_each(:conditions => {:public_body_id => public_body.id}) do |ir| + case ir.calculate_status + when 'waiting_response_very_overdue' + very_overdue_count += 1 + when 'waiting_response_overdue' + overdue_count += 1 + end + end + public_body.info_requests_overdue_count = overdue_count + very_overdue_count + public_body.save! + end + end end -- cgit v1.2.3 From e5855f7fd2657574c5af99890c63d530ca3bb5d0 Mon Sep 17 00:00:00 2001 From: Mark Longair Date: Mon, 19 Aug 2013 10:34:05 +0100 Subject: Improve calculation of PublicBody statistics columns On PublicBody, we don't need to update info_requests_count because that's already done with :counter_cache. On the other hand, info_requests_successful_count and info_requests_not_held_count can't be updated easily with counter_cache (since they need conditions to be attached). Instead we update them in post_save and post_destroy, as suggested here: http://blog.douglasfshearer.com/post/17495285851/custom-counter-cache-with-conditions This also adds tests to ensure that the after_(save|destroy) callbacks are called and that they modify the counts correctly. --- lib/tasks/stats.rake | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) (limited to 'lib/tasks/stats.rake') diff --git a/lib/tasks/stats.rake b/lib/tasks/stats.rake index f7a3b07a5..1242575fe 100644 --- a/lib/tasks/stats.rake +++ b/lib/tasks/stats.rake @@ -94,25 +94,10 @@ namespace :stats do desc 'Update statistics in the public_bodies table' task :update_public_bodies_stats => :environment do PublicBody.all.each do |public_body| - puts "Finding statistics for #{public_body.name}" - [["info_requests_count=", nil], - ["info_requests_successful_count=", ['successful', 'partially_successful']], - ["info_requests_not_held_count=", ['not_held']]].each do |column, states| - puts " Aggregating data for column #{column}" - where_clause = 'public_body_id = :pb' - parameters = {:pb => public_body.id} - if states - where_clause += " AND described_state in (:states)" - parameters[:states] = states - end - public_body.send(column, - InfoRequest.where(where_clause, - parameters).count.to_s) - end - # Now looking for values of 'waiting_response_overdue' and + puts "Counting overdue requests for #{public_body.name}" + # Look for values of 'waiting_response_overdue' and # 'waiting_response_very_overdue' which aren't directly in the - # described_state column, and instead need to - puts " Counting overdue requests" + # described_state column, and instead need to be calculated: overdue_count = 0 very_overdue_count = 0 InfoRequest.find_each(:conditions => {:public_body_id => public_body.id}) do |ir| -- cgit v1.2.3 From d27f435e2d624503e3284246c77e8c5931c14ec5 Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Thu, 5 Sep 2013 09:58:50 +0100 Subject: Add a verbosity flag to public body stats task --- lib/tasks/stats.rake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib/tasks/stats.rake') diff --git a/lib/tasks/stats.rake b/lib/tasks/stats.rake index 1242575fe..2a02b1716 100644 --- a/lib/tasks/stats.rake +++ b/lib/tasks/stats.rake @@ -93,8 +93,10 @@ namespace :stats do desc 'Update statistics in the public_bodies table' task :update_public_bodies_stats => :environment do + verbose = ENV['VERBOSE'] == '1' PublicBody.all.each do |public_body| - puts "Counting overdue requests for #{public_body.name}" + puts "Counting overdue requests for #{public_body.name}" if verbose + # Look for values of 'waiting_response_overdue' and # 'waiting_response_very_overdue' which aren't directly in the # described_state column, and instead need to be calculated: -- cgit v1.2.3 From 6518bfde0d974092777522c8dd883fd498b2972c Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Thu, 5 Sep 2013 12:23:52 +0100 Subject: Don't re-index body when updating stats --- lib/tasks/stats.rake | 1 + 1 file changed, 1 insertion(+) (limited to 'lib/tasks/stats.rake') diff --git a/lib/tasks/stats.rake b/lib/tasks/stats.rake index 2a02b1716..58d6e30fb 100644 --- a/lib/tasks/stats.rake +++ b/lib/tasks/stats.rake @@ -111,6 +111,7 @@ namespace :stats do end end public_body.info_requests_overdue_count = overdue_count + very_overdue_count + public_body.no_xapian_reindex = true public_body.save! end end -- cgit v1.2.3 From a84f1f84bbaf2c890b0f45bc0e9c460e0848e274 Mon Sep 17 00:00:00 2001 From: Louise Crow Date: Thu, 5 Sep 2013 16:28:57 +0100 Subject: Don't save a public body version when updating the stats. --- lib/tasks/stats.rake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib/tasks/stats.rake') diff --git a/lib/tasks/stats.rake b/lib/tasks/stats.rake index 58d6e30fb..4eda27289 100644 --- a/lib/tasks/stats.rake +++ b/lib/tasks/stats.rake @@ -112,7 +112,9 @@ namespace :stats do end public_body.info_requests_overdue_count = overdue_count + very_overdue_count public_body.no_xapian_reindex = true - public_body.save! + public_body.without_revision do + public_body.save! + end end end end -- cgit v1.2.3