diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/alaveteli_file_types.rb | 92 | ||||
| -rw-r--r-- | lib/external_command.rb | 135 | ||||
| -rw-r--r-- | lib/memory_profiler.rb | 2 | ||||
| -rw-r--r-- | lib/public_body_categories.rb | 35 | ||||
| -rw-r--r-- | lib/tasks/stats.rake | 48 | ||||
| -rw-r--r-- | lib/tasks/temp.rake | 15 |
6 files changed, 310 insertions, 17 deletions
diff --git a/lib/alaveteli_file_types.rb b/lib/alaveteli_file_types.rb new file mode 100644 index 000000000..076f8ffe8 --- /dev/null +++ b/lib/alaveteli_file_types.rb @@ -0,0 +1,92 @@ +require 'mahoro' + +class AlaveteliFileTypes + # To add an image, create a file with appropriate name corresponding to the + # mime type in public/images e.g. icon_image_tiff_large.png + FileExtensionToMimeType = { + "txt" => 'text/plain', + "pdf" => 'application/pdf', + "rtf" => 'application/rtf', + "doc" => 'application/vnd.ms-word', + "docx" => 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + "xls" => 'application/vnd.ms-excel', + "xlsx" => 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + "ppt" => 'application/vnd.ms-powerpoint', + "pptx" => 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + "oft" => 'application/vnd.ms-outlook', + "msg" => 'application/vnd.ms-outlook', + "tnef" => 'application/ms-tnef', + "tif" => 'image/tiff', + "gif" => 'image/gif', + "jpg" => 'image/jpeg', # XXX add jpeg + "png" => 'image/png', + "bmp" => 'image/bmp', + "html" => 'text/html', # XXX add htm + "vcf" => 'text/x-vcard', + "zip" => 'application/zip', + "delivery-status" => 'message/delivery-status' + } + # XXX doesn't have way of choosing default for inverse map - might want to add + # one when you need it + FileExtensionToMimeTypeRev = FileExtensionToMimeType.invert + + class << self + def all_extensions + return FileExtensionToMimeType.keys + end + + # Given file name and its content, return most likely type + def filename_and_content_to_mimetype(filename, content) + # Try filename + ret = filename_to_mimetype(filename) + if !ret.nil? + return ret + end + + # Otherwise look inside the file to work out the type. + # Mahoro is a Ruby binding for libmagic. + m = Mahoro.new(Mahoro::MIME) + mahoro_type = m.buffer(content) + mahoro_type.strip! + # XXX we shouldn't have to check empty? here, but Mahoro sometimes returns a blank line :( + # e.g. for InfoRequestEvent 17930 + if mahoro_type.nil? || mahoro_type.empty? + return nil + end + # text/plain types sometimes come with a charset + mahoro_type.match(/^(.*);/) + if $1 + mahoro_type = $1 + end + # see if looks like a content type, or has something in it that does + # and return that + # mahoro returns junk "\012- application/msword" as mime type. + mahoro_type.match(/([a-z0-9.-]+\/[a-z0-9.-]+)/) + if $1 + return $1 + end + # otherwise we got junk back from mahoro + return nil + end + + def filename_to_mimetype(filename) + if !filename + return nil + end + if filename.match(/\.([^.]+)$/i) + lext = $1.downcase + if FileExtensionToMimeType.include?(lext) + return FileExtensionToMimeType[lext] + end + end + return nil + end + + def mimetype_to_extension(mimetype) + if FileExtensionToMimeTypeRev.include?(mimetype) + return FileExtensionToMimeTypeRev[mimetype] + end + return nil + end + end +end
\ No newline at end of file diff --git a/lib/external_command.rb b/lib/external_command.rb new file mode 100644 index 000000000..96292854f --- /dev/null +++ b/lib/external_command.rb @@ -0,0 +1,135 @@ +# Run an external command, capturing its stdout and stderr +# streams into variables. +# +# So it’s rather like the `backtick` built-in, except that: +# - The command is run as-is, rather than being parsed by the shell; +# - Standard error is also captured. +# +# After the run() method has been called, the instance variables +# out, err and status contain the contents of the process’s stdout, +# the contents of its stderr, and the exit status. +# +# Example usage: +# require 'external_command' +# xc = ExternalCommand("ls", "-l").run() +# puts "Ran ls -l with exit status #{xc.status}" +# puts "===STDOUT===\n#{xc.out}" +# puts "===STDERR===\n#{xc.err}" +# +# The out and err attributes are writeable. If you assign +# a string, after calling the constructor and before calling +# run(), then the subprocess output/error will be appended +# to this string. + +# <rant author="robin"> +# In any sane language, this would be implemented with a +# single child process. The parent process would block on +# select(), and when the child process terminated, the +# select call would be interrupted by a CHLD signal +# and return EINTR. Unfortunately Ruby goes out of its +# way to prevent this from working, automatically restarting +# the select call if EINTR is returned. Therefore we +# use a parent-child-grandchild arrangement, where the +# parent blocks on select() and the child blocks on +# waitpid(). When the child detects that the grandchild +# has finished, it writes to a pipe that’s included in +# the parent’s select() for this purpose. +# </rant> + +class ExternalCommand + attr_accessor :out, :err + attr_reader :status + + def initialize(cmd, *args) + @cmd = cmd + @args = args + + # Strings to collect stdout and stderr from the child process + # These may be replaced by the caller, to append to existing strings. + @out = "" + @err = "" + @fin = "" + end + + def run() + # Pipes for parent-child communication + @out_read, @out_write = IO::pipe + @err_read, @err_write = IO::pipe + @fin_read, @fin_write = IO::pipe + + @pid = fork do + # Here we’re in the child process. + child_process + end + + # Here we’re in the parent process. + parent_process + + return self + end + + private + + def child_process() + # Reopen stdout and stderr to point at the pipes + STDOUT.reopen(@out_write) + STDERR.reopen(@err_write) + + # Close all the filehandles other than the ones we intend to use. + ObjectSpace.each_object(IO) do |fh| + fh.close unless ( + [STDOUT, STDERR, @fin_write].include?(fh) || fh.closed?) + end + + Process::waitpid(fork { grandchild_process }) + @fin_write.puts($?.exitstatus.to_s) + + exit! 0 + end + + def grandchild_process() + exec(@cmd, *@args) + + # This is only reached if the exec fails + @err_write.print("Failed to exec: #{[@cmd, *@args].join(' ')}") + exit! 99 + end + + def parent_process() + # Close the writing ends of the pipes + @out_write.close + @err_write.close + @fin_write.close + + @fhs = {@out_read => @out, @err_read => @err, @fin_read => @fin} + + while @fin.empty? + ok = read_data + if !ok + raise "select() timed out even with a nil (infinite) timeout" + end + end + + while read_data(0) + # Pull out any data that’s left in the pipes + end + + Process::waitpid(@pid) + @status = @fin.to_i + @out_read.close + @err_read.close + end + + def read_data(timeout=nil) + ready_array = IO.select(@fhs.keys, [], [], timeout) + return false if ready_array.nil? + ready_array[0].each do |fh| + begin + @fhs[fh] << fh.readpartial(8192) + rescue EOFError + @fhs.delete fh + end + end + return true + end +end diff --git a/lib/memory_profiler.rb b/lib/memory_profiler.rb index a1cb64d34..15e8457ee 100644 --- a/lib/memory_profiler.rb +++ b/lib/memory_profiler.rb @@ -65,7 +65,7 @@ class MemoryProfiler prev.update curr GC.start rescue Exception => err - STDERR.puts "** memory_profiler error: #{err}" + $stderr.puts "** memory_profiler error: #{err}" end sleep opt[:delay] end diff --git a/lib/public_body_categories.rb b/lib/public_body_categories.rb index e01ce8bf7..e30ce8707 100644 --- a/lib/public_body_categories.rb +++ b/lib/public_body_categories.rb @@ -12,27 +12,27 @@ module PublicBodyCategories "Miscellaneous", [ "other", "Miscellaneous", "miscellaneous" ], "Central government", - [ "department", "Ministerial departments", "a ministerial department" ], - [ "non_ministerial_department", "Non-ministerial departments", "a non-ministerial department" ], - [ "executive_agency", "Executive agencies", "an executive agency" ], - [ "government_office", "Government offices for the regions", "a government office for the regions" ], + [ "department", "Ministerial departments", "a ministerial department" ], + [ "non_ministerial_department", "Non-ministerial departments", "a non-ministerial department" ], + [ "executive_agency", "Executive agencies", "an executive agency" ], + [ "government_office", "Government offices for the regions", "a government office for the regions" ], [ "advisory_committee", "Advisory committees", "an advisory committee" ], [ "awc", "Agricultural wages committees", "an agriculatural wages committee" ], [ "adhac", "Agricultural dwelling house advisory committees", "an agriculatural dwelling house advisory committee" ], [ "newdeal", "New Deal for Communities partnership", "a New Deal for Communities partnership" ], "Local and regional", [ "local_council", "Local councils", "a local council" ], - [ "parish_council", "Town and Parish councils", "a town or parish council"], + [ "parish_council", "Town and Parish councils", "a town or parish council"], [ "housing_association", "Housing associations", "a housing association"], [ "almo", "Housing ALMOs", "a housing ALMO"], [ "municipal_bank", "Municipal bank", "a municipal bank"], [ "nsbody", "North/south bodies", "a north/south body"], [ "pbo", "Professional buying organisations", "a professional buying organisation"], - [ "regional_assembly", "Regional assemblies", "a regional assembly"], - [ "rda", "Regional development agencies", "a regional development agency" ], + [ "regional_assembly", "Regional assemblies", "a regional assembly"], + [ "rda", "Regional development agencies", "a regional development agency" ], "Education", [ "university", "Universities", "a university" ], - [ "university_college", "University colleges", "a university college" ], + [ "university_college", "University colleges", "a university college" ], [ "cambridge_college", "Cambridge colleges", "a Cambridge college" ], [ "durham_college", "Durham colleges", "a Durham college" ], [ "oxford_college", "Oxford colleges", "an Oxford college or permanent private hall" ], @@ -45,12 +45,11 @@ module PublicBodyCategories [ "lib_board", "Education and library boards", "an education and library board" ], [ "rbc", "Regional Broadband Consortia", "a Regional Broadband Consortium" ], "Environment", - [ "npa", "National park authorities", "a national park authority" ], + [ "npa", "National park authorities", "a national park authority" ], [ "rpa", "Regional park authorities", "a regional park authority" ], - [ "sea_fishery_committee", "Sea fisheries committees", "a sea fisheries committee" ], [ "watercompanies", "Water companies", "a water company" ], + [ "ifca", "Inshore fisheries and conservation authorities", "an inshore fisheries and conservation authority" ], [ "idb", "Internal drainage boards", "an internal drainage board" ], - [ "rfdc", "Regional flood defence committees", "a regional flood defence committee" ], [ "wda", "Waste disposal authorities", "a waste disposal authority" ], [ "zoo", "Zoos", "a zoo" ], "Health", @@ -70,22 +69,26 @@ module PublicBodyCategories [ "military_college", "Military colleges", "a military college" ], [ "security_services", "Security services", "a security services body" ], "Emergency services and the courts", - [ "police", "Police forces", "a police force" ], - [ "police_authority", "Police authorities", "a police authority" ], + [ "police", "Police forces", "a police force" ], + [ "police_authority", "Police authorities", "a police authority" ], [ "dpp", "District policing partnerships", "a district policing partnership" ], [ "fire_service", "Fire and rescue services", "a fire and rescue service" ], - [ "prob_board", "Probation boards", "a probation board" ], + [ "probation_trust", "Probation trusts", "a probation trust" ], [ "rules_committee", "Rules commitees", "a rules committee" ], [ "tribunal", "Tribunals", "a tribunal"], "Transport", [ "npte", "Passenger transport executives", "a passenger transport executive" ], [ "port_authority", "Port authorities", "a port authority" ], [ "scp", "Safety Camera Partnerships", "a safety camera partnership" ], - [ "srp", "Safer Roads Partnership", "a safer roads partnership" ] + [ "srp", "Safer Roads Partnership", "a safer roads partnership" ], + "Defunct", + [ "prob_board", "Probation boards", "a probation board" ], + [ "rfdc", "Regional flood defence committees", "a regional flood defence committee" ], + [ "sea_fishery_committee", "Sea fisheries committees", "a sea fisheries committee" ], ] # Arranged in different ways for different sorts of displaying - CATEGORIES_WITH_DESCRIPTION = CATEGORIES_WITH_HEADINGS.select() { |a| a.instance_of?(Array) } + CATEGORIES_WITH_DESCRIPTION = CATEGORIES_WITH_HEADINGS.select() { |a| a.instance_of?(Array) } CATEGORIES = CATEGORIES_WITH_DESCRIPTION.map() { |a| a[0] } CATEGORIES_BY_TAG = Hash[*CATEGORIES_WITH_DESCRIPTION.map() { |a| a[0..1] }.flatten] CATEGORY_SINGULAR_BY_TAG = Hash[*CATEGORIES_WITH_DESCRIPTION.map() { |a| [a[0],a[2]] }.flatten] diff --git a/lib/tasks/stats.rake b/lib/tasks/stats.rake new file mode 100644 index 000000000..e206ce951 --- /dev/null +++ b/lib/tasks/stats.rake @@ -0,0 +1,48 @@ +namespace :stats do + + desc 'Produce transaction stats' + task :show => :environment do + month_starts = (Date.new(2009, 1)..Date.new(2011, 5)).select { |d| d.day == 1 } + headers = ['Period', + 'Requests sent', + 'Annotations added', + 'Track this request email signups', + 'Comments on own requests', + 'Follow up messages sent'] + puts headers.join("\t") + month_starts.each do |month_start| + month_end = month_start.end_of_month + period = "#{month_start}-#{month_end}" + date_conditions = ['created_at >= ? + AND created_at < ?', + month_start, month_end+1] + request_count = InfoRequest.count(:conditions => date_conditions) + comment_count = Comment.count(:conditions => date_conditions) + track_conditions = ['track_type = ? + AND track_medium = ? + AND created_at >= ? + AND created_at < ?', + 'request_updates', 'email_daily', month_start, month_end+1] + email_request_track_count = TrackThing.count(:conditions => track_conditions) + comment_on_own_request_conditions = ['comments.user_id = info_requests.user_id + AND comments.created_at >= ? + AND comments.created_at < ?', + month_start, month_end+1] + comment_on_own_request_count = Comment.count(:conditions => comment_on_own_request_conditions, + :include => :info_request) + + followup_conditions = ['message_type = ? + AND created_at >= ? + AND created_at < ?', + 'followup', month_start, month_end+1] + follow_up_count = OutgoingMessage.count(:conditions => followup_conditions) + puts [period, + request_count, + comment_count, + email_request_track_count, + comment_on_own_request_count, + follow_up_count].join("\t") + end + end + +end
\ No newline at end of file diff --git a/lib/tasks/temp.rake b/lib/tasks/temp.rake new file mode 100644 index 000000000..ce04c7ddd --- /dev/null +++ b/lib/tasks/temp.rake @@ -0,0 +1,15 @@ +namespace :temp do + + desc "Remove plaintext passwords from post_redirect params" + task :remove_post_redirect_passwords => :environment do + PostRedirect.find_each(:conditions => ['post_params_yaml is not null']) do |post_redirect| + if post_redirect.post_params && post_redirect.post_params[:signchangeemail] && post_redirect.post_params[:signchangeemail][:password] + params = post_redirect.post_params + params[:signchangeemail].delete(:password) + post_redirect.post_params = params + post_redirect.save! + end + end + end + +end |