diff options
Diffstat (limited to 'app')
| -rw-r--r-- | app/controllers/admin_censor_rule_controller.rb | 2 | ||||
| -rw-r--r-- | app/controllers/admin_public_body_controller.rb | 97 | ||||
| -rw-r--r-- | app/models/censor_rule.rb | 47 | ||||
| -rw-r--r-- | app/models/info_request.rb | 24 | ||||
| -rw-r--r-- | app/models/public_body.rb | 7 | ||||
| -rw-r--r-- | app/views/admin_censor_rule/_form.rhtml | 2 | ||||
| -rw-r--r-- | app/views/admin_public_body/import_csv.rhtml | 33 |
7 files changed, 143 insertions, 69 deletions
diff --git a/app/controllers/admin_censor_rule_controller.rb b/app/controllers/admin_censor_rule_controller.rb index ec86cdf8e..52df8dfc1 100644 --- a/app/controllers/admin_censor_rule_controller.rb +++ b/app/controllers/admin_censor_rule_controller.rb @@ -31,8 +31,6 @@ class AdminCensorRuleController < AdminController redirect_to admin_url('request/show/' + @censor_rule.info_request.id.to_s) elsif !@censor_rule.user.nil? redirect_to admin_url('user/show/' + @censor_rule.user.id.to_s) - elsif @censor_rule.regexp? - redirect_to admin_url('') else raise "internal error" end diff --git a/app/controllers/admin_public_body_controller.rb b/app/controllers/admin_public_body_controller.rb index dd47c1a4f..30a43bb81 100644 --- a/app/controllers/admin_public_body_controller.rb +++ b/app/controllers/admin_public_body_controller.rb @@ -139,43 +139,80 @@ class AdminPublicBodyController < AdminController end def import_csv - dry_run_only = (params['commit'] == 'Upload' ? false : true) + @notes = "" + @errors = "" + if request.post? + dry_run_only = (params['commit'] == 'Upload' ? false : true) + # Read file from params + if params[:csv_file] + csv_contents = params[:csv_file].read + @original_csv_file = params[:csv_file].original_filename + # or from previous dry-run temporary file + elsif params[:temporary_csv_file] && params[:original_csv_file] + csv_contents = retrieve_csv_data(params[:temporary_csv_file]) + @original_csv_file = params[:original_csv_file] + end - if params[:csv_file] - csv_contents = params[:csv_file].read - else - csv_contents = session.delete(:previous_csv) - end - if !csv_contents.nil? - # Try with dry run first - en = PublicBody.import_csv(csv_contents, params[:tag], params[:tag_behaviour], true, admin_http_auth_user(), I18n.available_locales) - errors = en[0] - notes = en[1] - - if errors.size == 0 - if dry_run_only - notes.push("Dry run was successful, real run would do as above.") - session[:previous_csv] = csv_contents - else - # And if OK, with real run - en = PublicBody.import_csv(csv_contents, params[:tag], params[:tag_behaviour], false, admin_http_auth_user(), I18n.available_locales) - errors = en[0] - notes = en[1] - if errors.size != 0 - raise "dry run mismatched real run" + if !csv_contents.nil? + # Try with dry run first + errors, notes = PublicBody.import_csv(csv_contents, + params[:tag], + params[:tag_behaviour], + true, + admin_http_auth_user(), + I18n.available_locales) + + if errors.size == 0 + if dry_run_only + notes.push("Dry run was successful, real run would do as above.") + # Store the csv file for ease of performing the real run + @temporary_csv_file = store_csv_data(csv_contents) + else + # And if OK, with real run + errors, notes = PublicBody.import_csv(csv_contents, + params[:tag], + params[:tag_behaviour], + false, + admin_http_auth_user(), + I18n.available_locales) + if errors.size != 0 + raise "dry run mismatched real run" + end + notes.push("Import was successful.") end - notes.push("Import was successful.") end + @errors = errors.join("\n") + @notes = notes.join("\n") end - @errors = errors.join("\n") - @notes = notes.join("\n") - else - @errors = "" - @notes = "" end - end private + # Save the contents to a temporary file - not using Tempfile as we need + # the file to persist between requests. Return the name of the file. + def store_csv_data(csv_contents) + tempfile_name = "csv_upload-#{Time.now.strftime("%Y%m%d")}-#{SecureRandom.random_number(10000)}" + tempfile = File.new(File.join(Dir::tmpdir, tempfile_name), 'w') + tempfile.write(csv_contents) + tempfile.close + return tempfile_name + end + + # Get csv contents from the file whose name is passed, as long as the + # name is of the expected form. + # Delete the file, return the contents. + def retrieve_csv_data(tempfile_name) + if not /csv_upload-\d{8}-\d{1,5}/.match(tempfile_name) + raise "Invalid filename in upload_csv: #{tempfile_name}" + end + tempfile_path = File.join(Dir::tmpdir, tempfile_name) + if ! File.exist?(tempfile_path) + raise "Missing file in upload_csv: #{tempfile_name}" + end + csv_contents = File.read(tempfile_path) + File.delete(tempfile_path) + return csv_contents + end + end diff --git a/app/models/censor_rule.rb b/app/models/censor_rule.rb index cedbd767e..da3f49760 100644 --- a/app/models/censor_rule.rb +++ b/app/models/censor_rule.rb @@ -29,17 +29,39 @@ class CensorRule < ActiveRecord::Base belongs_to :user belongs_to :public_body - named_scope :regexps, {:conditions => {:regexp => true}} + # a flag to allow the require_user_request_or_public_body validation to be skipped + attr_accessor :allow_global + validate :require_user_request_or_public_body, :unless => proc{ |rule| rule.allow_global == true } + validate :require_valid_regexp, :if => proc{ |rule| rule.regexp? == true } + validates_presence_of :text - def binary_replacement - self.text.gsub(/./, 'x') + named_scope :global, {:conditions => {:info_request_id => nil, + :user_id => nil, + :public_body_id => nil}} + + def require_user_request_or_public_body + if self.info_request.nil? && self.user.nil? && self.public_body.nil? + errors.add("Censor must apply to an info request a user or a body; ") + end + end + + def require_valid_regexp + begin + self.make_regexp() + rescue RegexpError => e + errors.add(:text, e.message) + end + end + + def make_regexp + return Regexp.new(self.text, Regexp::MULTILINE) end def apply_to_text!(text) if text.nil? return nil end - to_replace = regexp? ? Regexp.new(self.text, Regexp::MULTILINE) : self.text + to_replace = regexp? ? self.make_regexp() : self.text text.gsub!(to_replace, self.replacement) end @@ -47,18 +69,19 @@ class CensorRule < ActiveRecord::Base if binary.nil? return nil end - binary.gsub!(self.text, self.binary_replacement) + to_replace = regexp? ? self.make_regexp() : self.text + binary.gsub!(to_replace){ |match| match.gsub(/./, 'x') } end - def validate - if !self.regexp? && self.info_request.nil? && self.user.nil? && self.public_body.nil? - errors.add("Censor must apply to an info request a user or a body; ") + def for_admin_column + self.class.content_columns.each do |column| + yield(column.human_name, self.send(column.name), column.type.to_s, column.name) end end - def for_admin_column - self.class.content_columns.each do |column| - yield(column.human_name, self.send(column.name), column.type.to_s, column.name) + def is_global? + return true if (info_request_id.nil? && user_id.nil? && public_body_id.nil?) + return false end - end + end diff --git a/app/models/info_request.rb b/app/models/info_request.rb index 19ec949ba..6f472c290 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -1001,24 +1001,28 @@ public return ret.reverse end + # Get the list of censor rules that apply to this request + def applicable_censor_rules + applicable_rules = [self.censor_rules, self.public_body.censor_rules, CensorRule.global.all] + if self.user && !self.user.censor_rules.empty? + applicable_rules << self.user.censor_rules + end + return applicable_rules.flatten + end + # Call groups of censor rules def apply_censor_rules_to_text!(text) - [self.censor_rules, self.user.try(:censor_rules), - CensorRule.regexps.all].flatten.compact.each do |censor_rule| - censor_rule.apply_to_text!(text) - end + self.applicable_censor_rules.each do |censor_rule| + censor_rule.apply_to_text!(text) + end return text end def apply_censor_rules_to_binary!(binary) - for censor_rule in self.censor_rules + self.applicable_censor_rules.each do |censor_rule| censor_rule.apply_to_binary!(binary) end - if self.user # requests during construction have no user - for censor_rule in self.user.censor_rules - censor_rule.apply_to_binary!(binary) - end - end + return binary end def is_owning_user?(user) diff --git a/app/models/public_body.rb b/app/models/public_body.rb index 9efeadf55..60ecb2781 100644 --- a/app/models/public_body.rb +++ b/app/models/public_body.rb @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # == Schema Information # # Table name: public_bodies @@ -41,6 +42,7 @@ class PublicBody < ActiveRecord::Base has_many :info_requests, :order => 'created_at desc' has_many :track_things, :order => 'created_at desc' + has_many :censor_rules, :order => 'created_at desc' has_tag_string @@ -91,8 +93,9 @@ class PublicBody < ActiveRecord::Base # Make sure publication_scheme gets the correct default value. # (This would work automatically, were publication_scheme not a translated attribute) self.publication_scheme = "" if self.publication_scheme.nil? - - # Set an API key if there isn’t one + end + + def before_save self.api_key = SecureRandom.base64(33) if self.api_key.nil? end diff --git a/app/views/admin_censor_rule/_form.rhtml b/app/views/admin_censor_rule/_form.rhtml index d8a8f05d7..ac43de704 100644 --- a/app/views/admin_censor_rule/_form.rhtml +++ b/app/views/admin_censor_rule/_form.rhtml @@ -35,4 +35,6 @@ things by individual request or by user by adding the censor rule from the appropriate page. If you need to redact across a whole authority, it will be easy enough to make code changes to add it, so do ask. </p> +<p><strong>Regexp rules that are hard to process will really slow down request display.</strong> Please only use regexps if you really need to. +</p> diff --git a/app/views/admin_public_body/import_csv.rhtml b/app/views/admin_public_body/import_csv.rhtml index fd652b370..4a03d0665 100644 --- a/app/views/admin_public_body/import_csv.rhtml +++ b/app/views/admin_public_body/import_csv.rhtml @@ -9,32 +9,39 @@ <pre id="error"><%=@errors %></pre> <% end %> - <% form_tag 'import_csv', :multipart => true do %> <p> - <label for="csv_file">CSV file:</label> - <%= file_field_tag :csv_file, :size => 40 %> + <% if @original_csv_file && @temporary_csv_file %> + CSV file: + <%= @original_csv_file %> + <%= hidden_field_tag :original_csv_file, @original_csv_file %> + <%= hidden_field_tag :temporary_csv_file, @temporary_csv_file %> + <%= link_to 'Clear current file', 'import_csv', :class => "btn btn-warning" %> + <% else %> + <label for="csv_file">CSV file:</label> + <%= file_field_tag :csv_file, :size => 40 %> + <% end %> </p> - + <p> <label for="tag">Optional: Tag to add entries to / alter entries for:</label> <%= text_field_tag 'tag', params[:tag] %> </p> - + <p> <label for="tag_behaviour">What to do with existing tags?</label> - <%= select_tag 'tag_behaviour', + <%= select_tag 'tag_behaviour', "<option value='add' selected>Add new tags to existing ones</option> - <option value='replace'>Replace existing tags with new ones</option>" + <option value='replace'>Replace existing tags with new ones</option>" %> </p> - <p><strong>CSV file format:</strong> A first row with the list of fields, + <p><strong>CSV file format:</strong> A first row with the list of fields, starting with '#', is optional but highly recommended. The fields 'name' and 'request_email' are required; additionally, translated values are supported by adding the locale name to the field name, e.g. 'name.es', 'name.de'... Example: </p> - + <blockquote> <p> #id,name,request_email,name.es,tag_string<br/> @@ -43,16 +50,16 @@ <p> </blockquote> - <p>Supported fields: name (i18n), short_name (i18n), request_email (i18n), notes (i18n), + <p>Supported fields: name (i18n), short_name (i18n), request_email (i18n), notes (i18n), publication_scheme (i18n), home_page, tag_string (tags separated by spaces).</p> - + <p><strong>Note:</strong> Choose <strong>dry run</strong> to test, without actually altering the database. Choose <strong>upload</strong> to actually make the changes. In either case, you will be shown any errors, or details of the changes. When uploading, any changes since last import will be overwritten - e.g. email addresses changed back. </p> - + <p><strong>Note:</strong> The import tag will also be added to the imported bodies if no tags are provided in the CSV file or if the import mode is set to "Add new tags to existing ones". @@ -63,7 +70,7 @@ <hr> -<p>Standard tags: +<p>Standard tags: <% for category, description in PublicBodyCategories::get().by_tag() %> <% if category != "other" %> <strong><%= category %></strong>=<%= description %>; |