diff options
Diffstat (limited to 'app')
-rw-r--r-- | app/controllers/admin_general_controller.rb | 4 | ||||
-rw-r--r-- | app/controllers/application_controller.rb | 40 | ||||
-rw-r--r-- | app/controllers/general_controller.rb | 12 | ||||
-rw-r--r-- | app/controllers/request_controller.rb | 15 | ||||
-rw-r--r-- | app/models/info_request_event.rb | 22 | ||||
-rw-r--r-- | app/models/raw_email.rb | 2 | ||||
-rw-r--r-- | app/models/track_thing.rb | 15 | ||||
-rw-r--r-- | app/views/admin_general/debug.rhtml | 4 | ||||
-rw-r--r-- | app/views/layouts/default.rhtml | 20 | ||||
-rw-r--r-- | app/views/public_body/_search_ahead.rhtml | 6 | ||||
-rw-r--r-- | app/views/request/_search_ahead.rhtml | 2 | ||||
-rw-r--r-- | app/views/request/list.rhtml | 4 | ||||
-rw-r--r-- | app/views/request/select_authority.rhtml | 1 |
13 files changed, 103 insertions, 44 deletions
diff --git a/app/controllers/admin_general_controller.rb b/app/controllers/admin_general_controller.rb index ae51e0923..0b7e9bec0 100644 --- a/app/controllers/admin_general_controller.rb +++ b/app/controllers/admin_general_controller.rb @@ -78,6 +78,10 @@ class AdminGeneralController < AdminController end def debug + @current_commit = `git log -1 --format="%H"` + @current_branch = `git branch | grep "\*" | awk '{print $2}'` + repo = `git remote show origin -n | grep Fetch | awk '{print $3}' | sed -re 's/.*:(.*).git/\\1/'` + @github_origin = "https://github.com/#{repo.strip}/tree/" @request_env = request.env end end diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 0c8544932..8fc6c3792 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -89,6 +89,7 @@ class ApplicationController < ActionController::Base def record_memory record_memory = MySociety::Config.get('DEBUG_RECORD_MEMORY', false) if record_memory + logger.info "Processing request for #{request.url} with Rails process #{Process.pid}" File.read("/proc/#{Process.pid}/status").match(/VmRSS:\s+(\d+)/) rss_before_action = $1.to_i yield @@ -180,7 +181,10 @@ class ApplicationController < ActionController::Base path = foi_fragment_cache_part_path(param) path = "/views" + path foi_cache_path = File.join(File.dirname(__FILE__), '../../cache') - return File.join(foi_cache_path, path) + max_file_length = 255 - 35 # we subtract 35 because tempfile + # adds on a variable number of + # characters + return File.join(foi_cache_path, path)[0...max_file_length] end def foi_fragment_cache_all_for_request(info_request) # return stub path so admin can expire it @@ -193,10 +197,12 @@ class ApplicationController < ActionController::Base return File.exists?(key_path) end def foi_fragment_cache_read(key_path) + logger.info "Reading from fragment cache #{key_path}" return File.read(key_path) end def foi_fragment_cache_write(key_path, content) FileUtils.mkdir_p(File.dirname(key_path)) + logger.info "Writing to fragment cache #{key_path}" File.atomic_write(key_path) do |f| f.write(content) end @@ -365,7 +371,10 @@ class ApplicationController < ActionController::Base def get_search_page_from_params return (params[:page] || "1").to_i end + def perform_search_typeahead(query, model) + @page = get_search_page_from_params + @per_page = 10 query_words = query.split(/ +(?![-+]+)/) if query_words.last.nil? || query_words.last.strip.length < 3 xapian_requests = nil @@ -376,8 +385,8 @@ class ApplicationController < ActionController::Base collapse = 'request_collapse' end options = { - :offset => 0, - :limit => 5, + :offset => (@page - 1) * @per_page, + :limit => @per_page, :sort_by_prefix => nil, :sort_by_ascending => true, :collapse_by_prefix => collapse, @@ -385,11 +394,24 @@ class ApplicationController < ActionController::Base ActsAsXapian.readable_init old_default_op = ActsAsXapian.query_parser.default_op ActsAsXapian.query_parser.default_op = Xapian::Query::OP_OR - user_query = ActsAsXapian.query_parser.parse_query( - query, - Xapian::QueryParser::FLAG_LOVEHATE | Xapian::QueryParser::FLAG_PARTIAL | - Xapian::QueryParser::FLAG_SPELLING_CORRECTION) - xapian_requests = ActsAsXapian::Search.new([model], query, options, user_query) + begin + user_query = ActsAsXapian.query_parser.parse_query( + query.strip + '*', + Xapian::QueryParser::FLAG_LOVEHATE | Xapian::QueryParser::FLAG_WILDCARD | + Xapian::QueryParser::FLAG_SPELLING_CORRECTION) + xapian_requests = ActsAsXapian::Search.new([model], query, options, user_query) + rescue RuntimeError => e + if e.message =~ /^QueryParserError: Wildcard/ + # Wildcard expands to too many terms + logger.info "Wildcard query '#{query.strip + '*'}' caused: #{e.message}" + + user_query = ActsAsXapian.query_parser.parse_query( + query, + Xapian::QueryParser::FLAG_LOVEHATE | + Xapian::QueryParser::FLAG_SPELLING_CORRECTION) + xapian_requests = ActsAsXapian::Search.new([model], query, options, user_query) + end + end ActsAsXapian.query_parser.default_op = old_default_op end return xapian_requests @@ -521,7 +543,7 @@ class ApplicationController < ActionController::Base def quietly_try_to_open(url) begin result = open(url).read.strip - rescue OpenURI::HTTPError, SocketError + rescue OpenURI::HTTPError, SocketError, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH logger.warn("Unable to open third-party URL #{url}") result = "" end diff --git a/app/controllers/general_controller.rb b/app/controllers/general_controller.rb index d28f4deec..c715b547d 100644 --- a/app/controllers/general_controller.rb +++ b/app/controllers/general_controller.rb @@ -32,12 +32,12 @@ class GeneralController < ApplicationController if body_short_names.empty? # This is too slow @popular_bodies = PublicBody.find(:all, - :select => "public_bodies.*, (select count(*) from info_requests where info_requests.public_body_id = public_bodies.id) as c", - :order => "c desc", - :limit => 32, - :conditions => conditions, - :joins => :translations - ) + :select => "public_bodies.*, (select count(*) from info_requests where info_requests.public_body_id = public_bodies.id) as c", + :order => "c desc", + :limit => 32, + :conditions => conditions, + :joins => :translations + ) else conditions[0] += " and public_bodies.url_name in (" + body_short_names + ")" @popular_bodies = PublicBody.find(:all, diff --git a/app/controllers/request_controller.rb b/app/controllers/request_controller.rb index 1c7aeedcc..11812b729 100644 --- a/app/controllers/request_controller.rb +++ b/app/controllers/request_controller.rb @@ -13,6 +13,9 @@ require 'open-uri' class RequestController < ApplicationController before_filter :check_read_only, :only => [ :new, :show_response, :describe_state, :upload_response ] protect_from_forgery :only => [ :new, :show_response, :describe_state, :upload_response ] # See ActionController::RequestForgeryProtection for details + + MAX_RESULTS = 500 + PER_PAGE = 25 @@custom_states_loaded = false begin @@ -155,14 +158,21 @@ class RequestController < ApplicationController if @view == "recent" return redirect_to request_list_all_path(:action => "list", :view => "all", :page => @page), :status => :moved_permanently end + + # Later pages are very expensive to load + if @page > MAX_RESULTS / PER_PAGE + raise ActiveRecord::RecordNotFound.new("Sorry. No pages after #{MAX_RESULTS / PER_PAGE}.") + end + params[:latest_status] = @view query = make_query_from_params @title = _("View and search requests") sortby = "newest" - behavior_cache :tag => [@view, @page] do + behavior_cache :tag => [@query, @page, I18n.locale] do xapian_object = perform_search([InfoRequestEvent], query, sortby, 'request_collapse') @list_results = xapian_object.results.map { |r| r[:model] } @matches_estimated = xapian_object.matches_estimated + @show_no_more_than = (@matches_estimated > MAX_RESULTS) ? MAX_RESULTS : @matches_estimated end @title = @title + " (page " + @page.to_s + ")" if (@page > 1) @@ -682,10 +692,11 @@ class RequestController < ApplicationController # Internal function def get_attachment_internal(html_conversion) @incoming_message = IncomingMessage.find(params[:incoming_message_id]) + @requested_request = InfoRequest.find(params[:id]) @incoming_message.parse_raw_email! @info_request = @incoming_message.info_request if @incoming_message.info_request_id != params[:id].to_i - raise sprintf("Incoming message %d does not belong to request %d", @incoming_message.info_request_id, params[:id]) + raise ActiveRecord::RecordNotFound.new(sprintf("Incoming message %d does not belong to request %d", @incoming_message.info_request_id, params[:id])) end @part_number = params[:part].to_i @filename = params[:file_name].join("/") diff --git a/app/models/info_request_event.rb b/app/models/info_request_event.rb index e99a0ae2f..99f34cf9e 100644 --- a/app/models/info_request_event.rb +++ b/app/models/info_request_event.rb @@ -190,6 +190,26 @@ class InfoRequestEvent < ActiveRecord::Base return message end + def get_clipped_response_efficiently + # XXX this ugly code is an attempt to not always load all the + # columns for an incoming message, which can be *very* large + # (due to all the cached text). We care particularly in this + # case because it's called for every search result on a page + # (to show the search snippet). Actually, we should review if we + # need all this data to be cached in the database at all, and + # then we won't need this horrid workaround. + message = self.incoming_message_selective_columns("cached_attachment_text_clipped, cached_main_body_text_folded") + clipped_body = message.cached_main_body_text_folded + clipped_attachment = message.cached_attachment_text_clipped + if clipped_body.nil? || clipped_attachment.nil? + # we're going to have to load it anyway + text = self.incoming_message.get_text_for_indexing_clipped + else + text = clipped_body.gsub("FOLDED_QUOTED_SECTION", " ").strip + "\n\n" + clipped_attachment + end + return text + "\n\n" + end + # clipped = true - means return shorter text. It is used for snippets fore # performance reasons. Xapian will take the full text. def search_text_main(clipped = false) @@ -200,7 +220,7 @@ class InfoRequestEvent < ActiveRecord::Base text = text + self.outgoing_message.get_text_for_indexing + "\n\n" elsif self.event_type == 'response' if clipped - text = text + self.incoming_message_selective_columns("cached_attachment_text_clipped").cached_attachment_text_clipped + "\n\n" + text = text + self.get_clipped_response_efficiently else text = text + self.incoming_message.get_text_for_indexing_full + "\n\n" end diff --git a/app/models/raw_email.rb b/app/models/raw_email.rb index c6f84318b..6e073aa27 100644 --- a/app/models/raw_email.rb +++ b/app/models/raw_email.rb @@ -43,7 +43,7 @@ class RawEmail < ActiveRecord::Base if !File.exists?(self.directory) FileUtils.mkdir_p self.directory end - File.open(self.filepath, "wb") { |file| + File.atomic_write(self.filepath) { |file| file.write d } end diff --git a/app/models/track_thing.rb b/app/models/track_thing.rb index 6938fade9..58d70ed86 100644 --- a/app/models/track_thing.rb +++ b/app/models/track_thing.rb @@ -71,14 +71,13 @@ class TrackThing < ActiveRecord::Base def track_query_description # XXX this is very brittle... we should probably ask users # simply to name their tracks when they make them? - self.track_query = self.track_query.gsub(/([()]|OR)/, "") - filters = self.track_query.scan /\b\S+:\S+\b/ - text = self.track_query + original_text = parsed_text = self.track_query.gsub(/([()]|OR)/, "") + filters = parsed_text.scan /\b\S+:\S+\b/ varieties = Set.new date = "" statuses = Set.new for filter in filters - text = text.sub(filter, "") + parsed_text = parsed_text.sub(filter, "") if filter =~ /variety:user/ varieties << _("users") end @@ -105,7 +104,7 @@ class TrackThing < ActiveRecord::Base end end if filters.empty? - text = self.track_query + parsed_text = original_text end descriptions = [] if varieties.include? _("requests") @@ -116,10 +115,10 @@ class TrackThing < ActiveRecord::Base varieties << _("anything") end descriptions += Array(varieties) - text = text.strip + parsed_text = parsed_text.strip descriptions = descriptions.join(_(" or ")) - if !text.empty? - descriptions += _("{{list_of_things}} matching text '{{search_query}}'", :list_of_things => "", :search_query => text) + if !parsed_text.empty? + descriptions += _("{{list_of_things}} matching text '{{search_query}}'", :list_of_things => "", :search_query => parsed_text) end return descriptions end diff --git a/app/views/admin_general/debug.rhtml b/app/views/admin_general/debug.rhtml index b3b06085f..40fe33616 100644 --- a/app/views/admin_general/debug.rhtml +++ b/app/views/admin_general/debug.rhtml @@ -7,6 +7,10 @@ <h2>Version numbers</h2> <p> +Alaveteli branch: <%= link_to @current_branch, @github_origin + @current_branch %> +<br> +Alaveteli commit: <%= link_to @current_commit, @github_origin + @current_commit %> +<br> RUBY_VERSION <%=h RUBY_VERSION %> <br> Rails::VERSION::STRING <%=h Rails::VERSION::STRING%> diff --git a/app/views/layouts/default.rhtml b/app/views/layouts/default.rhtml index ad0560baa..f439b27d2 100644 --- a/app/views/layouts/default.rhtml +++ b/app/views/layouts/default.rhtml @@ -10,13 +10,13 @@ </title> <link rel="shortcut icon" href="/favicon.ico"> - <%= stylesheet_link_tag 'main', :title => "Main", :rel => "stylesheet", :media => "all" %> - <%= stylesheet_link_tag 'fonts', :rel => "stylesheet", :media => "all" %> - <%= stylesheet_link_tag 'print', :rel => "stylesheet", :media => "print" %> - <% if !params[:print_stylesheet].nil? %> + <%= stylesheet_link_tag 'main', :title => "Main", :rel => "stylesheet", :media => "all" %> + <%= stylesheet_link_tag 'fonts', :rel => "stylesheet", :media => "all" %> + <%= stylesheet_link_tag 'print', :rel => "stylesheet", :media => "print" %> + <% if !params[:print_stylesheet].nil? %> <%= stylesheet_link_tag 'print', :rel => "stylesheet", :media => "all" %> - <% end %> - <%= javascript_include_tag 'jquery.js', 'jquery-ui.min','jquery.cookie.js', 'general.js' %> + <% end %> + <%= javascript_include_tag 'jquery.js', 'jquery-ui.min','jquery.cookie.js', 'general.js' %> <% if @profile_photo_javascript %> <script type="text/javascript" src="/javascripts/jquery.Jcrop.js"></script> @@ -24,7 +24,7 @@ <link rel="stylesheet" href="/stylesheets/jquery.Jcrop.css" type="text/css" > <% end %> - <%= stylesheet_link_tag 'admin-theme/jquery-ui-1.8.15.custom.css', :rel => 'stylesheet'%> + <%= stylesheet_link_tag 'admin-theme/jquery-ui-1.8.15.custom.css', :rel => 'stylesheet'%> <!--[if LT IE 7]> <style type="text/css">@import url("/stylesheets/ie6.css");</style> <![endif]--> @@ -34,8 +34,8 @@ <!--[if LT IE 8]> <style type="text/css">@import url("/stylesheets/ie7.css");</style> <![endif]--> - <!-- the following method for customising CSS is deprecated; see `doc/THEMES.md` for detail --> - <%= stylesheet_link_tag 'custom', :title => "Main", :rel => "stylesheet" %> + <!-- the following method for customising CSS is deprecated; see `doc/THEMES.md` for detail --> + <%= stylesheet_link_tag 'custom', :title => "Main", :rel => "stylesheet" %> <% if force_registration_on_new_request %> <%= stylesheet_link_tag 'jquery.fancybox-1.3.4', :rel => "stylesheet" %> <% end %> @@ -56,7 +56,7 @@ <meta name="robots" content="noindex, nofollow"> <% end %> - <%= render :partial => 'general/before_head_end' %> + <%= render :partial => 'general/before_head_end' %> </head> <body <%= "class='front'" if params[:action] == 'frontpage' %>> diff --git a/app/views/public_body/_search_ahead.rhtml b/app/views/public_body/_search_ahead.rhtml index 484d28256..7ade89b8e 100644 --- a/app/views/public_body/_search_ahead.rhtml +++ b/app/views/public_body/_search_ahead.rhtml @@ -1,4 +1,4 @@ -<p> +<div> <% if !@xapian_requests.nil? %> <% if @xapian_requests.results.size > 0 %> <h3><%= _('Top search results:') %></h3> @@ -13,9 +13,9 @@ <%= render :partial => 'body_listing_single', :locals => { :public_body => result[:model] } %> <% end %> </div> - <%= will_paginate WillPaginate::Collection.new(@page, 10, @xapian_requests.matches_estimated) %> + <%= will_paginate WillPaginate::Collection.new(@page, @per_page, @xapian_requests.matches_estimated) %> <% end %> -</p> +</div> diff --git a/app/views/request/_search_ahead.rhtml b/app/views/request/_search_ahead.rhtml index d0b19de7d..1e65a5458 100644 --- a/app/views/request/_search_ahead.rhtml +++ b/app/views/request/_search_ahead.rhtml @@ -8,7 +8,7 @@ <% end %> <p> - <a id="body-site-search-link" target="_blank"><%= _("Or search in their website for this information.") %></a> + <a id="body-site-search-link"><%= _("Or search in their website for this information.") %></a> </p> <% end %> </div> diff --git a/app/views/request/list.rhtml b/app/views/request/list.rhtml index 63faf3643..42cd41498 100644 --- a/app/views/request/list.rhtml +++ b/app/views/request/list.rhtml @@ -14,7 +14,7 @@ <div style="clear:both"></div> <div class="results_section"> - <% view_cache :ttl => 5.minutes.to_i, :tag => [@view, @page, I18n.locale] do %> + <% view_cache :ttl => 5.minutes.to_i, :tag => [@query, @page, I18n.locale] do %> <% if @list_results.empty? %> <p> <%= _('No requests of this sort yet.')%></p> <% else %> @@ -30,6 +30,6 @@ </div> <% end %> - <%= will_paginate WillPaginate::Collection.new(@page, @per_page, @matches_estimated) %> + <%= will_paginate WillPaginate::Collection.new(@page, @per_page, @show_no_more_than) %> <% end %> </div> diff --git a/app/views/request/select_authority.rhtml b/app/views/request/select_authority.rhtml index 0e8df872d..521136f8e 100644 --- a/app/views/request/select_authority.rhtml +++ b/app/views/request/select_authority.rhtml @@ -57,7 +57,6 @@ <%= render :partial => 'public_body/body_listing_single', :locals => { :public_body => result[:model] } %> <% end %> </div> - <%= will_paginate WillPaginate::Collection.new(@page, 10, @xapian_requests.matches_estimated) %> <% end %> |