aboutsummaryrefslogtreecommitdiffstats
path: root/app
diff options
context:
space:
mode:
Diffstat (limited to 'app')
-rw-r--r--app/controllers/admin_general_controller.rb4
-rw-r--r--app/controllers/application_controller.rb40
-rw-r--r--app/controllers/general_controller.rb12
-rw-r--r--app/controllers/request_controller.rb15
-rw-r--r--app/models/info_request_event.rb22
-rw-r--r--app/models/raw_email.rb2
-rw-r--r--app/models/track_thing.rb15
-rw-r--r--app/views/admin_general/debug.rhtml4
-rw-r--r--app/views/layouts/default.rhtml20
-rw-r--r--app/views/public_body/_search_ahead.rhtml6
-rw-r--r--app/views/request/_search_ahead.rhtml2
-rw-r--r--app/views/request/list.rhtml4
-rw-r--r--app/views/request/select_authority.rhtml1
13 files changed, 103 insertions, 44 deletions
diff --git a/app/controllers/admin_general_controller.rb b/app/controllers/admin_general_controller.rb
index ae51e0923..0b7e9bec0 100644
--- a/app/controllers/admin_general_controller.rb
+++ b/app/controllers/admin_general_controller.rb
@@ -78,6 +78,10 @@ class AdminGeneralController < AdminController
end
def debug
+ @current_commit = `git log -1 --format="%H"`
+ @current_branch = `git branch | grep "\*" | awk '{print $2}'`
+ repo = `git remote show origin -n | grep Fetch | awk '{print $3}' | sed -re 's/.*:(.*).git/\\1/'`
+ @github_origin = "https://github.com/#{repo.strip}/tree/"
@request_env = request.env
end
end
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 0c8544932..8fc6c3792 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -89,6 +89,7 @@ class ApplicationController < ActionController::Base
def record_memory
record_memory = MySociety::Config.get('DEBUG_RECORD_MEMORY', false)
if record_memory
+ logger.info "Processing request for #{request.url} with Rails process #{Process.pid}"
File.read("/proc/#{Process.pid}/status").match(/VmRSS:\s+(\d+)/)
rss_before_action = $1.to_i
yield
@@ -180,7 +181,10 @@ class ApplicationController < ActionController::Base
path = foi_fragment_cache_part_path(param)
path = "/views" + path
foi_cache_path = File.join(File.dirname(__FILE__), '../../cache')
- return File.join(foi_cache_path, path)
+ max_file_length = 255 - 35 # we subtract 35 because tempfile
+ # adds on a variable number of
+ # characters
+ return File.join(foi_cache_path, path)[0...max_file_length]
end
def foi_fragment_cache_all_for_request(info_request)
# return stub path so admin can expire it
@@ -193,10 +197,12 @@ class ApplicationController < ActionController::Base
return File.exists?(key_path)
end
def foi_fragment_cache_read(key_path)
+ logger.info "Reading from fragment cache #{key_path}"
return File.read(key_path)
end
def foi_fragment_cache_write(key_path, content)
FileUtils.mkdir_p(File.dirname(key_path))
+ logger.info "Writing to fragment cache #{key_path}"
File.atomic_write(key_path) do |f|
f.write(content)
end
@@ -365,7 +371,10 @@ class ApplicationController < ActionController::Base
def get_search_page_from_params
return (params[:page] || "1").to_i
end
+
def perform_search_typeahead(query, model)
+ @page = get_search_page_from_params
+ @per_page = 10
query_words = query.split(/ +(?![-+]+)/)
if query_words.last.nil? || query_words.last.strip.length < 3
xapian_requests = nil
@@ -376,8 +385,8 @@ class ApplicationController < ActionController::Base
collapse = 'request_collapse'
end
options = {
- :offset => 0,
- :limit => 5,
+ :offset => (@page - 1) * @per_page,
+ :limit => @per_page,
:sort_by_prefix => nil,
:sort_by_ascending => true,
:collapse_by_prefix => collapse,
@@ -385,11 +394,24 @@ class ApplicationController < ActionController::Base
ActsAsXapian.readable_init
old_default_op = ActsAsXapian.query_parser.default_op
ActsAsXapian.query_parser.default_op = Xapian::Query::OP_OR
- user_query = ActsAsXapian.query_parser.parse_query(
- query,
- Xapian::QueryParser::FLAG_LOVEHATE | Xapian::QueryParser::FLAG_PARTIAL |
- Xapian::QueryParser::FLAG_SPELLING_CORRECTION)
- xapian_requests = ActsAsXapian::Search.new([model], query, options, user_query)
+ begin
+ user_query = ActsAsXapian.query_parser.parse_query(
+ query.strip + '*',
+ Xapian::QueryParser::FLAG_LOVEHATE | Xapian::QueryParser::FLAG_WILDCARD |
+ Xapian::QueryParser::FLAG_SPELLING_CORRECTION)
+ xapian_requests = ActsAsXapian::Search.new([model], query, options, user_query)
+ rescue RuntimeError => e
+ if e.message =~ /^QueryParserError: Wildcard/
+ # Wildcard expands to too many terms
+ logger.info "Wildcard query '#{query.strip + '*'}' caused: #{e.message}"
+
+ user_query = ActsAsXapian.query_parser.parse_query(
+ query,
+ Xapian::QueryParser::FLAG_LOVEHATE |
+ Xapian::QueryParser::FLAG_SPELLING_CORRECTION)
+ xapian_requests = ActsAsXapian::Search.new([model], query, options, user_query)
+ end
+ end
ActsAsXapian.query_parser.default_op = old_default_op
end
return xapian_requests
@@ -521,7 +543,7 @@ class ApplicationController < ActionController::Base
def quietly_try_to_open(url)
begin
result = open(url).read.strip
- rescue OpenURI::HTTPError, SocketError
+ rescue OpenURI::HTTPError, SocketError, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH
logger.warn("Unable to open third-party URL #{url}")
result = ""
end
diff --git a/app/controllers/general_controller.rb b/app/controllers/general_controller.rb
index d28f4deec..c715b547d 100644
--- a/app/controllers/general_controller.rb
+++ b/app/controllers/general_controller.rb
@@ -32,12 +32,12 @@ class GeneralController < ApplicationController
if body_short_names.empty?
# This is too slow
@popular_bodies = PublicBody.find(:all,
- :select => "public_bodies.*, (select count(*) from info_requests where info_requests.public_body_id = public_bodies.id) as c",
- :order => "c desc",
- :limit => 32,
- :conditions => conditions,
- :joins => :translations
- )
+ :select => "public_bodies.*, (select count(*) from info_requests where info_requests.public_body_id = public_bodies.id) as c",
+ :order => "c desc",
+ :limit => 32,
+ :conditions => conditions,
+ :joins => :translations
+ )
else
conditions[0] += " and public_bodies.url_name in (" + body_short_names + ")"
@popular_bodies = PublicBody.find(:all,
diff --git a/app/controllers/request_controller.rb b/app/controllers/request_controller.rb
index 1c7aeedcc..11812b729 100644
--- a/app/controllers/request_controller.rb
+++ b/app/controllers/request_controller.rb
@@ -13,6 +13,9 @@ require 'open-uri'
class RequestController < ApplicationController
before_filter :check_read_only, :only => [ :new, :show_response, :describe_state, :upload_response ]
protect_from_forgery :only => [ :new, :show_response, :describe_state, :upload_response ] # See ActionController::RequestForgeryProtection for details
+
+ MAX_RESULTS = 500
+ PER_PAGE = 25
@@custom_states_loaded = false
begin
@@ -155,14 +158,21 @@ class RequestController < ApplicationController
if @view == "recent"
return redirect_to request_list_all_path(:action => "list", :view => "all", :page => @page), :status => :moved_permanently
end
+
+ # Later pages are very expensive to load
+ if @page > MAX_RESULTS / PER_PAGE
+ raise ActiveRecord::RecordNotFound.new("Sorry. No pages after #{MAX_RESULTS / PER_PAGE}.")
+ end
+
params[:latest_status] = @view
query = make_query_from_params
@title = _("View and search requests")
sortby = "newest"
- behavior_cache :tag => [@view, @page] do
+ behavior_cache :tag => [@query, @page, I18n.locale] do
xapian_object = perform_search([InfoRequestEvent], query, sortby, 'request_collapse')
@list_results = xapian_object.results.map { |r| r[:model] }
@matches_estimated = xapian_object.matches_estimated
+ @show_no_more_than = (@matches_estimated > MAX_RESULTS) ? MAX_RESULTS : @matches_estimated
end
@title = @title + " (page " + @page.to_s + ")" if (@page > 1)
@@ -682,10 +692,11 @@ class RequestController < ApplicationController
# Internal function
def get_attachment_internal(html_conversion)
@incoming_message = IncomingMessage.find(params[:incoming_message_id])
+ @requested_request = InfoRequest.find(params[:id])
@incoming_message.parse_raw_email!
@info_request = @incoming_message.info_request
if @incoming_message.info_request_id != params[:id].to_i
- raise sprintf("Incoming message %d does not belong to request %d", @incoming_message.info_request_id, params[:id])
+ raise ActiveRecord::RecordNotFound.new(sprintf("Incoming message %d does not belong to request %d", @incoming_message.info_request_id, params[:id]))
end
@part_number = params[:part].to_i
@filename = params[:file_name].join("/")
diff --git a/app/models/info_request_event.rb b/app/models/info_request_event.rb
index e99a0ae2f..99f34cf9e 100644
--- a/app/models/info_request_event.rb
+++ b/app/models/info_request_event.rb
@@ -190,6 +190,26 @@ class InfoRequestEvent < ActiveRecord::Base
return message
end
+ def get_clipped_response_efficiently
+ # XXX this ugly code is an attempt to not always load all the
+ # columns for an incoming message, which can be *very* large
+ # (due to all the cached text). We care particularly in this
+ # case because it's called for every search result on a page
+ # (to show the search snippet). Actually, we should review if we
+ # need all this data to be cached in the database at all, and
+ # then we won't need this horrid workaround.
+ message = self.incoming_message_selective_columns("cached_attachment_text_clipped, cached_main_body_text_folded")
+ clipped_body = message.cached_main_body_text_folded
+ clipped_attachment = message.cached_attachment_text_clipped
+ if clipped_body.nil? || clipped_attachment.nil?
+ # we're going to have to load it anyway
+ text = self.incoming_message.get_text_for_indexing_clipped
+ else
+ text = clipped_body.gsub("FOLDED_QUOTED_SECTION", " ").strip + "\n\n" + clipped_attachment
+ end
+ return text + "\n\n"
+ end
+
# clipped = true - means return shorter text. It is used for snippets fore
# performance reasons. Xapian will take the full text.
def search_text_main(clipped = false)
@@ -200,7 +220,7 @@ class InfoRequestEvent < ActiveRecord::Base
text = text + self.outgoing_message.get_text_for_indexing + "\n\n"
elsif self.event_type == 'response'
if clipped
- text = text + self.incoming_message_selective_columns("cached_attachment_text_clipped").cached_attachment_text_clipped + "\n\n"
+ text = text + self.get_clipped_response_efficiently
else
text = text + self.incoming_message.get_text_for_indexing_full + "\n\n"
end
diff --git a/app/models/raw_email.rb b/app/models/raw_email.rb
index c6f84318b..6e073aa27 100644
--- a/app/models/raw_email.rb
+++ b/app/models/raw_email.rb
@@ -43,7 +43,7 @@ class RawEmail < ActiveRecord::Base
if !File.exists?(self.directory)
FileUtils.mkdir_p self.directory
end
- File.open(self.filepath, "wb") { |file|
+ File.atomic_write(self.filepath) { |file|
file.write d
}
end
diff --git a/app/models/track_thing.rb b/app/models/track_thing.rb
index 6938fade9..58d70ed86 100644
--- a/app/models/track_thing.rb
+++ b/app/models/track_thing.rb
@@ -71,14 +71,13 @@ class TrackThing < ActiveRecord::Base
def track_query_description
# XXX this is very brittle... we should probably ask users
# simply to name their tracks when they make them?
- self.track_query = self.track_query.gsub(/([()]|OR)/, "")
- filters = self.track_query.scan /\b\S+:\S+\b/
- text = self.track_query
+ original_text = parsed_text = self.track_query.gsub(/([()]|OR)/, "")
+ filters = parsed_text.scan /\b\S+:\S+\b/
varieties = Set.new
date = ""
statuses = Set.new
for filter in filters
- text = text.sub(filter, "")
+ parsed_text = parsed_text.sub(filter, "")
if filter =~ /variety:user/
varieties << _("users")
end
@@ -105,7 +104,7 @@ class TrackThing < ActiveRecord::Base
end
end
if filters.empty?
- text = self.track_query
+ parsed_text = original_text
end
descriptions = []
if varieties.include? _("requests")
@@ -116,10 +115,10 @@ class TrackThing < ActiveRecord::Base
varieties << _("anything")
end
descriptions += Array(varieties)
- text = text.strip
+ parsed_text = parsed_text.strip
descriptions = descriptions.join(_(" or "))
- if !text.empty?
- descriptions += _("{{list_of_things}} matching text '{{search_query}}'", :list_of_things => "", :search_query => text)
+ if !parsed_text.empty?
+ descriptions += _("{{list_of_things}} matching text '{{search_query}}'", :list_of_things => "", :search_query => parsed_text)
end
return descriptions
end
diff --git a/app/views/admin_general/debug.rhtml b/app/views/admin_general/debug.rhtml
index b3b06085f..40fe33616 100644
--- a/app/views/admin_general/debug.rhtml
+++ b/app/views/admin_general/debug.rhtml
@@ -7,6 +7,10 @@
<h2>Version numbers</h2>
<p>
+Alaveteli branch: <%= link_to @current_branch, @github_origin + @current_branch %>
+<br>
+Alaveteli commit: <%= link_to @current_commit, @github_origin + @current_commit %>
+<br>
RUBY_VERSION <%=h RUBY_VERSION %>
<br>
Rails::VERSION::STRING <%=h Rails::VERSION::STRING%>
diff --git a/app/views/layouts/default.rhtml b/app/views/layouts/default.rhtml
index ad0560baa..f439b27d2 100644
--- a/app/views/layouts/default.rhtml
+++ b/app/views/layouts/default.rhtml
@@ -10,13 +10,13 @@
</title>
<link rel="shortcut icon" href="/favicon.ico">
- <%= stylesheet_link_tag 'main', :title => "Main", :rel => "stylesheet", :media => "all" %>
- <%= stylesheet_link_tag 'fonts', :rel => "stylesheet", :media => "all" %>
- <%= stylesheet_link_tag 'print', :rel => "stylesheet", :media => "print" %>
- <% if !params[:print_stylesheet].nil? %>
+ <%= stylesheet_link_tag 'main', :title => "Main", :rel => "stylesheet", :media => "all" %>
+ <%= stylesheet_link_tag 'fonts', :rel => "stylesheet", :media => "all" %>
+ <%= stylesheet_link_tag 'print', :rel => "stylesheet", :media => "print" %>
+ <% if !params[:print_stylesheet].nil? %>
<%= stylesheet_link_tag 'print', :rel => "stylesheet", :media => "all" %>
- <% end %>
- <%= javascript_include_tag 'jquery.js', 'jquery-ui.min','jquery.cookie.js', 'general.js' %>
+ <% end %>
+ <%= javascript_include_tag 'jquery.js', 'jquery-ui.min','jquery.cookie.js', 'general.js' %>
<% if @profile_photo_javascript %>
<script type="text/javascript" src="/javascripts/jquery.Jcrop.js"></script>
@@ -24,7 +24,7 @@
<link rel="stylesheet" href="/stylesheets/jquery.Jcrop.css" type="text/css" >
<% end %>
- <%= stylesheet_link_tag 'admin-theme/jquery-ui-1.8.15.custom.css', :rel => 'stylesheet'%>
+ <%= stylesheet_link_tag 'admin-theme/jquery-ui-1.8.15.custom.css', :rel => 'stylesheet'%>
<!--[if LT IE 7]>
<style type="text/css">@import url("/stylesheets/ie6.css");</style>
<![endif]-->
@@ -34,8 +34,8 @@
<!--[if LT IE 8]>
<style type="text/css">@import url("/stylesheets/ie7.css");</style>
<![endif]-->
- <!-- the following method for customising CSS is deprecated; see `doc/THEMES.md` for detail -->
- <%= stylesheet_link_tag 'custom', :title => "Main", :rel => "stylesheet" %>
+ <!-- the following method for customising CSS is deprecated; see `doc/THEMES.md` for detail -->
+ <%= stylesheet_link_tag 'custom', :title => "Main", :rel => "stylesheet" %>
<% if force_registration_on_new_request %>
<%= stylesheet_link_tag 'jquery.fancybox-1.3.4', :rel => "stylesheet" %>
<% end %>
@@ -56,7 +56,7 @@
<meta name="robots" content="noindex, nofollow">
<% end %>
- <%= render :partial => 'general/before_head_end' %>
+ <%= render :partial => 'general/before_head_end' %>
</head>
<body <%= "class='front'" if params[:action] == 'frontpage' %>>
diff --git a/app/views/public_body/_search_ahead.rhtml b/app/views/public_body/_search_ahead.rhtml
index 484d28256..7ade89b8e 100644
--- a/app/views/public_body/_search_ahead.rhtml
+++ b/app/views/public_body/_search_ahead.rhtml
@@ -1,4 +1,4 @@
-<p>
+<div>
<% if !@xapian_requests.nil? %>
<% if @xapian_requests.results.size > 0 %>
<h3><%= _('Top search results:') %></h3>
@@ -13,9 +13,9 @@
<%= render :partial => 'body_listing_single', :locals => { :public_body => result[:model] } %>
<% end %>
</div>
- <%= will_paginate WillPaginate::Collection.new(@page, 10, @xapian_requests.matches_estimated) %>
+ <%= will_paginate WillPaginate::Collection.new(@page, @per_page, @xapian_requests.matches_estimated) %>
<% end %>
-</p>
+</div>
diff --git a/app/views/request/_search_ahead.rhtml b/app/views/request/_search_ahead.rhtml
index d0b19de7d..1e65a5458 100644
--- a/app/views/request/_search_ahead.rhtml
+++ b/app/views/request/_search_ahead.rhtml
@@ -8,7 +8,7 @@
<% end %>
<p>
- <a id="body-site-search-link" target="_blank"><%= _("Or search in their website for this information.") %></a>
+ <a id="body-site-search-link"><%= _("Or search in their website for this information.") %></a>
</p>
<% end %>
</div>
diff --git a/app/views/request/list.rhtml b/app/views/request/list.rhtml
index 63faf3643..42cd41498 100644
--- a/app/views/request/list.rhtml
+++ b/app/views/request/list.rhtml
@@ -14,7 +14,7 @@
<div style="clear:both"></div>
<div class="results_section">
- <% view_cache :ttl => 5.minutes.to_i, :tag => [@view, @page, I18n.locale] do %>
+ <% view_cache :ttl => 5.minutes.to_i, :tag => [@query, @page, I18n.locale] do %>
<% if @list_results.empty? %>
<p> <%= _('No requests of this sort yet.')%></p>
<% else %>
@@ -30,6 +30,6 @@
</div>
<% end %>
- <%= will_paginate WillPaginate::Collection.new(@page, @per_page, @matches_estimated) %>
+ <%= will_paginate WillPaginate::Collection.new(@page, @per_page, @show_no_more_than) %>
<% end %>
</div>
diff --git a/app/views/request/select_authority.rhtml b/app/views/request/select_authority.rhtml
index 0e8df872d..521136f8e 100644
--- a/app/views/request/select_authority.rhtml
+++ b/app/views/request/select_authority.rhtml
@@ -57,7 +57,6 @@
<%= render :partial => 'public_body/body_listing_single', :locals => { :public_body => result[:model] } %>
<% end %>
</div>
- <%= will_paginate WillPaginate::Collection.new(@page, 10, @xapian_requests.matches_estimated) %>
<% end %>
generated by cgit v1.2.3 (git 2.25.1) at 2025-08-05 00:29:31 +0000