diff options
| author | Robin Houston <robin.houston@gmail.com> | 2012-01-19 13:50:26 +0000 |
|---|---|---|
| committer | Robin Houston <robin.houston@gmail.com> | 2012-01-19 13:50:26 +0000 |
| commit | 22b2db48c005be1d1b87dd6b58f830a2a74d2108 (patch) | |
| tree | 2355e7d73515b38ecd0edf1616498f4e62b82cd4 | |
| parent | 69a2ea41c7c723ba11690ee6ed148cda012f8358 (diff) | |
Limit Xapian wildcard expansions
Wildcard searches in Xapian can expand uncontrollably, consuming
all available RAM and crashing the server. This has been a real
problem on WhatDoTheyKnow.com. The underlying issue is tracked
in http://trac.xapian.org/ticket/350
This changeset imposes a limit on wildcard expansion. The type-ahead
search will first try a wildcard query, and if that fails because
of too much expansion will fall back to a plain non-wildcard search.
| -rw-r--r-- | app/controllers/application_controller.rb | 23 | ||||
| -rw-r--r-- | vendor/plugins/acts_as_xapian/lib/acts_as_xapian.rb | 1 |
2 files changed, 19 insertions, 5 deletions
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 0c8544932..eae27c667 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -385,11 +385,24 @@ class ApplicationController < ActionController::Base ActsAsXapian.readable_init old_default_op = ActsAsXapian.query_parser.default_op ActsAsXapian.query_parser.default_op = Xapian::Query::OP_OR - user_query = ActsAsXapian.query_parser.parse_query( - query, - Xapian::QueryParser::FLAG_LOVEHATE | Xapian::QueryParser::FLAG_PARTIAL | - Xapian::QueryParser::FLAG_SPELLING_CORRECTION) - xapian_requests = ActsAsXapian::Search.new([model], query, options, user_query) + begin + user_query = ActsAsXapian.query_parser.parse_query( + query.strip + '*', + Xapian::QueryParser::FLAG_LOVEHATE | Xapian::QueryParser::FLAG_WILDCARD | + Xapian::QueryParser::FLAG_SPELLING_CORRECTION) + xapian_requests = ActsAsXapian::Search.new([model], query, options, user_query) + rescue RuntimeError => e + if e.message =~ /^QueryParserError: Wildcard/ + # Wildcard expands to too many terms + logger.info "Wildcard query '#{query.strip + '*'}' caused: #{e.message}" + + user_query = ActsAsXapian.query_parser.parse_query( + query, + Xapian::QueryParser::FLAG_LOVEHATE | + Xapian::QueryParser::FLAG_SPELLING_CORRECTION) + xapian_requests = ActsAsXapian::Search.new([model], query, options, user_query) + end + end ActsAsXapian.query_parser.default_op = old_default_op end return xapian_requests diff --git a/vendor/plugins/acts_as_xapian/lib/acts_as_xapian.rb b/vendor/plugins/acts_as_xapian/lib/acts_as_xapian.rb index 38bfb7c98..dd033a112 100644 --- a/vendor/plugins/acts_as_xapian/lib/acts_as_xapian.rb +++ b/vendor/plugins/acts_as_xapian/lib/acts_as_xapian.rb @@ -143,6 +143,7 @@ module ActsAsXapian @@query_parser.stemming_strategy = Xapian::QueryParser::STEM_SOME @@query_parser.database = @@db @@query_parser.default_op = Xapian::Query::OP_AND + @@query_parser.set_max_wildcard_expansion(1000) @@stopper = Xapian::SimpleStopper.new @@stopper.add("and") |