diff options
| -rw-r--r-- | app/helpers/highlight_helper.rb | 1 | ||||
| -rw-r--r-- | config/initializers/alaveteli.rb | 2 | ||||
| -rw-r--r-- | lib/acts_as_xapian/acts_as_xapian.rb | 5 | ||||
| -rw-r--r-- | spec/lib/acts_as_xapian_spec.rb | 110 | ||||
| -rw-r--r-- | spec/models/info_request_spec.rb | 1 | ||||
| -rw-r--r-- | spec/models/xapian_spec.rb | 71 |
6 files changed, 117 insertions, 73 deletions
diff --git a/app/helpers/highlight_helper.rb b/app/helpers/highlight_helper.rb index a98f6f320..c72d5a64b 100644 --- a/app/helpers/highlight_helper.rb +++ b/app/helpers/highlight_helper.rb @@ -1,3 +1,4 @@ +# -*- encoding : utf-8 -*- module HighlightHelper include ERB::Util diff --git a/config/initializers/alaveteli.rb b/config/initializers/alaveteli.rb index 724ecbb47..d94515ae8 100644 --- a/config/initializers/alaveteli.rb +++ b/config/initializers/alaveteli.rb @@ -10,7 +10,7 @@ load "debug_helpers.rb" load "util.rb" # Application version -ALAVETELI_VERSION = '0.21.0.31' +ALAVETELI_VERSION = '0.21.0.32' # Add new inflection rules using the following format # (all these examples are active by default): diff --git a/lib/acts_as_xapian/acts_as_xapian.rb b/lib/acts_as_xapian/acts_as_xapian.rb index 6520a20a4..48d0b0554 100644 --- a/lib/acts_as_xapian/acts_as_xapian.rb +++ b/lib/acts_as_xapian/acts_as_xapian.rb @@ -375,7 +375,10 @@ module ActsAsXapian if correction.empty? return nil end - return correction + if correction.respond_to?(:force_encoding) + correction = correction.force_encoding('UTF-8') + end + correction end # Return array of models found diff --git a/spec/lib/acts_as_xapian_spec.rb b/spec/lib/acts_as_xapian_spec.rb new file mode 100644 index 000000000..1d9256441 --- /dev/null +++ b/spec/lib/acts_as_xapian_spec.rb @@ -0,0 +1,110 @@ +# -*- encoding : utf-8 -*- +require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') + +describe ActsAsXapian::Search do + + describe "#words_to_highlight" do + + before :all do + # make sure an index exists + @alice = FactoryGirl.create(:public_body, :name => 'alice') + ActsAsXapian.update_index + end + + after :all do + @alice.destroy + ActsAsXapian.update_index + end + + it "should return a list of words used in the search" do + s = ActsAsXapian::Search.new([PublicBody], "albatross words", :limit => 100) + s.words_to_highlight.should == ["albatross", "word"] + end + + it "should remove any operators" do + s = ActsAsXapian::Search.new([PublicBody], "albatross words tag:mice", :limit => 100) + s.words_to_highlight.should == ["albatross", "word"] + end + + it "should separate punctuation" do + s = ActsAsXapian::Search.new([PublicBody], "The doctor's patient", :limit => 100) + s.words_to_highlight.should == ["the", "doctor", "patient"].sort + end + + it "should handle non-ascii characters" do + s = ActsAsXapian::Search.new([PublicBody], "adatigénylés words tag:mice", :limit => 100) + s.words_to_highlight.should == ["adatigénylé", "word"] + end + + it "should ignore stopwords" do + s = ActsAsXapian::Search.new([PublicBody], "department of humpadinking", :limit => 100) + s.words_to_highlight.should_not include('of') + end + + it "uses stemming" do + s = ActsAsXapian::Search.new([PublicBody], 'department of humpadinking', :limit => 100) + s.words_to_highlight.should == ["depart", "humpadink"] + end + + it "doesn't stem proper nouns" do + s = ActsAsXapian::Search.new([PublicBody], 'department of Humpadinking', :limit => 1) + s.words_to_highlight.should == ["depart", "humpadinking"] + end + + it "includes the original search terms if requested" do + s = ActsAsXapian::Search.new([PublicBody], 'boring', :limit => 1) + s.words_to_highlight(:include_original => true).should == ['bore', 'boring'] + end + + it "does not return duplicate terms" do + s = ActsAsXapian::Search.new([PublicBody], 'boring boring', :limit => 1) + s.words_to_highlight.should == ['bore'] + end + + context 'the :regex option' do + + it 'wraps each words in a regex that matches the full word' do + expected = [/\b(albatross)\b/iu] + s = ActsAsXapian::Search.new([PublicBody], 'Albatross', :limit => 1) + s.words_to_highlight(:regex => true).should == expected + end + + it 'wraps each stem in a regex' do + expected = [/\b(depart)\w*\b/iu] + s = ActsAsXapian::Search.new([PublicBody], 'department', :limit => 1) + s.words_to_highlight(:regex => true).should == expected + end + + end + end + + describe :spelling_correction do + + before :all do + @alice = FactoryGirl.create(:public_body, :name => 'alice') + @bob = FactoryGirl.create(:public_body, :name => 'bôbby') + ActsAsXapian.update_index + end + + after :all do + @alice.destroy + @bob.destroy + ActsAsXapian.update_index + end + + it 'returns a UTF-8 encoded string' do + s = ActsAsXapian::Search.new([PublicBody], "alece", :limit => 100) + s.spelling_correction.should == "alice" + if s.spelling_correction.respond_to? :encoding + s.spelling_correction.encoding.to_s.should == 'UTF-8' + end + end + + it 'handles non-ASCII characters' do + s = ActsAsXapian::Search.new([PublicBody], "bobby", :limit => 100) + s.spelling_correction.should == "bôbby" + end + + end + +end
\ No newline at end of file diff --git a/spec/models/info_request_spec.rb b/spec/models/info_request_spec.rb index 70947584b..9d1e02442 100644 --- a/spec/models/info_request_spec.rb +++ b/spec/models/info_request_spec.rb @@ -1221,6 +1221,7 @@ describe InfoRequest do describe InfoRequest, "when constructing a list of requests by query" do before(:each) do + load_raw_emails_data get_fixtures_xapian_index end diff --git a/spec/models/xapian_spec.rb b/spec/models/xapian_spec.rb index 678e3a2dc..ca6cd7db7 100644 --- a/spec/models/xapian_spec.rb +++ b/spec/models/xapian_spec.rb @@ -370,77 +370,6 @@ describe PublicBody, " when only indexing selected things on a rebuild" do end end -# I would expect ActsAsXapian to have some tests under lib/acts_as_xapian, but -# it looks like this is not the case. Putting a test here instead. -describe ActsAsXapian::Search, "#words_to_highlight" do - before(:each) do - load_raw_emails_data - get_fixtures_xapian_index - end - - it "should return a list of words used in the search" do - s = ActsAsXapian::Search.new([PublicBody], "albatross words", :limit => 100) - s.words_to_highlight.should == ["albatross", "word"] - end - - it "should remove any operators" do - s = ActsAsXapian::Search.new([PublicBody], "albatross words tag:mice", :limit => 100) - s.words_to_highlight.should == ["albatross", "word"] - end - - it "should separate punctuation" do - s = ActsAsXapian::Search.new([PublicBody], "The doctor's patient", :limit => 100) - s.words_to_highlight.should == ["the", "doctor", "patient"].sort - end - - it "should handle non-ascii characters" do - s = ActsAsXapian::Search.new([PublicBody], "adatigénylés words tag:mice", :limit => 100) - s.words_to_highlight.should == ["adatigénylé", "word"] - end - - it "should ignore stopwords" do - s = ActsAsXapian::Search.new([PublicBody], "department of humpadinking", :limit => 100) - s.words_to_highlight.should_not include('of') - end - - it "uses stemming" do - s = ActsAsXapian::Search.new([PublicBody], 'department of humpadinking', :limit => 100) - s.words_to_highlight.should == ["depart", "humpadink"] - end - - it "doesn't stem proper nouns" do - s = ActsAsXapian::Search.new([PublicBody], 'department of Humpadinking', :limit => 1) - s.words_to_highlight.should == ["depart", "humpadinking"] - end - - it "includes the original search terms if requested" do - s = ActsAsXapian::Search.new([PublicBody], 'boring', :limit => 1) - s.words_to_highlight(:include_original => true).should == ['bore', 'boring'] - end - - it "does not return duplicate terms" do - s = ActsAsXapian::Search.new([PublicBody], 'boring boring', :limit => 1) - s.words_to_highlight.should == ['bore'] - end - - context 'the :regex option' do - - it 'wraps each words in a regex that matches the full word' do - expected = [/\b(albatross)\b/iu] - s = ActsAsXapian::Search.new([PublicBody], 'Albatross', :limit => 1) - s.words_to_highlight(:regex => true).should == expected - end - - it 'wraps each stem in a regex' do - expected = [/\b(depart)\w*\b/iu] - s = ActsAsXapian::Search.new([PublicBody], 'department', :limit => 1) - s.words_to_highlight(:regex => true).should == expected - end - - end - -end - describe InfoRequestEvent, " when faced with a race condition during xapian_mark_needs_index" do before(:each) do |