diff options
| -rw-r--r-- | config/crontab.ugly | 4 | ||||
| -rw-r--r-- | db/migrate/053_acts_as_xapian_migration.rb | 20 | ||||
| -rw-r--r-- | db/schema.rb | 12 | ||||
| -rw-r--r-- | public/stylesheets/main.css | 6 | ||||
| -rwxr-xr-x | script/update-xapian-index | 5 | ||||
| -rw-r--r-- | spec/controllers/general_controller_spec.rb | 23 | ||||
| -rw-r--r-- | spec/controllers/request_controller_spec.rb | 12 | ||||
| -rw-r--r-- | spec/fixtures/info_requests.yml | 2 | ||||
| -rw-r--r-- | todo.txt | 27 |
9 files changed, 87 insertions, 24 deletions
diff --git a/config/crontab.ugly b/config/crontab.ugly index 842c75c05..71563688e 100644 --- a/config/crontab.ugly +++ b/config/crontab.ugly @@ -4,13 +4,13 @@ # Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org. WWW: http://www.mysociety.org/ # -# $Id: crontab.ugly,v 1.9 2008-04-03 15:29:52 francis Exp $ +# $Id: crontab.ugly,v 1.10 2008-04-24 23:54:56 francis Exp $ PATH=/usr/local/bin:/usr/bin:/bin MAILTO=team@whatdotheyknow.com # Every 10 minutes -*/10 * * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/update-solr-index.lock /data/vhost/!!(*= $vhost *)!!/mysociety/foi/script/update-solr-index || echo "stalled?" +*/10 * * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/update-xapian-index.lock /data/vhost/!!(*= $vhost *)!!/mysociety/foi/script/update-xapian-index || echo "stalled?" 5,15,25,35,45,55 * * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/alert-tracks.lock /data/vhost/!!(*= $vhost *)!!/mysociety/foi/script/alert-tracks || echo "stalled?" # Once an hour diff --git a/db/migrate/053_acts_as_xapian_migration.rb b/db/migrate/053_acts_as_xapian_migration.rb new file mode 100644 index 000000000..77a298703 --- /dev/null +++ b/db/migrate/053_acts_as_xapian_migration.rb @@ -0,0 +1,20 @@ +class ActsAsXapianMigration < ActiveRecord::Migration + def self.up + create_table :acts_as_xapian_jobs do |t| + t.column :model, :string, :null => false + t.column :model_id, :integer, :null => false + + t.column :action, :string, :null => false + end + add_index :acts_as_xapian_jobs, [:model, :model_id], :unique => true + + remove_column :info_requests, :solr_up_to_date + end + + def self.down + drop_table :acts_as_xapian_jobs + + add_column :info_requests, :solr_up_to_date, :boolean, :default => false, :null => false + add_index :info_requests, :solr_up_to_date + end +end diff --git a/db/schema.rb b/db/schema.rb index 331f7adbe..b2466841d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -9,7 +9,15 @@ # # It's strongly recommended to check this file into your version control system. -ActiveRecord::Schema.define(:version => 52) do +ActiveRecord::Schema.define(:version => 53) do + + create_table "acts_as_xapian_jobs", :force => true do |t| + t.string "model", :null => false + t.integer "model_id", :null => false + t.string "action", :null => false + end + + add_index "acts_as_xapian_jobs", ["model", "model_id"], :name => "index_acts_as_xapian_jobs_on_model_and_model_id", :unique => true create_table "incoming_messages", :force => true do |t| t.integer "info_request_id", :null => false @@ -40,11 +48,9 @@ ActiveRecord::Schema.define(:version => 52) do t.boolean "awaiting_description", :default => false, :null => false t.string "prominence", :default => "normal", :null => false t.text "url_title", :null => false - t.boolean "solr_up_to_date", :default => false, :null => false end add_index "info_requests", ["created_at"], :name => "index_info_requests_on_created_at" - add_index "info_requests", ["solr_up_to_date"], :name => "index_info_requests_on_solr_up_to_date" add_index "info_requests", ["title"], :name => "index_info_requests_on_title" add_index "info_requests", ["url_title"], :name => "index_info_requests_on_url_title", :unique => true diff --git a/public/stylesheets/main.css b/public/stylesheets/main.css index 33fc09705..7eb544c0e 100644 --- a/public/stylesheets/main.css +++ b/public/stylesheets/main.css @@ -1,3 +1,9 @@ +/* XXX Francis put this here so he could check highlighting worked */ +.highlight { + background-color: #dddd00; + color: #000000; +} + /*------------------------------------------------ global */ body { diff --git a/script/update-xapian-index b/script/update-xapian-index new file mode 100755 index 000000000..378170317 --- /dev/null +++ b/script/update-xapian-index @@ -0,0 +1,5 @@ +#!/bin/bash + +cd `dirname $0` +rake xapian:update_index + diff --git a/spec/controllers/general_controller_spec.rb b/spec/controllers/general_controller_spec.rb index 9cda74ae7..7a9efcbb8 100644 --- a/spec/controllers/general_controller_spec.rb +++ b/spec/controllers/general_controller_spec.rb @@ -26,15 +26,16 @@ describe GeneralController, "when searching" do end it "should find info request when searching for '\"fancy dog\"'" do - InfoRequest.update_solr_index + #ActsAsXapian.rebuild_index([PublicBody, User, InfoRequestEvent]) + ActsAsXapian.update_index get :search, :query => '"fancy dog"' response.should render_template('search') assigns[:search_hits].should == 1 - assigns[:search_results].should == [ info_request_events(:useless_outgoing_message_event) ] + assigns[:search_results].size.should == 1 + assigns[:search_results][0][:model].should == info_request_events(:useless_outgoing_message_event) assigns[:highlight_words].should == ["fancy", "dog"] - assigns[:highlighting]["InfoRequestEvent"][900]["solr"][0].should include('Why do you have such a <span class="highlight">fancy</span> <span class="highlight">dog</span>?') end it "should show help when searching for nothing" do @@ -45,27 +46,27 @@ describe GeneralController, "when searching" do end it "should find public body and incoming message (in that order) when searching for 'geraldine quango'" do - InfoRequest.update_solr_index - PublicBody.rebuild_solr_index - User.rebuild_solr_index + ActsAsXapian.update_index get :search, :query => 'geraldine quango' response.should render_template('search') assigns[:search_hits].should == 2 - assigns[:search_results].should == [ public_bodies(:geraldine_public_body), info_request_events(:useless_incoming_message_event) ] + assigns[:search_results].size.should == 2 + assigns[:search_results][0][:model].should == public_bodies(:geraldine_public_body) + assigns[:search_results][1][:model].should == info_request_events(:useless_incoming_message_event) end it "should find incoming message and public body (in that order) when searching for 'geraldine quango', newest first" do - InfoRequest.update_solr_index - PublicBody.rebuild_solr_index - User.rebuild_solr_index + ActsAsXapian.update_index get :search, :query => 'geraldine quango', :sortby => 'newest' response.should render_template('search') assigns[:search_hits].should == 2 - assigns[:search_results].should == [ info_request_events(:useless_incoming_message_event), public_bodies(:geraldine_public_body) ] + assigns[:search_results].size.should == 2 + assigns[:search_results][0][:model].should == info_request_events(:useless_incoming_message_event) + assigns[:search_results][1][:model].should == public_bodies(:geraldine_public_body) end diff --git a/spec/controllers/request_controller_spec.rb b/spec/controllers/request_controller_spec.rb index c4fec9236..05e53b1f4 100644 --- a/spec/controllers/request_controller_spec.rb +++ b/spec/controllers/request_controller_spec.rb @@ -16,14 +16,14 @@ describe RequestController, "when listing all requests" do it "should assign the first page of results" do # XXX probably should load more than one page of requests into db here :) - - InfoRequest.update_solr_index + ActsAsXapian.update_index get :list - assigns[:search_results].should == [ - info_request_events(:silly_outgoing_message_event), # reverse-chronological order - info_request_events(:useless_outgoing_message_event) - ] + + # reverse-chronological order + assigns[:search_results].size.should == 2 + assigns[:search_results][0][:model].should == info_request_events(:silly_outgoing_message_event) + assigns[:search_results][1][:model].should == info_request_events(:useless_incoming_message_event) end end diff --git a/spec/fixtures/info_requests.yml b/spec/fixtures/info_requests.yml index dd296410b..a849542b8 100644 --- a/spec/fixtures/info_requests.yml +++ b/spec/fixtures/info_requests.yml @@ -8,7 +8,6 @@ fancy_dog_request: user_id: 1 described_state: waiting_response awaiting_description: true - solr_up_to_date: false naughty_chicken_request: id: 103 title: How much public money is wasted on breeding naughty chickens? @@ -19,6 +18,5 @@ naughty_chicken_request: user_id: 1 described_state: waiting_response awaiting_description: false - solr_up_to_date: false @@ -1,3 +1,30 @@ +deployment: + change the exec_after scripts to not do solr + rebuild the xapian index + install it on the server + +call info_request.calculate_event_states +check info request fully_destroy does remove it from index +solr_object +grep for solr in app + +check emails + +update the cron job + + +remove all the scripts in scripts/*solr* +remove all the solr plugins and stuff + + +highlight word docs text +full_search still has html_highlight parameter + +follow up E re. blog +needs tagline that v. quickly explains what site is on each page +needs development text box + + Watch this one: http://www.whatdotheyknow.com/request/crime_statistics_for_2007_champi |