diff options
| -rw-r--r-- | app/models/info_request.rb | 36 | ||||
| -rw-r--r-- | config/crontab.ugly | 5 | ||||
| -rw-r--r-- | db/schema.rb | 1 | ||||
| -rwxr-xr-x | script/restart-solr | 12 | ||||
| -rwxr-xr-x | script/update-solr-index | 7 | ||||
| -rw-r--r-- | todo.txt | 6 |
6 files changed, 53 insertions, 14 deletions
diff --git a/app/models/info_request.rb b/app/models/info_request.rb index 53f349535..2c38acd2c 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -21,7 +21,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: info_request.rb,v 1.54 2008-03-06 01:23:38 francis Exp $ +# $Id: info_request.rb,v 1.55 2008-03-06 12:17:21 francis Exp $ require 'digest/sha1' @@ -68,20 +68,34 @@ class InfoRequest < ActiveRecord::Base $do_solr_index = false def self.update_solr_index $do_solr_index = true - InfoRequest.rebuild_solr_index(0) do |ar, options| - ar.find(:all, :conditions => ["not solr_up_to_date"]) - end - OutgoingMessage.rebuild_solr_index(0) do |ar, options| - ar.find(:all, :conditions => ["not (select solr_up_to_date from info_requests where id = outgoing_messages.info_request_id)"]) - end - IncomingMessage.rebuild_solr_index(0) do |ar, options| - ar.find(:all, :conditions => ["not (select solr_up_to_date from info_requests where id = incoming_messages.info_request_id)"]) + + # Index each item separately in a transaction, so solr_up_to_date is right + ids_to_refresh = InfoRequest.find(:all, :conditions => ["not solr_up_to_date"]).map() { |i| i.id } + for id in ids_to_refresh + #puts "updating id " + id.to_s + ActiveRecord::Base.transaction do + info_request = InfoRequest.find(id, :lock =>true) + if not info_request.solr_save + raise "failed to solr_save" + end + for outgoing_message in info_request.outgoing_messages + outgoing_message.solr_save + end + for incoming_message in info_request.incoming_messages + incoming_message.solr_save + end + info_request.solr_up_to_date = true + info_request.save! + end end - InfoRequest.update_all("solr_up_to_date = 't'") + InfoRequest.solr_optimize $do_solr_index = false end def before_update - self.solr_up_to_date = false + # If we're not mid index, then mark we need to index later + if not $do_solr_index + self.solr_up_to_date = false + end true end diff --git a/config/crontab.ugly b/config/crontab.ugly index a580ce8ac..4d8580a51 100644 --- a/config/crontab.ugly +++ b/config/crontab.ugly @@ -4,11 +4,14 @@ # Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org. WWW: http://www.mysociety.org/ # -# $Id: crontab.ugly,v 1.3 2008-02-22 01:58:37 francis Exp $ +# $Id: crontab.ugly,v 1.4 2008-03-06 12:17:21 francis Exp $ PATH=/usr/local/bin:/usr/bin:/bin MAILTO=team@mysociety.org +# Every 10 minutes +*/10 0 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/update-solr-index.lock /data/vhost/!!(*= $vhost *)!!/mysociety/foi/script/update-solr-index || echo "stalled?" + # Once an hour 39 0 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/alert-overdue-requests.lock /data/vhost/!!(*= $vhost *)!!/mysociety/foi/script/alert-overdue-requests || echo "stalled?" diff --git a/db/schema.rb b/db/schema.rb index 2965f7090..49eec890b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -41,6 +41,7 @@ ActiveRecord::Schema.define(:version => 41) do end add_index "info_requests", ["created_at"], :name => "index_info_requests_on_created_at" + add_index "info_requests", ["solr_up_to_date"], :name => "index_info_requests_on_solr_up_to_date" add_index "info_requests", ["title"], :name => "index_info_requests_on_title" add_index "info_requests", ["url_title"], :name => "index_info_requests_on_url_title", :unique => true diff --git a/script/restart-solr b/script/restart-solr new file mode 100755 index 000000000..bf7b9194d --- /dev/null +++ b/script/restart-solr @@ -0,0 +1,12 @@ +#!/bin/bash + +# Stops and restarts the Solr (Lucene) search engine daemon. Redirects it +# to log. + +cd `dirname $0` +cd .. +rake solr:stop +nohup rake solr:start >log/solr.log 2>log/solr.err.log + + + diff --git a/script/update-solr-index b/script/update-solr-index new file mode 100755 index 000000000..33415aa9c --- /dev/null +++ b/script/update-solr-index @@ -0,0 +1,7 @@ +#!/bin/bash + +LOC=`dirname $0` + +$LOC/runner 'InfoRequest.update_solr_index()' + + @@ -1,7 +1,9 @@ Search: -Go to correct id -Add cron job +Deploy solr - do something with config file and access to ports + +Go to correct id for incoming / outgoing messages +Show correct extract for incoming / outgoing messages Add indexing of PDFs and DOCs etc. Pagination - http://www.quarkruby.com/2007/8/12/acts_as_solr-for-search-and-faceting Date ranges and types and stuff |