diff options
| -rw-r--r-- | app/models/exim_log.rb | 64 | ||||
| -rw-r--r-- | app/models/exim_log_done.rb | 14 | ||||
| -rw-r--r-- | app/models/info_request.rb | 3 | ||||
| -rw-r--r-- | app/views/admin_request/show.rhtml | 12 | ||||
| -rw-r--r-- | config/crontab.ugly | 5 | ||||
| -rw-r--r-- | db/migrate/071_add_exim_log.rb | 34 | ||||
| -rw-r--r-- | db/schema.rb | 20 | ||||
| -rw-r--r-- | public/stylesheets/admin.css | 8 | ||||
| -rwxr-xr-x | script/load-exim-logs | 21 | ||||
| -rw-r--r-- | todo.txt | 5 |
10 files changed, 178 insertions, 8 deletions
diff --git a/app/models/exim_log.rb b/app/models/exim_log.rb new file mode 100644 index 000000000..7399428f4 --- /dev/null +++ b/app/models/exim_log.rb @@ -0,0 +1,64 @@ +# models/exim_log.rb: +# We load log file lines for requests in here, for display in the admin interface. +# +# Copyright (c) 2009 UK Citizens Online Democracy. All rights reserved. +# Email: francis@mysociety.org; WWW: http://www.mysociety.org/ +# +# $Id: exim_log.rb,v 1.1 2009-01-27 17:12:31 francis Exp $ + +class EximLog < ActiveRecord::Base + belongs_to :info_request + belongs_to :exim_log_done + + # Load in exim log file from disk, or update if we already have it + # Assumes files are named with date, rather than cyclically. + # Doesn't do anything if file hasn't been modified since it was last loaded. + def EximLog.load_file(file_name) + modified = File::stat(file_name).mtime + raise "EximLog.load_file: file not found " + file_name if modified.nil? + + ActiveRecord::Base.transaction do + # see if we already have it + done = EximLogDone.find_by_filename(file_name) + if !done.nil? + if modified == done.last_stat + # already have that, nothing to do + return + end + ExmLog.delete_all "exim_log_dones.id = " + done.id + end + if !done + done = EximLogDone.new + done.filename = file_name + end + done.last_stat = modified + + # scan the file + f = File.open(file_name, 'r') + order = 0 + for line in f + order = order + 1 + email_domain = MySociety::Config.get("INCOMING_EMAIL_DOMAIN", "localhost") + emails = line.scan(/request-[^\s]+@#{email_domain}/).sort.uniq + for email in emails + info_request = InfoRequest.find_by_incoming_email(email) + if !info_request.nil? + STDERR.puts "adding log for " + info_request.url_title + " from " + file_name + " line " + line + exim_log = EximLog.new + exim_log.info_request = info_request + exim_log.exim_log_done = done + exim_log.line = line + exim_log.order = order + exim_log.save! + end + end + end + + # update done structure so we know when we last read this file + done.save! + end + end +end + + + diff --git a/app/models/exim_log_done.rb b/app/models/exim_log_done.rb new file mode 100644 index 000000000..a69f59194 --- /dev/null +++ b/app/models/exim_log_done.rb @@ -0,0 +1,14 @@ +# models/exim_log_done.rb: +# Stores that a particular exim file has been loaded in, see exim_log.rb +# +# Copyright (c) 2009 UK Citizens Online Democracy. All rights reserved. +# Email: francis@mysociety.org; WWW: http://www.mysociety.org/ +# +# $Id: exim_log_done.rb,v 1.1 2009-01-27 17:12:31 francis Exp $ + +class EximLogDone < ActiveRecord::Base + has_many :exim_logs +end + + + diff --git a/app/models/info_request.rb b/app/models/info_request.rb index 1a2b04410..0b02ae0b5 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -23,7 +23,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: info_request.rb,v 1.161 2009-01-26 12:12:17 francis Exp $ +# $Id: info_request.rb,v 1.162 2009-01-27 17:12:31 francis Exp $ require 'digest/sha1' require File.join(File.dirname(__FILE__),'../../vendor/plugins/acts_as_xapian/lib/acts_as_xapian') @@ -47,6 +47,7 @@ class InfoRequest < ActiveRecord::Base has_many :track_things, :order => 'created_at desc' has_many :comments, :order => 'created_at' has_many :censor_rules, :order => 'created_at desc' + has_many :exim_logs, :order => 'exim_log_done_id' # user described state (also update in info_request_event, admin_request/edit.rhtml) validates_inclusion_of :described_state, :in => [ diff --git a/app/views/admin_request/show.rhtml b/app/views/admin_request/show.rhtml index 8dad3ee91..eec9092b3 100644 --- a/app/views/admin_request/show.rhtml +++ b/app/views/admin_request/show.rhtml @@ -128,6 +128,18 @@ <% end %> </table> +<h2>Exim delivery logs</h2> + +<p>Lines containing the request incoming email address, updated hourly.</p> + +<% for exim_log_done_id, exim_logs in @info_request.exim_logs.group_by(&:exim_log_done_id) %> + <!-- <h3><%=h exim_logs[0].exim_log_done.filename %></h3> --> + <pre><% for exim_log in exim_logs %><%=h exim_log.line%><% end %></pre> +<% end %> +<% if @info_request.exim_logs.size == 0 %> + <p>None (perhaps this is an old or a very new request)</p> +<% end %> + <h2>Censor rules</h2> <%= render :partial => 'admin_censor_rule/show', :locals => { :censor_rules => @info_request.censor_rules, :info_request => @info_request } %> diff --git a/config/crontab.ugly b/config/crontab.ugly index 8c6d978eb..7b561b82c 100644 --- a/config/crontab.ugly +++ b/config/crontab.ugly @@ -4,7 +4,7 @@ # Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org. WWW: http://www.mysociety.org/ # -# $Id: crontab.ugly,v 1.18 2009-01-27 11:14:34 francis Exp $ +# $Id: crontab.ugly,v 1.19 2009-01-27 17:12:32 francis Exp $ PATH=/usr/local/bin:/usr/bin:/bin MAILTO=team@whatdotheyknow.com @@ -18,6 +18,9 @@ MAILTO=team@whatdotheyknow.com 39 * * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/alert-overdue-requests.lock /data/vhost/!!(*= $vhost *)!!/mysociety/foi/script/alert-overdue-requests || echo "stalled?" 09 * * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/alert-comment-on-request.lock /data/vhost/!!(*= $vhost *)!!/mysociety/foi/script/alert-comment-on-request || echo "stalled?" +# Only root can read the exim log files +#30 * * * * root run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/load-exim-logs.lock /data/vhost/!!(*= $vhost *)!!/mysociety/foi/script/load-exim-logs || echo "stalled?" + # Once a day, early morning 23 4 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/delete-old-post-redirects.lock /data/vhost/!!(*= $vhost *)!!/mysociety/foi/script/delete-old-post-redirects || echo "stalled?" 0 8 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/alert-new-response-reminders.lock /data/vhost/!!(*= $vhost *)!!/mysociety/foi/script/alert-new-response-reminders || echo "stalled?" diff --git a/db/migrate/071_add_exim_log.rb b/db/migrate/071_add_exim_log.rb new file mode 100644 index 000000000..d40383698 --- /dev/null +++ b/db/migrate/071_add_exim_log.rb @@ -0,0 +1,34 @@ +class AddEximLog < ActiveRecord::Migration + def self.up + create_table :exim_logs do |t| + t.column :exim_log_done_id, :integer + t.column :info_request_id, :integer + + t.column :order, :integer, :null => false + t.column :line, :text, :null => false + + t.column :created_at, :datetime, :null => false + t.column :updated_at, :datetime, :null => false + end + + create_table :exim_log_dones do |t| + t.column :filename, :text, :null => false, :unique => true + t.column :last_stat, :datetime, :null => false + + t.column :created_at, :datetime, :null => false + t.column :updated_at, :datetime, :null => false + end + add_index :exim_log_dones, :last_stat + + if ActiveRecord::Base.connection.adapter_name == "PostgreSQL" + execute "ALTER TABLE exim_logs ADD CONSTRAINT fk_exim_log_info_request FOREIGN KEY (info_request_id) REFERENCES info_requests(id)" + execute "ALTER TABLE exim_logs ADD CONSTRAINT fk_exim_log_done FOREIGN KEY (exim_log_done_id) REFERENCES exim_log_dones(id)" + end + end + + def self.down + drop_table :exim_logs + drop_table :exim_log_dones + end +end + diff --git a/db/schema.rb b/db/schema.rb index e0b8d6217..f9d59bc88 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -9,7 +9,7 @@ # # It's strongly recommended to check this file into your version control system. -ActiveRecord::Schema.define(:version => 70) do +ActiveRecord::Schema.define(:version => 71) do create_table "acts_as_xapian_jobs", :force => true do |t| t.string "model", :null => false @@ -41,6 +41,24 @@ ActiveRecord::Schema.define(:version => 70) do t.datetime "updated_at", :null => false end + create_table "exim_log_dones", :force => true do |t| + t.text "filename", :null => false + t.datetime "last_stat", :null => false + t.datetime "created_at", :null => false + t.datetime "updated_at", :null => false + end + + add_index "exim_log_dones", ["last_stat"], :name => "index_exim_log_dones_on_last_stat" + + create_table "exim_logs", :force => true do |t| + t.integer "exim_log_done_id" + t.integer "info_request_id" + t.integer "order", :null => false + t.text "line", :null => false + t.datetime "created_at", :null => false + t.datetime "updated_at", :null => false + end + create_table "incoming_messages", :force => true do |t| t.integer "info_request_id", :null => false t.datetime "created_at", :null => false diff --git a/public/stylesheets/admin.css b/public/stylesheets/admin.css index d35851b3b..114643fdb 100644 --- a/public/stylesheets/admin.css +++ b/public/stylesheets/admin.css @@ -1,7 +1,13 @@ /* FOIFA admin CSS */ +h2 { + margin-top: 0.2em; + margin-bottom: 0.5em; +} + table { border-collapse: collapse; + margin-bottom: 1em; } td, th { @@ -41,7 +47,7 @@ form { float: right; width: 25%; } -#tag_help, h2 { +#tag_help { margin-top: 0; } diff --git a/script/load-exim-logs b/script/load-exim-logs new file mode 100755 index 000000000..1802d6425 --- /dev/null +++ b/script/load-exim-logs @@ -0,0 +1,21 @@ +#!/bin/bash + +LOC=`dirname $0` + +# Specific file if specified +if [ x$1 != x ] +then + $LOC/runner 'EximLog.load_file("'$1'")' + exit +fi + +# Load in last week's worth of logs (if they've been modified) +LATEST=$( ls /var/log/exim4/mainlog-* 2>/dev/null | sort | tail -7 ) +for X in $LATEST +do + echo "doing $X" + $LOC/runner 'EximLog.load_file("'$X'")' +done + + + @@ -73,7 +73,7 @@ Search for text "internal review" in followups and add warning if they aren't using the internal review mode. CSS / design things - - Icon for internal_review + - Icon for internal_review, user_withdrawn - CSS error on "all councils" page on some browsers https://bugzilla.mozilla.org/show_bug.cgi?id=424194 - Spacing on error boxes round form elements. Matthew says: @@ -83,7 +83,6 @@ CSS / design things expect. But I had a look at the code and haven't got the slightest clue how you'd do that, sorry, given it appears new.rhtml is printing the <p> but some magic Ruby thing is printing the error span. - - icons for "Things to do with this request" ? - Improve CSS on IE7 for large images in docs http://www.whatdotheyknow.com/request/3289/response/7810/attach/html/3/20081023ReplyLetter.pdf.html @@ -105,8 +104,6 @@ Would be nice if you try and send or resend (from admin interface, or by sending followup as user) and original address is set to not_apply or "" that it deals with it better :) -Include log files from exim in admin interface for each request - When doing search, people often just want it to show the whole page. Perhaps all listing should just link to top of page, rather than # links for outgoing incoming, or perhaps just some of them. |