diff options
| -rw-r--r-- | app/models/censor_rule.rb | 4 | ||||
| -rw-r--r-- | app/models/comment.rb | 4 | ||||
| -rw-r--r-- | app/models/contact_validator.rb | 4 | ||||
| -rw-r--r-- | app/models/exim_log.rb | 53 | ||||
| -rw-r--r-- | app/models/exim_log_done.rb | 14 | ||||
| -rw-r--r-- | app/models/incoming_message.rb | 4 | ||||
| -rw-r--r-- | app/models/info_request.rb | 4 | ||||
| -rw-r--r-- | app/models/info_request_event.rb | 4 | ||||
| -rw-r--r-- | app/models/outgoing_message.rb | 4 | ||||
| -rw-r--r-- | app/models/post_redirect.rb | 4 | ||||
| -rw-r--r-- | app/models/public_body.rb | 4 | ||||
| -rw-r--r-- | app/models/public_body_tag.rb | 4 | ||||
| -rw-r--r-- | app/models/raw_email.rb | 4 | ||||
| -rw-r--r-- | app/models/track_thing.rb | 4 | ||||
| -rw-r--r-- | app/models/track_things_sent_email.rb | 4 | ||||
| -rw-r--r-- | app/models/user.rb | 4 | ||||
| -rw-r--r-- | app/models/user_info_request_sent_alert.rb | 4 | ||||
| -rw-r--r-- | config/crontab.ugly | 3 | ||||
| -rwxr-xr-x | script/check-recent-requests-sent | 8 | ||||
| -rw-r--r-- | todo.txt | 5 |
20 files changed, 110 insertions, 33 deletions
diff --git a/app/models/censor_rule.rb b/app/models/censor_rule.rb index 28785c5e0..65b9239f7 100644 --- a/app/models/censor_rule.rb +++ b/app/models/censor_rule.rb @@ -1,5 +1,5 @@ # == Schema Information -# Schema version: 70 +# Schema version: 71 # # Table name: censor_rules # @@ -21,7 +21,7 @@ # Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: censor_rule.rb,v 1.5 2009-01-26 12:12:17 francis Exp $ +# $Id: censor_rule.rb,v 1.6 2009-02-09 09:51:52 francis Exp $ class CensorRule < ActiveRecord::Base belongs_to :info_request diff --git a/app/models/comment.rb b/app/models/comment.rb index 8a905cf44..125b875ac 100644 --- a/app/models/comment.rb +++ b/app/models/comment.rb @@ -1,5 +1,5 @@ # == Schema Information -# Schema version: 70 +# Schema version: 71 # # Table name: comments # @@ -19,7 +19,7 @@ # Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: comment.rb,v 1.10 2009-01-26 12:12:17 francis Exp $ +# $Id: comment.rb,v 1.11 2009-02-09 09:51:52 francis Exp $ class Comment < ActiveRecord::Base strip_attributes! diff --git a/app/models/contact_validator.rb b/app/models/contact_validator.rb index 25c233462..e54766358 100644 --- a/app/models/contact_validator.rb +++ b/app/models/contact_validator.rb @@ -1,5 +1,5 @@ # == Schema Information -# Schema version: 70 +# Schema version: 71 # # Table name: contact_validators # @@ -15,7 +15,7 @@ # Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: contact_validator.rb,v 1.25 2009-01-26 12:12:17 francis Exp $ +# $Id: contact_validator.rb,v 1.26 2009-02-09 09:51:52 francis Exp $ class ContactValidator < ActiveRecord::BaseWithoutTable strip_attributes! diff --git a/app/models/exim_log.rb b/app/models/exim_log.rb index f7e23ed1c..864d5bf89 100644 --- a/app/models/exim_log.rb +++ b/app/models/exim_log.rb @@ -1,10 +1,24 @@ +# == Schema Information +# Schema version: 71 +# +# Table name: exim_logs +# +# id :integer not null, primary key +# exim_log_done_id :integer +# info_request_id :integer +# order :integer not null +# line :text not null +# created_at :datetime not null +# updated_at :datetime not null +# + # models/exim_log.rb: # We load log file lines for requests in here, for display in the admin interface. # # Copyright (c) 2009 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: exim_log.rb,v 1.3 2009-01-27 18:01:41 francis Exp $ +# $Id: exim_log.rb,v 1.4 2009-02-09 09:51:53 francis Exp $ class EximLog < ActiveRecord::Base belongs_to :info_request @@ -58,6 +72,43 @@ class EximLog < ActiveRecord::Base done.save! end end + + # Check that the last day of requests has been sent in Exim and we got the + # lines. Writes any errors to STDERR. This check is really mainly to + # check the envelope from is the request address, as Ruby is quite + # flaky with regard to that, and it is important for anti-spam reasons. + def EximLog.check_recent_requests_have_been_sent + # Get all requests sent for a period of 24 hours, ending a day ago. + # (the gap is because we load exim log lines via cron at best an hour + # after they are made) + irs = InfoRequest.find(:all, :conditions => [ "created_at < ? and created_at > ?", Time.now() - 1.day, Time.now() - 2.days ] ) + + # Go through each request and check it + for ir in irs + # Look for line showing request was sent + found = false + for exim_log in ir.exim_logs + test_outgoing = " <= " + ir.incoming_email + " " + if exim_log.line.include?(test_outgoing) + # Check the from value is the same (it always will be, but may as well + # be sure we are parsing the exim line right) + envelope_from = " from <" + ir.incoming_email + "> " + if !exim_log.line.include?(envelope_from) + raise "unexpected parsing of exim line" + end + + STDERR.puts exim_log.line # debugging + found = true + end + end + if !found + # It's very important the envelope from is set for avoiding spam filter reasons - this + # effectively acts as a check for that. + STDERR.puts("failed to find request sending Exim line for request id " + ir.id.to_s + " " + ir.url_title + " (check envelope from is being set to request address in Ruby, and load-exim-logs crontab is working)") # *** don't comment out this STDERR line, it is the point of the function! + end + end + end + end diff --git a/app/models/exim_log_done.rb b/app/models/exim_log_done.rb index a69f59194..6dced74f0 100644 --- a/app/models/exim_log_done.rb +++ b/app/models/exim_log_done.rb @@ -1,10 +1,22 @@ +# == Schema Information +# Schema version: 71 +# +# Table name: exim_log_dones +# +# id :integer not null, primary key +# filename :text not null +# last_stat :datetime not null +# created_at :datetime not null +# updated_at :datetime not null +# + # models/exim_log_done.rb: # Stores that a particular exim file has been loaded in, see exim_log.rb # # Copyright (c) 2009 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: exim_log_done.rb,v 1.1 2009-01-27 17:12:31 francis Exp $ +# $Id: exim_log_done.rb,v 1.2 2009-02-09 09:51:53 francis Exp $ class EximLogDone < ActiveRecord::Base has_many :exim_logs diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb index 4a0c1ab5b..3b9922f79 100644 --- a/app/models/incoming_message.rb +++ b/app/models/incoming_message.rb @@ -1,5 +1,5 @@ # == Schema Information -# Schema version: 70 +# Schema version: 71 # # Table name: incoming_messages # @@ -19,7 +19,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: incoming_message.rb,v 1.187 2009-02-03 03:46:46 francis Exp $ +# $Id: incoming_message.rb,v 1.188 2009-02-09 09:51:53 francis Exp $ # TODO # Move some of the (e.g. quoting) functions here into rblib, as they feel diff --git a/app/models/info_request.rb b/app/models/info_request.rb index ee811d9c0..c7ff72208 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -1,5 +1,5 @@ # == Schema Information -# Schema version: 70 +# Schema version: 71 # # Table name: info_requests # @@ -23,7 +23,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: info_request.rb,v 1.163 2009-02-03 01:45:08 francis Exp $ +# $Id: info_request.rb,v 1.164 2009-02-09 09:51:53 francis Exp $ require 'digest/sha1' require File.join(File.dirname(__FILE__),'../../vendor/plugins/acts_as_xapian/lib/acts_as_xapian') diff --git a/app/models/info_request_event.rb b/app/models/info_request_event.rb index 855af4fc3..f4a3849f3 100644 --- a/app/models/info_request_event.rb +++ b/app/models/info_request_event.rb @@ -1,5 +1,5 @@ # == Schema Information -# Schema version: 70 +# Schema version: 71 # # Table name: info_request_events # @@ -21,7 +21,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: info_request_event.rb,v 1.71 2009-01-26 12:12:17 francis Exp $ +# $Id: info_request_event.rb,v 1.72 2009-02-09 09:51:53 francis Exp $ class InfoRequestEvent < ActiveRecord::Base belongs_to :info_request diff --git a/app/models/outgoing_message.rb b/app/models/outgoing_message.rb index f23dfb1dd..323c11cc0 100644 --- a/app/models/outgoing_message.rb +++ b/app/models/outgoing_message.rb @@ -1,5 +1,5 @@ # == Schema Information -# Schema version: 70 +# Schema version: 71 # # Table name: outgoing_messages # @@ -22,7 +22,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: outgoing_message.rb,v 1.77 2009-01-26 12:12:17 francis Exp $ +# $Id: outgoing_message.rb,v 1.78 2009-02-09 09:51:53 francis Exp $ class OutgoingMessage < ActiveRecord::Base strip_attributes! diff --git a/app/models/post_redirect.rb b/app/models/post_redirect.rb index 2f7dce783..4cdee8415 100644 --- a/app/models/post_redirect.rb +++ b/app/models/post_redirect.rb @@ -1,5 +1,5 @@ # == Schema Information -# Schema version: 70 +# Schema version: 71 # # Table name: post_redirects # @@ -26,7 +26,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: post_redirect.rb,v 1.43 2009-01-26 12:12:17 francis Exp $ +# $Id: post_redirect.rb,v 1.44 2009-02-09 09:51:53 francis Exp $ require 'openssl' # for random bytes function diff --git a/app/models/public_body.rb b/app/models/public_body.rb index 58b180db4..d0e49f2d6 100644 --- a/app/models/public_body.rb +++ b/app/models/public_body.rb @@ -1,5 +1,5 @@ # == Schema Information -# Schema version: 70 +# Schema version: 71 # # Table name: public_bodies # @@ -24,7 +24,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: public_body.rb,v 1.128 2009-01-29 11:48:46 francis Exp $ +# $Id: public_body.rb,v 1.129 2009-02-09 09:51:53 francis Exp $ require 'csv' require 'set' diff --git a/app/models/public_body_tag.rb b/app/models/public_body_tag.rb index 5687a239d..62410da60 100644 --- a/app/models/public_body_tag.rb +++ b/app/models/public_body_tag.rb @@ -1,5 +1,5 @@ # == Schema Information -# Schema version: 70 +# Schema version: 71 # # Table name: public_body_tags # @@ -15,7 +15,7 @@ # Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: public_body_tag.rb,v 1.22 2009-01-26 12:12:17 francis Exp $ +# $Id: public_body_tag.rb,v 1.23 2009-02-09 09:51:53 francis Exp $ class PublicBodyTag < ActiveRecord::Base strip_attributes! diff --git a/app/models/raw_email.rb b/app/models/raw_email.rb index 9eb40f395..867fd5f9e 100644 --- a/app/models/raw_email.rb +++ b/app/models/raw_email.rb @@ -1,5 +1,5 @@ # == Schema Information -# Schema version: 70 +# Schema version: 71 # # Table name: raw_emails # @@ -13,7 +13,7 @@ # Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: raw_email.rb,v 1.5 2009-01-26 12:12:17 francis Exp $ +# $Id: raw_email.rb,v 1.6 2009-02-09 09:51:53 francis Exp $ class RawEmail < ActiveRecord::Base # deliberately don't strip_attributes, so keeps raw email properly diff --git a/app/models/track_thing.rb b/app/models/track_thing.rb index 7b1b41ade..8d605bd6a 100644 --- a/app/models/track_thing.rb +++ b/app/models/track_thing.rb @@ -1,5 +1,5 @@ # == Schema Information -# Schema version: 70 +# Schema version: 71 # # Table name: track_things # @@ -21,7 +21,7 @@ # Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: track_thing.rb,v 1.46 2009-01-26 12:12:17 francis Exp $ +# $Id: track_thing.rb,v 1.47 2009-02-09 09:51:53 francis Exp $ class TrackThing < ActiveRecord::Base belongs_to :tracking_user, :class_name => 'User' diff --git a/app/models/track_things_sent_email.rb b/app/models/track_things_sent_email.rb index 5719304f5..af5c51b48 100644 --- a/app/models/track_things_sent_email.rb +++ b/app/models/track_things_sent_email.rb @@ -1,5 +1,5 @@ # == Schema Information -# Schema version: 70 +# Schema version: 71 # # Table name: track_things_sent_emails # @@ -18,7 +18,7 @@ # Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: track_things_sent_email.rb,v 1.15 2009-01-26 12:12:17 francis Exp $ +# $Id: track_things_sent_email.rb,v 1.16 2009-02-09 09:51:53 francis Exp $ class TrackThingsSentEmail < ActiveRecord::Base belongs_to :info_request_event diff --git a/app/models/user.rb b/app/models/user.rb index 2232d71ec..ed7104450 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -1,5 +1,5 @@ # == Schema Information -# Schema version: 70 +# Schema version: 71 # # Table name: users # @@ -22,7 +22,7 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: user.rb,v 1.79 2009-01-26 12:12:17 francis Exp $ +# $Id: user.rb,v 1.80 2009-02-09 09:51:53 francis Exp $ require 'digest/sha1' diff --git a/app/models/user_info_request_sent_alert.rb b/app/models/user_info_request_sent_alert.rb index b3b935852..b5a6d09db 100644 --- a/app/models/user_info_request_sent_alert.rb +++ b/app/models/user_info_request_sent_alert.rb @@ -1,5 +1,5 @@ # == Schema Information -# Schema version: 70 +# Schema version: 71 # # Table name: user_info_request_sent_alerts # @@ -17,7 +17,7 @@ # Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org; WWW: http://www.mysociety.org/ # -# $Id: user_info_request_sent_alert.rb,v 1.26 2009-01-26 12:12:17 francis Exp $ +# $Id: user_info_request_sent_alert.rb,v 1.27 2009-02-09 09:51:53 francis Exp $ class UserInfoRequestSentAlert < ActiveRecord::Base belongs_to :user diff --git a/config/crontab.ugly b/config/crontab.ugly index 537104ee5..7d68a2b07 100644 --- a/config/crontab.ugly +++ b/config/crontab.ugly @@ -4,7 +4,7 @@ # Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved. # Email: francis@mysociety.org. WWW: http://www.mysociety.org/ # -# $Id: crontab.ugly,v 1.20 2009-01-27 21:09:07 francis Exp $ +# $Id: crontab.ugly,v 1.21 2009-02-09 09:51:53 francis Exp $ PATH=/usr/local/bin:/usr/bin:/bin MAILTO=team@whatdotheyknow.com @@ -25,6 +25,7 @@ MAILTO=team@whatdotheyknow.com 23 4 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/delete-old-post-redirects.lock /data/vhost/!!(*= $vhost *)!!/mysociety/foi/script/delete-old-post-redirects || echo "stalled?" 0 8 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/alert-new-response-reminders.lock /data/vhost/!!(*= $vhost *)!!/mysociety/foi/script/alert-new-response-reminders || echo "stalled?" 0 8 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/alert-not-clarified-request.lock /data/vhost/!!(*= $vhost *)!!/mysociety/foi/script/alert-not-clarified-request || echo "stalled?" +2 4 * * * !!(*= $user *)!! run-with-lockfile -n /data/vhost/!!(*= $vhost *)!!/check-recent-requests-sent.lock /data/vhost/!!(*= $vhost *)!!/mysociety/foi/script/check-recent-requests-sent || echo "stalled?" # Once a day on all servers 43 2 * * * !!(*= $user *)!! /data/vhost/!!(*= $vhost *)!!/mysociety/foi/script/request-creation-graph diff --git a/script/check-recent-requests-sent b/script/check-recent-requests-sent new file mode 100755 index 000000000..351ffaaed --- /dev/null +++ b/script/check-recent-requests-sent @@ -0,0 +1,8 @@ +#!/bin/bash + +LOC=`dirname $0` + +$LOC/runner 'EximLog.check_recent_requests_have_been_sent' + + + @@ -1,3 +1,5 @@ +check-recent-requests-sent works + Next (things that will reduce admin time mainly) ==== @@ -95,6 +97,9 @@ CSS / design things - Get Atom feed of search results to include stylesheet for highlighting words in yellow somehow +What happens if you view an address with the captcha, and it is not_apply or +blank? + Would be nice if you try and send or resend (from admin interface, or by sending followup as user) and original address is set to not_apply or "" that it deals with it better :) |