diff options
Diffstat (limited to 'script')
| -rwxr-xr-x | script/alert-comment-on-request | 4 | ||||
| -rwxr-xr-x | script/alert-new-response-reminders | 4 | ||||
| -rwxr-xr-x | script/alert-not-clarified-request | 4 | ||||
| -rwxr-xr-x | script/alert-overdue-requests | 6 | ||||
| -rwxr-xr-x | script/alert-tracks-daemon | 4 | ||||
| -rw-r--r-- | script/cache-incoming-emails | 4 | ||||
| -rwxr-xr-x | script/check-recent-requests-sent | 4 | ||||
| -rwxr-xr-x | script/clear-caches | 4 | ||||
| -rwxr-xr-x | script/delete-old-things | 9 | ||||
| -rwxr-xr-x | script/handle-mail-replies | 182 | ||||
| -rwxr-xr-x | script/handle-mail-replies.rb | 180 | ||||
| -rwxr-xr-x | script/load-exim-logs | 13 | ||||
| -rwxr-xr-x | script/rails-post-deploy | 19 | ||||
| -rwxr-xr-x | script/runner | 2 | ||||
| -rwxr-xr-x | script/stop-new-responses-on-old-requests | 4 |
15 files changed, 221 insertions, 222 deletions
diff --git a/script/alert-comment-on-request b/script/alert-comment-on-request index bfca2663f..74782aa13 100755 --- a/script/alert-comment-on-request +++ b/script/alert-comment-on-request @@ -1,7 +1,7 @@ #!/bin/bash -LOC=`dirname $0` +cd "`dirname "$0"`" -"$LOC/runner" 'RequestMailer.alert_comment_on_request' +bundle exec ./runner 'RequestMailer.alert_comment_on_request' diff --git a/script/alert-new-response-reminders b/script/alert-new-response-reminders index d52381750..59fa76df1 100755 --- a/script/alert-new-response-reminders +++ b/script/alert-new-response-reminders @@ -1,7 +1,7 @@ #!/bin/bash -LOC=`dirname $0` +cd "`dirname "$0"`" -"$LOC/runner" 'RequestMailer.alert_new_response_reminders' +bundle exec ./runner 'RequestMailer.alert_new_response_reminders' diff --git a/script/alert-not-clarified-request b/script/alert-not-clarified-request index bfdbeaff5..a38fbc70e 100755 --- a/script/alert-not-clarified-request +++ b/script/alert-not-clarified-request @@ -1,7 +1,7 @@ #!/bin/bash -LOC=`dirname $0` +cd "`dirname "$0"`" -"$LOC/runner" 'RequestMailer.alert_not_clarified_request' +bundle exec ./runner 'RequestMailer.alert_not_clarified_request' diff --git a/script/alert-overdue-requests b/script/alert-overdue-requests index f6549e5b6..46450f1f2 100755 --- a/script/alert-overdue-requests +++ b/script/alert-overdue-requests @@ -1,7 +1,5 @@ #!/bin/bash -LOC=`dirname $0` - -"$LOC/runner" 'RequestMailer.alert_overdue_requests()' - +cd "`dirname "$0"`" +bundle exec ./runner 'RequestMailer.alert_overdue_requests()' diff --git a/script/alert-tracks-daemon b/script/alert-tracks-daemon index 9064eaa05..43712ae68 100755 --- a/script/alert-tracks-daemon +++ b/script/alert-tracks-daemon @@ -1,6 +1,6 @@ #!/bin/sh -LOC=`dirname $0` +cd "`dirname "$0"`" -"$LOC/runner" --daemon TrackMailer.alert_tracks_loop +bundle exec ./runner --daemon TrackMailer.alert_tracks_loop diff --git a/script/cache-incoming-emails b/script/cache-incoming-emails index a84a713d6..0b3069bd7 100644 --- a/script/cache-incoming-emails +++ b/script/cache-incoming-emails @@ -4,6 +4,6 @@ # Will take a while to run! Can use after clear-caches to refresh the database # level caches if you like. -LOC=`dirname $0` +cd "`dirname "$0"`" -"$LOC/runner" 'IncomingMessage.find_each() { |im| print "info request " + im.info_request.id.to_s + ", incoming message " + im.id.to_s + ": " + im.extract_attachments!.count.to_s + " attachments extracted to " + im.foi_attachments[0].directory + "; main body folded: " + im.get_main_body_text_folded.size.to_s + " attachment clipped:" + im.get_attachment_text_clipped.size.to_s + "\n" }' +bundle exec ./runner 'IncomingMessage.find_each() { |im| print "info request " + im.info_request.id.to_s + ", incoming message " + im.id.to_s + ": " + im.extract_attachments!.count.to_s + " attachments extracted to " + im.foi_attachments[0].directory + "; main body folded: " + im.get_main_body_text_folded.size.to_s + " attachment clipped:" + im.get_attachment_text_clipped.size.to_s + "\n" }' diff --git a/script/check-recent-requests-sent b/script/check-recent-requests-sent index df5baf10a..588c6b73a 100755 --- a/script/check-recent-requests-sent +++ b/script/check-recent-requests-sent @@ -1,8 +1,8 @@ #!/bin/bash -LOC=`dirname $0` +cd "`dirname "$0"`" -"$LOC/runner" 'EximLog.check_recent_requests_have_been_sent' +bundle exec ./runner 'EximLog.check_recent_requests_have_been_sent' diff --git a/script/clear-caches b/script/clear-caches index be1d3d017..43fec2811 100755 --- a/script/clear-caches +++ b/script/clear-caches @@ -2,9 +2,9 @@ # Clear the cache of attachment and body text. -LOC=`dirname $0` +cd "`dirname "$0"`" -"$LOC/runner" "ActiveRecord::Base.connection.execute(\"update incoming_messages set cached_attachment_text_clipped = null, cached_main_body_text_unfolded = null, cached_main_body_text_folded = null, sent_at = null, subject = null, mail_from = null, mail_from_domain = null, valid_to_reply_to = null, last_parsed = null\")" +bundle exec ./runner "ActiveRecord::Base.connection.execute(\"update incoming_messages set cached_attachment_text_clipped = null, cached_main_body_text_unfolded = null, cached_main_body_text_folded = null, sent_at = null, subject = null, mail_from = null, mail_from_domain = null, valid_to_reply_to = null, last_parsed = null\")" # Remove page cache (do it in two stages so live site gets cache cleared faster) rm -fr $LOC/../old-cache diff --git a/script/delete-old-things b/script/delete-old-things index 1d8cf645b..063d85e1f 100755 --- a/script/delete-old-things +++ b/script/delete-old-things @@ -1,9 +1,6 @@ #!/bin/bash -LOC=`dirname $0` - -"$LOC/runner" 'PostRedirect.delete_old_post_redirects()' - -"$LOC/runner" 'TrackThingsSentEmail.delete_old_track_things_sent_email()' - +cd "`dirname "$0"`" +bundle exec ./runner 'PostRedirect.delete_old_post_redirects()' +bundle exec ./runner 'TrackThingsSentEmail.delete_old_track_things_sent_email()' diff --git a/script/handle-mail-replies b/script/handle-mail-replies index 7590f5848..ad4b3719e 100755 --- a/script/handle-mail-replies +++ b/script/handle-mail-replies @@ -1,180 +1,4 @@ -#!/usr/bin/env ruby -# -*- coding: utf-8 -*- +#!/bin/bash -# Handle email responses sent to us. -# -# This script is invoked as a pipe command, i.e. with the raw email message on stdin. -# - If a message is identified as a permanent bounce, the user is marked as having a -# bounced address, and will not be sent any more messages. -# - If a message is identified as an out-of-office autoreply, it is discarded. -# - Any other messages are forwarded to config.get("FORWARD_NONBOUNCE_RESPONSES_TO") - - -# We want to avoid loading rails unless we need it, so we start by just loading the -# config file ourselves. -$alaveteli_dir = File.join(File.dirname(__FILE__), '..') -$:.push(File.join($alaveteli_dir, "commonlib", "rblib")) -load "config.rb" -MySociety::Config.set_file(File.join($alaveteli_dir, 'config', 'general'), true) -MySociety::Config.load_default - -require 'action_mailer' - -def main(in_test_mode) - Dir.chdir($alaveteli_dir) do - raw_message = $stdin.read - begin - message = TMail::Mail.parse(raw_message) - rescue - # Error parsing message. Just pass it on, to be on the safe side. - forward_on(raw_message) unless in_test_mode - return 0 - end - - pfas = permanently_failed_addresses(message) - if !pfas.empty? - if in_test_mode - puts pfas - else - pfas.each do |pfa| - record_bounce(pfa, raw_message) - end - end - return 1 - end - - # If we are still here, there are no permanent failures, - # so if the message is a multipart/report then it must be - # reporting a temporary failure. In this case we discard it - if message.content_type == "multipart/report" - return 1 - end - - # Another style of temporary failure message - subject = message.header_string("Subject") - if message.content_type == "multipart/mixed" && subject == "Delivery Status Notification (Delay)" - return 1 - end - - # Discard out-of-office messages - if is_oof?(message) - return 2 # Use a different return code, to distinguish OOFs from bounces - end - - # Otherwise forward the message on - forward_on(raw_message) unless in_test_mode - return 0 - end -end - -def permanently_failed_addresses(message) - if message.header_string("Return-Path") == "<>" - # Some sort of auto-response - - # Check for Exim’s X-Failed-Recipients header - failed_recipients = message.header_string("X-Failed-Recipients") - if !failed_recipients.nil? - # The X-Failed-Recipients header contains the email address that failed - # Check for the words "This is a permanent error." in the body, to indicate - # a permanent failure - if message.body =~ /This is a permanent error./ - return failed_recipients.split(/,\s*/) - end - end - - # Next, look for multipart/report - if message.content_type == "multipart/report" - permanently_failed_recipients = [] - message.parts.each do |part| - if part.content_type == "message/delivery-status" - sections = part.body.split(/\r?\n\r?\n/) - # The first section is a generic header; subsequent sections - # represent a particular recipient. Since we - sections[1..-1].each do |section| - if section !~ /^Status: (\d)/ || $1 != '5' - # Either we couldn’t find the Status field, or it was a transient failure - break - end - if section =~ /^Final-Recipient: rfc822;(.+)/ - permanently_failed_recipients.push($1) - end - end - end - end - if !permanently_failed_recipients.empty? - return permanently_failed_recipients - end - end - end - - subject = message.header_string("Subject") - # Then look for the style we’ve seen in WebShield bounces - # (These do not have a return path of <> in the cases I have seen.) - if subject == "Returned Mail: Error During Delivery" - if message.body =~ /^\s*---- Failed Recipients ----\s*((?:<[^>]+>\n)+)/ - return $1.scan(/<([^>]+)>/).flatten - end - end - - return [] -end - -def is_oof?(message) - # Check for out-of-office - - if message.header_string("X-POST-MessageClass") == "9; Autoresponder" - return true - end - - subject = message.header_string("Subject").downcase - if message.header_string("Return-Path") == "<>" - if subject.start_with? "out of office: " - return true - end - if subject.start_with? "automatic reply: " - return true - end - end - - if message.header_string("Auto-Submitted") == "auto-generated" - if subject =~ /out of( the)? office/ - return true - end - end - - if subject.start_with? "out of office autoreply:" - return true - end - if subject == "out of office" - return true - end - if subject == "out of office reply" - return true - end - if subject.end_with? "is out of the office" - return true - end - return false -end - -def forward_on(raw_message) - forward_non_bounces_to = MySociety::Config.get("FORWARD_NONBOUNCE_RESPONSES_TO", "user-support@localhost") - IO.popen("/usr/sbin/sendmail -i #{forward_non_bounces_to}", "w") do |f| - f.write(raw_message); - f.close; - end -end - -def load_rails - require File.join('config', 'boot') - require Rails.root + '/config/environment' -end - -def record_bounce(email_address, bounce_message) - load_rails - User.record_bounce_for_email(email_address, bounce_message) -end - -in_test_mode = (ARGV[0] == "--test") -status = main(in_test_mode) -exit(status) if in_test_mode +cd "`dirname "$0"`" +exec bundle exec ./handle-mail-replies.rb diff --git a/script/handle-mail-replies.rb b/script/handle-mail-replies.rb new file mode 100755 index 000000000..cc15fe35a --- /dev/null +++ b/script/handle-mail-replies.rb @@ -0,0 +1,180 @@ +#!/usr/bin/env ruby +# -*- coding: utf-8 -*- + +# Handle email responses sent to us. +# +# This script is invoked as a pipe command, i.e. with the raw email message on stdin. +# - If a message is identified as a permanent bounce, the user is marked as having a +# bounced address, and will not be sent any more messages. +# - If a message is identified as an out-of-office autoreply, it is discarded. +# - Any other messages are forwarded to config.get("FORWARD_NONBOUNCE_RESPONSES_TO") + + +# We want to avoid loading rails unless we need it, so we start by just loading the +# config file ourselves. +$alaveteli_dir = File.expand_path(File.join(File.dirname(__FILE__), '..')) +$:.push(File.join($alaveteli_dir, "commonlib", "rblib")) +load "config.rb" +MySociety::Config.set_file(File.join($alaveteli_dir, 'config', 'general'), true) +MySociety::Config.load_default + +require 'action_mailer' + +def main(in_test_mode) + Dir.chdir($alaveteli_dir) do + raw_message = $stdin.read + begin + message = TMail::Mail.parse(raw_message) + rescue + # Error parsing message. Just pass it on, to be on the safe side. + forward_on(raw_message) unless in_test_mode + return 0 + end + + pfas = permanently_failed_addresses(message) + if !pfas.empty? + if in_test_mode + puts pfas + else + pfas.each do |pfa| + record_bounce(pfa, raw_message) + end + end + return 1 + end + + # If we are still here, there are no permanent failures, + # so if the message is a multipart/report then it must be + # reporting a temporary failure. In this case we discard it + if message.content_type == "multipart/report" + return 1 + end + + # Another style of temporary failure message + subject = message.header_string("Subject") + if message.content_type == "multipart/mixed" && subject == "Delivery Status Notification (Delay)" + return 1 + end + + # Discard out-of-office messages + if is_oof?(message) + return 2 # Use a different return code, to distinguish OOFs from bounces + end + + # Otherwise forward the message on + forward_on(raw_message) unless in_test_mode + return 0 + end +end + +def permanently_failed_addresses(message) + if message.header_string("Return-Path") == "<>" + # Some sort of auto-response + + # Check for Exim’s X-Failed-Recipients header + failed_recipients = message.header_string("X-Failed-Recipients") + if !failed_recipients.nil? + # The X-Failed-Recipients header contains the email address that failed + # Check for the words "This is a permanent error." in the body, to indicate + # a permanent failure + if message.body =~ /This is a permanent error./ + return failed_recipients.split(/,\s*/) + end + end + + # Next, look for multipart/report + if message.content_type == "multipart/report" + permanently_failed_recipients = [] + message.parts.each do |part| + if part.content_type == "message/delivery-status" + sections = part.body.split(/\r?\n\r?\n/) + # The first section is a generic header; subsequent sections + # represent a particular recipient. Since we + sections[1..-1].each do |section| + if section !~ /^Status: (\d)/ || $1 != '5' + # Either we couldn’t find the Status field, or it was a transient failure + break + end + if section =~ /^Final-Recipient: rfc822;(.+)/ + permanently_failed_recipients.push($1) + end + end + end + end + if !permanently_failed_recipients.empty? + return permanently_failed_recipients + end + end + end + + subject = message.header_string("Subject") + # Then look for the style we’ve seen in WebShield bounces + # (These do not have a return path of <> in the cases I have seen.) + if subject == "Returned Mail: Error During Delivery" + if message.body =~ /^\s*---- Failed Recipients ----\s*((?:<[^>]+>\n)+)/ + return $1.scan(/<([^>]+)>/).flatten + end + end + + return [] +end + +def is_oof?(message) + # Check for out-of-office + + if message.header_string("X-POST-MessageClass") == "9; Autoresponder" + return true + end + + subject = message.header_string("Subject").downcase + if message.header_string("Return-Path") == "<>" + if subject.start_with? "out of office: " + return true + end + if subject.start_with? "automatic reply: " + return true + end + end + + if message.header_string("Auto-Submitted") == "auto-generated" + if subject =~ /out of( the)? office/ + return true + end + end + + if subject.start_with? "out of office autoreply:" + return true + end + if subject == "out of office" + return true + end + if subject == "out of office reply" + return true + end + if subject.end_with? "is out of the office" + return true + end + return false +end + +def forward_on(raw_message) + forward_non_bounces_to = MySociety::Config.get("FORWARD_NONBOUNCE_RESPONSES_TO", "user-support@localhost") + IO.popen("/usr/sbin/sendmail -i #{forward_non_bounces_to}", "w") do |f| + f.write(raw_message); + f.close; + end +end + +def load_rails + require File.join('config', 'boot') + require Rails.root + '/config/environment' +end + +def record_bounce(email_address, bounce_message) + load_rails + User.record_bounce_for_email(email_address, bounce_message) +end + +in_test_mode = (ARGV[0] == "--test") +status = main(in_test_mode) +exit(status) if in_test_mode diff --git a/script/load-exim-logs b/script/load-exim-logs index 7e7f37a35..5ca0c66f8 100755 --- a/script/load-exim-logs +++ b/script/load-exim-logs @@ -1,21 +1,20 @@ #!/bin/bash -LOC=`dirname $0` +LOC=`dirname "$0"` # Specific file if specified if [ x$1 != x ] then - "$LOC/runner" 'EximLog.load_file("'$1'")' + f=`abspath "$1"` + cd "$LOC" + bundle exec ./runner 'EximLog.load_file("'$f'")' exit fi # Load in last three days worth of logs (if they've been modified) +cd "$LOC" LATEST=$( ls /var/log/exim4/exim-mainlog-* 2>/dev/null | sort | tail -3 ) for X in $LATEST do - # echo "doing $X" - "$LOC/runner" 'EximLog.load_file("'$X'")' + bundle exec ./runner 'EximLog.load_file("'$X'")' done - - - diff --git a/script/rails-post-deploy b/script/rails-post-deploy index 575b995f9..193d0bbec 100755 --- a/script/rails-post-deploy +++ b/script/rails-post-deploy @@ -45,10 +45,10 @@ else mkdir -p log fi # link the "downloads" directory in the cache to somewhere it can be served -if [ ! -e $APP_DIR/public/download ] +if [ ! -e "$APP_DIR/public/download" ] then - mkdir -p $APP_DIR/cache/zips/download - ln -s $APP_DIR/cache/zips/download $APP_DIR/public/ + mkdir -p "$APP_DIR/cache/zips/download" + ln -s "$APP_DIR/cache/zips/download" "$APP_DIR/public/" fi cd log @@ -74,19 +74,20 @@ then echo "ENV['RAILS_ENV'] ||= 'production'" > config/rails_env.rb fi +bundle_install_options="" if [ "$OPTION_STAGING_SITE" = "0" ] then - bundle exec bundle install --without development:test --deployment -else - bundle exec bundle install + bundle_install_options="--without development:test --deployment" fi +bundle install $bundle_install_options + if [ -n "$OPTION_THEME_URLS" ] then for THEME in "${OPTION_THEME_URLS[@]}" do - echo "Installing $THEME..." - script/plugin install --force $THEME + echo "Installing $THEME..." + script/plugin install --force $THEME done fi @@ -94,7 +95,7 @@ fi if [ -n "$OPTION_THEME_URL" ] then echo "Installing $OPTION_THEME_URL using deprecated THEME_URL..." - script/plugin install --force $OPTION_THEME_URL + script/plugin install --force $OPTION_THEME_URL fi # upgrade database diff --git a/script/runner b/script/runner index 6b0bc0a08..2bd5e9aa7 100755 --- a/script/runner +++ b/script/runner @@ -3,7 +3,7 @@ daemon_mode = !ARGV.empty? && ARGV[0] == "--daemon" script_dir = File.dirname(__FILE__) -alaveteli_dir = File.join(script_dir, "..") +alaveteli_dir = File.expand_path(File.join(script_dir, "..")) Dir.chdir(alaveteli_dir) do require File.join(alaveteli_dir, 'config', 'boot') diff --git a/script/stop-new-responses-on-old-requests b/script/stop-new-responses-on-old-requests index 77189cf0b..1e325403e 100755 --- a/script/stop-new-responses-on-old-requests +++ b/script/stop-new-responses-on-old-requests @@ -1,7 +1,7 @@ #!/bin/bash -LOC=`dirname $0` +cd "`dirname "$0"`" -"$LOC/runner" 'InfoRequest.stop_new_responses_on_old_requests()' +bundle exec ./runner 'InfoRequest.stop_new_responses_on_old_requests()' |