10 files changed, 129 insertions, 10 deletions
diff --git a/app/controllers/api_controller.rb b/app/controllers/api_controller.rb
index 6f83d89d6..8d7de3140 100644
--- a/app/controllers/api_controller.rb
+++ b/app/controllers/api_controller.rb
@@ -203,14 +203,28 @@ class ApiController < ApplicationController
 
         if since_date_str
             begin
-                since_date = Date.strptime(since_date_str, "%Y-%m-%d")
+              since_date = Date.strptime(since_date_str, "%Y-%m-%d")
             rescue ArgumentError
+              render :json => {"errors" => [
+                      "Parameter since_date must be in format yyyy-mm-dd (not '#{since_date_str}')" ] },
+                  :status => 500
+              return
+            end
+            @events = @events.where("info_request_events.created_at >= ?", since_date)
+        end
+
+        # We take a "since" parameter that allows the client
+        # to restrict to events more recent than a certain other event
+        if since_event_id
+            begin
+                event = InfoRequestEvent.find(since_event_id)
+            rescue ActiveRecord::RecordNotFound
                 render :json => {"errors" => [
-                                    "Parameter since_date must be in format yyyy-mm-dd (not '#{since_date_str}')" ] },
+                                    "Event ID #{since_event_id} not found" ] },
                        :status => 500
                 return
             end
-            @events = @events.where("info_request_events.created_at >= ?", since_date)
+            @events = @events.where("info_request_events.created_at > ?", event.created_at)
         end
 
         # We take a "since" parameter that allows the client
diff --git a/app/models/info_request_event.rb b/app/models/info_request_event.rb
index 635ba8f58..0ee82d30c 100644
--- a/app/models/info_request_event.rb
+++ b/app/models/info_request_event.rb
@@ -278,9 +278,15 @@ class InfoRequestEvent < ActiveRecord::Base
         end
         self.params_yaml = params.to_yaml
     end
+
     def params
-        YAML.load(self.params_yaml)
+        param_hash = YAML.load(params_yaml)
+        param_hash.each do |key, value|
+            param_hash[key] = value.force_encoding('UTF-8') if value.respond_to?(:force_encoding)
+        end
+        param_hash
     end
+
     def params_yaml_as_html
         ret = ''
         # split out parameters into old/new diffs, and other ones
diff --git a/app/models/post_redirect.rb b/app/models/post_redirect.rb
index 8049349d0..59160381c 100644
--- a/app/models/post_redirect.rb
+++ b/app/models/post_redirect.rb
@@ -71,7 +71,11 @@ class PostRedirect < ActiveRecord::Base
     end
 
     def reason_params
-        YAML.load(reason_params_yaml)
+        param_hash = YAML.load(reason_params_yaml)
+        param_hash.each do |key, value|
+            param_hash[key] = value.force_encoding('UTF-8') if value.respond_to?(:force_encoding)
+        end
+        param_hash
     end
 
     # Extract just local path part, without domain or #
diff --git a/config/application.rb b/config/application.rb
index 3c01e26c4..a89ccaa9e 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -60,7 +60,8 @@ module Alaveteli
     config.time_zone = ::AlaveteliConfiguration::time_zone
 
     # Set the cache to use a memcached backend
-    config.cache_store = :mem_cache_store, { :namespace => AlaveteliConfiguration::domain }
+    config.cache_store = :mem_cache_store,
+                        { :namespace => "#{AlaveteliConfiguration::domain}_#{RUBY_VERSION}" }
     config.action_dispatch.rack_cache = nil
 
     config.after_initialize do |app|
diff --git a/config/general.yml-example b/config/general.yml-example
index 8acea374b..3cef075cf 100644
--- a/config/general.yml-example
+++ b/config/general.yml-example
@@ -571,6 +571,16 @@ VARNISH_HOST: null
 # ---
 GA_CODE: ''
 
+
+# We need to add the WDTK survey variables here, or else the deployment
+# system will cry.
+SURVEY_SECRET: ''
+# Blank this out if the survey host is unavailable - people going to the
+# survey URL will get a message that the survey is unavailable
+SURVEY_URL: ''
+# Set this to false if you don't want survey emails to be sent
+SEND_SURVEY_MAILS: true
+
 # If you want to override all the public body request emails with your own
 # email address so that request emails that would normally go to the public
 # body go to you, use this setting. This is useful for a staging server, so you
diff --git a/lib/configuration.rb b/lib/configuration.rb
index 90fd30d5f..9fcd346cd 100644
--- a/lib/configuration.rb
+++ b/lib/configuration.rb
@@ -32,8 +32,8 @@ module AlaveteliConfiguration
             :DISABLE_EMERGENCY_USER => false,
             :DOMAIN => 'localhost:3000',
             :DONATION_URL => '',
-            :EXCEPTION_NOTIFICATIONS_FROM => '',
-            :EXCEPTION_NOTIFICATIONS_TO => '',
+            :EXCEPTION_NOTIFICATIONS_FROM => 'errors@localhost',
+            :EXCEPTION_NOTIFICATIONS_TO => 'user-support@localhost',
             :FORCE_REGISTRATION_ON_NEW_REQUEST => false,
             :FORCE_SSL => true,
             :FORWARD_NONBOUNCE_RESPONSES_TO => 'user-support@localhost',
@@ -64,6 +64,8 @@ module AlaveteliConfiguration
             :SITE_NAME => 'Alaveteli',
             :SKIP_ADMIN_AUTH => false,
             :SPECIAL_REPLY_VERY_LATE_AFTER_DAYS => 60,
+            :SURVEY_URL => '',
+            :SEND_SURVEY_MAILS => true,
             :THEME_BRANCH => false,
             :THEME_URL => "",
             :THEME_URLS => [],
diff --git a/lib/tasks/temp.rake b/lib/tasks/temp.rake
index 67fa10174..094bf4492 100644
--- a/lib/tasks/temp.rake
+++ b/lib/tasks/temp.rake
@@ -1,5 +1,39 @@
+# -*- coding: utf-8 -*-
 namespace :temp do
 
+    desc 'Rewrite cached HTML attachment headers to use responsive CSS'
+    task :responsive_attachments => :environment do
+        example = 'rake responsive_attachments PATTERN="./cache/views/request/*/*/response/*/attach/html/*/*.html"'
+        check_for_env_vars(['PATTERN'],example)
+        pattern = ENV['PATTERN']
+        replacement_head_content = <<-EOF
+<!--[if LTE IE 7]>
+<link href="/assets/responsive/application-lte-ie7.css" media="all" rel="stylesheet" title="Main" type="text/css" />
+<![endif]-->
+
+<!--[if IE 8]>
+<link href="/assets/responsive/application-ie8.css" media="all" rel="stylesheet" title="Main" type="text/css" />
+<![endif]-->
+
+<!--[if GT IE 8]><!-->
+<link href="/assets/responsive/application.css" media="all" rel="stylesheet" title="Main" type="text/css" />
+<!--<![endif]-->
+
+<script type="text/javascript" src="//use.typekit.net/csi1ugd.js"></script>
+<script type="text/javascript">try{Typekit.load();}catch(e){}</script>
+EOF
+
+
+        Dir.glob(pattern) do |cached_html_file|
+            puts cached_html_file
+            text = File.read(cached_html_file)
+            text.sub!(/<link [^>]*href="(\/assets\/application.css|\/stylesheets\/main.css|https?:\/\/www.whatdotheyknow.com\/stylesheets\/main.css)[^>]*>/, replacement_head_content)
+            text.sub!(/<\/div>(\s*This is an HTML version of an attachment to the Freedom of Information request.*?)<\/div>/m, '</div><p class="view_html_description">\1</p></div>')
+            text.sub!(/<iframe src='http:\/\/docs.google.com\/viewer/, "<iframe src='https://docs.google.com/viewer")
+            text.sub!(/<\/head>/, '<meta name="viewport" content="width=device-width, initial-scale=1.0" /></head>')
+            File.open(cached_html_file, 'w') { |file| file.write(text) }
+        end
+    end
 
     desc 'Analyse rails log specified by LOG_FILE to produce a list of request volume'
     task :request_volume => :environment do
@@ -37,4 +71,38 @@ namespace :temp do
 
     end
 
+    desc 'Look for broken UTF-8 text in IncomingMessage cached_attachment_text_clipped'
+    task :find_broken_cached_utf8 => :environment do
+        PublicBody.find_each do |public_body|
+            begin
+                public_body.name.split(' ')
+            rescue
+                puts "Bad encoding in public_body #{public_body.id} #{public_body.name}"
+                public_body.name = public_body.name.force_encoding("cp1252").encode('UTF-8').gsub('’', "'")
+                public_body.last_edit_editor = 'system'
+                public_body.last_edit_comment = 'Broken utf-8 encoding fixed by temp:find_broken_cached_utf8'
+                public_body.save!
+                public_body.name.split(' ')
+                puts "Fixed #{public_body.id}"
+            end
+        end
+
+        IncomingMessage.find_each do |incoming_message|
+            begin
+                incoming_message.get_attachment_text_full.split(' ')
+                incoming_message.get_attachment_text_clipped.split(' ')
+                incoming_message.get_main_body_text_folded.split(' ')
+                incoming_message.get_main_body_text_unfolded.split(' ')
+            rescue ArgumentError => e
+                puts "Bad encoding in incoming message #{incoming_message.id}"
+                incoming_message.clear_in_database_caches!
+                incoming_message.get_attachment_text_full.split(' ')
+                incoming_message.get_attachment_text_clipped.split(' ')
+                incoming_message.get_main_body_text_folded.split(' ')
+                incoming_message.get_main_body_text_unfolded.split(' ')
+                puts "Fixed #{incoming_message.id}"
+            end
+        end
+
+    end
 end
diff --git a/script/mailin b/script/mailin
index f6b7d1194..48e236cfe 100755
--- a/script/mailin
+++ b/script/mailin
@@ -23,7 +23,7 @@ then
     SUBJ="Mail import error for $OPTION_DOMAIN"
     BODY="There was an error code $ERROR_CODE returned by the RequestMailer.receive command in script/mailin. See attached for details. This might be quite serious, such as the database was down, or might be an email with corrupt headers that Rails is choking on. We returned the email with an exit code 75, which for Exim at least instructs the MTA to try again later. A well configured installation of this code will separately have had Exim make a backup copy of the email in a separate mailbox, just in case."
     FROM="$OPTION_BLACKHOLE_PREFIX@$OPTION_INCOMING_EMAIL_DOMAIN"
-    /usr/bin/mutt -e "set use_envelope_from" -e "set envelope_from_address=$FROM" -s "$SUBJ" -a "$OUTPUT" "$INPUT" -- "$OPTION_FORWARD_NONBOUNCE_RESPONSES_TO" <<<"$BODY"
+    /usr/bin/mutt -e "set use_envelope_from" -e "set envelope_from_address=$FROM" -s "$SUBJ" -a "$OUTPUT" "$INPUT" -- "$OPTION_EXCEPTION_NOTIFICATIONS_TO" <<<"$BODY"
 
     # tell exim error was temporary, so try again later (no point bouncing message to authority)
     rm -f "$INPUT" "$OUTPUT"
diff --git a/spec/models/info_request_event_spec.rb b/spec/models/info_request_event_spec.rb
index 53c83bd46..1299dfb63 100644
--- a/spec/models/info_request_event_spec.rb
+++ b/spec/models/info_request_event_spec.rb
@@ -30,6 +30,12 @@ describe InfoRequestEvent do
             ire.params.should == example_params
         end
 
+        it "should restore UTF8-heavy params stored under ruby 1.8 as UTF-8" do
+            ire = InfoRequestEvent.new
+            utf8_params = "--- \n:foo: !binary |\n  0KLQvtCz0LDRiCDR\n"
+            ire.params_yaml = utf8_params
+            ire.params[:foo].encoding.to_s.should == 'UTF-8' if ire.params[:foo].respond_to?(:encoding)
+        end
     end
 
     describe 'when deciding if it is indexed by search' do
diff --git a/spec/models/post_redirect_spec.rb b/spec/models/post_redirect_spec.rb
index 73740e914..70b221f10 100644
--- a/spec/models/post_redirect_spec.rb
+++ b/spec/models/post_redirect_spec.rb
@@ -65,11 +65,19 @@ describe PostRedirect, " when accessing values" do
     end
 
     it "should convert reason parameters into YAML and back successfully" do
-        pr = PostRedirect.new 
+        pr = PostRedirect.new
         example_reason_params = { :foo => 'this is stuff', :bar => 83, :humbug => "yikes!!!" }
         pr.reason_params = example_reason_params
         pr.reason_params_yaml.should == example_reason_params.to_yaml
         pr.reason_params.should == example_reason_params
     end
+
+    it "should restore UTF8-heavy params stored under ruby 1.8 as UTF-8" do
+        pr = PostRedirect.new
+        utf8_params = "--- \n:foo: !binary |\n  0KLQvtCz0LDRiCDR\n"
+        pr.reason_params_yaml = utf8_params
+        puts pr.reason_params
+        pr.reason_params[:foo].encoding.to_s.should == 'UTF-8' if pr.reason_params[:foo].respond_to?(:encoding)
+    end
 end