aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.gitignore3
-rw-r--r--Gemfile5
-rw-r--r--Gemfile.lock21
-rw-r--r--app/controllers/application_controller.rb11
-rw-r--r--app/models/info_request.rb21
-rw-r--r--app/models/purge_request.rb27
-rw-r--r--config/environment.rb2
-rw-r--r--config/environments/development.rb2
-rw-r--r--config/general.yml-example4
-rw-r--r--config/test.yml2
-rw-r--r--config/varnish-alaveteli.vcl24
-rw-r--r--lib/quiet_opener.rb34
-rw-r--r--lib/varnish_purge.rb11
-rwxr-xr-xscript/purge-varnish11
-rw-r--r--spec/controllers/admin_censor_rule_controller_spec.rb2
-rw-r--r--spec/controllers/request_controller_spec.rb21
-rw-r--r--spec/models/purge_request_spec.rb32
-rw-r--r--spec/spec_helper.rb2
18 files changed, 176 insertions, 59 deletions
diff --git a/.gitignore b/.gitignore
index 527bccb44..1155b055d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,4 +18,5 @@ TAGS
/public/download
/public/*theme
/vendor/bundle
-.bundle \ No newline at end of file
+.bundle
+bin/ \ No newline at end of file
diff --git a/Gemfile b/Gemfile
index 39469aaf6..29319e41a 100644
--- a/Gemfile
+++ b/Gemfile
@@ -16,6 +16,7 @@ gem 'json', '~> 1.5.1'
gem 'mahoro'
gem 'memcache-client', :require => 'memcache'
gem 'locale', '>= 2.0.5'
+gem 'net-purge'
gem 'rack', '~> 1.1.0'
gem 'rdoc', '~> 2.4.3'
gem 'recaptcha', '~> 0.3.1', :require => 'recaptcha/rails'
@@ -37,3 +38,7 @@ group :test do
gem 'fakeweb'
gem 'rspec-rails', '~> 1.3.4'
end
+
+group :develop do
+ gem 'ruby-debug'
+end
diff --git a/Gemfile.lock b/Gemfile.lock
index 8cf57fc79..f92be20a9 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,3 +1,9 @@
+GIT
+ remote: git://github.com/sebbacon/xapian-full.git
+ revision: 8f0f827d5964b28daa72c756e40caabfa2981fd0
+ specs:
+ xapian-full (1.2.9)
+
GEM
remote: http://rubygems.org/
specs:
@@ -11,14 +17,18 @@ GEM
activeresource (2.3.14)
activesupport (= 2.3.14)
activesupport (2.3.14)
+ columnize (0.3.6)
fakeweb (1.3.0)
fast_gettext (0.6.1)
gettext (2.1.0)
locale (>= 2.0.5)
json (1.5.4)
+ linecache (0.46)
+ rbx-require-relative (> 0.0.4)
locale (2.0.5)
mahoro (0.3)
memcache-client (1.8.5)
+ net-purge (0.1.0)
pg (0.11.0)
rack (1.1.0)
rails (2.3.14)
@@ -29,6 +39,7 @@ GEM
activesupport (= 2.3.14)
rake (>= 0.8.3)
rake (0.9.2)
+ rbx-require-relative (0.0.9)
rdoc (2.4.3)
recaptcha (0.3.1)
rmagick (2.13.1)
@@ -38,13 +49,17 @@ GEM
rspec-rails (1.3.4)
rack (>= 1.0.0)
rspec (~> 1.3.1)
+ ruby-debug (0.10.4)
+ columnize (>= 0.1)
+ ruby-debug-base (~> 0.10.4.0)
+ ruby-debug-base (0.10.4)
+ linecache (>= 0.3)
ruby-msg (1.5.0)
ruby-ole (>= 1.2.8)
vpim (>= 0.360)
ruby-ole (1.2.11.2)
vpim (0.695)
will_paginate (2.3.16)
- xapian-full (1.2.3)
xml-simple (1.1.0)
zip (2.0.2)
@@ -59,6 +74,7 @@ DEPENDENCIES
locale (>= 2.0.5)
mahoro
memcache-client
+ net-purge
pg
rack (~> 1.1.0)
rails (= 2.3.14)
@@ -68,9 +84,10 @@ DEPENDENCIES
routing-filter (~> 0.2.4)
rspec (~> 1.3.2)
rspec-rails (~> 1.3.4)
+ ruby-debug
ruby-msg (~> 1.5.0)
vpim
will_paginate (~> 2.3.11)
- xapian-full
+ xapian-full (~> 1.2.9)!
xml-simple
zip
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 0ec8e206e..0d0cca3e4 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
# controllers/application.rb:
# Parent class of all controllers in FOI site. Filters added to this controller
# apply to all controllers in the application. Likewise, all the methods added
@@ -543,16 +544,6 @@ class ApplicationController < ActionController::Base
return country
end
- def quietly_try_to_open(url)
- begin
- result = open(url).read.strip
- rescue OpenURI::HTTPError, SocketError, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH
- logger.warn("Unable to open third-party URL #{url}")
- result = ""
- end
- return result
- end
-
# URL generating functions are needed by all controllers (for redirects),
# views (for links) and mailers (for use in emails), so include them into
# all of all.
diff --git a/app/models/info_request.rb b/app/models/info_request.rb
index 3a1f4b9f3..9b129e74b 100644
--- a/app/models/info_request.rb
+++ b/app/models/info_request.rb
@@ -453,7 +453,6 @@ public
# An annotation (comment) is made
def add_comment(body, user)
comment = Comment.new
-
ActiveRecord::Base.transaction do
comment.body = body
comment.user = user
@@ -1043,18 +1042,14 @@ public
return ret
end
- before_save(:mark_view_is_dirty)
- def mark_view_is_dirty
- self.view_is_dirty = true
- self.save!
- end
-
- def self.purge_varnish
- for info_request in InfoRequest.find_by_view_is_dirty(true)
- url = "/request/#{info_request.url_title}"
- purge(url)
- info_request.view_is_dirty = true
- info_request.save!
+ before_save :purge_in_cache
+ def purge_in_cache
+ if !MySociety::Config.get('VARNISH_HOST').nil? && !self.id.nil?
+ # we only do this for existing info_requests (new ones have a nil id)
+ req = PurgeRequest.new(:url => "/request/#{self.url_title}",
+ :model => self.class.base_class.to_s,
+ :model_id => self.id)
+ req.save()
end
end
end
diff --git a/app/models/purge_request.rb b/app/models/purge_request.rb
new file mode 100644
index 000000000..a96d0f39e
--- /dev/null
+++ b/app/models/purge_request.rb
@@ -0,0 +1,27 @@
+# models/purge_request.rb:
+# A queue of URLs to purge
+#
+# Copyright (c) 2008 UK Citizens Online Democracy. All rights reserved.
+# Email: francis@mysociety.org; WWW: http://www.mysociety.org/
+#
+
+class PurgeRequest < ActiveRecord::Base
+ require 'open-uri'
+ def self.purge_all
+ for item in PurgeRequest.all()
+ item.purge
+ end
+ end
+
+ def purge
+ config = MySociety::Config.load_default()
+ varnish_url = config['VARNISH_HOST']
+ result = quietly_try_to_purge(varnish_url, self.url)
+ if result == "200"
+ self.delete()
+ end
+ end
+end
+
+
+
diff --git a/config/environment.rb b/config/environment.rb
index 7366179bf..b958c6475 100644
--- a/config/environment.rb
+++ b/config/environment.rb
@@ -135,7 +135,7 @@ require 'i18n_fixes.rb'
require 'rack_quote_monkeypatch.rb'
require 'world_foi_websites.rb'
require 'alaveteli_external_command.rb'
-require 'varnish_purge.rb'
+require 'quiet_opener.rb'
ExceptionNotification::Notifier.sender_address = MySociety::Config::get('EXCEPTION_NOTIFICATIONS_FROM')
ExceptionNotification::Notifier.exception_recipients = MySociety::Config::get('EXCEPTION_NOTIFICATIONS_TO')
diff --git a/config/environments/development.rb b/config/environments/development.rb
index d5f2f5772..a1e8133a8 100644
--- a/config/environments/development.rb
+++ b/config/environments/development.rb
@@ -1,5 +1,7 @@
# Settings specified here will take precedence over those in config/environment.rb
+config.log_level = :info
+
# In the development environment your application's code is reloaded on
# every request. This slows down response time but is perfect for development
# since you don't have to restart the webserver when you make code changes.
diff --git a/config/general.yml-example b/config/general.yml-example
index ed04e0fd5..84980c353 100644
--- a/config/general.yml-example
+++ b/config/general.yml-example
@@ -142,3 +142,7 @@ EXCEPTION_NOTIFICATIONS_TO:
# This rate limiting can be turned off per-user via the admin interface
MAX_REQUESTS_PER_USER_PER_DAY: 6
+
+# This is used to work out where to send purge requests. Should be
+# unset if you aren't running behind varnish
+VARNISH_HOST: localhost
diff --git a/config/test.yml b/config/test.yml
index 991588f81..c35001747 100644
--- a/config/test.yml
+++ b/config/test.yml
@@ -124,4 +124,4 @@ EXCEPTION_NOTIFICATIONS_TO:
MAX_REQUESTS_PER_USER_PER_DAY: 2
-VARNISH_URL: http://varnish
+VARNISH_HOST: varnish.localdomain
diff --git a/config/varnish-alaveteli.vcl b/config/varnish-alaveteli.vcl
index 7eedf83fc..f81ec2295 100644
--- a/config/varnish-alaveteli.vcl
+++ b/config/varnish-alaveteli.vcl
@@ -9,12 +9,18 @@
backend default {
.host = "127.0.0.1";
- .port = "80";
+ .port = "3000";
.connect_timeout = 600s;
.first_byte_timeout = 600s;
.between_bytes_timeout = 600s;
}
+// set the servers alaveteli can issue a purge from
+acl purge {
+ "localhost";
+ "127.0.0.1";
+}
+
sub vcl_recv {
# Handle IPv6
@@ -54,12 +60,13 @@ sub vcl_recv {
req.request != "HEAD" &&
req.request != "POST" &&
req.request != "PUT" &&
+ req.request != "PURGE" &&
req.request != "DELETE" ) {
# We don't allow any other methods.
error 405 "Method Not Allowed";
}
- if (req.request != "GET" && req.request != "HEAD") {
+ if (req.request != "GET" && req.request != "HEAD" && req.request != "PURGE") {
/* We only deal with GET and HEAD by default, the rest get passed direct to backend */
return (pass);
}
@@ -73,15 +80,21 @@ sub vcl_recv {
if (req.http.Authorization || req.http.Cookie) {
return (pass);
}
-
# Let's have a little grace
set req.grace = 30s;
+ # Handle PURGE requests
+ if (req.request == "PURGE") {
+ if (!client.ip ~ purge) {
+ error 405 "Not allowed.";
+ }
+ ban("obj.http.x-url ~ " + req.url);
+ error 200 "Banned";
+ }
return (lookup);
}
-
sub vcl_fetch {
-
+ set beresp.http.x-url = req.url;
if (req.url ~ "\.(png|gif|jpg|jpeg|swf|css|js|rdf|ico|txt)(\?.*|)$") {
# Ignore backend headers..
remove beresp.http.set-Cookie;
@@ -94,3 +107,4 @@ sub vcl_fetch {
return (deliver);
}
}
+
diff --git a/lib/quiet_opener.rb b/lib/quiet_opener.rb
new file mode 100644
index 000000000..a077ca323
--- /dev/null
+++ b/lib/quiet_opener.rb
@@ -0,0 +1,34 @@
+require 'open-uri'
+require 'net-purge'
+
+def quietly_try_to_open(url)
+ begin
+ result = open(url).read.strip
+ rescue OpenURI::HTTPError, SocketError, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH
+ logger.warn("Unable to open third-party URL #{url}")
+ result = ""
+ end
+ return result
+end
+
+def quietly_try_to_purge(host, url)
+ begin
+ result = ""
+ result_body = ""
+ Net::HTTP.start(host) {|http|
+ request = Net::HTTP::Purge.new(url)
+ response = http.request(request)
+ result = response.code
+ result_body = response.body
+ }
+ rescue OpenURI::HTTPError, SocketError, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH
+ logger.warn("Unable to reach host #{host}")
+ end
+ if result == "200"
+ logger.info("Purged URL #{url} at #{host}: #{result}")
+ else
+ logger.warn("Unable to purge URL #{url} at #{host}: status #{result}")
+ end
+ return result
+end
+
diff --git a/lib/varnish_purge.rb b/lib/varnish_purge.rb
deleted file mode 100644
index ef0cbd7ea..000000000
--- a/lib/varnish_purge.rb
+++ /dev/null
@@ -1,11 +0,0 @@
-require 'open-uri'
-
-def purge(url)
- config = MySociety::Config.load_default()
- varnish_url = config['VARNISH_URL']
- url = "#{varnish_url}#{url}"
- result = open(url).read
- if result != "OK"
- raise
- end
-end
diff --git a/script/purge-varnish b/script/purge-varnish
new file mode 100755
index 000000000..932cf6635
--- /dev/null
+++ b/script/purge-varnish
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+LOC=`dirname $0`
+
+if [ "$1" == "--loop" ]
+then
+ "$LOC/runner" 'PurgeRequest.purge_all_loop'
+else
+ "$LOC/runner" 'PurgeRequest.purge_all'
+fi
+
diff --git a/spec/controllers/admin_censor_rule_controller_spec.rb b/spec/controllers/admin_censor_rule_controller_spec.rb
index 952830f02..8893a858b 100644
--- a/spec/controllers/admin_censor_rule_controller_spec.rb
+++ b/spec/controllers/admin_censor_rule_controller_spec.rb
@@ -6,13 +6,13 @@ describe AdminCensorRuleController, "when making censor rules from the admin int
it "should create a censor rule and purge the corresponding request from varnish" do
ir = info_requests(:fancy_dog_request)
- ir.should_receive(:purge_in_cache)
post :create, :censor_rule => {
:text => "meat",
:replacement => "tofu",
:last_edit_comment => "none",
:info_request => ir
}
+ PurgeRequest.all().first.model_id.should == ir.id
end
diff --git a/spec/controllers/request_controller_spec.rb b/spec/controllers/request_controller_spec.rb
index 89d165587..9090bc26f 100644
--- a/spec/controllers/request_controller_spec.rb
+++ b/spec/controllers/request_controller_spec.rb
@@ -121,55 +121,50 @@ describe RequestController, "when changing things that appear on the request pag
integrate_views
- before(:each) do
- FakeWeb.last_request = nil
- end
-
it "should purge the downstream cache when mail is received" do
ir = info_requests(:fancy_dog_request)
receive_incoming_mail('incoming-request-plain.email', ir.incoming_email)
- FakeWeb.last_request.path.should include(ir.url_title)
+ PurgeRequest.all().first.model_id.should == ir.id
end
it "should purge the downstream cache when a comment is added" do
ir = info_requests(:fancy_dog_request)
- ir.should_receive(:purge_in_cache)
new_comment = info_requests(:fancy_dog_request).add_comment('I also love making annotations.', users(:bob_smith_user))
+ PurgeRequest.all().first.model_id.should == ir.id
end
it "should purge the downstream cache when a followup is made" do
session[:user_id] = users(:bob_smith_user).id
ir = info_requests(:fancy_dog_request)
post :show_response, :outgoing_message => { :body => "What a useless response! You suck.", :what_doing => 'normal_sort' }, :id => ir.id, :incoming_message_id => incoming_messages(:useless_incoming_message), :submitted_followup => 1
- FakeWeb.last_request.path.should include(ir.url_title)
+ PurgeRequest.all().first.model_id.should == ir.id
end
it "should purge the downstream cache when the request is categorised" do
ir = info_requests(:fancy_dog_request)
- ir.should_receive(:purge_in_cache)
ir.set_described_state('waiting_clarification')
+ PurgeRequest.all().first.model_id.should == ir.id
end
it "should purge the downstream cache when the authority data is changed" do
ir = info_requests(:fancy_dog_request)
ir.public_body.name = "Something new"
ir.public_body.save!
- FakeWeb.last_request.path.should include(ir.url_title)
+ PurgeRequest.all().map{|x| x.model_id}.should =~ ir.public_body.info_requests.map{|x| x.id}
end
it "should purge the downstream cache when the user details are changed" do
ir = info_requests(:fancy_dog_request)
ir.user.name = "Something new"
- FakeWeb.last_request.should == nil
ir.user.save!
- FakeWeb.last_request.path.should include(ir.url_title)
+ PurgeRequest.all().map{|x| x.model_id}.should =~ ir.user.info_requests.map{|x| x.id}
end
it "should purge the downstream cache when censor rules have changed" do
# XXX really, CensorRules should execute expiry logic as part
# of the after_save of the model. Currently this is part of
- # the AdminController logic, so must be tested from
+ # the AdminCensorRuleController logic, so must be tested from
# there. Leaving this stub test in place as a reminder
end
it "should purge the downstream cache when something is hidden by an admin" do
ir = info_requests(:fancy_dog_request)
- ir.should_receive(:purge_in_cache)
ir.prominence = 'hidden'
ir.save!
+ PurgeRequest.all().first.model_id.should == ir.id
end
end
diff --git a/spec/models/purge_request_spec.rb b/spec/models/purge_request_spec.rb
new file mode 100644
index 000000000..f7d01f784
--- /dev/null
+++ b/spec/models/purge_request_spec.rb
@@ -0,0 +1,32 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require 'fakeweb'
+
+describe PurgeRequest, "purging things" do
+ before do
+ FakeWeb.last_request = nil
+ end
+
+ it 'should issue purge requests to the server' do
+ req = PurgeRequest.new(:url => "/begone_from_here",
+ :model => "don't care",
+ :model_id => "don't care")
+ req.save()
+ PurgeRequest.all().count.should == 1
+ PurgeRequest.purge_all()
+ PurgeRequest.all().count.should == 0
+ end
+
+ it 'should fail silently for a misconfigured server' do
+ FakeWeb.register_uri(:get, %r|brokenv|, :body => "BROKEN")
+ config = MySociety::Config.load_default()
+ config['VARNISH_HOST'] = "brokencache"
+ req = PurgeRequest.new(:url => "/begone_from_here",
+ :model => "don't care",
+ :model_id => "don't care")
+ req.save()
+ PurgeRequest.all().count.should == 1
+ PurgeRequest.purge_all()
+ PurgeRequest.all().count.should == 1
+ end
+end
+
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 5ca4b6de9..a98a5113d 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -15,7 +15,7 @@ config['REPLY_LATE_AFTER_DAYS'] = 20
# register a fake Varnish server
require 'fakeweb'
-FakeWeb.register_uri(:get, %r|varnish|, :body => "OK")
+FakeWeb.register_uri(:purge, %r|varnish.localdomain|, :body => "OK")
# Uncomment the next line to use webrat's matchers
#require 'webrat/integrations/rspec-rails'
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-27 05:27:05 +0000