diff options
102 files changed, 1310 insertions, 1528 deletions
diff --git a/.gitignore b/.gitignore index a4ec2380e..994f9a3a1 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ bin/ config/aliases config/httpd.conf config/general*.yml +config/deploy.yml.* .sass-cache alaveteli.sublime* webrat.log diff --git a/.travis.yml b/.travis.yml index 6a6b73e90..3351bed2a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,9 @@ before_install: - gem update --system 2.1.11 - gem install rake --version=0.9.2.2 - git submodule update --init --recursive - - psql -c "create database foi_test template template0 encoding 'SQL_ASCII';" -U postgres + - psql -c "create database template_utf8 template template0 encoding 'UTF-8';" -U postgres + - psql -c "update pg_database set datistemplate=true where datname='template_utf8';" -U postgres + - psql -c "create database foi_test template template_utf8;" -U postgres - cp config/database.yml-test config/database.yml - cp config/general.yml-example config/general.yml - cp config/newrelic.yml-example config/newrelic.yml @@ -17,7 +17,6 @@ gem 'charlock_holmes' gem 'dynamic_form' gem 'exception_notification' gem 'fancybox-rails' -gem 'fastercsv', '>=1.5.5' gem 'foundation-rails' gem 'jquery-rails', '~> 3.0.4' gem 'jquery-ui-rails' @@ -82,6 +81,7 @@ end group :development do gem 'capistrano' gem 'mailcatcher' + gem 'quiet_assets' gem 'rdoc' end diff --git a/Gemfile.lock b/Gemfile.lock index 32a1e77f5..8a7695810 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -104,7 +104,6 @@ GEM fancybox-rails (0.2.1) railties (>= 3.1.0) fast_gettext (0.7.0) - fastercsv (1.5.5) foundation-rails (5.2.1.0) railties (>= 3.1.0) sass (>= 3.2.0) @@ -162,6 +161,8 @@ GEM railties (~> 3.0) pg (0.15.1) polyglot (0.3.4) + quiet_assets (1.0.2) + railties (>= 3.1, < 5.0) rack (1.4.5) rack-cache (1.2) rack (>= 0.4) @@ -243,7 +244,7 @@ GEM hike (~> 1.2) multi_json (~> 1.0) rack (~> 1.0) - tilt (!= 1.3.0, ~> 1.1) + tilt (~> 1.1, != 1.3.0) sqlite3 (1.3.7) statistics2 (0.54) syslog_protocol (0.9.2) @@ -295,7 +296,6 @@ DEPENDENCIES fakeweb fancybox-rails fast_gettext - fastercsv (>= 1.5.5) foundation-rails gettext gettext_i18n_rails @@ -312,6 +312,7 @@ DEPENDENCIES newrelic_rpm nokogiri pg + quiet_assets rack rails (= 3.2.18) rails-i18n @@ -17,12 +17,12 @@ Please join our mailing list at https://groups.google.com/group/alaveteli-dev and introduce yourself, or drop a line to hello@alaveteli.org to let us know that you're using Alaveteli. -Some documentation can be found in the -[`doc/` folder](https://github.com/mysociety/alaveteli/tree/master/doc). -There's background information and more documentation on -[our wiki](https://github.com/mysociety/alaveteli/wiki/Home/), and lots -of useful information (including a blog) on -[the project website](http://alaveteli.org) +There's lots of useful information and documentation (including a blog) +on [the project website](http://alaveteli.org). There's background +information and notes on [our +wiki](https://github.com/mysociety/alaveteli/wiki/Home/), and upgrade +notes in the [`doc/` +folder](https://github.com/mysociety/alaveteli/tree/master/doc/CHANGES.md) ## How to contribute diff --git a/Vagrantfile b/Vagrantfile index 5d56914a5..31b2553a1 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -46,6 +46,15 @@ # Both have the same effect, but exporting will retain the variable for the # duration of your shell session. # +# Using Themes +# ------------ +# +# You can also use the built in theme switcher (script/switch-theme.rb). The +# ALAVETELI_THEMES_DIR will be shared in to /home/vagrant/alaveteli-themes so +# that the default location is used on the guest. You can use the env var +# ALAVETELI_THEMES_DIR to change where this Vagrantfile looks for the themes +# directory on the host. +# # Customization Options # ===================== ALAVETELI_FQDN = ENV['ALAVETELI_VAGRANT_FQDN'] || "alaveteli.10.10.10.30.xip.io" diff --git a/app/assets/stylesheets/responsive/_footer_layout.scss b/app/assets/stylesheets/responsive/_footer_layout.scss index 2b0c956fa..55b6839c2 100644 --- a/app/assets/stylesheets/responsive/_footer_layout.scss +++ b/app/assets/stylesheets/responsive/_footer_layout.scss @@ -45,6 +45,10 @@ img { display: inherit; + @include lte-ie7 { + display: block; + } } + } } diff --git a/app/assets/stylesheets/responsive/_global_style.scss b/app/assets/stylesheets/responsive/_global_style.scss index 5b268f3a6..290591b5f 100644 --- a/app/assets/stylesheets/responsive/_global_style.scss +++ b/app/assets/stylesheets/responsive/_global_style.scss @@ -222,4 +222,7 @@ div.pagination { } - +/* Search result highlighting */ +.highlight { + background:#FF0; +} diff --git a/app/controllers/admin_request_controller.rb b/app/controllers/admin_request_controller.rb index fc291d998..5c45a6e6e 100644 --- a/app/controllers/admin_request_controller.rb +++ b/app/controllers/admin_request_controller.rb @@ -199,7 +199,7 @@ class AdminRequestController < AdminController end # Bejeeps, look, sometimes a URL is something that belongs in a controller, jesus. - # XXX hammer this square peg into the round MVC hole + # TODO: hammer this square peg into the round MVC hole post_redirect = PostRedirect.new( :uri => upload_response_url(:url_title => info_request.url_title), :user_id => user.id) @@ -253,7 +253,7 @@ class AdminRequestController < AdminController end info_request_event.described_state = 'waiting_clarification' info_request_event.calculated_state = 'waiting_clarification' - # XXX deliberately don't update described_at so doesn't reenter search? + # TODO: deliberately don't update described_at so doesn't reenter search? info_request_event.save! flash[:notice] = "Old response marked as having been a clarification" diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 78a82316a..0c5f5bd02 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -278,10 +278,10 @@ class ApplicationController < ActionController::Base session[:post_redirect_token] = post_redirect.token - # XXX what is the built in Ruby URI munging function that can do this + # TODO: what is the built in Ruby URI munging function that can do this # choice of & vs. ? more elegantly than this dumb if statement? if uri.include?("?") - # XXX This looks odd. What would a fragment identifier be doing server-side? + # TODO: This looks odd. What would a fragment identifier be doing server-side? # But it also looks harmless, so I’ll leave it just in case. if uri.include?("#") uri.sub!("#", "&post_redirect=1#") diff --git a/app/controllers/comment_controller.rb b/app/controllers/comment_controller.rb index 5e39c3a2c..2c0037577 100644 --- a/app/controllers/comment_controller.rb +++ b/app/controllers/comment_controller.rb @@ -21,7 +21,7 @@ class CommentController < ApplicationController end if params[:comment] - # XXX this check should theoretically be a validation rule in the model + # TODO: this check should theoretically be a validation rule in the model @existing_comment = Comment.find_existing(@info_request.id, params[:comment][:body]) else # Default to subscribing to request when first viewing form diff --git a/app/controllers/general_controller.rb b/app/controllers/general_controller.rb index 6f0d29889..158492eb2 100644 --- a/app/controllers/general_controller.rb +++ b/app/controllers/general_controller.rb @@ -59,7 +59,7 @@ class GeneralController < ApplicationController # Actual search def search - # XXX Why is this so complicated with arrays and stuff? Look at the route + # TODO: Why is this so complicated with arrays and stuff? Look at the route # in config/routes.rb for comments. combined = params[:combined].split("/") @sortby = nil @@ -70,7 +70,7 @@ class GeneralController < ApplicationController else @advanced = false end - # XXX currently /described isn't linked to anywhere, just used in RSS and for /list/successful + # TODO: currently /described isn't linked to anywhere, just used in RSS and for /list/successful # This is because it's confusingly different from /newest - but still useful for power users. if combined.size > 0 && (['newest', 'described', 'relevant'].include?(combined[-1])) @sort_postfix = combined.pop @@ -124,7 +124,7 @@ class GeneralController < ApplicationController end end - # Query each type separately for separate display (XXX we are calling + # Query each type separately for separate display (TODO: we are calling # perform_search multiple times and it clobbers per_page for each one, # so set as separate var) requests_per_page = params[:requests_per_page] ? params[:requests_per_page].to_i : 25 @@ -159,7 +159,7 @@ class GeneralController < ApplicationController end # Spelling and highight words are same for all three queries - @highlight_words = @request_for_spelling.words_to_highlight + @highlight_words = @request_for_spelling.words_to_highlight(:regex => true, :include_original => true) if !(@request_for_spelling.spelling_correction =~ /[a-z]+:/) @spelling_correction = @request_for_spelling.spelling_correction end @@ -178,7 +178,9 @@ class GeneralController < ApplicationController format.json { render :json => { :alaveteli_git_commit => alaveteli_git_commit, :alaveteli_version => ALAVETELI_VERSION, - :ruby_version => RUBY_VERSION + :ruby_version => RUBY_VERSION, + :visible_request_count => InfoRequest.visible.count, + :confirmed_user_count => User.where(:email_confirmed => true).count }} end end diff --git a/app/controllers/public_body_controller.rb b/app/controllers/public_body_controller.rb index 862f4b318..d2c84d820 100644 --- a/app/controllers/public_body_controller.rb +++ b/app/controllers/public_body_controller.rb @@ -5,12 +5,11 @@ # Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved. # Email: hello@mysociety.org; WWW: http://www.mysociety.org/ -require 'fastercsv' require 'confidence_intervals' require 'tempfile' class PublicBodyController < ApplicationController - # XXX tidy this up with better error messages, and a more standard infrastructure for the redirect to canonical URL + # TODO: tidy this up with better error messages, and a more standard infrastructure for the redirect to canonical URL def show long_cache if MySociety::Format.simplify_url_part(params[:url_name], 'body') != params[:url_name] @@ -43,7 +42,7 @@ class PublicBodyController < ApplicationController query = InfoRequestEvent.make_query_from_params(params.merge(:latest_status => @view)) query += " requested_from:#{@public_body.url_name}" # Use search query for this so can collapse and paginate easily - # XXX really should just use SQL query here rather than Xapian. + # TODO: really should just use SQL query here rather than Xapian. sortby = "described" begin @xapian_requests = perform_search([InfoRequestEvent], query, sortby, 'request_collapse') @@ -86,7 +85,7 @@ class PublicBodyController < ApplicationController def list long_cache - # XXX move some of these tag SQL queries into has_tag_string.rb + # TODO: move some of these tag SQL queries into has_tag_string.rb like_query = params[:public_body_query] like_query = "" if like_query.nil? @@ -109,17 +108,17 @@ class PublicBodyController < ApplicationController # Restrict the public bodies shown according to the tag # parameter supplied in the URL: - if @tag.nil? or @tag == "all" - @tag = "all" + if @tag.nil? || @tag == 'all' + @tag = 'all' elsif @tag == 'other' - category_list = PublicBodyCategories::get().tags().map{|c| "'"+c+"'"}.join(",") + category_list = PublicBodyCategories.get.tags.map{ |c| %Q('#{ c }') }.join(",") where_condition += base_tag_condition + " AND has_tag_string_tags.name in (#{category_list})) = 0" elsif @tag.scan(/./mu).size == 1 - @tag = Unicode.upcase @tag + @tag = Unicode.upcase(@tag) # The first letter queries have to be done on # translations, so just indicate to add that later: first_letter = true - elsif @tag.include?(":") + elsif @tag.include?(':') name, value = HasTagString::HasTagStringTag.split_tag_into_name_value(@tag) where_condition += base_tag_condition + " AND has_tag_string_tags.name = ? AND has_tag_string_tags.value = ?) > 0" where_parameters.concat [name, value] @@ -128,16 +127,16 @@ class PublicBodyController < ApplicationController where_parameters.concat [@tag] end - if @tag == "all" - @description = "" + if @tag == 'all' + @description = '' elsif @tag.size == 1 - @description = _("beginning with ‘{{first_letter}}’", :first_letter=>@tag) + @description = _("beginning with ‘{{first_letter}}’", :first_letter => @tag) else - category_name = PublicBodyCategories::get().by_tag()[@tag] + category_name = PublicBodyCategories.get.by_tag[@tag] if category_name.nil? - @description = _("matching the tag ‘{{tag_name}}’", :tag_name=>@tag) + @description = _("matching the tag ‘{{tag_name}}’", :tag_name => @tag) else - @description = _("in the category ‘{{category_name}}’", :category_name=>category_name) + @description = _("in the category ‘{{category_name}}’", :category_name => category_name) end end @@ -151,15 +150,15 @@ class PublicBodyController < ApplicationController FROM public_bodies LEFT OUTER JOIN public_body_translations as current_locale ON (public_bodies.id = current_locale.public_body_id - AND current_locale.locale = ? AND #{get_public_body_list_translated_condition 'current_locale', first_letter}) + AND current_locale.locale = ? AND #{ get_public_body_list_translated_condition('current_locale', first_letter) }) LEFT OUTER JOIN public_body_translations as default_locale ON (public_bodies.id = default_locale.public_body_id - AND default_locale.locale = ? AND #{get_public_body_list_translated_condition 'default_locale', first_letter}) - WHERE #{where_condition} AND COALESCE(current_locale.name, default_locale.name) IS NOT NULL + AND default_locale.locale = ? AND #{ get_public_body_list_translated_condition('default_locale', first_letter) }) + WHERE #{ where_condition } AND COALESCE(current_locale.name, default_locale.name) IS NOT NULL ORDER BY display_name} - sql = [query, underscore_locale, like_query, like_query] + sql = [query, underscore_locale, like_query, like_query, like_query] sql.push @tag if first_letter - sql += [underscore_default_locale, like_query, like_query] + sql += [underscore_default_locale, like_query, like_query, like_query] sql.push @tag if first_letter sql += where_parameters @public_bodies = PublicBody.paginate_by_sql( @@ -170,17 +169,17 @@ class PublicBodyController < ApplicationController # The simpler case where we're just searching in the current locale: where_condition = get_public_body_list_translated_condition('public_body_translations', first_letter, true) + ' AND ' + where_condition - where_sql = [where_condition, like_query, like_query] + where_sql = [where_condition, like_query, like_query, like_query] where_sql.push @tag if first_letter where_sql += [underscore_locale] + where_parameters - @public_bodies = PublicBody.where(where_sql) \ - .joins(:translations) \ - .order("public_body_translations.name") \ - .paginate(:page => params[:page], :per_page => 100) + @public_bodies = PublicBody.where(where_sql). + joins(:translations). + order("public_body_translations.name"). + paginate(:page => params[:page], :per_page => 100) end respond_to do |format| - format.html { render :template => "public_body/list" } + format.html { render :template => 'public_body/list' } end end end @@ -191,6 +190,9 @@ class PublicBodyController < ApplicationController redirect_to list_public_bodies_url(:tag => @tag) end + # GET /body/all-authorities.csv + # + # Returns all public bodies (except for the internal admin authority) as CSV def list_all_csv # FIXME: this is just using the download directory for zip # archives, since we know that is allowed for X-Sendfile and @@ -198,21 +200,29 @@ class PublicBodyController < ApplicationController # used for the zips. However, really there should be a # generically named downloads directory that contains all # kinds of downloadable assets. - download_directory = File.join(InfoRequest.download_zip_dir(), - 'download') - FileUtils.mkdir_p download_directory + download_directory = File.join(InfoRequest.download_zip_dir, 'download') + FileUtils.mkdir_p(download_directory) output_leafname = 'all-authorities.csv' - output_filename = File.join download_directory, output_leafname + output_filename = File.join(download_directory, output_leafname) # Create a temporary file in the same directory, so we can # rename it atomically to the intended filename: - tmp = Tempfile.new output_leafname, download_directory + tmp = Tempfile.new(output_leafname, download_directory) tmp.close - # Export all the public bodies to that temporary path and make - # it readable: - PublicBody.export_csv tmp.path - FileUtils.chmod 0644, tmp.path - # Rename into place and send the file: - File.rename tmp.path, output_filename + + # Create the CSV + csv = PublicBodyCSV.new + PublicBody.visible.find_each(:include => [:translations, :tags]) do |public_body| + next if public_body.site_administration? + csv << public_body + end + + # Export all the public bodies to that temporary path, make it readable, + # and rename it + File.open(tmp.path, 'w') { |file| file.write(csv.generate) } + FileUtils.chmod(0644, tmp.path) + File.rename(tmp.path, output_filename) + + # Send the file send_file(output_filename, :type => 'text/csv; charset=utf-8; header=present', :filename => 'all-authorities.csv', @@ -344,9 +354,11 @@ class PublicBodyController < ApplicationController end private + def get_public_body_list_translated_condition(table, first_letter=false, locale=nil) result = "(upper(#{table}.name) LIKE upper(?)" \ - " OR upper(#{table}.notes) LIKE upper (?))" + " OR upper(#{table}.notes) LIKE upper(?)" \ + " OR upper(#{table}.short_name) LIKE upper(?))" if first_letter result += " AND #{table}.first_letter = ?" end diff --git a/app/controllers/request_controller.rb b/app/controllers/request_controller.rb index d66c28275..6281959fb 100644 --- a/app/controllers/request_controller.rb +++ b/app/controllers/request_controller.rb @@ -303,8 +303,14 @@ class RequestController < ApplicationController return render_new_compose(batch=false) end + # Check we have :public_body_id - spammers seem to be using :public_body + # erroneously instead + if params[:info_request][:public_body_id].blank? + redirect_to frontpage_path and return + end + # See if the exact same request has already been submitted - # XXX this check should theoretically be a validation rule in the + # TODO: this check should theoretically be a validation rule in the # model, except we really want to pass @existing_request to the view so # it can link to it. @existing_request = InfoRequest.find_existing(params[:info_request][:title], params[:info_request][:public_body_id], params[:outgoing_message][:body]) @@ -359,7 +365,7 @@ class RequestController < ApplicationController end # This automatically saves dependent objects, such as @outgoing_message, in the same transaction @info_request.save! - # XXX send_message needs the database id, so we send after saving, which isn't ideal if the request broke here. + # TODO: send_message needs the database id, so we send after saving, which isn't ideal if the request broke here. @outgoing_message.send_message flash[:notice] = _("<p>Your {{law_used_full}} request has been <strong>sent on its way</strong>!</p> <p><strong>We will email you</strong> when there is a response, or after {{late_number_of_days}} working days if the authority still hasn't @@ -537,7 +543,7 @@ class RequestController < ApplicationController elsif @info_request_event.is_outgoing_message? redirect_to outgoing_message_url(@info_request_event.outgoing_message), :status => :moved_permanently else - # XXX maybe there are better URLs for some events than this + # TODO: maybe there are better URLs for some events than this redirect_to request_url(@info_request_event.info_request), :status => :moved_permanently end end @@ -1006,7 +1012,7 @@ class RequestController < ApplicationController params[:info_request][:public_body] = PublicBody.find(params[:url_name]) else public_body = PublicBody.find_by_url_name_with_historic(params[:url_name]) - raise ActiveRecord::RecordNotFound.new("None found") if public_body.nil? # XXX proper 404 + raise ActiveRecord::RecordNotFound.new("None found") if public_body.nil? # TODO: proper 404 params[:info_request][:public_body] = public_body end elsif params[:public_body_id] diff --git a/app/controllers/services_controller.rb b/app/controllers/services_controller.rb index 97c47c448..dc4f783a6 100644 --- a/app/controllers/services_controller.rb +++ b/app/controllers/services_controller.rb @@ -31,7 +31,7 @@ class ServicesController < ApplicationController FastGettext.locale = old_fgt_locale end end - render :text => text, :content_type => "text/plain" # XXX workaround the HTML validation in test suite + render :text => text, :content_type => "text/plain" # TODO: workaround the HTML validation in test suite end def hidden_user_explanation diff --git a/app/controllers/track_controller.rb b/app/controllers/track_controller.rb index dccc52efc..83700a55b 100644 --- a/app/controllers/track_controller.rb +++ b/app/controllers/track_controller.rb @@ -82,7 +82,7 @@ class TrackController < ApplicationController def track_search_query @query = params[:query_array] - # XXX more hackery to make alternate formats still work with query_array + # TODO: more hackery to make alternate formats still work with query_array if /^(.*)\.json$/.match(@query) @query = $1 params[:format] = "json" @@ -154,7 +154,15 @@ class TrackController < ApplicationController request.format = 'xml' unless params[:format] respond_to do |format| format.json { render :json => @xapian_object.results.map { |r| r[:model].json_for_api(true, - lambda { |t| view_context.highlight_and_excerpt(t, @xapian_object.words_to_highlight, 150) } + lambda do |t| + view_context.highlight_and_excerpt( + t, + @xapian_object.words_to_highlight( + :regex => true, + :include_original => true), + 150 + ) + end ) } } format.any { render :template => 'track/atom_feed', :formats => ['atom'], diff --git a/app/controllers/user_controller.rb b/app/controllers/user_controller.rb index 8d6522923..fcc500e06 100644 --- a/app/controllers/user_controller.rb +++ b/app/controllers/user_controller.rb @@ -46,7 +46,7 @@ class UserController < ApplicationController @is_you = !@user.nil? && @user.id == @display_user.id # Use search query for this so can collapse and paginate easily - # XXX really should just use SQL query here rather than Xapian. + # TODO: really should just use SQL query here rather than Xapian. if @show_requests begin requests_query = 'requested_by:' + @display_user.url_name @@ -102,11 +102,11 @@ class UserController < ApplicationController @is_you = !@user.nil? && @user.id == @display_user.id feed_results = Set.new # Use search query for this so can collapse and paginate easily - # XXX really should just use SQL query here rather than Xapian. + # TODO: really should just use SQL query here rather than Xapian. begin requests_query = 'requested_by:' + @display_user.url_name comments_query = 'commented_by:' + @display_user.url_name - # XXX combine these as OR query + # TODO: combine these as OR query @xapian_requests = perform_search([InfoRequestEvent], requests_query, 'newest', 'request_collapse') @xapian_comments = perform_search([InfoRequestEvent], comments_query, 'newest', nil) rescue @@ -121,7 +121,7 @@ class UserController < ApplicationController if @is_you @track_things = TrackThing.find(:all, :conditions => ["tracking_user_id = ? and track_medium = ?", @display_user.id, 'email_daily'], :order => 'created_at desc') for track_thing in @track_things - # XXX factor out of track_mailer.rb + # TODO: factor out of track_mailer.rb xapian_object = ActsAsXapian::Search.new([InfoRequestEvent], track_thing.track_query, :sort_by_prefix => 'described_at', :sort_by_ascending => true, @@ -262,7 +262,7 @@ class UserController < ApplicationController end end - # Change password (XXX and perhaps later email) - requires email authentication + # Change password (TODO: and perhaps later email) - requires email authentication def signchangepassword if @user and ((not session[:user_circumstance]) or (session[:user_circumstance] != "change_password")) # Not logged in via email, so send confirmation @@ -288,7 +288,7 @@ class UserController < ApplicationController :reason_params => { :web => "", :email => _("Then you can change your password on {{site_name}}",:site_name=>site_name), - :email_subject => _("Change your password {{site_name}}",:site_name=>site_name) + :email_subject => _("Change your password on {{site_name}}",:site_name=>site_name) }, :circumstance => "change_password" # special login that lets you change your password ) diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 33525cb3d..49ce94951 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -13,12 +13,18 @@ module ApplicationHelper # all of all. include LinkToHelper + # Some extra date and time formatters + include DateTimeHelper + # Site-wide access to configuration settings include ConfigHelper # Useful for sending emails include MailerHelper + # Extra highlight helpers + include HighlightHelper + # Copied from error_messages_for in active_record_helper.rb def foi_error_messages_for(*params) options = params.last.is_a?(Hash) ? params.pop.symbolize_keys : {} @@ -51,25 +57,6 @@ module ApplicationHelper end end - # Highlight words, also escapes HTML (other than spans that we add) - def highlight_words(t, words, html = true) - if html - highlight(h(t), words, :highlighter => '<span class="highlight">\1</span>').html_safe - else - highlight(t, words, :highlighter => '*\1*') - end - end - - def highlight_and_excerpt(t, words, excount, html = true) - newt = excerpt(t, words[0], :radius => excount) - if not newt - newt = excerpt(t, '', :radius => excount) - end - t = newt - t = highlight_words(t, words, html) - return t - end - def locale_name(locale) return LanguageNames::get_language_name(locale) end diff --git a/app/helpers/date_time_helper.rb b/app/helpers/date_time_helper.rb new file mode 100644 index 000000000..5f129e590 --- /dev/null +++ b/app/helpers/date_time_helper.rb @@ -0,0 +1,69 @@ +module DateTimeHelper + # Public: Usually-correct format for a DateTime-ish object + # To define a new new format define the `simple_date_{FORMAT}` method + # + # date - a DateTime, Date or Time + # opts - a Hash of options (default: { format: :html}) + # :format - :html returns a HTML <time> tag + # :text returns a plain String + # + # Examples + # + # simple_date(Time.now) + # # => "<time>..." + # + # simple_date(Time.now, :format => :text) + # # => "March 10, 2014" + # + # Returns a String + # Raises ArgumentError if the format is unrecognized + def simple_date(date, opts = {}) + opts = { :format => :html }.merge(opts) + date_formatter = "simple_date_#{ opts[:format] }" + + if respond_to?(date_formatter) + send(date_formatter, date) + else + raise ArgumentError, "Unrecognized format :#{ opts[:format] }" + end + end + + # Usually-correct HTML formatting of a DateTime-ish object + # Use LinkToHelper#simple_date with desired formatting options + # + # date - a DateTime, Date or Time + # + # Returns a String + def simple_date_html(date) + date = date.in_time_zone unless date.is_a?(Date) + time_tag date, simple_date_text(date), :title => date.to_s + end + + # Usually-correct plain text formatting of a DateTime-ish object + # Use LinkToHelper#simple_date with desired formatting options + # + # date - a DateTime, Date or Time + # + # Returns a String + def simple_date_text(date) + date = date.in_time_zone.to_date unless date.is_a? Date + + date_format = _('simple_date_format') + date_format = :long if date_format == 'simple_date_format' + I18n.l(date, :format => date_format) + end + + # Strips the date from a DateTime + # + # date - a DateTime, Date or Time + # + # Examples + # + # simple_time(Time.now) + # # => "10:46:54" + # + # Returns a String + def simple_time(date) + date.strftime("%H:%M:%S").strip + end +end diff --git a/app/helpers/highlight_helper.rb b/app/helpers/highlight_helper.rb new file mode 100644 index 000000000..a98f6f320 --- /dev/null +++ b/app/helpers/highlight_helper.rb @@ -0,0 +1,98 @@ +module HighlightHelper + include ERB::Util + + # Implementation of rails' highlight that allows regex to be passed to + # the phrases parameter. + # https://github.com/rails/rails/pull/11793 + def highlight_matches(text, phrases, options = {}) + text = ActionController::Base.helpers.sanitize(text).try(:html_safe) if options.fetch(:sanitize, true) + + if text.blank? || phrases.blank? + text + else + match = Array(phrases).map do |p| + Regexp === p ? p.to_s : Regexp.escape(p) + end.join('|') + + if block_given? + text.gsub(/(#{match})(?![^<]*?>)/i) { |found| yield found } + else + highlighter = options.fetch(:highlighter, '<mark>\1</mark>') + text.gsub(/(#{match})(?![^<]*?>)/i, highlighter) + end + end.html_safe + end + + # Highlight words, also escapes HTML (other than spans that we add) + def highlight_words(t, words, html = true) + if html + highlight_matches(h(t), words, :highlighter => '<span class="highlight">\1</span>').html_safe + else + highlight_matches(t, words, :highlighter => '*\1*') + end + end + + def highlight_and_excerpt(t, words, excount, html = true) + newt = excerpt(t, words[0], :radius => excount) + if not newt + newt = excerpt(t, '', :radius => excount) + end + t = newt + t = highlight_words(t, words, html) + return t + end + + def excerpt(text, phrase, options = {}) + return unless text && phrase + + separator = options.fetch(:separator, nil) || "" + case phrase + when Regexp + regex = phrase + else + regex = /#{Regexp.escape(phrase)}/i + end + + return unless matches = text.match(regex) + phrase = matches[0] + + unless separator.empty? + text.split(separator).each do |value| + if value.match(regex) + regex = phrase = value + break + end + end + end + + first_part, second_part = text.split(phrase, 2) + + prefix, first_part = cut_excerpt_part(:first, first_part, separator, options) + postfix, second_part = cut_excerpt_part(:second, second_part, separator, options) + + affix = [first_part, separator, phrase, separator, second_part].join.strip + [prefix, affix, postfix].join + end + + private + + def cut_excerpt_part(part_position, part, separator, options) + return "", "" unless part + + radius = options.fetch(:radius, 100) + omission = options.fetch(:omission, "...") + + part = part.split(separator) + part.delete("") + affix = part.size > radius ? omission : "" + + part = if part_position == :first + drop_index = [part.length - radius, 0].max + part.drop(drop_index) + else + part.first(radius) + end + + return affix, part.join(separator) + end +end diff --git a/app/helpers/link_to_helper.rb b/app/helpers/link_to_helper.rb index dd6ffa805..3709469cf 100755 --- a/app/helpers/link_to_helper.rb +++ b/app/helpers/link_to_helper.rb @@ -28,19 +28,19 @@ module LinkToHelper # Incoming / outgoing messages def incoming_message_url(incoming_message, options = {}) - return request_url(incoming_message.info_request, options.merge(:anchor => "incoming-#{incoming_message.id}")) + message_url(incoming_message, options) end def incoming_message_path(incoming_message) - incoming_message_url(incoming_message, :only_path => true) + message_path(incoming_message) end def outgoing_message_url(outgoing_message, options = {}) - request_url(outgoing_message.info_request, options.merge(:anchor => "outgoing-#{outgoing_message.id}")) + message_url(outgoing_message, options) end def outgoing_message_path(outgoing_message) - outgoing_message_url(outgoing_message, :only_path => true) + message_path(outgoing_message) end def comment_url(comment, options = {}) @@ -279,73 +279,30 @@ module LinkToHelper end end - # Public: Usually-correct format for a DateTime-ish object - # To define a new new format define the `simple_date_{FORMAT}` method - # - # date - a DateTime, Date or Time - # opts - a Hash of options (default: { format: :html}) - # :format - :html returns a HTML <time> tag - # :text returns a plain String - # - # Examples - # - # simple_date(Time.now) - # # => "<time>..." - # - # simple_date(Time.now, :format => :text) - # # => "March 10, 2014" - # - # Returns a String - # Raises ArgumentError if the format is unrecognized - def simple_date(date, opts = {}) - opts = { :format => :html }.merge(opts) - date_formatter = "simple_date_#{ opts[:format] }" - - if respond_to?(date_formatter) - send(date_formatter, date) - else - raise ArgumentError, "Unrecognised format :#{ opts[:format] }" - end - end + #I18n locale switcher - # Usually-correct HTML formatting of a DateTime-ish object - # Use LinkToHelper#simple_date with desired formatting options - # - # date - a DateTime, Date or Time - # - # Returns a String - def simple_date_html(date) - date = date.in_time_zone unless date.is_a? Date - time_tag date, simple_date_text(date), :title => date.to_s + def locale_switcher(locale, params) + params['locale'] = locale + return url_for(params) end - # Usually-correct plain text formatting of a DateTime-ish object - # Use LinkToHelper#simple_date with desired formatting options - # - # date - a DateTime, Date or Time - # - # Returns a String - def simple_date_text(date) - date = date.in_time_zone.to_date unless date.is_a? Date + private - date_format = _("simple_date_format") - date_format = :long if date_format == "simple_date_format" - I18n.l(date, :format => date_format) - end + # Private: Generate a request_url linking to the new correspondence + def message_url(message, options = {}) + message_type = message.class.to_s.gsub('Message', '').downcase - def simple_time(date) - return date.strftime("%H:%M:%S").strip - end + default_options = { :anchor => "#{ message_type }-#{ message.id }" } - def year_from_date(date) - return date.strftime("%Y").strip - end + if options.delete(:cachebust) + default_options.merge!(:nocache => "#{ message_type }-#{ message.id }") + end - #I18n locale switcher + request_url(message.info_request, options.merge(default_options)) + end - def locale_switcher(locale, params) - params['locale'] = locale - return url_for(params) + def message_path(message) + message_url(message, :only_path => true) end end diff --git a/app/mailers/outgoing_mailer.rb b/app/mailers/outgoing_mailer.rb index 083c05a7c..797bf9fdd 100644 --- a/app/mailers/outgoing_mailer.rb +++ b/app/mailers/outgoing_mailer.rb @@ -8,7 +8,7 @@ # separated) paragraphs, as is the convention for all the other mailers. This # turned out to fit better with user exepectations when formatting messages. # -# XXX The other mail templates are written to use blank line separated +# TODO: The other mail templates are written to use blank line separated # paragraphs. They could be rewritten, and the wrapping method made uniform # throughout the application. @@ -35,10 +35,10 @@ class OutgoingMailer < ApplicationMailer :subject => OutgoingMailer.subject_for_followup(info_request, outgoing_message)) end - # XXX the condition checking valid_to_reply_to? also appears in views/request/_followup.html.erb, + # TODO: the condition checking valid_to_reply_to? also appears in views/request/_followup.html.erb, # it shouldn't really, should call something here. - # XXX also OutgoingMessage.get_salutation - # XXX these look like they should be members of IncomingMessage, but logically they + # TODO: also OutgoingMessage.get_salutation + # TODO: these look like they should be members of IncomingMessage, but logically they # need to work even when IncomingMessage is nil def OutgoingMailer.name_and_email_for_followup(info_request, incoming_message_followup) if incoming_message_followup.nil? || !incoming_message_followup.valid_to_reply_to? diff --git a/app/mailers/request_mailer.rb b/app/mailers/request_mailer.rb index 1fd5b9ba7..768257ba8 100644 --- a/app/mailers/request_mailer.rb +++ b/app/mailers/request_mailer.rb @@ -71,7 +71,7 @@ class RequestMailer < ApplicationMailer def new_response(info_request, incoming_message) # Don't use login link here, just send actual URL. This is # because people tend to forward these emails amongst themselves. - @url = incoming_message_url(incoming_message) + @url = incoming_message_url(incoming_message, :cachebust => true) @incoming_message, @info_request = incoming_message, info_request headers('Return-Path' => blackhole_email, @@ -234,7 +234,7 @@ class RequestMailer < ApplicationMailer def requests_matching_email(email) # We deliberately don't use Envelope-to here, so ones that are BCC # drop into the holding pen for checking. - reply_info_requests = [] # XXX should be set? + reply_info_requests = [] # TODO: should be set? for address in (email.to || []) + (email.cc || []) reply_info_request = InfoRequest.find_by_incoming_email(address) reply_info_requests.push(reply_info_request) if reply_info_request @@ -362,7 +362,7 @@ class RequestMailer < ApplicationMailer store_sent.user = info_request.user store_sent.alert_type = type_code store_sent.info_request_event_id = alert_event_id - # XXX uses same template for reminder 1 and reminder 2 right now. + # TODO: uses same template for reminder 1 and reminder 2 right now. RequestMailer.new_response_reminder_alert(info_request, last_response_message).deliver store_sent.save! end @@ -405,7 +405,7 @@ class RequestMailer < ApplicationMailer # cron jobs broke for more than a month events would be lost, but no # matter. I suspect the performance gain will be needed (with an index on updated_at) - # XXX the :order part info_request_events.created_at is a work around + # TODO: the :order part info_request_events.created_at is a work around # for a very old Rails bug which means eager loading does not respect # association orders. # http://dev.rubyonrails.org/ticket/3438 diff --git a/app/models/comment.rb b/app/models/comment.rb index b4c099123..a62c086d5 100644 --- a/app/models/comment.rb +++ b/app/models/comment.rb @@ -63,7 +63,7 @@ class Comment < ActiveRecord::Base # When posting a new comment, use this to check user hasn't double submitted. def Comment.find_existing(info_request_id, body) - # XXX can add other databases here which have regexp_replace + # TODO: can add other databases here which have regexp_replace if ActiveRecord::Base.connection.adapter_name == "PostgreSQL" # Exclude spaces from the body comparison using regexp_replace return Comment.find(:first, :conditions => [ "info_request_id = ? and regexp_replace(body, '[[:space:]]', '', 'g') = regexp_replace(?, '[[:space:]]', '', 'g')", info_request_id, body ]) diff --git a/app/models/foi_attachment.rb b/app/models/foi_attachment.rb index 6f198249a..a8d105f52 100644 --- a/app/models/foi_attachment.rb +++ b/app/models/foi_attachment.rb @@ -178,7 +178,7 @@ class FoiAttachment < ActiveRecord::Base return filename end - # XXX changing this will break existing URLs, so have a care - maybe + # TODO: changing this will break existing URLs, so have a care - maybe # make another old_display_filename see above def display_filename filename = self.filename diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb index 124db8d4a..135a6bdaf 100644 --- a/app/models/incoming_message.rb +++ b/app/models/incoming_message.rb @@ -150,7 +150,7 @@ class IncomingMessage < ActiveRecord::Base end # The cached fields mentioned in the previous comment - # XXX there must be a nicer way to do this without all that + # TODO: there must be a nicer way to do this without all that # repetition. I tried overriding method_missing but got some # unpredictable results. def valid_to_reply_to @@ -194,7 +194,7 @@ class IncomingMessage < ActiveRecord::Base end # And look up by URL part number and display filename to get an attachment - # XXX relies on extract_attachments calling MailHandler.ensure_parts_counted + # TODO: relies on extract_attachments calling MailHandler.ensure_parts_counted # The filename here is passed from the URL parameter, so it's the # display_filename rather than the real filename. def self.get_attachment_by_url_part_number_and_filename(attachments, found_url_part_number, display_filename) @@ -220,7 +220,7 @@ class IncomingMessage < ActiveRecord::Base # Converts email addresses we know about into textual descriptions of them def mask_special_emails!(text) - # XXX can later display some of these special emails as actual emails, + # TODO: can later display some of these special emails as actual emails, # if they are public anyway. For now just be precautionary and only # put in descriptions of them in square brackets. if self.info_request.public_body.is_followupable? @@ -368,8 +368,8 @@ class IncomingMessage < ActiveRecord::Base # Remove quoted sections from emails (eventually the aim would be for this - # to do as good a job as GMail does) XXX bet it needs a proper parser - # XXX and this FOLDED_QUOTED_SECTION stuff is a mess + # to do as good a job as GMail does) TODO: bet it needs a proper parser + # TODO: and this FOLDED_QUOTED_SECTION stuff is a mess def self.remove_quoted_sections(text, replacement = "FOLDED_QUOTED_SECTION") text = text.dup replacement = "\n" + replacement + "\n" @@ -399,7 +399,7 @@ class IncomingMessage < ActiveRecord::Base ( \s*#{score}\n(?:(?!#{score}\n).)*? # top line (disclaimer:\n|confidential|received\sthis\semail\sin\serror|virus|intended\s+recipient|monitored\s+centrally|intended\s+(for\s+|only\s+for\s+use\s+by\s+)the\s+addressee|routinely\s+monitored|MessageLabs|unauthorised\s+use) - .*?(?:#{score}|\z) # bottom line OR end of whole string (for ones with no terminator XXX risky) + .*?(?:#{score}|\z) # bottom line OR end of whole string (for ones with no terminator TODO: risky) ) /imx, replacement) end @@ -480,7 +480,7 @@ class IncomingMessage < ActiveRecord::Base # Returns body text from main text part of email, converted to UTF-8, with uudecode removed, # emails and privacy sensitive things remove, censored, and folded to remove excess quoted text # (marked with FOLDED_QUOTED_SECTION) - # XXX returns a .dup of the text, so calling functions can in place modify it + # TODO: returns a .dup of the text, so calling functions can in place modify it def get_main_body_text_folded if self.cached_main_body_text_folded.nil? self._cache_main_body_text @@ -511,7 +511,7 @@ class IncomingMessage < ActiveRecord::Base source_charset = part.charset if part.content_type == 'text/html' # e.g. http://www.whatdotheyknow.com/request/35/response/177 - # XXX This is a bit of a hack as it is calling a + # TODO: This is a bit of a hack as it is calling a # convert to text routine. Could instead call a # sanitize HTML one. @@ -627,7 +627,7 @@ class IncomingMessage < ActiveRecord::Base return nil end # otherwise return it assuming it is text (sometimes you get things - # like binary/octet-stream, or the like, which are really text - XXX if + # like binary/octet-stream, or the like, which are really text - TODO: if # you find an example, put URL here - perhaps we should be always returning # nil in this case) return p @@ -722,7 +722,7 @@ class IncomingMessage < ActiveRecord::Base text = get_main_body_text_unfolded folded_quoted_text = get_main_body_text_folded - # Remove quoted sections, adding HTML. XXX The FOLDED_QUOTED_SECTION is + # Remove quoted sections, adding HTML. TODO: The FOLDED_QUOTED_SECTION is # a nasty hack so we can escape other HTML before adding the unfold # links, without escaping them. Rather than using some proper parser # making a tree structure (I don't know of one that is to hand, that diff --git a/app/models/info_request.rb b/app/models/info_request.rb index 47ad435cb..a2112a210 100644 --- a/app/models/info_request.rb +++ b/app/models/info_request.rb @@ -387,16 +387,16 @@ public # When constructing a new request, use this to check user hasn't double submitted. - # XXX could have a date range here, so say only check last month's worth of new requests. If somebody is making + # TODO: could have a date range here, so say only check last month's worth of new requests. If somebody is making # repeated requests, say once a quarter for time information, then might need to do that. - # XXX this *should* also check outgoing message joined to is an initial + # TODO: this *should* also check outgoing message joined to is an initial # request (rather than follow up) def InfoRequest.find_existing(title, public_body_id, body) return InfoRequest.find(:first, :conditions => [ "title = ? and public_body_id = ? and outgoing_messages.body = ?", title, public_body_id, body ], :include => [ :outgoing_messages ] ) end def find_existing_outgoing_message(body) - # XXX can add other databases here which have regexp_replace + # TODO: can add other databases here which have regexp_replace if ActiveRecord::Base.connection.adapter_name == "PostgreSQL" # Exclude spaces from the body comparison using regexp_replace return self.outgoing_messages.find(:first, :conditions => [ "regexp_replace(outgoing_messages.body, '[[:space:]]', '', 'g') = regexp_replace(?, '[[:space:]]', '', 'g')", body ]) @@ -658,7 +658,7 @@ public event.last_described_at = Time.now() event.save! end - if event.last_described_at.nil? # XXX actually maybe this isn't needed + if event.last_described_at.nil? # TODO: actually maybe this isn't needed event.last_described_at = Time.now() event.save! end @@ -713,7 +713,7 @@ public elsif event.event_type == 'resent' last_sent = event elsif expecting_clarification and event.event_type == 'followup_sent' - # XXX this needs to cope with followup_resent, which it doesn't. + # TODO: this needs to cope with followup_resent, which it doesn't. # Not really easy to do, and only affects cases where followups # were resent after a clarification. last_sent = event diff --git a/app/models/info_request_event.rb b/app/models/info_request_event.rb index 5eed5ba83..9dde3ba80 100644 --- a/app/models/info_request_event.rb +++ b/app/models/info_request_event.rb @@ -75,7 +75,7 @@ class InfoRequestEvent < ActiveRecord::Base :values => [ [ :created_at, 0, "range_search", :date ], # for QueryParser range searches e.g. 01/01/2008..14/01/2008 [ :created_at_numeric, 1, "created_at", :number ], # for sorting - [ :described_at_numeric, 2, "described_at", :number ], # XXX using :number for lack of :datetime support in Xapian values + [ :described_at_numeric, 2, "described_at", :number ], # TODO: using :number for lack of :datetime support in Xapian values [ :request, 3, "request_collapse", :string ], [ :request_title_collapse, 4, "request_title_collapse", :string ], ], @@ -174,7 +174,7 @@ class InfoRequestEvent < ActiveRecord::Base end def get_clipped_response_efficiently - # XXX this ugly code is an attempt to not always load all the + # TODO: this ugly code is an attempt to not always load all the # columns for an incoming message, which can be *very* large # (due to all the cached text). We care particularly in this # case because it's called for every search result on a page @@ -266,7 +266,7 @@ class InfoRequestEvent < ActiveRecord::Base # We store YAML version of parameters in the database def params=(params) - # XXX should really set these explicitly, and stop storing them in + # TODO: should really set these explicitly, and stop storing them in # here, but keep it for compatibility with old way for now if not params[:incoming_message_id].nil? self.incoming_message_id = params[:incoming_message_id] @@ -392,7 +392,7 @@ class InfoRequestEvent < ActiveRecord::Base :outgoing_message_id => self.outgoing_message_id, :comment_id => self.comment_id, - # XXX would be nice to add links here, but alas the + # TODO: would be nice to add links here, but alas the # code to make them is in views only. See views/request/details.html.erb # perhaps can call with @template somehow } diff --git a/app/models/mail_server_log.rb b/app/models/mail_server_log.rb index 0e5b60ff1..07d2fdac0 100644 --- a/app/models/mail_server_log.rb +++ b/app/models/mail_server_log.rb @@ -166,7 +166,7 @@ class MailServerLog < ActiveRecord::Base # lines. Writes any errors to STDERR. This check is really mainly to # check the envelope from is the request address, as Ruby is quite # flaky with regard to that, and it is important for anti-spam reasons. - # XXX does this really check that, as the log just wouldn't pick + # TODO: does this really check that, as the log just wouldn't pick # up at all if the requests weren't sent that way as there would be # no request- email in it? # diff --git a/app/models/outgoing_message.rb b/app/models/outgoing_message.rb index a435511d3..160f69d0b 100644 --- a/app/models/outgoing_message.rb +++ b/app/models/outgoing_message.rb @@ -125,7 +125,7 @@ class OutgoingMessage < ActiveRecord::Base get_salutation + "\n\n" + get_default_letter + "\n\n" + get_signoff + "\n\n" end def set_signature_name(name) - # XXX We use raw_body here to get unstripped one + # TODO: We use raw_body here to get unstripped one if self.raw_body == self.get_default_message self.body = self.raw_body + name end diff --git a/app/models/post_redirect.rb b/app/models/post_redirect.rb index 5da3d2742..6f288b471 100644 --- a/app/models/post_redirect.rb +++ b/app/models/post_redirect.rb @@ -65,7 +65,7 @@ class PostRedirect < ActiveRecord::Base # Used by (rspec) test code only def self.get_last_post_redirect - # XXX yeuch - no other easy way of getting the token so we can check + # TODO: yeuch - no other easy way of getting the token so we can check # the redirect URL, as it is by definition opaque to the controller # apart from in the place that it redirects to. post_redirects = PostRedirect.find_by_sql("select * from post_redirects order by id desc limit 1") diff --git a/app/models/profile_photo.rb b/app/models/profile_photo.rb index 6c3b2cfa0..3c0be222c 100644 --- a/app/models/profile_photo.rb +++ b/app/models/profile_photo.rb @@ -115,7 +115,7 @@ class ProfilePhoto < ActiveRecord::Base return end - self.image = image_list[0] # XXX perhaps take largest image or somesuch if there were multiple in the file? + self.image = image_list[0] # TODO: perhaps take largest image or somesuch if there were multiple in the file? self.convert_image end end diff --git a/app/models/public_body.rb b/app/models/public_body.rb index 03ec270ee..b22482541 100644 --- a/app/models/public_body.rb +++ b/app/models/public_body.rb @@ -93,7 +93,7 @@ class PublicBody < ActiveRecord::Base self.translations.find_by_locale(locale) end - # XXX - Don't like repeating this! + # TODO: - Don't like repeating this! def calculate_cached_fields(t) PublicBody.set_first_letter(t) short_long_name = t.name @@ -329,7 +329,7 @@ class PublicBody < ActiveRecord::Base first = false end if html - # XXX this should call proper route helpers, but is in model sigh + # TODO: this should call proper route helpers, but is in model sigh desc = '<a href="/body/list/' + tag.name + '">' + desc + '</a>' end types.push(desc) @@ -399,6 +399,9 @@ class PublicBody < ActiveRecord::Base end end + def site_administration? + has_tag?('site_administration') + end class ImportCSVDryRun < StandardError end @@ -569,45 +572,6 @@ class PublicBody < ActiveRecord::Base return [errors, notes] end - # Returns all public bodies (except for the internal admin authority) as csv - def self.export_csv(output_filename) - CSV.open(output_filename, "w") do |csv| - csv << [ - 'Name', - 'Short name', - # deliberately not including 'Request email' - 'URL name', - 'Tags', - 'Home page', - 'Publication scheme', - 'Disclosure log', - 'Notes', - 'Created at', - 'Updated at', - 'Version', - ] - PublicBody.visible.find_each(:include => [:translations, :tags]) do |public_body| - # Skip bodies we use only for site admin - next if public_body.has_tag?('site_administration') - csv << [ - public_body.name, - public_body.short_name, - # DO NOT include request_email (we don't want to make it - # easy to spam all authorities with requests) - public_body.url_name, - public_body.tag_string, - public_body.calculated_home_page, - public_body.publication_scheme, - public_body.disclosure_log, - public_body.notes, - public_body.created_at, - public_body.updated_at, - public_body.version, - ] - end - end - end - # Does this user have the power of FOI officer for this body? def is_foi_officer?(user) user_domain = user.email_domain diff --git a/app/models/track_thing.rb b/app/models/track_thing.rb index 13b6f78dd..10ba28f4a 100644 --- a/app/models/track_thing.rb +++ b/app/models/track_thing.rb @@ -149,7 +149,7 @@ class TrackThing < ActiveRecord::Base end end track_thing.track_query = query - # XXX should extract requested_by:, request:, requested_from: + # TODO: should extract requested_by:, request:, requested_from: # and stick their values into the respective relations. # Should also update "params" to make the list_description # nicer and more generic. It will need to do some clever @@ -271,7 +271,7 @@ class TrackThing < ActiveRecord::Base :web => _("To follow requests and responses matching your search"), :email => _("Then you will be notified whenever a new request or response matches your search."), :email_subject => _("Confirm you want to follow new requests or responses matching your search"), - # RSS sorting - XXX hmmm, we don't really know which to use + # RSS sorting - TODO: hmmm, we don't really know which to use # here for sorting. Might be a query term (e.g. 'cctv'), in # which case newest is good, or might be something like # all refused requests in which case want to sort by diff --git a/app/models/user.rb b/app/models/user.rb index d75622b37..4b83d8572 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -99,7 +99,7 @@ class User < ActiveRecord::Base end # Don't display any leading/trailing spaces - # XXX we have strip_attributes! now, so perhaps this can be removed (might + # TODO: we have strip_attributes! now, so perhaps this can be removed (might # be still needed for existing cases) def name name = read_attribute(:name) @@ -222,7 +222,7 @@ class User < ActiveRecord::Base # Can the user make new requests, without having to describe state of (most) existing ones? def can_leave_requests_undescribed? - # XXX should be flag in database really + # TODO: should be flag in database really if self.url_name == "heather_brooke" || self.url_name == "heather_brooke_2" return true end @@ -425,7 +425,7 @@ class User < ActiveRecord::Base ## Class methods def User.encrypted_password(password, salt) - string_to_hash = password + salt # XXX need to add a secret here too? + string_to_hash = password + salt # TODO: need to add a secret here too? Digest::SHA1.hexdigest(string_to_hash) end diff --git a/app/views/admin_request/edit.html.erb b/app/views/admin_request/edit.html.erb index 0e9c68aea..552b31bbb 100644 --- a/app/views/admin_request/edit.html.erb +++ b/app/views/admin_request/edit.html.erb @@ -33,7 +33,7 @@ <p><label for="info_request_tag_string"><strong>Tags</strong> <small>(space separated, can use key:value)</small></label><br/> <%= text_field 'info_request', 'tag_string', :size => 60 %></p> - <p><%= submit_tag 'Save changes', :accesskey => 's' %> + <p><%= submit_tag 'Save changes', :accesskey => 's', :class => 'btn btn-primary' %> </p> <p><strong>Note:</strong> To edit the actual request body text, click edit @@ -49,13 +49,17 @@ <hr> +<div class="well"> + <%= form_tag admin_request_destroy_path(@info_request) do %> <p> - <strong>This is permanent and irreversible!</strong> <%= submit_tag 'Destroy request entirely' %> + <strong>This is permanent and irreversible!</strong> <%= submit_tag 'Destroy request entirely', :class => 'btn btn-danger' %> <br>Use it mainly if someone posts private information, e.g. made a Data Protection request. It destroys all responses and tracks as well. </p> <% end %> +</div> + <hr> diff --git a/app/views/general/_frontpage_bodies_list.html.erb b/app/views/general/_frontpage_bodies_list.html.erb index 44321f14a..1c35c55ad 100644 --- a/app/views/general/_frontpage_bodies_list.html.erb +++ b/app/views/general/_frontpage_bodies_list.html.erb @@ -3,9 +3,10 @@ <div id="examples_0"> <h3><%= _("Who can I request information from?") %></h3> <%= _("{{site_name}} covers requests to {{number_of_authorities}} authorities, including:", - :site_name => site_name, :number_of_authorities => PublicBody.visible.count) %> + :site_name => site_name, + :number_of_authorities => number_with_delimiter(PublicBody.visible.count)) %> <ul> - <% for popular_body in popular_bodies %> + <% popular_bodies.each do |popular_body| %> <li><%=public_body_link(popular_body)%> <%= n_('{{count}} request', '{{count}} requests', popular_body.info_requests_count, :count => popular_body.info_requests_count) %> </li> diff --git a/app/views/general/_frontpage_new_request.html.erb b/app/views/general/_frontpage_new_request.html.erb index 499b60eb5..5b987b320 100644 --- a/app/views/general/_frontpage_new_request.html.erb +++ b/app/views/general/_frontpage_new_request.html.erb @@ -4,4 +4,4 @@ Information<br/> request</strong>") %> </h1> -<a class="link_button_green_large" href="<%= select_authority_path %>"><%= _("Start now »") %></a> +<a class="link_button_green_large" href="<%= select_authority_path %>"><%= _("Make a request »") %></a> diff --git a/app/views/general/_frontpage_requests_list.html.erb b/app/views/general/_frontpage_requests_list.html.erb index d7d9184c4..39187f3f0 100644 --- a/app/views/general/_frontpage_requests_list.html.erb +++ b/app/views/general/_frontpage_requests_list.html.erb @@ -8,9 +8,10 @@ <% end %> </h3> <%= _("{{site_name}} users have made {{number_of_requests}} requests, including:", - :site_name => site_name, :number_of_requests => InfoRequest.visible.count) %> + :site_name => site_name, + :number_of_requests => number_with_delimiter(InfoRequest.visible.count)) %> <ul> - <% for event in @request_events %> + <% @request_events.each do |event| %> <li> <% if @request_events_all_successful %> <%= _("{{public_body_link}} answered a request about", diff --git a/app/views/general/_frontpage_search_box.html.erb b/app/views/general/_frontpage_search_box.html.erb index 890602416..f77bd97fc 100644 --- a/app/views/general/_frontpage_search_box.html.erb +++ b/app/views/general/_frontpage_search_box.html.erb @@ -2,7 +2,8 @@ <%= _("Search over<br/> <strong>{{number_of_requests}} requests</strong> <span>and</span><br/> <strong>{{number_of_authorities}} authorities</strong>", - :number_of_requests => InfoRequest.visible.count, :number_of_authorities => PublicBody.visible.count) %> + :number_of_requests => number_with_delimiter(InfoRequest.visible.count), + :number_of_authorities => number_with_delimiter(PublicBody.visible.count)) %> </h2> <form id="search_form" method="post" action="<%= search_redirect_path %>"> <div> diff --git a/app/views/general/_locale_switcher.html.erb b/app/views/general/_locale_switcher.html.erb index a318f61f3..7b6377665 100644 --- a/app/views/general/_locale_switcher.html.erb +++ b/app/views/general/_locale_switcher.html.erb @@ -1,7 +1,7 @@ <% if FastGettext.default_available_locales.length > 1 && !params.empty? %> <div id="user_locale_switcher"> <div class="btn-group"> - <% for possible_locale in FastGettext.default_available_locales %> + <% FastGettext.default_available_locales.each do |possible_locale| %> <% if possible_locale == FastGettext.locale %> <a href="#" class="btn disabled"><%= locale_name(possible_locale) %></a> <% else %> diff --git a/app/views/public_body/_list_sidebar_extra.html.erb b/app/views/public_body/_list_sidebar_extra.html.erb index 290593d6a..6e683d7a1 100644 --- a/app/views/public_body/_list_sidebar_extra.html.erb +++ b/app/views/public_body/_list_sidebar_extra.html.erb @@ -1,3 +1,8 @@ +<% if AlaveteliConfiguration::public_body_statistics_page %> + <p> + <%= link_to _('Public authority statistics'), public_bodies_statistics_path %> + </p> +<% end %> <p> <%= link_to _('Are we missing a public authority?'), help_requesting_path + '#missing_body' %> </p> diff --git a/app/views/public_body/show.html.erb b/app/views/public_body/show.html.erb index a9c50e657..9352747ea 100644 --- a/app/views/public_body/show.html.erb +++ b/app/views/public_body/show.html.erb @@ -127,7 +127,10 @@ <% if @xapian_requests.results.empty? %> <p><% _('There were no requests matching your query.') %></p> <% else %> - <p> <%= _('Only requests made using {{site_name}} are shown.', :site_name => site_name) %></p> + <p> + <%= _('Only requests made using {{site_name}} are shown.', :site_name => site_name) %> + <%= link_to _('?'), help_about_path %> + </p> <% end %> <% else %> diff --git a/app/views/public_body/statistics.html.erb b/app/views/public_body/statistics.html.erb index d935a9e47..0e7e7424e 100644 --- a/app/views/public_body/statistics.html.erb +++ b/app/views/public_body/statistics.html.erb @@ -1,6 +1,6 @@ <% @title = _("Public Body Statistics") %> <div id="main_content"> - <h1>Public Body Statistics</h1> + <h1><%= @title %></h1> <p><%= _("This page of public body statistics is currently \ experimental, so there are some caveats that should be borne \ @@ -33,6 +33,8 @@ requests to the authority through this site, rather than, \ say, all requests that have been made to the public body by \ any means.") %></li> + <li><%= _("Unclassified or hidden requests are not counted.") %></li> + </ul> <p><%= _("These graphs were partly inspired by \ @@ -47,7 +49,7 @@ are due to him.") %></p> <table border=0> <thead> <tr> - <th>Public Body</th> + <th><%= _('Public Body') %></th> <th><%= graph_data['y_axis'] %></th> </tr> </thead> diff --git a/app/views/request/upload_response.html.erb b/app/views/request/upload_response.html.erb index f5fd6f000..f07af34d5 100644 --- a/app/views/request/upload_response.html.erb +++ b/app/views/request/upload_response.html.erb @@ -9,7 +9,9 @@ <%= foi_error_messages_for :comment %> - <h1><%= _('Respond to the FOI request')%> '<%=request_link(@info_request)%>'<% _(' made by ')%><%=user_link(@info_request.user) %></h1> + <h1><%= _("Respond to the FOI request '{{request}}' made by {{user}}", + :request => request_link(@info_request), + :user => user_link(@info_request.user)) %></h1> <p> <%= raw(_('Your response will <strong>appear on the Internet</strong>, <a href="{{url}}">read why</a> and answers to other questions.', :url => help_officers_path.html_safe)) %> @@ -48,5 +50,3 @@ </p> <% end %> <% end %> - - diff --git a/app/views/track_mailer/event_digest.text.erb b/app/views/track_mailer/event_digest.text.erb index b83c184f0..f6e699e41 100644 --- a/app/views/track_mailer/event_digest.text.erb +++ b/app/views/track_mailer/event_digest.text.erb @@ -4,7 +4,7 @@ for track_thing, alert_results, xapian_object in @email_about_things main_text += track_thing.params[:title_in_email] + "\n" main_text += ("=" * track_thing.params[:title_in_email].size) + "\n\n" - @highlight_words = xapian_object.words_to_highlight + @highlight_words = xapian_object.words_to_highlight(:regex => true) for result in alert_results.reverse if result[:model].class.to_s == "InfoRequestEvent" event = result[:model] @@ -17,14 +17,14 @@ # e.g. Julian Burgess sent a request to Royal Mail Group (15 May 2008) if event.event_type == 'response' - url = incoming_message_url(event.incoming_message) + url = incoming_message_url(event.incoming_message, :cachebust => true) main_text += _("{{public_body}} sent a response to {{user_name}}", :public_body => event.info_request.public_body.name, :user_name => event.info_request.user_name) elsif event.event_type == 'followup_sent' - url = outgoing_message_url(event.outgoing_message) + url = outgoing_message_url(event.outgoing_message, :cachebust => true) main_text += _("{{user_name}} sent a follow up message to {{public_body}}", :user_name => event.info_request.user_name, :public_body => event.info_request.public_body.name) elsif event.event_type == 'sent' # this is unlikely to happen in real life, but happens in the test code - url = outgoing_message_url(event.outgoing_message) + url = outgoing_message_url(event.outgoing_message, :cachebust => true) main_text += _("{{user_name}} sent a request to {{public_body}}", :user_name => event.info_request.user_name, :public_body => event.info_request.public_body.name) elsif event.event_type == 'comment' url = comment_url(event.comment) diff --git a/app/views/user/_user_listing_single.html.erb b/app/views/user/_user_listing_single.html.erb index ed1b95718..3cb0d283f 100644 --- a/app/views/user/_user_listing_single.html.erb +++ b/app/views/user/_user_listing_single.html.erb @@ -18,7 +18,7 @@ end %> <span class="bottomline"> <%= pluralize(display_user.info_requests.size, "request") %> <%= _('made.')%> <%= pluralize(display_user.visible_comments.size, "annotation") %> <%= _('made.')%> - <%= _('Joined in')%> <%= year_from_date(display_user.created_at) %>. + <%= _('Joined in')%> <%= display_user.created_at.year %>. </span> </div> diff --git a/app/views/user/show.html.erb b/app/views/user/show.html.erb index ce328b46f..7ae577565 100644 --- a/app/views/user/show.html.erb +++ b/app/views/user/show.html.erb @@ -64,7 +64,7 @@ <h1> <%= h(@display_user.name) + (@is_you ? _(" (you)") : "") %></h1> <p class="subtitle"> - <%= _('Joined {{site_name}} in', :site_name=>site_name) %> <%= year_from_date(@display_user.created_at) %> + <%= _('Joined {{site_name}} in', :site_name=>site_name) %> <%= @display_user.created_at.year %> <% if !@user.nil? && @user.admin_page_links? %> (<%= link_to "admin", admin_user_show_path(@display_user) %>) <% end %> diff --git a/config/crontab-example b/config/crontab-example index 8fe13151b..212a4a689 100644 --- a/config/crontab-example +++ b/config/crontab-example @@ -5,32 +5,32 @@ # Email: hello@mysociety.org. WWW: http://www.mysociety.org/ PATH=/usr/local/bin:/usr/bin:/bin -MAILTO=cron-!!(*= $site *)!!@mysociety.org +MAILTO=!!(*= $mailto *)!! # Every 5 minutes -*/5 * * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/change-xapian-database.lock "!!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/update-xapian-index verbose=true" >> !!(*= $vhost_dir *)!!/logs/update-xapian-index.log || echo "stalled?" +*/5 * * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/change-xapian-database.lock "!!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/update-xapian-index verbose=true" >> !!(*= $vhost_dir *)!!/logs/update-xapian-index.log || echo "stalled?" # Every 10 minutes 5,15,25,35,45,55 * * * * !!(*= $user *)!! /etc/init.d/foi-alert-tracks check 5,15,25,35,45,55 * * * * !!(*= $user *)!! /etc/init.d/foi-purge-varnish check -0,10,20,30,40,50 * * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/send-batch-requests.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/send-batch-requests || echo "stalled?" +0,10,20,30,40,50 * * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/send-batch-requests.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/send-batch-requests || echo "stalled?" # Once an hour -09 * * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/alert-comment-on-request.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-comment-on-request || echo "stalled?" +09 * * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/alert-comment-on-request.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-comment-on-request || echo "stalled?" # Only root can read the log files -31 * * * * root run-with-lockfile -n !!(*= $vhost_dir *)!!/load-mail-server-logs.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/load-mail-server-logs || echo "stalled?" +31 * * * * root !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/load-mail-server-logs.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/load-mail-server-logs || echo "stalled?" # Once a day, early morning -23 4 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/delete-old-things.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/delete-old-things || echo "stalled?" -0 6 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/alert-overdue-requests.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-overdue-requests || echo "stalled?" -0 7 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/alert-new-response-reminders.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-new-response-reminders || echo "stalled?" -0 8 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/alert-not-clarified-request.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-not-clarified-request || echo "stalled?" -2 4 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/check-recent-requests-sent.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/check-recent-requests-sent || echo "stalled?" -45 3 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/stop-new-responses-on-old-requests.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/stop-new-responses-on-old-requests || echo "stalled?" -55 4 * * * !!(*= $user *)!! run-with-lockfile -n !!(*= $vhost_dir *)!!/update-public-body-stats.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/update-public-body-stats || echo "stalled?" +23 4 * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/delete-old-things.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/delete-old-things || echo "stalled?" +0 6 * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/alert-overdue-requests.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-overdue-requests || echo "stalled?" +0 7 * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/alert-new-response-reminders.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-new-response-reminders || echo "stalled?" +0 8 * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/alert-not-clarified-request.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/alert-not-clarified-request || echo "stalled?" +2 4 * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/check-recent-requests-sent.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/check-recent-requests-sent || echo "stalled?" +45 3 * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/stop-new-responses-on-old-requests.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/stop-new-responses-on-old-requests || echo "stalled?" +55 4 * * * !!(*= $user *)!! !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/update-public-body-stats.lock !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/update-public-body-stats || echo "stalled?" # Only root can restart apache -31 1 * * * root run-with-lockfile -n !!(*= $vhost_dir *)!!/change-xapian-database.lock "!!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/compact-xapian-database production" || echo "stalled?" +31 1 * * * root !!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/commonlib/bin/run-with-lockfile.sh -n !!(*= $vhost_dir *)!!/change-xapian-database.lock "!!(*= $vhost_dir *)!!/!!(*= $vcspath *)!!/script/compact-xapian-database production" || echo "stalled?" # Once a day on all servers diff --git a/config/database.yml-example b/config/database.yml-example index e48577f23..126e5a264 100644 --- a/config/database.yml-example +++ b/config/database.yml-example @@ -3,7 +3,8 @@ development: adapter: postgresql - database: foi_development + template: template_utf8 + database: alaveteli_development username: <username> password: <password> host: localhost @@ -14,7 +15,8 @@ development: # Do not set this db to the same as development or production. test: adapter: postgresql - database: foi_test + template: template_utf8 + database: alaveteli_test username: <username> password: <password> host: localhost @@ -24,9 +26,9 @@ test: production: adapter: postgresql - database: foi_production + template: template_utf8 + database: alaveteli_production username: <username> password: <password> host: localhost port: 5432 - diff --git a/config/database.yml-test b/config/database.yml-test index ff6e2013a..64eafd82d 100644 --- a/config/database.yml-test +++ b/config/database.yml-test @@ -1,4 +1,5 @@ test: adapter: postgresql + template: template_utf8 database: foi_test username: postgres diff --git a/config/general.yml-example b/config/general.yml-example index 6e223406e..803c1c6ae 100644 --- a/config/general.yml-example +++ b/config/general.yml-example @@ -26,7 +26,7 @@ ISO_COUNTRY_CODE: GB TIME_ZONE: Australia/Sydney # These feeds are displayed accordingly on the Alaveteli "blog" page: -BLOG_FEED: 'http://www.mysociety.org/category/projects/whatdotheyknow/feed/' +BLOG_FEED: 'https://www.mysociety.org/category/projects/whatdotheyknow/feed/' TWITTER_USERNAME: 'whatdotheyknow' # Set the widget_id to get the Twitter sidebar on the blog page. # To get one https://twitter.com/settings/widgets diff --git a/config/httpd.conf-example b/config/httpd.conf-example index dc2e4966e..8d549d363 100644 --- a/config/httpd.conf-example +++ b/config/httpd.conf-example @@ -16,7 +16,7 @@ RewriteEngine On #RewriteLog /var/log/apache2/rewrite.log #RewriteLogLevel 9 -# XXX do we need this now we use Passenger? +# TODO: do we need this now we use Passenger? # Pass through the HTTP basic authentication to mongrel. See also # admin_http_auth_user in app/controllers/application.rb # Note: Apache 2 only. Doesn't work in Apache 1.3, you'll need to live without diff --git a/config/initializers/alaveteli.rb b/config/initializers/alaveteli.rb index 6fb6b1420..d9879b5d1 100644 --- a/config/initializers/alaveteli.rb +++ b/config/initializers/alaveteli.rb @@ -53,6 +53,7 @@ require 'message_prominence' require 'theme' require 'xapian_queries' require 'date_quarter' +require 'public_body_csv' AlaveteliLocalization.set_locales(AlaveteliConfiguration::available_locales, AlaveteliConfiguration::default_locale) diff --git a/config/packages b/config/packages index 9a07c5f20..fda09cbc1 100644 --- a/config/packages +++ b/config/packages @@ -39,3 +39,4 @@ sqlite3 libsqlite3-dev libicu-dev memcached +ttf-bitstream-vera diff --git a/config/packages.debian-squeeze b/config/packages.debian-squeeze index 6cdf2f9d6..d82d66324 100644 --- a/config/packages.debian-squeeze +++ b/config/packages.debian-squeeze @@ -37,3 +37,4 @@ libsqlite3-dev libicu-dev postgresql postgresql-client +ttf-bitstream-vera diff --git a/config/packages.ubuntu-precise b/config/packages.ubuntu-precise index 177d504e2..68911359a 100644 --- a/config/packages.ubuntu-precise +++ b/config/packages.ubuntu-precise @@ -33,3 +33,4 @@ libsqlite3-dev libicu-dev postgresql postgresql-client +ttf-bitstream-vera diff --git a/config/routes.rb b/config/routes.rb index d9d21f0bd..7cc85c974 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -16,7 +16,7 @@ Alaveteli::Application.routes.draw do match '/blog' => 'general#blog', :as => :blog match '/search' => 'general#search_redirect', :as => :search_redirect match '/search/all' => 'general#search_redirect', :as => :search_redirect - # XXX combined is the search query, and then if sorted a "/newest" at the end. + # `combined` is the search query, and then if sorted a "/newest" at the end. # Couldn't find a way to do this in routes which also picked up multiple other slashes # and dots and other characters that can appear in search query. So we sort it all # out in the controller. @@ -130,7 +130,7 @@ Alaveteli::Application.routes.draw do match '/:feed/list/:view' => 'track#track_list', :as => :track_list, :view => nil, :feed => /(track|feed)/ match '/:feed/body/:url_name' => 'track#track_public_body', :as => :track_public_body, :feed => /(track|feed)/ match '/:feed/user/:url_name' => 'track#track_user', :as => :track_user, :feed => /(track|feed)/ - # XXX :format doesn't work. See hacky code in the controller that makes up for this. + # TODO: :format doesn't work. See hacky code in the controller that makes up for this. match '/:feed/search/:query_array' => 'track#track_search_query', :as => :track_search, :feed => /(track|feed)/, diff --git a/config/varnish-alaveteli.vcl b/config/varnish-alaveteli.vcl index 5dd0ac83c..d3726682c 100644 --- a/config/varnish-alaveteli.vcl +++ b/config/varnish-alaveteli.vcl @@ -92,7 +92,7 @@ sub vcl_recv { # ban lists, see # http://kristianlyng.wordpress.com/2010/07/28/smart-bans-with-varnish/ - # XXX in Varnish 2.x, the following would be + # TODO: in Varnish 2.x, the following would be # purge("obj.http.x-url ~ " req.url); ban("obj.http.x-url ~ " + req.url); error 200 "Banned"; diff --git a/db/migrate/028_give_incoming_messages_events.rb b/db/migrate/028_give_incoming_messages_events.rb index 831068562..46acd831e 100644 --- a/db/migrate/028_give_incoming_messages_events.rb +++ b/db/migrate/028_give_incoming_messages_events.rb @@ -1,4 +1,4 @@ -# XXX If this one fails with errors about described_state on save, then you need +# TODO: If this one fails with errors about described_state on save, then you need # to temporarily modify the model for InfoRequestEvents to remove this part: # validates_inclusion_of :described_state, :in => [ # Or do some nice hack in here to make it happen permanently :) diff --git a/db/migrate/036_add_public_body_tags.rb b/db/migrate/036_add_public_body_tags.rb index 99a507f13..f7fefdf48 100644 --- a/db/migrate/036_add_public_body_tags.rb +++ b/db/migrate/036_add_public_body_tags.rb @@ -11,7 +11,7 @@ class AddPublicBodyTags < ActiveRecord::Migration end # MySQL cannot index text blobs like this - # XXX perhaps should change :name to be a :string + # TODO: perhaps should change :name to be a :string if ActiveRecord::Base.connection.adapter_name != "MySQL" add_index :public_body_tags, [:public_body_id, :name], :unique => true end diff --git a/db/migrate/061_include_responses_in_tracks.rb b/db/migrate/061_include_responses_in_tracks.rb index f357e57c2..c7a3b26cf 100644 --- a/db/migrate/061_include_responses_in_tracks.rb +++ b/db/migrate/061_include_responses_in_tracks.rb @@ -4,6 +4,6 @@ class IncludeResponsesInTracks < ActiveRecord::Migration end def self.down - # XXX forget it + # TODO: forget it end end diff --git a/db/migrate/065_add_comments_to_user_track.rb b/db/migrate/065_add_comments_to_user_track.rb index 9c4ff2936..50d1f9d5d 100644 --- a/db/migrate/065_add_comments_to_user_track.rb +++ b/db/migrate/065_add_comments_to_user_track.rb @@ -9,6 +9,6 @@ class AddCommentsToUserTrack < ActiveRecord::Migration end def self.down - # XXX forget it + # TODO: forget it end end diff --git a/db/migrate/088_public_body_machine_tags.rb b/db/migrate/088_public_body_machine_tags.rb index 0089607c6..6a0815568 100644 --- a/db/migrate/088_public_body_machine_tags.rb +++ b/db/migrate/088_public_body_machine_tags.rb @@ -3,7 +3,7 @@ class PublicBodyMachineTags < ActiveRecord::Migration add_column :public_body_tags, :value, :text # MySQL cannot index text blobs like this - # XXX perhaps should change :name/:value to be a :string + # TODO: perhaps should change :name/:value to be a :string if ActiveRecord::Base.connection.adapter_name != "MySQL" add_index :public_body_tags, :name end diff --git a/db/migrate/090_remove_tag_uniqueness.rb b/db/migrate/090_remove_tag_uniqueness.rb index 1c06de439..d1affade3 100644 --- a/db/migrate/090_remove_tag_uniqueness.rb +++ b/db/migrate/090_remove_tag_uniqueness.rb @@ -1,7 +1,7 @@ class RemoveTagUniqueness < ActiveRecord::Migration def self.up # MySQL cannot index text blobs like this - # XXX perhaps should change :name/:value to be a :string + # TODO: perhaps should change :name/:value to be a :string if ActiveRecord::Base.connection.adapter_name != "MySQL" remove_index :public_body_tags, [:public_body_id, :name] # allow the key to repeat, but not the value also diff --git a/db/migrate/20140528110536_update_track_things_index.rb b/db/migrate/20140528110536_update_track_things_index.rb new file mode 100644 index 000000000..55ee0b70b --- /dev/null +++ b/db/migrate/20140528110536_update_track_things_index.rb @@ -0,0 +1,17 @@ +class UpdateTrackThingsIndex < ActiveRecord::Migration + + def up + if ActiveRecord::Base.connection.adapter_name == "PostgreSQL" + execute "ALTER TABLE track_things_sent_emails DROP CONSTRAINT fk_track_request_public_body" + execute "ALTER TABLE track_things_sent_emails ADD CONSTRAINT fk_track_request_public_body FOREIGN KEY (public_body_id) REFERENCES public_bodies(id)" + end + end + + def down + if ActiveRecord::Base.connection.adapter_name == "PostgreSQL" + execute "ALTER TABLE track_things_sent_emails DROP CONSTRAINT fk_track_request_public_body" + execute "ALTER TABLE track_things_sent_emails ADD CONSTRAINT fk_track_request_public_body FOREIGN KEY (user_id) REFERENCES users(id)" + end + end + +end diff --git a/doc/DEPLOY.md b/doc/DEPLOY.md deleted file mode 100644 index f2e643ec5..000000000 --- a/doc/DEPLOY.md +++ /dev/null @@ -1,41 +0,0 @@ -# Deployment - -mySociety uses a custom deployment and buildout system however Capistrano is included as part of Alaveteli as a standard deployment system. - -## Capistrano - -### Set up - -First you need to customise your deployment settings, e.g. the name of the server you're deploying to. This is done by copying the example file `config/deploy.yml.example` to `config/deploy.yml` and editing the settings to suit you. - -TODO: The following instructions could be greatly improved - -These are the general steps required to get your staging server up and running: - -* Install packages from `config/packages` -* Install Postgres and configure a user -* Create a directory to deploy to and make sure your deployment user can write to it -* Run `cap deploy:setup` to create directories, etc. -* Run `cap deploy:update_code` so that we've got a copy of the example config on the server. This process will take a long time installing gems, etc. it will also fail on `rake:themes:install` but that's OK -* SSH to the server, change to the `deploy_to` directory -* `cp releases/[SOME_DATE]/config/general.yml-example shared/general.yml` -* `cp releases/[SOME_DATE]/config/database.yml-example shared/database.yml` -* Edit those files to match your required settings -* Back on your machine run `cap deploy` and it should successfully deploy -* Run the DB migrations `cap deploy:migrate` -* Build the Xapian DB `cap xapian:rebuild_index` -* Configure Apache/Passenger with a DocumentRoot of `your_deploy_to/current/public` -* Phew. Time to admire your work by browsing to the server! - -### Usage - -Ensure you've got a `config/deploy.yml` file with the correct settings for your site. You'll need to share this with everyone in your team that deploys so it might be a good idea to keep the latest version in a [Gist](http://gist.github.com/). - -To deploy to staging just run `cap deploy` but if you want to deploy to production you need to run `cap -S stage=production deploy`. - -For additional usage instructions, see the [Capistrano wiki](https://github.com/capistrano/capistrano/wiki/). - -### TODO - -* Get `cap deploy:setup` to do most of the work described above in the *Set up* section -* Use [Whenever](https://github.com/javan/whenever) to set up cronjobs diff --git a/doc/INSTALL-exim4.md b/doc/INSTALL-exim4.md deleted file mode 100644 index 796fb295c..000000000 --- a/doc/INSTALL-exim4.md +++ /dev/null @@ -1,99 +0,0 @@ -As an example of how to set up your MTA, in exim on Ubuntu, you might -add the following to its configuration. - -In `/etc/exim4/conf.d/main/04_alaveteli_options`: - - ALAVETELI_HOME=/path/to/alaveteli/software - ALAVETELI_USER=www-data - log_file_path=/var/log/exim4/exim-%slog-%D - MAIN_LOG_SELECTOR==+all -retry_defer - extract_addresses_remove_arguments=false - -(The user ALAVETELI_USER should have write permissions on ALAVETELI_HOME). - -Note that the name and location of the log files created by Exim must match -what the `load-mail-server-logs` script expects, hence the need for the extra -`log_file_path` setting. And the `check-recent-requests-sent` scripts expects -the logs to contain the `from=<...>` envelope information, so we make the -logs more verbose with `log_selector`. The ALAVETELI_USER may need to also -need to be added to the `trusted_users` list in your Exim config in order to -set the return path on outgoing mail, depending on your setup. - -In `/etc/exim4/conf.d/router/04_alaveteli`: - - alaveteli_request: - debug_print = "R: alaveteli for $local_part@$domain" - driver = redirect - data = ${lookup{$local_part}wildlsearch{ALAVETELI_HOME/config/aliases}} - pipe_transport = alaveteli_mailin_transport - -In `/etc/exim4/conf.d/transport/04_alaveteli`: - - alaveteli_mailin_transport: - driver = pipe - command = $address_pipe ${lc:$local_part} - current_directory = ALAVETELI_HOME - home_directory = ALAVETELI_HOME - user = ALAVETELI_USER - group = ALAVETELI_USER - -And, assuming you set `INCOMING_EMAIL_PREFIX` in your config at -`config/general` to "foi+", create `config/aliases` with the following -content: - - ^foi\\+.*: |/path/to/alaveteli/software/script/mailin - -You should also configure exim to discard any messages sent to the -`BLACKHOLE_PREFIX` address, whose default value is -'do-not-reply-to-this-address'. For example, add the following to -config/aliases: - - # We use this for envelope from for some messages where we don't care about delivery - do-not-reply-to-this-address: :blackhole: - -If you want to make use of the automatic bounce-message handling, then -set the `TRACK_SENDER_EMAIL` address to be filtered through -`script/handle-mail-replies`. Messages that are not bounces or -out-of-office autoreplies will be forwarded to -`FORWARD_NONBOUNCE_RESPONSES_TO`. For example, in WhatDoTheyKnow the -configuration looks like this: - - raw_team: [a list of people on the team] - team: |/path/to/alaveteli/software/script/handle-mail-replies - -with `FORWARD_NONBOUNCE_RESPONSES_TO: 'raw_team@whatdotheyknow.com'` - -Finally, make sure you have `dc_use_split_config='true'` in -`/etc/exim4/update-exim4.conf.conf`, and execute the command -`update-exim4.conf`. - -NB: if the file `/etc/exim4/exim4.conf` exists then `update-exim4.conf` -will silently do nothing. Some distributions include this file. If -yours does, you will need to rename it before running `update-exim4.conf`. - -(You may also want to set `dc_eximconfig_configtype='internet'`, -`dc_local_interfaces='0.0.0.0 ; ::1'`, and -`dc_other_hostnames='<your-host-name>'`) - -# Troubleshooting - -To test mail delivery, run: - - exim -bt foi+request-1234@localhost - -This should tell you which routers are being processed. You should -see something like: - - $ exim -bt foi+request-1234@localhost - R: alaveteli pipe for snafflerequest-234@localhost - snafflerequest-234@localhost -> |/home/alaveteli/alaveteli/script/mailin - transport = alaveteli_mailin_transport - -This tells you that the routing part (making emails to -`foi\+.*@localhost` be forwarded to Alaveteli's `mailin` script) is -working. - -There is a great -[Exim Cheatsheet](http://bradthemad.org/tech/notes/exim_cheatsheet.php) -online that you may find useful. - diff --git a/doc/INSTALL-postfix.md b/doc/INSTALL-postfix.md deleted file mode 100644 index a73d67ce1..000000000 --- a/doc/INSTALL-postfix.md +++ /dev/null @@ -1,68 +0,0 @@ -As an example of how to set up your MTA, in postfix on Ubuntu, you might -add the following to its configuration. - -In /etc/postfix/master.cf: - - alaveteli unix - n n - 50 pipe - flags=R user=ALAVETELI_USER argv=ALAVETELI_HOME/script/mailin - -In /etc/postfix/main.cf - - virtual_alias_maps = regexp:/etc/postfix/regexp - -For example - -ALAVETELI_HOME=/path/to/alaveteli/software -ALAVETELI_USER=www-data - -The user ALAVETELI_USER should have write permissions on ALAVETELI_HOME. - -And, assuming you set `OPTION_INCOMING_EMAIL_PREFIX` in your config at -`config/general` to "foi+", create `/etc/postfix/regexp` with the following -content: - - /^foi.*/ alaveteli - - -You should also configure postfix to discard any messages sent to the `BLACKHOLE_PREFIX` -address, whose default value is 'do-not-reply-to-this-address'. For example, add the -following to /etc/aliases: - - # We use this for envelope from for some messages where we don't care about delivery - do-not-reply-to-this-address: :blackhole: - -# Logging - -For the postfix logs to be read by the script 'load-mail-server-logs' succesfully they need to be log rotated with a date in the filename. Since that will create a lot of rotated log files (one for each day), it's good to have them in their own directory. For example (on Ubuntu) /etc/rsyslog.d/50-default.conf - - mail.* -/var/log/mail/mail.log - -And also edit /etc/logrotate.d/rsyslog: - - /var/log/mail/mail.log - { - rotate 30 - daily - dateext - missingok - notifempty - compress - delaycompress - sharedscripts - postrotate - reload rsyslog >/dev/null 2>&1 || true - endscript - } - -You'll also need to tell Alaveteli where the log files are stored and that they're in postfix format. Update config/general.yml with: - - MTA_LOG_PATH: '/var/log/mail/mail.log-*' - MTA_LOG_TYPE: "postfix" - -# Troubleshooting - -To test mail delivery, run: - - $ /usr/sbin/sendmail -bv foi+requrest-1234@localhost - -This tells you if sending the emails to 'foi\+.*localhost' is working. diff --git a/doc/INSTALL.md b/doc/INSTALL.md deleted file mode 100644 index 04cdb1352..000000000 --- a/doc/INSTALL.md +++ /dev/null @@ -1,651 +0,0 @@ -# Installation Script and AMI - -The easiest options for installating Alaveteli for evaluation -are to use our install script or to use the AMI (Amazon Machine -Image) to create an instance on Amazon EC2. These options are -described below. If you would prefer to install the site -manually, please go to the Manual Installation section below. - -## Installing from an AMI (Amazon Machine Image) - -To help people try out Alaveteli, we have created an AMI (Amazon -Machine Image) with a basic installation of Alaveteli, which you -can use to create a running server on an Amazon EC2 instance. -This creates an instance that runs in development mode, so we -wouldn't recommend you use it for a production system without -changing the configuration. - -Unfortunately, Alaveteli will not run properly on a free Micro -instance due to the low amount of memory available on those -instances; you will need to use at least a Small instance, which -Amazon will charge for. - -The AMI can be found in the EU West (Ireland) region, with the -ID ami-8603f4f1 and name “Basic Alaveteli installation -2014-01-29”. You can launch an instance based on that AMI with -[this link](https://console.aws.amazon.com/ec2/home?region=eu-west-1#launchAmi=ami-8603f4f1). - -When you create an EC2 instance based on that AMI, make sure -that you choose Security Groups that allows at least inbound -HTTP, HTTPS, SSH and, if you want to test incoming mail as well, -SMTP. - -When your EC2 instance is launched, you will be able to log in -as the `ubuntu` user. This user can `sudo` freely to run -commands as root. However, the code is actually owned by (and -runs as) the `alaveteli` user. After creating the instance, you -may want to edit a configuration file to customize the site's -configuration. That configuration file is -`/var/www/alaveteli/alaveteli/config/general.yml`, which can be -edited with: - - ubuntu@ip-10-58-191-98:~$ sudo su - alaveteli - alaveteli@ip-10-58-191-98:~$ cd alaveteli - alaveteli@ip-10-58-191-98:~/alaveteli$ nano config/general.yml - -Then you should restart the Thin webserver with: - - alaveteli@ip-10-58-191-98:~/alaveteli$ logout - ubuntu@ip-10-58-191-98:~$ sudo /etc/init.d/alaveteli restart - -If you find the hostname of your EC2 instance from the AWS -console, you should then be able to see the site at -`http://your-ec2-hostname.eu-west-1.compute.amazonaws.com` - -If you have any problems or questions, please ask on the -[Alaveteli Google Group](https://groups.google.com/forum/#!forum/alaveteli-dev) -or [report an issue](https://github.com/mysociety/alaveteli/issues?state=open). - -## Installing with the Installation Script - -If you have a clean installation of Debian squeeze or Ubuntu -precise, you can use an install script in our commonlib -repository to set up a working instance of Alaveteli. This is -not suitable for production (it runs in development mode, for -example) but should set up a functional installation of the -site. - -**Warning: only use this script on a newly installed server – it -will make significant changes to your server’s setup, including -modifying your nginx setup, creating a user account, creating a -database, installing new packages etc.** - -To download the script, run the following command: - - curl -O https://raw.github.com/mysociety/commonlib/master/bin/install-site.sh - -If you run this script with `sh install-site.sh`, you'll see its -usage message: - - Usage: ./install-site.sh [--default] <SITE-NAME> <UNIX-USER> [HOST] - HOST is only optional if you are running this on an EC2 instance. - --default means to install as the default site for this server, - rather than a virtualhost for HOST. - -In this case `<SITE-NAME>` should be `alaveteli`. `<UNIX-USER>` -is the name of the Unix user that you want to own and run the -code. (This user will be created by the script.) - -The `HOST` parameter is a hostname for the server that will be -usable externally – a virtualhost for this name will be created -by the script, unless you specified the `--default` option. This -parameter is optional if you are on an EC2 instance, in which -case the hostname of that instance will be used. - -For example, if you wish to use a new user called `alaveteli` -and the hostname `alaveteli.127.0.0.1.xip.io`, creating a -virtualhost just for that hostname, you could download and run -the script with: - - sudo sh install-site.sh alaveteli alaveteli alaveteli.127.0.0.1.xip.io - -([xip.io](http://xip.io/) is a helpful domain for development.) - -Or, if you want to set this up as the default site on an EC2 -instance, you could download the script, make it executable and -then invoke it with: - - sudo ./install-site.sh --default alaveteli alaveteli - -When the script has finished, you should have a working copy of -the website, accessible via the hostname you supplied to the -script. - -If you have any problems or questions, please ask on the -[Alaveteli Google Group](https://groups.google.com/forum/#!forum/alaveteli-dev) -or [report an issue](https://github.com/mysociety/alaveteli/issues?state=open). - -# Manual Installation - -These instructions assume Debian Squeeze (64-bit) or Ubuntu 12.04 LTS (precise). -[Install instructions for OS X](https://github.com/mysociety/alaveteli/wiki/OS-X-Quickstart) -are under development. Debian Squeeze is the best supported -deployment platform. - -Commands are intended to be run via the terminal or over ssh. - -As an aid to evaluation, there is an -[Amazon AMI](https://github.com/mysociety/alaveteli/wiki/Alaveteli-ec2-ami) -with all these steps configured. It is *not* production-ready. - -## Get Alaveteli - -To start with, you may need to install git, e.g. with `sudo apt-get -install git-core` - -Next, get hold of the Alaveteli source code from github: - - git clone https://github.com/mysociety/alaveteli.git - cd alaveteli - -This will get the development branch, which has the latest (possibly -buggy) code. If you don't want to add or try new features, swap to the -master branch (which always contains the latest stable release): - - git checkout master - -## Package pinning - -You need to configure [apt-pinning](http://wiki.debian.org/AptPreferences#Pinning-1) preferences in order to prevent packages being pulled from the debian wheezy distribution in preference to the stable distribution once you have added the wheezy repository as described below. - -In order to configure apt-pinning and to keep most packages coming from the Debian stable repository while installing the ones required from wheezy and the mySociety repository you need to run the following commands: - - echo "Package: *" >> /tmp/preferences - echo "Pin: release a=squeeze-backports">> /tmp/preferences - echo "Pin-Priority: 200" >> /tmp/preferences - echo "" >> /tmp/preferences - echo "Package: *" >> /tmp/preferences - echo "Pin: release a=wheezy">> /tmp/preferences - echo "Pin-Priority: 50" >> /tmp/preferences - sudo cp /tmp/preferences /etc/apt/ - rm /tmp/preferences - -## Install system dependencies - -These are packages that the software depends on: third-party software -used to parse documents, host the site, etc. There are also packages -that contain headers necessary to compile some of the gem dependencies -in the next step. - -If you are running Debian, add the following repositories to -`/etc/apt/sources.list` and run `apt-get update`: - - deb http://debian.mysociety.org squeeze main - deb http://ftp.debian.org/debian/ wheezy main non-free contrib - deb http://backports.debian.org/debian-backports squeeze-backports main contrib non-free - -The repositories above allow us to install the packages -`wkhtmltopdf-static` and `bundler` using `apt`; so if you're running -Ubuntu, you won't be able to use the above repositories, and you will -need to comment out those two lines in `config/packages` before -following the next step (and install bundler manually). - -Now install the packages that are listed in config/packages using apt-get -e.g.: - - sudo apt-get install `cut -d " " -f 1 config/packages | grep -v "^#"` - -Some of the files also have a version number listed in config/packages -- check that you have appropriate versions installed. Some also list -"|" and offer a choice of packages. - -## Install Ruby dependencies - -To install Alaveteli's Ruby dependencies, we need to install -bundler. In Debian, this is provided as a package (installed as part -of the package install process above). You could also install it as a -gem: - - sudo gem1.8 install bundler - -## Install mySociety libraries - -You will also want to install mySociety's common ruby libraries and the Rails -code. Run: - - git submodule update --init - -to fetch the contents of the submodules. - -### Packages customised by mySociety - -Debian users should add the mySociety debian archive to their -`/etc/apt/sources.list` as described above. Doing this and following -the above instructions should install a couple of custom -dependencies. Users of other platforms can optionally install these -dependencies manually, as follows: - -1. If you would like users to be able to download pretty PDFs as part of -the downloadable zipfile of their request history, you should install -[wkhtmltopdf](http://code.google.com/p/wkhtmltopdf/downloads/list). -We recommend downloading the latest, statically compiled version from -the project website, as this allows running headless (i.e. without a -graphical interface running) on Linux. If you do install -`wkhtmltopdf`, you need to edit a setting in the config file to point -to it (see below). If you don't install it, everything will still -work, but users will get ugly, plain text versions of their requests -when they download them. - -2. Version 1.44 of `pdftk` contains a bug which makes it to loop forever -in certain edge conditions. Until it's incorporated into an official -release, you can either hope you don't encounter the bug (it ties up a -rails process until you kill it) you'll need to patch it yourself or -use the Debian package compiled by mySociety (see link in -[issue 305](https://github.com/mysociety/alaveteli/issues/305)) - - -## Configure Database - -There has been a little work done in trying to make the code work with -other databases (e.g. SQLite), but the currently supported database is -PostgreSQL. - -If you don't have it installed: - - apt-get install postgresql postgresql-client - -Now we need to set up the database config file to contain the name, -username and password of your postgres database. - -* copy `database.yml-example` to `database.yml` in `alaveteli/config` -* edit it to point to your local postgresql database in the development - and test sections and create the databases: - -Make sure that the user specified in database.yml exists, and has full -permissions on these databases. As they need the ability to turn off -constraints whilst running the tests they also need to be a superuser. -If you don't want your database user to be a superuser, you can add a line -`disable_constraints: false` to the test config in database.yml, as seen in database.yml-example - -You can create a `foi` user from the command line, thus: - - # su - postgres - $ createuser -s -P foi - -And you can create a database thus: - - $ createdb -T template0 -E SQL_ASCII -O foi foi_production - $ createdb -T template0 -E SQL_ASCII -O foi foi_test - $ createdb -T template0 -E SQL_ASCII -O foi foi_development - -We create using the ``SQL_ASCII`` encoding, because in postgres this -is means "no encoding"; and because we handle and store all kinds of -data that may not be valid UTF (for example, data originating from -various broken email clients that's not 8-bit clean), it's safer to be -able to store *anything*, than reject data at runtime. - -## Configure email - -You will need to set up an email server (MTA) to send and receive -emails. Full configuration for an MTA is beyond the scope of this -document, though we describe an example configuration for Exim in -`INSTALL-exim4.md`. - -Note that in development mode, mail is handled by default by mailcatcher -so that you can see the mails in a browser - see http://mailcatcher.me/ -for more details. Start mailcatcher by running `bundle exec mailcatcher` -in your application directory. - -### Minimal - -If you just want to get the tests to pass, you will at a minimum need -to allow sending emails via a `sendmail` command (a requirement met, -for example, with `sudo apt-get install exim4`). - -### Detailed - -When an authority receives an email, the email's `reply-to` field is a -magic address which is parsed and consumed by the Rails app. - -To receive such email in a production setup, you will need to -configure your MTA to pipe incoming emails to the Alaveteli script -`script/mailin`. Therefore, you will need to configure your MTA to -accept emails to magic addresses, and to pipe such emails to this -script. - -Magic email addresses are of the form: - - <foi+request-3-691c8388@example.com> - -The respective parts of this address are controlled with options in -config/general.yml, thus: - - INCOMING_EMAIL_PREFIX = 'foi+' - INCOMING_EMAIL_DOMAIN = 'example.com' - -When you set up your MTA, note that if there is some error inside -Rails, the email is returned with an exit code 75, which for Exim at -least means the MTA will try again later. Additionally, a stacktrace -is emailed to `CONTACT_EMAIL`. - -`INSTALL-exim4.md` describes one possible configuration for Exim (>= -1.9). - -A well-configured installation of this code will separately have had -Exim make a backup copy of the email in a separate mailbox, just in -case. - -## Set up configs - -Copy `config/general.yml-example` to `config/general.yml` and edit to -your taste. - -Note that the default settings for frontpage examples are designed to -work with the dummy data shipped with Alaveteli; once you have real -data, you should certainly edit these. - -The default theme is the "Alaveteli" theme. When you run -`rails-post-deploy` (see below), that theme gets installed -automatically. - -Finally, copy `config/newrelic.yml-example` to `config/newrelic.yml`. -This file contains configuration information for the New Relic -performance management system. By default, monitoring is switched off -by the `agent_enabled: false` setting. See https://github.com/newrelic/rpm -for instructions on switching on local and remote performance analysis. - -## Deployment - -In the 'alaveteli' directory, run: - - script/rails-post-deploy - -(This will need execute privs so `chmod 755` if necessary.) This sets -up directory structures, creates logs, installs/updates themes, runs -database migrations, etc. You should run it after each new software -update. - -One of the things the script does is install dependencies (using -`bundle install`). Note that the first time you run it, part of the -`bundle install` that compiles `xapian-full` takes a *long* time! - -If you want some dummy data to play with, you can try loading the -fixtures that the test suite uses into your development database. You -can do this with: - - script/load-sample-data - -Next we need to create the index for the search engine (Xapian): - - script/rebuild-xapian-index - -If this fails, the site should still mostly run, but it's a core -component so you should really try to get this working. - -## Run the Tests - -Make sure everything looks OK: - - bundle exec rake spec - -If there are failures here, something has gone wrong with the -preceding steps (see the next section for a common problem and -workaround). You might be able to move on to the next step, depending -on how serious they are, but ideally you should try to find out what's -gone wrong. - -### glibc bug workaround - -There's a -[bug in glibc](http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=637239) -which causes Xapian to segfault when running the tests. Although the -bug report linked to claims it's fixed in the current Debian stable, -it's not as of version `2.11.3-2`. - -Until it's fixed (e.g. `libc6 2.13-26` does work), you can get the -tests to pass by setting `export LD_PRELOAD=/lib/libuuid.so.1`. - -## Run the Server - -Run the following to get the server running: - - bundle exec rails server --environment=development - -By default the server listens on all interfaces. You can restrict it to the -localhost interface by adding ` --binding=127.0.0.1` - -The server should have told you the URL to access in your browser to see -the site in action. - -## Administrator privileges - -The administrative interface is at the URL `/admin`. - -Only users with the `super` admin level can access the admin -interface. Users create their own accounts in the usual way, and then -administrators can give them `super` privileges. - -There is an emergency user account which can be accessed via -`/admin?emergency=1`, using the credentials `ADMIN_USERNAME` and -`ADMIN_PASSWORD`, which are set in `general.yml`. To bootstrap the -first `super` level accounts, you will need to log in as the emergency -user. You can disable the emergency user account by setting `DISABLE_EMERGENCY_USER` to `true` in `general.yml`. - -Users with the superuser role also have extra privileges in the -website frontend, such as being able to categorise any request, being -able to view items that have been hidden from the search, and being -presented with "admin" links next to individual requests and comments -in the front end. - -It is possible completely to override the administrator authentication -by setting `SKIP_ADMIN_AUTH` to `true` in `general.yml`. - -## Cron jobs and init scripts - -`config/crontab-example` contains the cronjobs run on WhatDoTheyKnow. -It's in a strange templating format they use in mySociety. mySociety -render the example file to reference absolute paths, and then drop it -in `/etc/cron.d/` on the server. - -The `ugly` format uses simple variable substitution. A variable looks -like `!!(*= $this *)!!`. The variables are: - -* `vhost`: part of the path to the directory where the software is - served from. In the mySociety files, it usually comes as - `/data/vhost/!!(*= $vhost *)!!` -- you should replace that whole - port with a path to the directory where your Alaveteli software - installation lives, e.g. `/var/www/` -* `vhost_dir`: the entire path to the directory where the software is - served from. -- you should replace this with a path to the - directory where your Alaveteli software installation lives, - e.g. `/var/www/` -* `vcspath`: the name of the alaveteli checkout, e.g. `alaveteli`. - Thus, `/data/vhost/!!(*= $vhost *)!!/!!(*= $vcspath *)!!` might be - replaced with `/var/www/alaveteli` in your cron tab -* `user`: the user that the software runs as -* `site`: a string to identify your alaveteli instance - -There is a rake task that will help to rewrite this file into -one that is useful to you, which can be invoked with: - - bundle exec rake config_files:convert_crontab \ - DEPLOY_USER=deploy \ - VHOST_DIR=/dir/above/alaveteli \ - VCSPATH=alaveteli \ - SITE=alaveteli \ - CRONTAB=config/crontab-example > crontab - -You should change the `DEPLOY_USER`, `VHOST_DIR`, `VCSPATH` and -`SITE` environment variables to match your server and -installation. You should also edit the resulting `crontab` file -to customize the `MAILTO` variable. - -One of the cron jobs refers to a script at -`/etc/init.d/foi-alert-tracks`. This is an init script, a copy of -which lives in `config/alert-tracks-debian.ugly`. As with the cron -jobs above, replace the variables (and/or bits near the variables) -with paths to your software. You can use the rake task `rake -config_files:convert_init_script` to do this. -`config/purge-varnish-debian.ugly` is a -similar init script, which is optional and not required if you choose -not to run your site behind Varnish (see below). Either tweak the file -permissions to make the scripts executable by your deploy user, or add the -following line to your sudoers file to allow these to be run by your deploy -user (named `deploy` in this case): - - deploy ALL = NOPASSWD: /etc/init.d/foi-alert-tracks, /etc/init.d/foi-purge-varnish - -The cron jobs refer to a program `run-with-lockfile`. See -[this issue](https://github.com/mysociety/alaveteli/issues/112) for a -discussion of where to find this program, and how you might replace -it. This [one line script](https://gist.github.com/3741194) can install -this program system-wide. - -## Set up production web server - -It is not recommended to run the website using the default Rails web -server. There are various recommendations here: -http://rubyonrails.org/deploy - -We usually use Passenger / mod_rails. The file at `conf/httpd.conf-example` -gives you an example config file for WhatDoTheyKnow. At a minimum, you should -include the following in an Apache configuration file: - - PassengerResolveSymlinksInDocumentRoot on - PassengerMaxPoolSize 6 # Recommend setting this to 3 or less on servers with 512MB RAM - -Under all but light loads, it is strongly recommended to run the -server behind an http accelerator like Varnish. A sample varnish VCL -is supplied in `../conf/varnish-alaveteli.vcl`. - -It's strongly recommended that you run the site over SSL. (Set FORCE_SSL to true in -config/general.yml). For this you will need an SSL certificate for your domain and you will -need to configure an SSL terminator to sit in front of Varnish. If you're already using Apache -as a web server you could simply use Apache as the SSL terminator. A minimal configuration -would look something like this: - -<VirtualHost *:443> - ServerName www.yourdomain - - ProxyRequests Off - ProxyPreserveHost On - ProxyPass / http://localhost:80/ - ProxyPassReverse / http://localhost:80/ - RequestHeader set X-Forwarded-Proto 'https' - - SSLEngine on - SSLProtocol all -SSLv2 - SSLCipherSuite ALL:!ADH:!EXPORT:!SSLv2:RC4+RSA:+HIGH:+MEDIUM - - SSLCertificateFile /etc/apache2/ssl/ssl.crt - SSLCertificateKeyFile /etc/apache2/ssl/ssl.key - SSLCertificateChainFile /etc/apache2/ssl/sub.class2.server.ca.pem - SSLCACertificateFile /etc/apache2/ssl/ca.pem - SetEnvIf User-Agent ".*MSIE.*" nokeepalive ssl-unclean-shutdown -</VirtualHost> - -Notice the line "RequestHeader" that sets the X-Forwarded-Proto header. This is important. This ultimately tells Rails that it's serving a page over https and so it knows to include that in any absolute urls it serves. - -Some -[production server best practice notes](https://github.com/mysociety/alaveteli/wiki/Production-Server-Best-Practices) -are evolving on the wiki. - -## Upgrading Alaveteli - -The developer team policy is that the master branch in git should -always contain the latest stable release. Therefore, in production, -you should usually have your software deployed from the master branch, -and an upgrade can be simply `git pull`. - -Patch version increases (e.g. 1.2.3 -> 1.2.4) should not require any -further action on your part. - -Minor version increases (e.g. 1.2.4 -> 1.3.0) will usually require -further action. You should read the `CHANGES.md` document to see -what's changed since your last deployment, paying special attention to -anything in the "Updgrading" sections. - -Any upgrade may include new translations strings, i.e. new or altered -messages to the user that need translating to your locale. You should -visit Transifex and try to get your translation up to 100% on each new -release. Failure to do so means that any new words added to the -Alaveteli source code will appear in your website in English by -default. If your translations didn't make it to the latest release, -you will need to download the updated `app.po` for your locale from -Transifex and save it in the `locale/` folder. - -You should always run the script `scripts/rails-post-deploy` after -each deployment. This runs any database migrations for you, plus -various other things that can be automated for deployment. - -## Troubleshooting - -* **Incoming emails aren't appearing in my Alaveteli install** - - First, you need to check that your MTA is delivering relevant - incoming emails to the `script/mailin` command. There are various - ways of setting your MTA up to do this; we have documented one way - of doing it in Exim at `doc/INSTALL-exim4.conf`, including a - command you can use to check that the email routing is set up - correctly. - - Second, you need to test that the mailin script itself is working - correctly, by running it from the command line, First, find a - valid "To" address for a request in your system. You can do this - through your site's admin interface, or from the command line, - like so: - - $ ./script/console - Loading development environment (Rails 2.3.14) - >> InfoRequest.find_by_url_title("why_do_you_have_such_a_fancy_dog").incoming_email - => "request-101-50929748@localhost" - - Now take the source of a valid email (there are some sample emails in - `spec/fixtures/files/`); edit the `To:` header to match this address; - and then pipe it through the mailin script. A non-zero exit code - means there was a problem. For example: - - $ cp spec/fixtures/files/incoming-request-plain.email /tmp/ - $ perl -pi -e 's/^To:.*/To: <request-101-50929748@localhost>/' /tmp/incoming-request-plain.email - $ ./script/mailin < /tmp/incoming-request-plain.email - $ echo $? - 75 - - The `mailin` script emails the details of any errors to - `CONTACT_EMAIL` (from your `general.yml` file). A common problem is - for the user that the MTA runs as not to have write access to - `files/raw_emails/`. - -* **Various tests fail with "*Your PostgreSQL connection does not support - unescape_bytea. Try upgrading to pg 0.9.0 or later.*"** - - You have an old version of `pg`, the ruby postgres driver. In - Ubuntu, for example, this is provided by the package `libdbd-pg-ruby`. - - Try upgrading your system's `pg` installation, or installing the pg - gem with `gem install pg` - -* **Some of the tests relating to mail are failing, with messages like - "*when using TMail should load an email with funny MIME settings' - FAILED*"** - - This sounds like the tests are running using the `production` - environment, rather than the `test` environment, for some reason. - -* **Non-ASCII characters are being displayed as asterisks in my incoming messages** - - We rely on `elinks` to convert HTML email to plain text. - Normally, the encoding should just work, but under some - circumstances it appears that `elinks` ignores the parameters - passed to it from Alaveteli. - - To force `elinks` always to treat input as UTF8, add the following - to `/etc/elinks/elinks.conf`: - - set document.codepage.assume = "utf-8" - set document.codepage.force_assumed = 1 - - You should also check that your locale is set up correctly. See - [https://github.com/mysociety/alaveteli/issues/128#issuecomment-1814845](this issue followup) - for further discussion. - -* **I'm seeing `rake: command not found` when running the post install script - - The script uses `rake`. - - It may be that the binaries installed by bundler are not put in the - system `PATH`; therefore, in order to run `rake` (needed for - deployments), you may need to do something like: - - ln -s /usr/lib/ruby/gems/1.8/bin/rake /usr/local/bin/ - diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 000000000..181076f2c --- /dev/null +++ b/doc/README.md @@ -0,0 +1,9 @@ +The main Alaveteli documentation now lives on the [Alaveteli site](http://alaveteli.org/docs/) + +Specifically: + +* [Installation](http://alaveteli.org/docs/installing/) +* [Themes](http://alaveteli.org/docs/customising/themes/) +* [Translation](http://alaveteli.org/docs/customising/translation/) +* [Email](http://alaveteli.org/docs/installing/email/) +* [Deployment](http://alaveteli.org/docs/installing/deploy/) diff --git a/doc/THEMES.md b/doc/THEMES.md deleted file mode 100644 index 8a4828a99..000000000 --- a/doc/THEMES.md +++ /dev/null @@ -1,165 +0,0 @@ -When installing an Alaveteli site, there are a few things that you -might want to do to customise it, beyond the available settings in the -`config/general` file. - -The most common requirement is to brand the site: at a minimum, -inserting your own logo and colour scheme. You may also want to tweak -the different states that a request can go through. You'll also want -to edit the categories that public bodies can appear in (i.e. the -groupings on the left hand side of the -"[View authorities](https://www.whatdotheyknow.com/body/list/all)" page -on WhatDoTheyKnow. - -There may also be other things you want to customise; drop a line on -the developer's mailing list to discuss, if so. We're still working -out the best way of doing these kinds of customisations! - -In any case, the important principle to bear in mind is that the less -you override and customise the code, the easier your site will be to -maintain in the long term. Any customisation is possible, but for -each customisation beyond the simple cases documented here, ask -yourself or your client, "can we possibly live without this?" If the -answer is "no", then consider starting a discussion about a pluggable -way of achieving your goals, rather than overriding any of the core -code. - -# General principles - -We try to encapsulate all site-specific functionality in one of these -places: - -* Site configuration (e.g. the name of your site, the available - languages etc -- all in `config/general`) -* Data (e.g. the public bodies to whom requests should be addressed) -* A rails "plugin", installed in `vendor/plugins/`. We call these - "themes". - -This document is about what you can do in a theme. - -By default, the sample theme ("alavetelitheme") has already been -installed. See the setting `THEME_URLS` in `general.yml` for an -explanation. - -You can also install the sample theme by hand, by running: - - bundle exec rails plugin install git://github.com/mysociety/alavetelitheme.git - -The sample theme contains examples for nearly everything you might -want to customise. You should probably make a copy, rename it, and -use that as the basis for your own theme. - -# Make sure your theme is as lightweight as possible - -The more you put in your theme, the harder it will be to upgrade to -future versions of Alaveteli. Everything you place in your theme -overrides things in the core theme, so if you make a new "main -template", then new widgets that appear in the core theme won't appear -on your website. - -Therefore, you should consider how you can brand your website without -changing much in the core theme. The ideal would be if you are able -to rebrand the site by only changing the CSS. You will also need to -add custom help pages, as described below. - -# Branding the site - -The core templates that comprise the layout and user interface of an -Alaveteli site live in `app/views/`. They use Rails' ERB syntax. -For example, the template for the home page lives at -`app/views/general/frontpage.html.erb`, and the template for the "about -us" page is at `app/views/help/about.html.erb`. - -Obviously, you *could* edit those core files directly, but this would -be a Bad Idea, because you would find it increasingly hard to do -upgrades. Having said that, sometimes you may want to change the core -templates in a way that would benefit everyone, in which case, discuss -the changes on the mailing list, make them in a fork of Alaveteli, and -then issue a pull request. - -Normally, however, you should override these pages **in your own -theme**, by placing them at a corresponding location within your -theme's `lib/` directory. These means that a file at -`vendor/plugins/alavetelitheme/lib/help/about.rhml` will appear -instead of the core "about us" file. - -Rails expects all its stylesheets to live at `<railshome>/public`, -which presents a problem for plugins. Here's how we solve it: the -stylesheet and associated resources for your theme live (by -convention) in `alavetelitheme/public/`. This is symlinked from -the main Rails app -- see `alavetelitheme/install.rb` to see how this -happens. - -The partial at -`alavetelitheme/lib/views/general/_before_head_end.html.erb` includes the -custom CSS in your theme's stylesheet folder (by convention, in -`alavetelitheme/public/stylesheets/`), with: - - <%= stylesheet_link_tag "/alavetelitheme/stylesheets/custom" %> - -...which will, usually, need changing for your theme. - -# Adding your own categories for public bodies - -Categories are implemented in Alaveteli using tags. Specific tags can -be designated to group authorities together as a category. - -There's a file in the sample theme, -`alavetelitheme/lib/public_body_categories_en.rb`, which contains a -nested structure that defines categories. It contains a comment -describing its structure. You should make a copy of this file for each -locale you support. - -# Customising the request states - -As mentioned above, if you can possibly live with the -[default Alaveteli request statuses](https://github.com/mysociety/alaveteli/wiki/Alaveteli's-request-statuses), -it would be good to do so. Note that you can set how many days counts -as "overdue" in the main site config file. - -If you can't live with the states as they are, there's a very basic -way to add to them (which will get improved over time). There's not -currently a way to remove any easily. There is an example of how to -do this in the `alavetelitheme`. - -To do add states, create two modules in your theme, -`InfoRequestCustomStates` and `RequestControllerCustomStates`. The -former must have these methods: - -* `theme_calculate_status`: return a tag to identify the current state of the request -* `theme_extra_states`: return a list of tags which identify the extra states you'd like to support -* `theme_display_status`: return human-readable strings corresponding with these tags - -The latter must have one method: - -* `theme_describe_state`: Return a notice for the user suitable for - displaying after they've categorised a request; and redirect them to - a suitable next page - -When you've added your extra states, you also need to create the following files in your theme: - -* `lib/views/general/_custom_state_descriptions.html.erb`: Descriptions - of your new states, suitable for displaying to end users -* `lib/views/general/_custom_state_transitions_complete.html.erb`: - Descriptions for any new states that you might characterise as - 'completion' states, for displaying on the categorisation form that - we ask requestors to fill out -* `lib/views/general/_custom_state_transitions_pending.html.erb`: As - above, but for new states you might characterise as 'pending' - states. - -You can see examples of these customisations in -[this commit](https://github.com/sebbacon/informatazyrtare-theme/commit/2b240491237bd72415990399904361ce9bfa431d) -for the Kosovan version of Alaveteli, Informata Zyrtare (ignore the -file `lib/views/general/_custom_state_transitions.html.erb`, which is -unused). - -# Adding new pages in the navigation - -`alavetelitheme/lib/config/custom-routes.rb` allows you to extend the base routes in -Alaveteli. The example in `alavetelitheme` adds an extra help page. -You can also use this to override the behaviour of specific pages if -necessary. - -# Adding or overriding models and controllers - -If you need to extend the behaviour of Alaveteli at the controller or model level, see `alavetelitheme/lib/controller_patches.rb` and `alavetelitheme/lib/model_patches.rb` for examples. diff --git a/doc/TRANSLATE.md b/doc/TRANSLATE.md deleted file mode 100644 index aef2cfdc9..000000000 --- a/doc/TRANSLATE.md +++ /dev/null @@ -1,106 +0,0 @@ -The software translations are implemented using GNU gettext, and the -resource files are managed in Transifex. - -The Transifex project is at -https://www.transifex.net/projects/p/alaveteli/; you'll probably want -an account there (ask on the mailing list). It has a fairly easy to -use interface for contributing translations. - -# Translation process: translator's view - -When a developer adds a new feature to the user interface in -Alaveteli, they use some code to mark sentences or words ("strings") -that they think will need to be translated. - -When the Alaveteli release manager is planning a release, they upload -a template containing all the strings to be translated (called a POT) -to Transifex. This causes your own translations in Transifex to be -updated with the latest strings. - -When you visit Transifex, it will prompt you to fill out values for -all new strings, and all strings that have been modified. In the case -where a string has only been slightly modified, such as with -punctuation ("Hello" has become "Hello!"), Transifex will suggest a -suitable translation for you (look for the "suggestions" tab under the -source string). - -In order for this feature to work properly, the release manager has to -download your translations, run a program that inserts the -suggestions, and then upload them again. Therefore, when a release -candidate is announced, make sure you have uploaded any outstanding -translations, or you will lose them. - -When a release candidate has been annouced, there is a **translation -freeze**: during this period, developers must not add any new strings -to the software, so you can be confident that you're translating -everything that will be in the final release. - -The release manager will also give you a **translation deadline**. After -this date, you can continue to contribute new translations, but they -won't make it into the release. - -## General notes on translation in Transifex - -Some bits of text will have comments attached to them from the Alaveteli -application developers about the context in which the text appears in the -application - these comments will appear under the 'Details' tab for the -text in Transifex. - -Some texts will have placeholders in them to indicate that another bit of text generated by Alaveteli will be inserted into them when they're displayed. They will look like `some text with a {{placeholder}}`. For these texts, don't translate the placeholder. It needs to stay exactly the same for the text to be inserted properly. - -Similarly, some texts will have some code in angle brackets in them to turn bits of the text into links, and other small bits of HTML formatting. e.g. `please <a href=\"{{url}}\">send it to us</a>`. Again, don't edit the bits in angle brackets, preserve them in your translation, just edit the text around them. - -Some bits of text are in the form of two bits of text separated by a `|` -character e.g. `IncomingMessage|Subject`. These represent attribute names, so -`IncomingMessage|Subject` is the subject attribute of an incoming message on -the site. You should not prioritise these types of text when translating - -they do not appear on the site anywhere at the moment, and when they do, -they will only be used in the admin interface. If you do translate them, only -translate the text after the `|`. - -# Translation process: release manager's view - -Before the Alaveteli release manager cuts a new release branch, they -must: - -* pick a date for the release branch to be cut ("release candidate date") -* make an announcement to the translators (using the "announcements" - feature in Transifex) that they should ensure they have any pending - translations saved in Transifex before the release candidate date -* make an announcement to the developers that all new strings should - be committed before the release candidate date -* on the release candidate date: - * download (`tx pull -a -f`) and commit all the current translations (important: - there's no revision history in Transifex!) - * you should also commit these translations to a hotfix for the - previous version, so they are preserved against the last known - good msgids - * regenerate the POT file and individual PO files for each - language, using `bundle exec rake - gettext:store_model_attributes`, followed by `bundle exec rake - gettext:find` - * careful of including msgids from themes in `lib/themes`; - you might want to move them out of the way before running - the above commands - * this updates the PO template, but also merges it with the - individual PO files, marking strings that have only changed - slightly as "fuzzy" - * reupload (`tx push -s -t`) the POT and PO files to Transifex from the - current release branch - * The point of uploading the PO files is that Transifex - converts the "fuzzy" suggestions from Transifex into - "suggestions" under each source string - * Note that Transifex *does not* preserve fuzzy strings in the - PO files it makes available for download, on the grounds - that Transifex supports multiple suggestions, whereas - gettext only allows one fuzzy suggestion per msgid. - * remove the fuzzy strings from the local PO files (because they - make Rails very noisy), and then commit the result. You can do - this by re-pulling from Transifex. -* on the release date: - * download and commit all the current translations to the current - release branch - -# Translations: developers' view - -See the [I18n guide](https://github.com/mysociety/alaveteli/wiki/I18n-guide) on the wiki. diff --git a/lib/acts_as_xapian/acts_as_xapian.rb b/lib/acts_as_xapian/acts_as_xapian.rb index b30bb4d10..6520a20a4 100644 --- a/lib/acts_as_xapian/acts_as_xapian.rb +++ b/lib/acts_as_xapian/acts_as_xapian.rb @@ -21,10 +21,24 @@ rescue LoadError $acts_as_xapian_bindings_available = false end +module Xapian + class QueryParser + def unstem(term) + words = [] + + Xapian._safelyIterate(unstem_begin(term), unstem_end(term)) do |item| + words << item.term + end + + words + end + end +end + module ActsAsXapian ###################################################################### # Module level variables - # XXX must be some kind of cattr_accessor that can do this better + # TODO: must be some kind of cattr_accessor that can do this better def ActsAsXapian.bindings_available $acts_as_xapian_bindings_available end @@ -109,12 +123,12 @@ module ActsAsXapian @@db_path = File.join(db_parent_path, environment) # make some things that don't depend on the db - # XXX this gets made once for each acts_as_xapian. Oh well. + # TODO: this gets made once for each acts_as_xapian. Oh well. @@stemmer = Xapian::Stem.new('english') end # Opens / reopens the db for reading - # XXX we perhaps don't need to rebuild database and enquire and queryparser - + # TODO: we perhaps don't need to rebuild database and enquire and queryparser - # but db.reopen wasn't enough by itself, so just do everything it's easier. def ActsAsXapian.readable_init raise NoXapianRubyBindingsError.new("Xapian Ruby bindings not installed") unless ActsAsXapian.bindings_available @@ -188,7 +202,7 @@ module ActsAsXapian raise "Z is reserved for stemming terms" if term[1] == "Z" raise "Already have code '" + term[1] + "' in another model but with different prefix '" + @@terms_by_capital[term[1]] + "'" if @@terms_by_capital.include?(term[1]) && @@terms_by_capital[term[1]] != term[2] @@terms_by_capital[term[1]] = term[2] - # XXX use boolean here so doesn't stem our URL names in WhatDoTheyKnow + # TODO: use boolean here so doesn't stem our URL names in WhatDoTheyKnow # If making acts_as_xapian generic, would really need to make the :terms have # another option that lets people choose non-boolean for terms that need it # (i.e. searching explicitly within a free text field) @@ -231,7 +245,7 @@ module ActsAsXapian raise "acts_as_xapian hasn't been called in any models" if @@init_values.empty? # if DB is not nil, then we're already initialised, so don't do it - # again XXX reopen it each time, xapian_spec.rb needs this so database + # again TODO: reopen it each time, xapian_spec.rb needs this so database # gets written twice correctly. # return unless @@writable_db.nil? @@ -472,16 +486,42 @@ module ActsAsXapian # Return just normal words in the query i.e. Not operators, ones in # date ranges or similar. Use this for cheap highlighting with # TextHelper::highlight, and excerpt. - def words_to_highlight - # TODO: In Ruby 1.9 we can do matching of any unicode letter with \p{L} - # But we still need to support ruby 1.8 for the time being so... - query_nopunc = self.query_string.gsub(/[^ёЁа-яА-Яa-zA-Zà-üÀ-Ü0-9:\.\/_]/iu, " ") - query_nopunc = query_nopunc.gsub(/\s+/, " ") - words = query_nopunc.split(" ") - # Remove anything with a :, . or / in it - words = words.find_all {|o| !o.match(/(:|\.|\/)/) } - words = words.find_all {|o| !o.match(/^(AND|NOT|OR|XOR)$/) } - return words + def words_to_highlight(opts = {}) + default_opts = { :include_original => false, :regex => false } + opts = default_opts.merge(opts) + + # Reject all prefixes other than Z, which we know is reserved for stems + terms = query.terms.reject { |t| t.term.first.match(/^[A-Y]$/) } + # Collect the stems including the Z prefix + raw_stems = terms.map { |t| t.term if t.term.start_with?('Z') }.compact.uniq.sort + # Collect stems, chopping the Z prefix off + stems = raw_stems.map { |t| t[1..-1] }.compact.sort + # Collect the non-stem terms + words = terms.map { |t| t.term unless t.term.start_with?('Z') }.compact.sort + + # Add the unstemmed words from the original query + # Sometimes stems can be unhelpful with the :regex option, for example + # stemming 'boring' results in us trying to highlight 'bore'. + if opts[:include_original] + raw_stems.each do |raw_stem| + words << ActsAsXapian.query_parser.unstem(raw_stem).uniq + end + + words = words.any? ? words.flatten.uniq : [] + end + + if opts[:regex] + stems.map! { |w| /\b(#{ w })\w*\b/iu } + words.map! { |w| /\b(#{ w })\b/iu } + end + + if RUBY_VERSION.to_f >= 1.9 + (stems + words).map! do |term| + term.is_a?(String) ? term.force_encoding('UTF-8') : term + end + else + stems + words + end end # Text for lines in log file @@ -510,7 +550,7 @@ module ActsAsXapian # Find the documents by their unique term input_models_query = Xapian::Query.new(Xapian::Query::OP_OR, query_models.map{|m| "I" + m.xapian_document_term}) ActsAsXapian.enquire.query = input_models_query - matches = ActsAsXapian.enquire.mset(0, 100, 100) # XXX so this whole method will only work with 100 docs + matches = ActsAsXapian.enquire.mset(0, 100, 100) # TODO: so this whole method will only work with 100 docs # Get set of relevant terms for those documents selection = Xapian::RSet.new() @@ -601,7 +641,7 @@ module ActsAsXapian begin if job.action == 'update' - # XXX Index functions may reference other models, so we could eager load here too? + # TODO: Index functions may reference other models, so we could eager load here too? model = job.model.constantize.find(job.model_id) # :include => cls.constantize.xapian_options[:include] model.xapian_index elsif job.action == 'destroy' @@ -717,7 +757,7 @@ module ActsAsXapian ActiveRecord::Base.connection.disconnect! - pid = Process.fork # XXX this will only work on Unix, tough + pid = Process.fork # TODO: this will only work on Unix, tough if pid Process.waitpid(pid) if not $?.success? @@ -898,7 +938,7 @@ module ActsAsXapian ActsAsXapian.term_generator.document = doc for text in texts_to_index ActsAsXapian.term_generator.increase_termpos # stop phrases spanning different text fields - # XXX the "1" here is a weight that could be varied for a boost function + # TODO: the "1" here is a weight that could be varied for a boost function ActsAsXapian.term_generator.index_text(xapian_value(text, nil, true), 1) end end @@ -975,5 +1015,3 @@ end # Reopen ActiveRecord and include the acts_as_xapian method ActiveRecord::Base.extend ActsAsXapian::ActsMethods - - diff --git a/lib/alaveteli_file_types.rb b/lib/alaveteli_file_types.rb index e89bc0c78..617048c05 100644 --- a/lib/alaveteli_file_types.rb +++ b/lib/alaveteli_file_types.rb @@ -16,15 +16,15 @@ class AlaveteliFileTypes "tnef" => 'application/ms-tnef', "tif" => 'image/tiff', "gif" => 'image/gif', - "jpg" => 'image/jpeg', # XXX add jpeg + "jpg" => 'image/jpeg', # TODO: add jpeg "png" => 'image/png', "bmp" => 'image/bmp', - "html" => 'text/html', # XXX add htm + "html" => 'text/html', # TODO: add htm "vcf" => 'text/x-vcard', "zip" => 'application/zip', "delivery-status" => 'message/delivery-status' } - # XXX doesn't have way of choosing default for inverse map - might want to add + # TODO: doesn't have way of choosing default for inverse map - might want to add # one when you need it FileExtensionToMimeTypeRev = FileExtensionToMimeType.invert @@ -46,7 +46,7 @@ class AlaveteliFileTypes m = Mahoro.new(Mahoro::MIME) mahoro_type = m.buffer(content) mahoro_type.strip! - # XXX we shouldn't have to check empty? here, but Mahoro sometimes returns a blank line :( + # TODO: we shouldn't have to check empty? here, but Mahoro sometimes returns a blank line :( # e.g. for InfoRequestEvent 17930 if mahoro_type.nil? || mahoro_type.empty? return nil diff --git a/lib/has_tag_string/has_tag_string.rb b/lib/has_tag_string/has_tag_string.rb index 4022faaac..c28720f04 100644 --- a/lib/has_tag_string/has_tag_string.rb +++ b/lib/has_tag_string/has_tag_string.rb @@ -10,7 +10,7 @@ module HasTagString # Represents one tag of one model. # The migration to make this is currently only in WDTK code. class HasTagStringTag < ActiveRecord::Base - # XXX strip_attributes! + # TODO: strip_attributes! validates_presence_of :name @@ -46,7 +46,7 @@ module HasTagString # Methods which are added to the model instances being tagged module InstanceMethods # Given an input string of tags, sets all tags to that string. - # XXX This immediately saves the new tags. + # TODO: This immediately saves the new tags. def tag_string=(tag_string) if tag_string.nil? tag_string = "" diff --git a/lib/mail_handler/backends/mail_backend.rb b/lib/mail_handler/backends/mail_backend.rb index e019eba97..190e79e97 100644 --- a/lib/mail_handler/backends/mail_backend.rb +++ b/lib/mail_handler/backends/mail_backend.rb @@ -323,7 +323,7 @@ module MailHandler end end end - # XXX call _convert_part_body_to_text here, but need to get charset somehow + # TODO: call _convert_part_body_to_text here, but need to get charset somehow # e.g. http://www.whatdotheyknow.com/request/1593/response/3088/attach/4/Freedom%20of%20Information%20request%20-%20car%20oval%20sticker:%20Article%2020,%20Convention%20on%20Road%20Traffic%201949.txt body = headers + "\n" + body end diff --git a/lib/mail_handler/mail_handler.rb b/lib/mail_handler/mail_handler.rb index 53033d440..47015f207 100644 --- a/lib/mail_handler/mail_handler.rb +++ b/lib/mail_handler/mail_handler.rb @@ -70,7 +70,7 @@ module MailHandler # note re. charset: TMail always tries to convert email bodies # to UTF8 by default, so normally it should already be that. text = '' - # XXX - tell all these command line tools to return utf-8 + # TODO: - tell all these command line tools to return utf-8 if content_type == 'text/plain' text += body + "\n\n" else @@ -151,7 +151,7 @@ module MailHandler body = entry.get_input_stream.read rescue # move to next attachment silently if there were problems - # XXX really should reduce this to specific exceptions? + # TODO: really should reduce this to specific exceptions? # e.g. password protected next end diff --git a/lib/public_body_csv.rb b/lib/public_body_csv.rb new file mode 100644 index 000000000..afb5d9043 --- /dev/null +++ b/lib/public_body_csv.rb @@ -0,0 +1,95 @@ +require 'csv' + +# Public: Generate a CSV representation of PublicBody instances +# +# Examples +# +# bodies = PublicBody.search('useless') +# +# csv = PublicBodyCSV.new(:fields => [:name, :calculated_home_page], +# :headers => ['Name', 'Home Page']) +# +# bodies.each { |body| csv << body } +# +# csv.generate +# # => Name,Home Page +# Department for Humpadinking,http://localhost +# Ministry of Silly Walks,http://www.localhost +# Department of Loneliness,http://localhost +class PublicBodyCSV + + def self.default_fields + [:name, + :short_name, + :url_name, + :tag_string, + :calculated_home_page, + :publication_scheme, + :disclosure_log, + :notes, + :created_at, + :updated_at, + :version] + end + + # TODO: Generate headers from fields + def self.default_headers + ['Name', + 'Short name', + 'URL name', + 'Tags', + 'Home page', + 'Publication scheme', + 'Disclosure log', + 'Notes', + 'Created at', + 'Updated at', + 'Version'] + end + + attr_reader :fields, :headers, :rows + + def initialize(args = {}) + @fields = args.fetch(:fields, self.class.default_fields) + @headers = args.fetch(:headers, self.class.default_headers) + @rows = [] + end + + def <<(public_body) + # Allow join_rows to handle newlines because of differences between + # CSV.generate_line in 1.8 / 1.9+ + if RUBY_VERSION.to_f >= 1.9 + rows << CSV.generate_line(collect_public_body_attributes(public_body), :row_sep => '') + else + rows << CSV.generate_line(collect_public_body_attributes(public_body)) + end + end + + # TODO: Just use CSV.generate when Ruby 1.8.7 support is dropped + def generate + csv = generate_header_row + csv << join_rows + csv << "\n" + end + + private + + def join_rows + rows.join("\n") + end + + def generate_header_row + # Add a newline because of differences between + # CSV.generate_line in 1.8 / 1.9+ + row = CSV.generate_line(headers) + row += "\n" unless RUBY_VERSION.to_f >= 1.9 + row + end + + def collect_public_body_attributes(public_body) + fields.map do |field| + public_body.respond_to?(field) ? public_body.send(field) : '' + end + end + +end diff --git a/lib/strip_attributes/strip_attributes.rb b/lib/strip_attributes/strip_attributes.rb index 130d10185..12350277d 100644 --- a/lib/strip_attributes/strip_attributes.rb +++ b/lib/strip_attributes/strip_attributes.rb @@ -1,6 +1,6 @@ module StripAttributes # Strips whitespace from model fields and leaves nil values as nil. - # XXX this differs from official StripAttributes, as it doesn't make blank cells null. + # TODO: this differs from official StripAttributes, as it doesn't make blank cells null. def strip_attributes!(options = nil) before_validation do |record| attribute_names = StripAttributes.narrow(record.attribute_names, options) diff --git a/lib/tasks/cleanup.rake b/lib/tasks/cleanup.rake new file mode 100644 index 000000000..9a8be9521 --- /dev/null +++ b/lib/tasks/cleanup.rake @@ -0,0 +1,20 @@ +namespace :cleanup do + + desc 'Clean up all message redelivery and destroy actions from the holding pen to make admin actions there faster' + task :holding_pen => :environment do + dryrun = ENV['DRYRUN'] != '0' + if dryrun + $stderr.puts "This is a dryrun - nothing will be deleted" + end + holding_pen = InfoRequest.find_by_url_title('holding_pen') + old_events = holding_pen.info_request_events.find_each(:conditions => ['event_type in (?)', + ['redeliver_incoming', + 'destroy_incoming']]) do |event| + puts event.inspect + if ! dryrun + event.destroy + end + end + end + +end diff --git a/lib/tasks/config_files.rake b/lib/tasks/config_files.rake index d0e4001f0..60814cb27 100644 --- a/lib/tasks/config_files.rake +++ b/lib/tasks/config_files.rake @@ -50,7 +50,7 @@ namespace :config_files do desc 'Convert Debian .ugly crontab file in config to a form suitable for installing in /etc/cron.d' task :convert_crontab => :environment do - example = 'rake config_files:convert_crontab DEPLOY_USER=deploy VHOST_DIR=/dir/above/alaveteli VCSPATH=alaveteli SITE=alaveteli CRONTAB=config/crontab-example' + example = 'rake config_files:convert_crontab DEPLOY_USER=deploy VHOST_DIR=/dir/above/alaveteli VCSPATH=alaveteli SITE=alaveteli CRONTAB=config/crontab-example MAILTO=cron-alaveteli@example.org' check_for_env_vars(['DEPLOY_USER', 'VHOST_DIR', 'VCSPATH', @@ -60,7 +60,8 @@ namespace :config_files do :user => ENV['DEPLOY_USER'], :vhost_dir => ENV['VHOST_DIR'], :vcspath => ENV['VCSPATH'], - :site => ENV['SITE'] + :site => ENV['SITE'], + :mailto => ENV.fetch('MAILTO') { "cron-#{ ENV['SITE'] }@mysociety.org" } } convert_ugly(ENV['CRONTAB'], replacements).each do |line| puts line diff --git a/script/request-creation-graph b/script/request-creation-graph index ef1d2cf73..7d347a7d2 100755 --- a/script/request-creation-graph +++ b/script/request-creation-graph @@ -17,7 +17,7 @@ cd `dirname $0` cd ../ source commonlib/shlib/deployfns -# XXX this is nasty :) +# TODO: this is nasty :) OPTION_FOI_DB_HOST=`grep "host:" config/database.yml | head --lines=1 | cut -d ":" -f 2` OPTION_FOI_DB_PORT=`grep "port:" config/database.yml | head --lines=1 | cut -d ":" -f 2` OPTION_FOI_DB_NAME=`grep "database:" config/database.yml | head --lines=1 | cut -d ":" -f 2` diff --git a/script/site-specific-install.sh b/script/site-specific-install.sh index 7d47a5990..fce230822 100755 --- a/script/site-specific-install.sh +++ b/script/site-specific-install.sh @@ -148,7 +148,6 @@ echo -n "Creating /etc/cron.d/alaveteli... " sed -r \ -e "/foi-purge-varnish/d" \ -e "s,^(MAILTO=).*,\1root@$HOST," \ - -e "s,run-with-lockfile,$REPOSITORY/commonlib/bin/run-with-lockfile.sh,g" \ -i /etc/cron.d/alaveteli echo $DONE_MSG diff --git a/script/user-use-graph b/script/user-use-graph index f508c9cb6..00eeb36f8 100755 --- a/script/user-use-graph +++ b/script/user-use-graph @@ -16,7 +16,7 @@ cd `dirname $0` cd ../ source commonlib/shlib/deployfns -# XXX this is nasty :) +# TODO: this is nasty :) OPTION_FOI_DB_HOST=`grep "host:" config/database.yml | head --lines=1 | cut -d ":" -f 2` OPTION_FOI_DB_PORT=`grep "port:" config/database.yml | head --lines=1 | cut -d ":" -f 2` OPTION_FOI_DB_NAME=`grep "database:" config/database.yml | head --lines=1 | cut -d ":" -f 2` diff --git a/spec/controllers/comment_controller_spec.rb b/spec/controllers/comment_controller_spec.rb index 5e250f689..480c85ad7 100644 --- a/spec/controllers/comment_controller_spec.rb +++ b/spec/controllers/comment_controller_spec.rb @@ -26,7 +26,7 @@ describe CommentController, "when commenting on a request" do post :new, params post_redirect = PostRedirect.get_last_post_redirect response.should redirect_to(:controller => 'user', :action => 'signin', :token => post_redirect.token) - # post_redirect.post_params.should == params # XXX get this working. there's a : vs '' problem amongst others + # post_redirect.post_params.should == params # TODO: get this working. there's a : vs '' problem amongst others end it "should create the comment, and redirect to request page when input is good and somebody is logged in" do diff --git a/spec/controllers/general_controller_spec.rb b/spec/controllers/general_controller_spec.rb index 7590a5b42..c0a9d57d3 100644 --- a/spec/controllers/general_controller_spec.rb +++ b/spec/controllers/general_controller_spec.rb @@ -188,7 +188,7 @@ describe GeneralController, 'when using xapian search' do it 'should highlight words for a user-only request' do get :search, :combined => "bob/users" - assigns[:highlight_words].should == ['bob'] + assigns[:highlight_words].should == [/\b(bob)\w*\b/iu, /\b(bob)\b/iu] end it 'should show spelling corrections for a user-only request' do diff --git a/spec/controllers/public_body_controller_spec.rb b/spec/controllers/public_body_controller_spec.rb index 63989baaa..f64975580 100644 --- a/spec/controllers/public_body_controller_spec.rb +++ b/spec/controllers/public_body_controller_spec.rb @@ -184,6 +184,11 @@ describe PublicBodyController, "when listing bodies" do assigns[:public_bodies].should == [ public_bodies(:geraldine_public_body) ] end + it "should support simple searching of bodies by short_name" do + get :list, :public_body_query => 'DfH' + assigns[:public_bodies].should == [ public_bodies(:humpadink_public_body) ] + end + it "should support simple searching of bodies by notes" do get :list, :public_body_query => 'Albatross' assigns[:public_bodies].should == [ public_bodies(:humpadink_public_body) ] @@ -287,6 +292,23 @@ describe PublicBodyController, "when asked to export public bodies as CSV" do all_data[1].length.should == 11 end + it "only includes visible bodies" do + get :list_all_csv + all_data = CSV.parse(response.body) + all_data.any?{ |row| row.include?('Internal admin authority') }.should be_false + end + + it "does not include site_administration bodies" do + FactoryGirl.create(:public_body, + :name => 'Site Admin Body', + :tag_string => 'site_administration') + + get :list_all_csv + + all_data = CSV.parse(response.body) + all_data.any?{ |row| row.include?('Site Admin Body') }.should be_false + end + end describe PublicBodyController, "when showing public body statistics" do diff --git a/spec/controllers/request_controller_spec.rb b/spec/controllers/request_controller_spec.rb index 9353efcb3..f7c935af3 100644 --- a/spec/controllers/request_controller_spec.rb +++ b/spec/controllers/request_controller_spec.rb @@ -77,7 +77,7 @@ describe RequestController, "when changing things that appear on the request pag PurgeRequest.all().count.should == 0 end it "should purge the downstream cache when censor rules have changed" do - # XXX really, CensorRules should execute expiry logic as part + # TODO: really, CensorRules should execute expiry logic as part # of the after_save of the model. Currently this is part of # the AdminCensorRuleController logic, so must be tested from # there. Leaving this stub test in place as a reminder @@ -643,7 +643,7 @@ describe RequestController, "when showing one request" do ir = info_requests(:fancy_dog_request) receive_incoming_mail('incoming-request-two-same-name.email', ir.incoming_email) - # XXX this is horrid, but don't know a better way. If we + # TODO: this is horrid, but don't know a better way. If we # don't do this, the info_request_event to which the # info_request is attached still uses the unmodified # version from the fixture. @@ -900,7 +900,7 @@ describe RequestController, "when handling prominence" do end -# XXX do this for invalid ids +# TODO: do this for invalid ids # it "should render 404 file" do # response.should render_template("#{Rails.root}/public/404.html") # response.headers["Status"].should == "404 Not Found" @@ -923,7 +923,6 @@ describe RequestController, "when searching for an authority" do end it "should return matching bodies" do - session[:user_id] = @user.id get :select_authority, :query => "Quango" @@ -1004,7 +1003,18 @@ describe RequestController, "when creating a new request" do post :new, params post_redirect = PostRedirect.get_last_post_redirect response.should redirect_to(:controller => 'user', :action => 'signin', :token => post_redirect.token) - # post_redirect.post_params.should == params # XXX get this working. there's a : vs '' problem amongst others + # post_redirect.post_params.should == params # TODO: get this working. there's a : vs '' problem amongst others + end + + it 'redirects to the frontpage if the action is sent the invalid + public_body param' do + post :new, :info_request => { :public_body => @body.id, + :title => 'Why Geraldine?', + :tag_string => '' }, + :outgoing_message => { :body => 'This is a silly letter.' }, + :submitted_new_request => 1, + :preview => 1 + response.should redirect_to frontpage_url end it "should show preview when input is good" do @@ -1793,7 +1803,7 @@ describe RequestController, "when sending a followup message" do session[:user_id] = users(:bob_smith_user).id post :show_response, :outgoing_message => { :body => "", :what_doing => 'normal_sort'}, :id => info_requests(:fancy_dog_request).id, :incoming_message_id => incoming_messages(:useless_incoming_message), :submitted_followup => 1 - # XXX how do I check the error message here? + # TODO: how do I check the error message here? response.should render_template('show_response') end @@ -1843,13 +1853,13 @@ describe RequestController, "when sending a followup message" do # second time should give an error post :show_response, :outgoing_message => { :body => "Stop repeating yourself!", :what_doing => 'normal_sort' }, :id => info_requests(:fancy_dog_request).id, :incoming_message_id => incoming_messages(:useless_incoming_message), :submitted_followup => 1 - # XXX how do I check the error message here? + # TODO: how do I check the error message here? response.should render_template('show_response') end end -# XXX Stuff after here should probably be in request_mailer_spec.rb - but then +# TODO: Stuff after here should probably be in request_mailer_spec.rb - but then # it can't check the URLs in the emails I don't think, ugh. describe RequestController, "sending overdue request alerts" do @@ -1878,7 +1888,7 @@ describe RequestController, "sending overdue request alerts" do mail_token = $2 session[:user_id].should be_nil - controller.test_code_redirect_by_email_token(mail_token, self) # XXX hack to avoid having to call User controller for email link + controller.test_code_redirect_by_email_token(mail_token, self) # TODO: hack to avoid having to call User controller for email link session[:user_id].should == info_requests(:naughty_chicken_request).user.id response.should render_template('show_response') @@ -1935,7 +1945,7 @@ describe RequestController, "sending overdue request alerts" do mail_token = $2 session[:user_id].should be_nil - controller.test_code_redirect_by_email_token(mail_token, self) # XXX hack to avoid having to call User controller for email link + controller.test_code_redirect_by_email_token(mail_token, self) # TODO: hack to avoid having to call User controller for email link session[:user_id].should == info_requests(:naughty_chicken_request).user.id response.should render_template('show_response') @@ -2017,12 +2027,12 @@ describe RequestController, "sending unclassified new response reminder alerts" mail_token = $2 session[:user_id].should be_nil - controller.test_code_redirect_by_email_token(mail_token, self) # XXX hack to avoid having to call User controller for email link + controller.test_code_redirect_by_email_token(mail_token, self) # TODO: hack to avoid having to call User controller for email link session[:user_id].should == info_requests(:fancy_dog_request).user.id response.should render_template('show') assigns[:info_request].should == info_requests(:fancy_dog_request) - # XXX should check anchor tag here :) that it goes to last new response + # TODO: should check anchor tag here :) that it goes to last new response end end @@ -2053,7 +2063,7 @@ describe RequestController, "clarification required alerts" do mail_token = $2 session[:user_id].should be_nil - controller.test_code_redirect_by_email_token(mail_token, self) # XXX hack to avoid having to call User controller for email link + controller.test_code_redirect_by_email_token(mail_token, self) # TODO: hack to avoid having to call User controller for email link session[:user_id].should == info_requests(:fancy_dog_request).user.id response.should render_template('show_response') diff --git a/spec/controllers/track_controller_spec.rb b/spec/controllers/track_controller_spec.rb index d2b45b6bf..29f5c7fe1 100644 --- a/spec/controllers/track_controller_spec.rb +++ b/spec/controllers/track_controller_spec.rb @@ -122,11 +122,11 @@ describe TrackController, "when sending alerts for a track" do mail.body.should =~ /This a the daftest comment the world has ever seen/ # comment text included # Check subscription managing link -# XXX We can't do this, as it is redirecting to another controller. I'm +# TODO: We can't do this, as it is redirecting to another controller. I'm # apparently meant to be writing controller unit tests here, not functional # tests. Bah, I so don't care, bit of an obsessive constraint. # session[:user_id].should be_nil -# controller.test_code_redirect_by_email_token(mail_token, self) # XXX hack to avoid having to call User controller for email link +# controller.test_code_redirect_by_email_token(mail_token, self) # TODO: hack to avoid having to call User controller for email link # session[:user_id].should == users(:silly_name_user).id # # response.should render_template('users/show') @@ -173,7 +173,7 @@ describe TrackController, "when viewing RSS feed for a track" do get :track_request, :feed => 'feed', :url_title => track_thing.info_request.url_title response.should render_template('track/atom_feed') response.content_type.should == 'application/atom+xml' - # XXX should check it is an atom.builder type being rendered, not sure how to + # TODO: should check it is an atom.builder type being rendered, not sure how to assigns[:xapian_object].matches_estimated.should == 3 assigns[:xapian_object].results.size.should == 3 diff --git a/spec/controllers/user_controller_spec.rb b/spec/controllers/user_controller_spec.rb index cf361d898..6ecdf1ad4 100644 --- a/spec/controllers/user_controller_spec.rb +++ b/spec/controllers/user_controller_spec.rb @@ -1,7 +1,7 @@ # coding: utf-8 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') -# XXX Use route_for or params_from to check /c/ links better +# TODO: Use route_for or params_from to check /c/ links better # http://rspec.rubyforge.org/rspec-rails/1.1.12/classes/Spec/Rails/Example/ControllerExampleGroup.html describe UserController, "when redirecting a show request to a canonical url" do @@ -327,7 +327,7 @@ describe UserController, "when signing up" do deliveries[0].body.should match(/when\s+you\s+already\s+have\s+an/) end - # XXX need to do bob@localhost signup and check that sends different email + # TODO: need to do bob@localhost signup and check that sends different email end describe UserController, "when signing out" do @@ -380,7 +380,7 @@ describe UserController, "when sending another user a message" do mail = deliveries[0] mail.body.should include("Bob Smith has used #{AlaveteliConfiguration::site_name} to send you the message below") mail.body.should include("Just a test!") - #mail.to_addrs.first.to_s.should == users(:silly_name_user).name_and_email # XXX fix some nastiness with quoting name_and_email + #mail.to_addrs.first.to_s.should == users(:silly_name_user).name_and_email # TODO: fix some nastiness with quoting name_and_email mail.from_addrs.first.to_s.should == users(:bob_smith_user).email end @@ -651,7 +651,7 @@ describe UserController, "when using profile photos" do @user.profile_photo.should_not be_nil end - # XXX todo check the two stage javascript cropping (above only tests one stage non-javascript one) + # TODO: todo check the two stage javascript cropping (above only tests one stage non-javascript one) end describe UserController, "when showing JSON version for API" do diff --git a/spec/helpers/date_time_helper_spec.rb b/spec/helpers/date_time_helper_spec.rb new file mode 100644 index 000000000..c4fdee1d1 --- /dev/null +++ b/spec/helpers/date_time_helper_spec.rb @@ -0,0 +1,71 @@ +require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') + +describe DateTimeHelper do + + include DateTimeHelper + + describe :simple_date do + + it 'formats a date in html by default' do + time = Time.utc(2012, 11, 07, 21, 30, 26) + self.should_receive(:simple_date_html).with(time) + simple_date(time) + end + + it 'formats a date in the specified format' do + time = Time.utc(2012, 11, 07, 21, 30, 26) + self.should_receive(:simple_date_text).with(time) + simple_date(time, :format => :text) + end + + it 'raises an argument error if given an unrecognized format' do + time = Time.utc(2012, 11, 07, 21, 30, 26) + expect { simple_date(time, :format => :unknown) }.to raise_error(ArgumentError) + end + + end + + describe :simple_date_html do + + it 'formats a date in a time tag' do + Time.use_zone('London') do + time = Time.utc(2012, 11, 07, 21, 30, 26) + expected = %Q(<time datetime="2012-11-07T21:30:26+00:00" title="2012-11-07 21:30:26 +0000">November 07, 2012</time>) + simple_date_html(time).should == expected + end + end + + end + + describe :simple_date_text do + + it 'should respect time zones' do + Time.use_zone('Australia/Sydney') do + simple_date_text(Time.utc(2012, 11, 07, 21, 30, 26)).should == 'November 08, 2012' + end + end + + it 'should handle Date objects' do + simple_date_text(Date.new(2012, 11, 21)).should == 'November 21, 2012' + end + + end + + describe :simple_time do + + it 'returns 00:00:00 for a date' do + simple_time(Date.new(2012, 11, 21)).should == '00:00:00' + end + + it 'returns the time component of a datetime' do + date = DateTime.new(2012, 11, 21, 10, 34, 56) + simple_time(date).should == '10:34:56' + end + + it 'returns the time component of a time' do + time = Time.utc(2000, 'jan', 1, 20, 15, 1) + simple_time(time).should == '20:15:01' + end + + end +end diff --git a/spec/helpers/highlight_helper_spec.rb b/spec/helpers/highlight_helper_spec.rb new file mode 100644 index 000000000..e1be7e153 --- /dev/null +++ b/spec/helpers/highlight_helper_spec.rb @@ -0,0 +1,247 @@ +require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') + +describe HighlightHelper do + include HighlightHelper + + describe :highlight_and_excerpt do + + it 'excerpts text and highlights phrases' do + text = "Quentin Nobble-Boston, Permanent Under-Secretary, Department for Humpadinking" + phrases = ['humpadinking'] + expected = '...Department for <span class="highlight">Humpadinking</span>' + highlight_and_excerpt(text, phrases, 15).should == expected + end + + it 'excerpts text and highlights matches' do + text = "Quentin Nobble-Boston, Permanent Under-Secretary, Department for Humpadinking" + matches = [/\bhumpadink\w*\b/iu] + expected = '...Department for <span class="highlight">Humpadinking</span>' + highlight_and_excerpt(text, matches, 15).should == expected + end + + context 'multiple matches' do + + it 'highlights multiple matches' do + text = <<-EOF +Quentin Nobble-Boston, Permanent Under-Secretary, Department for Humpadinking +decided to visit Humpadink so that he could be with the Humpadinks +EOF + + expected = <<-EOF +Quentin Nobble-Boston, Permanent Under-Secretary, Department for <span class="highlight">Humpadinking</span> +decided to visit <span class="highlight">Humpadink</span> so that he could be with the <span class="highlight">Humpadinks</span> +EOF + text.chomp! + expected.chomp! + matches = [/\b(humpadink\w*)\b/iu] + highlight_and_excerpt(text, matches, 1000).should == expected + end + + it 'bases the split on the first match' do + text = "Quentin Nobble-Boston, Permanent Under-Secretary," \ + "Department for Humpadinking decided to visit Humpadink" \ + "so that he could be with the Humpadinks" + + expected = "...Department for <span class=\"highlight\">" \ + "Humpadinking</span> decided to vis..." + + matches = [/\b(humpadink\w*)\b/iu] + highlight_and_excerpt(text, matches, 15).should == expected + end + + end + + end + + describe :highlight_matches do + + it 'highlights' do + assert_equal( + "This is a <mark>beautiful</mark> morning", + highlight_matches("This is a beautiful morning", "beautiful") + ) + + assert_equal( + "This is a <mark>beautiful</mark> morning, but also a <mark>beautiful</mark> day", + highlight_matches("This is a beautiful morning, but also a beautiful day", "beautiful") + ) + + assert_equal( + "This is a <b>beautiful</b> morning, but also a <b>beautiful</b> day", + highlight_matches("This is a beautiful morning, but also a beautiful day", "beautiful", :highlighter => '<b>\1</b>') + ) + + assert_equal( + "This text is not changed because we supplied an empty phrase", + highlight_matches("This text is not changed because we supplied an empty phrase", nil) + ) + + assert_equal ' ', highlight_matches(' ', 'blank text is returned verbatim') + end + + it 'sanitizes input' do + assert_equal( + "This is a <mark>beautiful</mark> morning", + highlight_matches("This is a beautiful morning<script>code!</script>", "beautiful") + ) + end + + it 'doesnt sanitize when the sanitize option is false' do + assert_equal( + "This is a <mark>beautiful</mark> morning<script>code!</script>", + highlight_matches("This is a beautiful morning<script>code!</script>", "beautiful", :sanitize => false) + ) + end + + it 'highlights using regexp' do + assert_equal( + "This is a <mark>beautiful!</mark> morning", + highlight_matches("This is a beautiful! morning", "beautiful!") + ) + + assert_equal( + "This is a <mark>beautiful! morning</mark>", + highlight_matches("This is a beautiful! morning", "beautiful! morning") + ) + + assert_equal( + "This is a <mark>beautiful? morning</mark>", + highlight_matches("This is a beautiful? morning", "beautiful? morning") + ) + end + + it 'accepts regex' do + assert_equal("This day was challenging for judge <mark>Allen</mark> and his colleagues.", + highlight_matches("This day was challenging for judge Allen and his colleagues.", /\ballen\b/i)) + end + + it 'highlights multiple phrases in one pass' do + assert_equal %(<em>wow</em> <em>em</em>), highlight_matches('wow em', %w(wow em), :highlighter => '<em>\1</em>') + end + + it 'highlights with html' do + assert_equal( + "<p>This is a <mark>beautiful</mark> morning, but also a <mark>beautiful</mark> day</p>", + highlight_matches("<p>This is a beautiful morning, but also a beautiful day</p>", "beautiful") + ) + assert_equal( + "<p>This is a <em><mark>beautiful</mark></em> morning, but also a <mark>beautiful</mark> day</p>", + highlight_matches("<p>This is a <em>beautiful</em> morning, but also a beautiful day</p>", "beautiful") + ) + assert_equal( + "<p>This is a <em class=\"error\"><mark>beautiful</mark></em> morning, but also a <mark>beautiful</mark> <span class=\"last\">day</span></p>", + highlight_matches("<p>This is a <em class=\"error\">beautiful</em> morning, but also a beautiful <span class=\"last\">day</span></p>", "beautiful") + ) + assert_equal( + "<p class=\"beautiful\">This is a <mark>beautiful</mark> morning, but also a <mark>beautiful</mark> day</p>", + highlight_matches("<p class=\"beautiful\">This is a beautiful morning, but also a beautiful day</p>", "beautiful") + ) + assert_equal( + "<p>This is a <mark>beautiful</mark> <a href=\"http://example.com/beautiful#top?what=beautiful%20morning&when=now+then\">morning</a>, but also a <mark>beautiful</mark> day</p>", + highlight_matches("<p>This is a beautiful <a href=\"http://example.com/beautiful\#top?what=beautiful%20morning&when=now+then\">morning</a>, but also a beautiful day</p>", "beautiful") + ) + assert_equal( + "<div>abc <b>div</b></div>", + highlight_matches("<div>abc div</div>", "div", :highlighter => '<b>\1</b>') + ) + end + + it 'doesnt modify the options hash' do + options = { :highlighter => '<b>\1</b>', :sanitize => false } + passed_options = options.dup + highlight_matches("<div>abc div</div>", "div", passed_options) + assert_equal options, passed_options + end + + it 'highlights with a block' do + assert_equal( + "<b>one</b> <b>two</b> <b>three</b>", + highlight_matches("one two three", ["one", "two", "three"]) { |word| "<b>#{word}</b>" } + ) + end + + end + + describe :excerpt do + + it 'excerpts' do + assert_equal("...is a beautiful morn...", excerpt("This is a beautiful morning", "beautiful", :radius => 5)) + assert_equal("This is a...", excerpt("This is a beautiful morning", "this", :radius => 5)) + assert_equal("...iful morning", excerpt("This is a beautiful morning", "morning", :radius => 5)) + assert_nil excerpt("This is a beautiful morning", "day") + end + + it 'is not html safe' do + assert !excerpt('This is a beautiful! morning', 'beautiful', :radius => 5).html_safe? + end + + it 'excerpts borderline cases' do + assert_equal("", excerpt("", "", :radius => 0)) + assert_equal("a", excerpt("a", "a", :radius => 0)) + assert_equal("...b...", excerpt("abc", "b", :radius => 0)) + assert_equal("abc", excerpt("abc", "b", :radius => 1)) + assert_equal("abc...", excerpt("abcd", "b", :radius => 1)) + assert_equal("...abc", excerpt("zabc", "b", :radius => 1)) + assert_equal("...abc...", excerpt("zabcd", "b", :radius => 1)) + assert_equal("zabcd", excerpt("zabcd", "b", :radius => 2)) + + # excerpt strips the resulting string before ap-/prepending excerpt_string. + # whether this behavior is meaningful when excerpt_string is not to be + # appended is questionable. + assert_equal("zabcd", excerpt(" zabcd ", "b", :radius => 4)) + assert_equal("...abc...", excerpt("z abc d", "b", :radius => 1)) + end + + it 'excerpts with regex' do + assert_equal('...is a beautiful! mor...', excerpt('This is a beautiful! morning', 'beautiful', :radius => 5)) + assert_equal('...is a beautiful? mor...', excerpt('This is a beautiful? morning', 'beautiful', :radius => 5)) + assert_equal('...is a beautiful? mor...', excerpt('This is a beautiful? morning', /\bbeau\w*\b/i, :radius => 5)) + assert_equal('...is a beautiful? mor...', excerpt('This is a beautiful? morning', /\b(beau\w*)\b/i, :radius => 5)) + assert_equal("...udge Allen and...", excerpt("This day was challenging for judge Allen and his colleagues.", /\ballen\b/i, :radius => 5)) + assert_equal("...judge Allen and...", excerpt("This day was challenging for judge Allen and his colleagues.", /\ballen\b/i, :radius => 1, :separator => ' ')) + assert_equal("...was challenging for...", excerpt("This day was challenging for judge Allen and his colleagues.", /\b(\w*allen\w*)\b/i, :radius => 5)) + end + + it 'excerpts with omission' do + assert_equal("[...]is a beautiful morn[...]", excerpt("This is a beautiful morning", "beautiful", :omission => "[...]",:radius => 5)) + assert_equal( + "This is the ultimate supercalifragilisticexpialidoceous very looooooooooooooooooong looooooooooooong beautiful morning with amazing sunshine and awesome tempera[...]", + excerpt("This is the ultimate supercalifragilisticexpialidoceous very looooooooooooooooooong looooooooooooong beautiful morning with amazing sunshine and awesome temperatures. So what are you gonna do about it?", "very", + :omission => "[...]") + ) + end + + it 'excerpts with utf8' do + if RUBY_VERSION.to_f >= 1.9 + assert_equal("...\357\254\203ciency could not be...".force_encoding(Encoding::UTF_8), excerpt("That's why e\357\254\203ciency could not be helped".force_encoding(Encoding::UTF_8), 'could', :radius => 8)) + else + assert_equal("...\357\254\203ciency could not be...", excerpt("That's why e\357\254\203ciency could not be helped", 'could', :radius => 8)) + end + end + + it 'doesnt modify the options hash' do + options = { :omission => "[...]",:radius => 5 } + passed_options = options.dup + excerpt("This is a beautiful morning", "beautiful", passed_options) + assert_equal options, passed_options + end + + it 'excerpts with separator' do + options = { :separator => ' ', :radius => 1 } + assert_equal('...a very beautiful...', excerpt('This is a very beautiful morning', 'very', options)) + assert_equal('This is...', excerpt('This is a very beautiful morning', 'this', options)) + assert_equal('...beautiful morning', excerpt('This is a very beautiful morning', 'morning', options)) + + options = { :separator => "\n", :radius => 0 } + assert_equal("...very long...", excerpt("my very\nvery\nvery long\nstring", 'long', options)) + + options = { :separator => "\n", :radius => 1 } + assert_equal("...very\nvery long\nstring", excerpt("my very\nvery\nvery long\nstring", 'long', options)) + + assert_equal excerpt('This is a beautiful morning', 'a'), + excerpt('This is a beautiful morning', 'a', :separator => nil) + end + + end + +end diff --git a/spec/helpers/link_to_helper_spec.rb b/spec/helpers/link_to_helper_spec.rb index 4a01ec683..261e1ef3e 100644 --- a/spec/helpers/link_to_helper_spec.rb +++ b/spec/helpers/link_to_helper_spec.rb @@ -20,6 +20,82 @@ describe LinkToHelper do end + describe 'when linking to new incoming messages' do + + before do + @info_request = mock_model(InfoRequest, :id => 123, :url_title => 'test_title') + @incoming_message = mock_model(IncomingMessage, :id => 32, :info_request => @info_request) + end + + context 'for external links' do + + it 'generates the url to the info request of the message' do + incoming_message_url(@incoming_message).should include('http://test.host/request/test_title') + end + + it 'includes an anchor to the new message' do + incoming_message_url(@incoming_message).should include('#incoming-32') + end + + it 'does not cache by default' do + incoming_message_url(@incoming_message).should_not include('nocache=incoming-32') + end + + it 'includes a cache busting parameter if set' do + incoming_message_url(@incoming_message, :cachebust => true).should include('nocache=incoming-32') + end + + end + + context 'for internal links' do + + it 'generates the incoming_message_url with the path only' do + expected = '/request/test_title#incoming-32' + incoming_message_path(@incoming_message).should == expected + end + + end + + end + + describe 'when linking to new outgoing messages' do + + before do + @info_request = mock_model(InfoRequest, :id => 123, :url_title => 'test_title') + @outgoing_message = mock_model(OutgoingMessage, :id => 32, :info_request => @info_request) + end + + context 'for external links' do + + it 'generates the url to the info request of the message' do + outgoing_message_url(@outgoing_message).should include('http://test.host/request/test_title') + end + + it 'includes an anchor to the new message' do + outgoing_message_url(@outgoing_message).should include('#outgoing-32') + end + + it 'does not cache by default' do + outgoing_message_url(@outgoing_message).should_not include('nocache=outgoing-32') + end + + it 'includes a cache busting parameter if set' do + outgoing_message_url(@outgoing_message, :cachebust => true).should include('nocache=outgoing-32') + end + + end + + context 'for internal links' do + + it 'generates the outgoing_message_url with the path only' do + expected = '/request/test_title#outgoing-32' + outgoing_message_path(@outgoing_message).should == expected + end + + end + + end + describe 'when displaying a user link for a request' do context "for external requests" do @@ -69,51 +145,4 @@ describe LinkToHelper do end - describe 'simple_date' do - - it 'formats a date in html by default' do - time = Time.utc(2012, 11, 07, 21, 30, 26) - self.should_receive(:simple_date_html).with(time) - simple_date(time) - end - - it 'formats a date in the specified format' do - time = Time.utc(2012, 11, 07, 21, 30, 26) - self.should_receive(:simple_date_text).with(time) - simple_date(time, :format => :text) - end - - it 'raises an argument error if given an unrecognized format' do - time = Time.utc(2012, 11, 07, 21, 30, 26) - expect { simple_date(time, :format => :unknown) }.to raise_error(ArgumentError) - end - - end - - describe 'simple_date_html' do - - it 'formats a date in a time tag' do - Time.use_zone('London') do - time = Time.utc(2012, 11, 07, 21, 30, 26) - expected = "<time datetime=\"2012-11-07T21:30:26+00:00\" title=\"2012-11-07 21:30:26 +0000\">November 07, 2012</time>" - simple_date_html(time).should == expected - end - end - - end - - describe 'simple_date_text' do - - it 'should respect time zones' do - Time.use_zone('Australia/Sydney') do - simple_date_text(Time.utc(2012, 11, 07, 21, 30, 26)).should == 'November 08, 2012' - end - end - - it 'should handle Date objects' do - simple_date_text(Date.new(2012, 11, 21)).should == 'November 21, 2012' - end - - end - end diff --git a/spec/integration/xapian_search_highlighting_spec.rb b/spec/integration/xapian_search_highlighting_spec.rb new file mode 100644 index 000000000..65a34cf91 --- /dev/null +++ b/spec/integration/xapian_search_highlighting_spec.rb @@ -0,0 +1,39 @@ +require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') + +describe 'highlighting search results' do + include HighlightHelper + + it 'ignores stopwords' do + phrase = 'department of humpadinking' + search = ActsAsXapian::Search.new([PublicBody], phrase, :limit => 1) + matches = search.words_to_highlight(:regex => true) + highlight_matches(phrase, matches).should == '<mark>department</mark> of <mark>humpadinking</mark>' + end + + it 'ignores case' do + search_phrase = 'department of humpadinking' + search = ActsAsXapian::Search.new([PublicBody], search_phrase, :limit => 1) + matches = search.words_to_highlight(:regex => true) + highlight_matches('Department of Humpadinking', matches).should == '<mark>Department</mark> of <mark>Humpadinking</mark>' + end + + it 'highlights stemmed words' do + phrase = 'department' + search = ActsAsXapian::Search.new([PublicBody], phrase, :limit => 1) + matches = search.words_to_highlight(:regex => true) + + search.words_to_highlight(:regex => false).should == ['depart'] + highlight_matches(phrase, matches).should == '<mark>department</mark>' + end + + it 'highlights stemmed words even if the stem is unhelpful' do + # Stemming returns 'bore' as the word to highlight which can't be + # matched in the original phrase. + phrase = 'boring' + search = ActsAsXapian::Search.new([PublicBody], phrase, :limit => 1) + matches = search.words_to_highlight(:regex => true, :include_original => true) + + highlight_matches(phrase, matches).should == '<mark>boring</mark>' + end + +end diff --git a/spec/lib/public_body_csv_spec.rb b/spec/lib/public_body_csv_spec.rb new file mode 100644 index 000000000..e3cc4be6e --- /dev/null +++ b/spec/lib/public_body_csv_spec.rb @@ -0,0 +1,114 @@ +require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') + +describe PublicBodyCSV do + + describe '.default_fields' do + defaults = [:name, + :short_name, + :url_name, + :tag_string, + :calculated_home_page, + :publication_scheme, + :disclosure_log, + :notes, + :created_at, + :updated_at, + :version] + PublicBodyCSV.default_fields.should == defaults + end + + describe '.default_headers' do + defaults = ['Name', + 'Short name', + 'URL name', + 'Tags', + 'Home page', + 'Publication scheme', + 'Disclosure log', + 'Notes', + 'Created at', + 'Updated at', + 'Version'] + PublicBodyCSV.default_headers.should == defaults + end + + describe :fields do + + it 'has a default set of fields' do + csv = PublicBodyCSV.new + csv.fields.should == PublicBodyCSV.default_fields + end + + # DO NOT include request_email (we don't want to make it + # easy to spam all authorities with requests) + it 'does not include the request_email attribute' do + csv = PublicBodyCSV.new + csv.fields.should_not include(:request_email) + end + + it 'allows custom fields to be set on instantiation' do + custom_fields = [:name, :short_name] + csv = PublicBodyCSV.new(:fields => custom_fields) + csv.fields.should == custom_fields + end + + end + + describe :headers do + + it 'has a default set of headers' do + csv = PublicBodyCSV.new + csv.headers.should == PublicBodyCSV.default_headers + end + + it 'allows custom headers to be set on instantiation' do + custom_headers = ['Name', 'Short Name'] + csv = PublicBodyCSV.new(:headers => custom_headers) + csv.headers.should == custom_headers + end + + end + + describe :rows do + + it 'is empty on instantiation' do + csv = PublicBodyCSV.new + csv.rows.should be_empty + end + + end + + describe :<< do + + it 'adds an elements attributes to the rows collection' do + csv = PublicBodyCSV.new + expected = ["Ministry of Silly Walks,MSW,msw,useless_agency,http://www.localhost,\"\",\"\",You know the one.,2007-10-25 10:51:01 UTC,2007-10-25 10:51:01 UTC,1"] + csv << PublicBody.find(5) + csv.rows.should == expected + end + + end + + describe :generate do + + it 'generates the csv' do + expected = <<-CSV +Name,Short name,URL name,Home page,Publication scheme,Disclosure log,Notes,Created at,Updated at,Version +Department for Humpadinking,DfH,dfh,http://www.localhost,"","",An albatross told me!!!,2007-10-25 10:51:01 UTC,2007-10-25 10:51:01 UTC,2 +Department of Loneliness,DoL,lonely,http://www.localhost,"","",A very lonely public body that no one has corresponded with,2011-01-26 14:11:02 UTC,2011-01-26 14:11:02 UTC,1 +CSV + + # Miss out the tags field because the specs keep changing the order + # that the tags are returned in + fields = [:name, :short_name, :url_name, :calculated_home_page, :publication_scheme, :disclosure_log, :notes, :created_at, :updated_at, :version] + headers = ['Name', 'Short name', 'URL name', 'Home page', 'Publication scheme', 'Disclosure log', 'Notes', 'Created at', 'Updated at', 'Version'] + + csv = PublicBodyCSV.new(:fields => fields, :headers => headers) + csv << PublicBody.where(:name => 'Department for Humpadinking').first + csv << PublicBody.where(:name => 'Department of Loneliness').first + csv.generate.should == expected + end + + end + +end diff --git a/spec/models/customstates.rb b/spec/models/customstates.rb index bffbe86fb..942e1fcde 100644 --- a/spec/models/customstates.rb +++ b/spec/models/customstates.rb @@ -24,7 +24,7 @@ module InfoRequestCustomStates end def date_deadline_extended - # XXX shouldn't this be 15 days after the date the status was + # TODO: shouldn't this be 15 days after the date the status was # changed to "deadline extended"? Or perhaps 15 days ater the # initial request due date? return Holiday.due_date_from_working_days(self.date_response_required_by, 15) diff --git a/spec/models/public_body_spec.rb b/spec/models/public_body_spec.rb index 38e31783d..a7544c218 100644 --- a/spec/models/public_body_spec.rb +++ b/spec/models/public_body_spec.rb @@ -493,7 +493,7 @@ describe PublicBody, " when loading CSV files" do PublicBody.count.should == original_count + 3 - # XXX Not sure why trying to do a I18n.with_locale fails here. Seems related to + # TODO: Not sure why trying to do a I18n.with_locale fails here. Seems related to # the way categories are loaded every time from the PublicBody class. For now we just # test some translation was done. body = PublicBody.find_by_name('North West Fake Authority') @@ -594,6 +594,20 @@ describe PublicBody do end + describe :site_administration? do + + it 'is true when the body has the site_administration tag' do + p = FactoryGirl.build(:public_body, :tag_string => 'site_administration') + p.site_administration?.should be_true + end + + it 'is false when the body does not have the site_administration tag' do + p = FactoryGirl.build(:public_body) + p.site_administration?.should be_false + end + + end + end describe PublicBody, " when override all public body request emails set" do diff --git a/spec/models/raw_email_spec.rb b/spec/models/raw_email_spec.rb index f86b35e99..aa82b0bc3 100644 --- a/spec/models/raw_email_spec.rb +++ b/spec/models/raw_email_spec.rb @@ -23,7 +23,7 @@ describe User, "manipulating a raw email" do @raw_email.data.should == "Hello, world!" end - # XXX this test fails, hopefully will be fixed in later Rails. + # TODO: this test fails, hopefully will be fixed in later Rails. # Doesn't matter too much for us for storing raw_emails, it would seem, # but keep an eye out. diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index c54043092..7dcd3ab8a 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -284,7 +284,7 @@ describe User, " when making name and email address" do end end -# XXX not finished +# TODO: not finished describe User, "when setting a profile photo" do before do @user = User.new diff --git a/spec/models/xapian_spec.rb b/spec/models/xapian_spec.rb index a1e060d8e..678e3a2dc 100644 --- a/spec/models/xapian_spec.rb +++ b/spec/models/xapian_spec.rb @@ -380,23 +380,63 @@ describe ActsAsXapian::Search, "#words_to_highlight" do it "should return a list of words used in the search" do s = ActsAsXapian::Search.new([PublicBody], "albatross words", :limit => 100) - s.words_to_highlight.should == ["albatross", "words"] + s.words_to_highlight.should == ["albatross", "word"] end it "should remove any operators" do s = ActsAsXapian::Search.new([PublicBody], "albatross words tag:mice", :limit => 100) - s.words_to_highlight.should == ["albatross", "words"] + s.words_to_highlight.should == ["albatross", "word"] end - # This is the current behaviour but it seems a little simplistic to me it "should separate punctuation" do s = ActsAsXapian::Search.new([PublicBody], "The doctor's patient", :limit => 100) - s.words_to_highlight.should == ["The", "doctor", "s", "patient"] + s.words_to_highlight.should == ["the", "doctor", "patient"].sort end it "should handle non-ascii characters" do s = ActsAsXapian::Search.new([PublicBody], "adatigénylés words tag:mice", :limit => 100) - s.words_to_highlight.should == ["adatigénylés", "words"] + s.words_to_highlight.should == ["adatigénylé", "word"] + end + + it "should ignore stopwords" do + s = ActsAsXapian::Search.new([PublicBody], "department of humpadinking", :limit => 100) + s.words_to_highlight.should_not include('of') + end + + it "uses stemming" do + s = ActsAsXapian::Search.new([PublicBody], 'department of humpadinking', :limit => 100) + s.words_to_highlight.should == ["depart", "humpadink"] + end + + it "doesn't stem proper nouns" do + s = ActsAsXapian::Search.new([PublicBody], 'department of Humpadinking', :limit => 1) + s.words_to_highlight.should == ["depart", "humpadinking"] + end + + it "includes the original search terms if requested" do + s = ActsAsXapian::Search.new([PublicBody], 'boring', :limit => 1) + s.words_to_highlight(:include_original => true).should == ['bore', 'boring'] + end + + it "does not return duplicate terms" do + s = ActsAsXapian::Search.new([PublicBody], 'boring boring', :limit => 1) + s.words_to_highlight.should == ['bore'] + end + + context 'the :regex option' do + + it 'wraps each words in a regex that matches the full word' do + expected = [/\b(albatross)\b/iu] + s = ActsAsXapian::Search.new([PublicBody], 'Albatross', :limit => 1) + s.words_to_highlight(:regex => true).should == expected + end + + it 'wraps each stem in a regex' do + expected = [/\b(depart)\w*\b/iu] + s = ActsAsXapian::Search.new([PublicBody], 'department', :limit => 1) + s.words_to_highlight(:regex => true).should == expected + end + end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index e391c97d3..0e3fe35c7 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -123,7 +123,7 @@ Spork.prefork do end end - # XXX No idea what namespace/class/module to put this in + # TODO: No idea what namespace/class/module to put this in # Create a clean xapian index based on the fixture files and the raw_email data. def create_fixtures_xapian_index load_raw_emails_data |