diff options
42 files changed, 590 insertions, 617 deletions
@@ -5,7 +5,7 @@ require(File.join(File.dirname(__FILE__), 'config', 'boot')) require 'rake' require 'rake/testtask' -require 'rake/rdoctask' +require 'rdoc/task' require 'tasks/rails' diff --git a/app/models/incoming_message.rb b/app/models/incoming_message.rb index 2b795ddf5..97e4a6c09 100644 --- a/app/models/incoming_message.rb +++ b/app/models/incoming_message.rb @@ -606,21 +606,13 @@ class IncomingMessage < ActiveRecord::Base text.gsub!(/^(>.*\n)/, replacement) text.gsub!(/^(On .+ (wrote|said):\n)/, replacement) - # Multiple line sections - # http://www.whatdotheyknow.com/request/identity_card_scheme_expenditure - # http://www.whatdotheyknow.com/request/parliament_protest_actions - # http://www.whatdotheyknow.com/request/64/response/102 - # http://www.whatdotheyknow.com/request/47/response/283 - # http://www.whatdotheyknow.com/request/30/response/166 - # http://www.whatdotheyknow.com/request/52/response/238 - # http://www.whatdotheyknow.com/request/224/response/328 # example with * * * * * - # http://www.whatdotheyknow.com/request/297/response/506 - ['-', '_', '*', '#'].each do |score| + ['-', '_', '*', '#'].each do |scorechar| + score = /(?:[#{scorechar}]\s*){8,}/ text.sub!(/(Disclaimer\s+)? # appears just before ( - \s*(?:[#{score}]\s*){8,}\s*\n.*? # top line + \s*#{score}\n(?:(?!#{score}\n).)*? # top line (disclaimer:\n|confidential|received\sthis\semail\sin\serror|virus|intended\s+recipient|monitored\s+centrally|intended\s+(for\s+|only\s+for\s+use\s+by\s+)the\s+addressee|routinely\s+monitored|MessageLabs|unauthorised\s+use) - .*?((?:[#{score}]\s*){8,}\s*\n|\z) # bottom line OR end of whole string (for ones with no terminator XXX risky) + .*?(?:#{score}|\z) # bottom line OR end of whole string (for ones with no terminator XXX risky) ) /imx, replacement) end diff --git a/config/boot.rb b/config/boot.rb index 90fd9cd4d..0f5e661ed 100644 --- a/config/boot.rb +++ b/config/boot.rb @@ -31,7 +31,7 @@ module Rails end def vendor_rails? - File.exist?("#{RAILS_ROOT}/vendor/rails") + File.exist?("#{RAILS_ROOT}/vendor/rails/Rakefile") end def preinitialize diff --git a/config/environment.rb b/config/environment.rb index 0af465049..91839c043 100644 --- a/config/environment.rb +++ b/config/environment.rb @@ -6,7 +6,7 @@ # ENV['RAILS_ENV'] ||= 'production' # Specifies gem version of Rails to use when vendor/rails is not present -RAILS_GEM_VERSION = '2.3.11' unless defined? RAILS_GEM_VERSION +RAILS_GEM_VERSION = '2.3.14' unless defined? RAILS_GEM_VERSION # Bootstrap the Rails environment, frameworks, and default configuration require File.join(File.dirname(__FILE__), 'boot') diff --git a/db/.gitignore b/db/.gitignore index 7b6eceda6..f3aee50a1 100644 --- a/db/.gitignore +++ b/db/.gitignore @@ -1 +1,2 @@ +schema.rb development_structure.sql diff --git a/doc/INSTALL.md b/doc/INSTALL.md index 656b96467..bb8c7de21 100644 --- a/doc/INSTALL.md +++ b/doc/INSTALL.md @@ -266,3 +266,16 @@ is supplied in `../conf/varnish-alaveteli.vcl`. Did you remember to remove the file `alaveteli/config/rails_env.rb` as described above? It's created every time you run `script/rails-post-deploy` + +* **Non-ASCII characters are being displayed as asterisks in my incoming messages** + + We rely on `elinks` to convert HTML email to plain text. + Normally, the encoding should just work, but under some + circumstances it appears that `elinks` ignores the parameters + passed to it from Alaveteli. + + To force `elinks` always to treat input as UTF8, add the following + to `/etc/elinks/elinks.conf`: + + set document.codepage.assume = "utf-8" + diff --git a/doc/todo.txt b/doc/todo.txt deleted file mode 100644 index e78429960..000000000 --- a/doc/todo.txt +++ /dev/null @@ -1,559 +0,0 @@ - - -Next (things that will reduce admin time mainly) -==== - -- "Can I help out" a bit invisible - -Destroy request - does it remove the tags? -Richard says he wants the internationalisation to be so it could be one site -with combined search. Why obey the notion of a country? I'm not sure, but -it might be prudent to write it so it can run multiple jurisdictions in -one deploy, if only for administrative reasoins. - - path maybe: lib/juris/uk, lib/juris/eu etc. - - consider Single Table Inheritance (harder to back out of though) - - http://code.alexreisner.com/articles/single-table-inheritance-in-rails.html - - use mixins with explicit include otherwise - -Add links to these tags where possible: - ch:* - Bodies that appear on the Register of Companies. '*' is replaced by the company number, which is eight characters long and consists of optional upper-case letters followed by digits. - coins:* Bodies appearing in COINS database followed by reference code e.g. coins:BRL048 (British Library) - dpr:* - Bodies that appear on the Register of Data Controllers. '*' is replaced by the registration number. - urn:* - Bodies that appear on EduBase. '*' is replaced by the institution's Unique Reference Number. - VAT:* - Bodies with a VAT (UK Value Added Tax) registration number. '*' is replaced by the VAT registration number (no spaces) e.g. VAT:895108987 -http://foiwiki.com/foiwiki/index.php/WhatDoTheyKnow.com_Tags - -Merge with New Zealand code base properly - -Handle bounce messages from alerts automatically - -Make it so when you make followups the whole request is shown on the page. -Remove all show_response URLs, and replace with a special version of the -request URL with a new input box at the bottom and a hash link to it? -<< when following links such as "I'm about to send clarification", a -form appears into which the reply can be typed. However, the -previous correspondence in that thread is not shown. -I usually open a new tab to see what was written previously before -writing in the form. It might be useful if the previous -correspondence were instead shown on the page in which the form -appears. >> - -Make profile photo on comments slightly larger - -Ask people for annotation immediately after they have submitted their request -Ask for annotation about what they learnt from request? - -Private request premium feature -http://www.activemerchant.org/ - -Froze during reindex, is the doc files inside the .zip here: - http://www.whatdotheyknow.com/request/last_collection_times#incoming-8405 - ActsAsXapian.rebuild_index InfoRequestEvent 16061 - foi 23175 0.0 0.0 5176 1472 pts/1 S+ 14:16 0:00 /bin/sh /usr/bin/wvText /tmp/foiextract20100619-20578-1gcbuqz-0 /tmp/foiextract20100619-20578-1gcbuqz-0.txt - foi 23180 0.0 0.0 4664 1220 pts/1 S+ 14:16 0:00 /bin/sh /usr/bin/wvHtml -1 /tmp/foiextract20100619-20578-1gcbuqz-0 --targetdir=/tmp wv-XeJwGT -Also freezes Abiword, but not catdoc - - -Performance -=========== - -Reduce storing the number of bogus post redirects that aren't people - -Receiving email can be resource drain starting app instance each time - use daemon instead - -Cache /feed/list/successful -Cache /body/list/a - -Cache parts of /body/xxxxx -Cache parts of /user/xxxxx - -Finish migration to Rails 1.9 - for uncached requests, seems to be twice as fast. - -Regular expression library - change to faster one. Oniguruma isn't enough. -This shows slowness: - e = InfoRequestEvent.find(213700) - text = e.incoming_message.get_main_body_text (XXX alter to call internal not cache) - IncomingMessage.remove_quoted_sections(text, "") - -This is slow: -http://www.whatdotheyknow.com/request/renumeration_committee - -Varnish config -http://www.varnish-cache.org/wiki/VCLExampleCachingLoggedInUsers - -Some requests to lower memory use of still: -PID: 676 CONSUME MEMORY: 16968 KB Now: 102604 KB http://www.whatdotheyknow.com/request/parking_ticket_data_81 -PID: 2036 CONSUME MEMORY: 129368 KB Now: 179652 KB http://www.whatdotheyknow.com/request/14186/response/33740 -- search engines shouldn't be going for those URLs. and do they really need to - unpack so much? could use snippet cache. - -Things to make bots not crawl somehow: - /request/13683/response?internal_review=1 - /request/febrile_neutropenia_154?unfold=1 - -Renaming of a body, or changing its domain, should clear the cached bubbles of -all requests to that body. - -Change it to store emails as files in the filesystem? For speed of backup if nothing else. -Should have simpler system for us to upload files sent to us via CD etc. -Currently we have to manually put them in the files directory on the vhost. -Make it so web upload interface copes gracefully with arbitarily large messages -(it causes speed trouble having them in the database right now) -Maybe have flag saying "don't try to make snippet for this it is just too big". - -Compress the emails in the database, or on disk - -Keep cached columns in database in their own table, might give performance benefits - - - - -Letting you hide individual events (incoming/outgoing messages, annotations) -================================== - -*** this needs either removing or finishing, it is half done. Has the -database entries but doesn't use them yet. - -Move comments to use new system first -Show message to signed in user that others can't see this part -Make sure hidden things don't show in search snippets - in models/comment.rb: # So when made invisble it vanishes -Remove comments visible - - -Later -===== - -JSON API: - Pagination on the Atom feed JSON, so you can get later pages, and/or choose > 25 items - Information about attachments in event JSON - Allow Javascript callbacks (JSONP) - -Spelling correction not working if you search for "comission", as described here: - http://comments.gmane.org/gmane.comp.search.xapian.general/8384 -When this patch from Xapian is in stable version, check that it all works. - http://trac.xapian.org/changeset/14859 - -Make outgoing requests and follow ups get CCed to our backup mailbox, so that -can do data recovery more easily - -Admin button to resend request one off to particular address - -Stop search for users working on unconfirmed accounts - -Make zip file contents browseable, so each document in them "appears in Google". -http://www.whatdotheyknow.com/request/transport_direct_cycle_journey_p#incoming-78421 - -Ability to move requests to other bodies. Useful in these two places - anywhere else? -http://www.whatdotheyknow.com/body/suffolk_primary_care_trust_pct_duplicate -http://www.whatdotheyknow.com/body/colchester_hospital_university_nhs_foundation_trust_duplicate - -In admin interface, move a response from the holding pen to a request which is -closed to new responses. The message disappears into the ether. Should either -stop or allow such moves. - -PDF that gets corrupted by email censoring - have only seen this once, watch for it again - http://www.whatdotheyknow.com/request/information_on_traffic_flows_in -The image in a "stream" section get corrupted: - _#p!/DB]eER4cPAPm&W7;-]L!e(*U=7"h^X7hYXqSI][9UZJV+>hr2:&c@S.lRr.ndm)2]b$-lU+#lg - _#p!/DB]eER4cPAPm&W7;-]L!e(*U=7"h^X7hYXqSI][9UZJV+>hr2:&x@x.xxx.xxx)2]b$-lU+#lg -Needs a fancy PDF library (which doesn't exist yet) that can tell when it is -binary or text stream within the file. See thread in email "corrupted pdf" for -more details. Maybe have option in admin to turn off censoring on a particular file. -Maybe just do an MX check to see if it is really an email :) - -check-recent-requests-sent probably doesn't work, as exim log lines wouldn't -be load in case where the envelope from gets broken? - -Point all MX records to one server, so can see incoming messages in exim logs also. -Hmmm, but less robust. Run the exim log grabber across all mail servers? - -When on a small screen, the actual form when making a new request is below -the fold, and it isn't obvious what you need to do. (Seen while watching -a new user try to make a request) - -Put public body name in search text for each info request, so that people -typing in a word and a body name in the search (have seen people new -to the site doing that) do find things - -Completely remove the "feed" option in the database from tracks (it is only -there for historical reasons, as feeds used to have to be explicitly in the -database) - -Show the Subject: lines on request pages. Perhaps only show them where they are -substantively different (modulo Re: and Fwd:) from the title and other subjects -- so you can see any FOI code number the authority has put in the subject. - -For Scotland, don't need to say "normally" equivocally when it is taking more than 20 days -(as there is no public interest test). -Arguably, and to simplify things with Alaveteli, we should just be campaigning about this whole public interest test thing anyway, and not pussy foot about it. Not sure what form of wording to use that does that that is true though. - -Add explicit option for user to select "misdelivered to the wrong request" -and let people move them to the right place. -(Julian wants that too) - -Perhaps fold up request pages more by default - don't show known acknowledgements until -you click and some (javascript) expands them. - -Some people want the fold/unfold of quoted sections to be javascripty as well. -Esp. when filling in a form on the same page. - -Somehow make clear that a "rejection because referring to info already -in public domain" should really be marked sucessful. - -Emails sent to stopped requests should follow RFC: http://tools.ietf.org/html/rfc3834 - Shouldn't bounce message back to Auto-Submitted - Should check from address being replied to is valid - Should set In-Reply-To and References fields - Reconsider message content given that section in RFC - -When registering a new user, give a warning if they only enter one name. Link -to the help about pseudonymous requests, that you need at least initial -as well. - -Let requesters view the uncensored versions of their correspondence (e.g. with -emails in it). Let other people do so with a CAPTCHA? -Perhaps easy thing to do is just to do it for email addresses (and not the request email address) - -For followups, have radio button to say is it a new request or followup -Do by uncommenting the "new information" option when writing a followup, so -that it makes a new request - -When a user says that a response is an error message, it prompts them to say -what the problem is to administrators. It could at that time show them the -email of the authority that their request was sent to for them to check -then and there, and tell the administrators. - -If you've already conducted an internal review, at all places - - when on unhappy/url - - when on not held link - - on the page for the request -don't offer it again, as they've already done it. -Example of completed review: -http://www.whatdotheyknow.com/request/request_for_full_disclosure_of_b#incoming-9267 - -Don't allow sending internal review boiler plate text twice (although make sure they -can write followups to internal review) -e.g. http://www.whatdotheyknow.com/request/reply_to_letter_from_historic_ro - -Clock for internal review - The Information Commissioner has issued a "Good Practice Guidance" document: http://www.ico.gov.uk/upload/documents/library/freedom_of_information/detailed_specialist_guides/foi_good_practice_guidance_5.pdf - 20 days is late - 40 days max. -Fix up the text: "The internal review should take 2-3 weeks for simple cases, -and up to 6 weeks even for complex reviews." -Awaiting internal review overdue state? - -Sort requests on user page by status. - "For sorting I was just thinking of a generic sort/filter by clicking - on the header or some such -- I'd probably want to sort open requests - in order of 'last action'... to quickly see what was most overdue." -Group list on user page by authority - -I have several email alerts set up. Is there any chance they could include part -(or, preferably, all) of the search criterion in the Subject: line? :o) -(Perhaps do it in the case when only one search criterion makes the mail) - -Search for text "internal review" in followups and add warning if they aren't -using the internal review mode. - -CSS / design things - - The stepwise instruction boxes "Next, select the public authority ... " - need to look better, and have icons associated with them etc. - - CSS error on "all councils" page on some browsers - https://bugzilla.mozilla.org/show_bug.cgi?id=424194 - - Spacing on error boxes round form elements. Matthew says: - Well, the correct thing to do is have the class="fieldWithErrors" on the - <p> containing the Summary: label and text input box, not have the - pointless <span> at all, and then it all looks perfect and as you'd - expect. But I had a look at the code and haven't got the slightest clue - how you'd do that, sorry, given it appears new.rhtml is printing the <p> - but some magic Ruby thing is printing the error span. - - Improve CSS on IE7 for large images in docs - http://www.whatdotheyknow.com/request/3289/response/7810/attach/html/3/20081023ReplyLetter.pdf.html - - Get Atom feed of search results to include stylesheet for highlighting words in - yellow somehow - -When doing search, people often just want it to show the whole page. Perhaps -all listing should just link to top of page, rather than # links for outgoing -incoming, or perhaps just some of them. - -Some more traditional help (in a new section in the help) such as: - * Information about how to track requests and RSS feeds - * Information about how to contacting other users - * How to change your email address - -Show similar requests after you have filed yours - maybe on preview too. - -Test code for rendering lots of different attachments and filetypes -Test code for internal review submitting - -Look at quote_address_if_necessary in actionmailer's quoting.rb - why did it -not work for the email address with "@" in its name part? - -Something to check which tags are used but aren't in PublicBody category lists - -Other references to title field changes don't get search index updated when title is altered - (e.g. when a public body is renamed) -Maybe just reindex all once a week, but it is a bit slow now, so perhaps do it properly. -$ ./script/rebuild-xapian-index - -Renaming public authorities will break alerts on them. For basic alerts the -structured info is there so this should just be fixed. For searches, perhaps -Xapian index should search url_name history as well? -We have a policy of not renaming them in some cases, which helps a bit. - -Never updates cached attachment text unless cache is explicitly cleared (which - might matter with software updates, or code changes). Should we clear the - cache automatically every month in the middle of the night or something? -$ ./script/clear-caches -Alternative - embed some identifier in the code and the cache directory, and -detect it changes and automatically clear the cache. - - -Display and indexing of response emails/attachments ---------------------------------------------------- - -Install more recent poppler-utils - e.g. 0.12.0 can definitely convert this to HTML, extacting the images: - http://www.whatdotheyknow.com/request/13903/response/36117/attach/html/4/FOI%20beaver%20site%20species%20audit%20SNH%20review%20of%20proposal%20redact.pdf.html -Really need a "pdftk -nodrm" to remove compression from encrypted PDFs, so strips emails from e.g.: - http://www.whatdotheyknow.com/request/14414/response/38590/attach/html/3/090807%20FOI.pdf.html - ... this misses a whole page out (someone emailed us) - http://www.whatdotheyknow.com/request/unredacted_expense_claims_for_jo#incoming-49674 - -Worth doing View as HTML ourselves for .docx, .ppt, .tif (covered now by Google Docs) -View as HTML for .txt requested - -Failed to detect attachments are emails and decode them: - http://www.whatdotheyknow.com/request/malicious_communication_act#incoming-12964 - -When indexing .docx do you need to index docProps/custom.xml and docProps/app.xml -as well as word/document.xml ? (thread on xapian-discuss does so) - -Consider using odt2txt or unoconv -http://www-verimag.imag.fr/~moy/opendocument/ - -Mime type / extension wrong on these .docx's -http://www.whatdotheyknow.com/request/bridleway_classifications -http://www.whatdotheyknow.com/request/19976/response/51468/attach/3/TU%20MembershipTeachers%20SolidarityTU%20231009.docx.doc (thinks it is doc when it is docx) -Search for "OIC" for some more examples - -VSD files vsdump - example in zip file - http://www.whatdotheyknow.com/request/dog_control_orders#incoming-3510 - doing file RESPONSE/Internal documents/Briefing with Contact Islington/Contact Islington Flowchart Jul 08.vsd content type - -Search for other file extensions that we have now and look for ones we could -and should be indexing - (call IncomingMessage.find_all_unknown_mime_types to find them - needs - updating to do it in clumps as all requests won't load in RAM now ) - -Render HTML alternative rather than text (so tables look good) e.g.: - http://www.whatdotheyknow.com/request/parking_policy - -These attachment.bin files should come out as winmail.dat and be parsed -by existing TNEF code. For some reason though TMail doesn't get the right -content-type out of them. Not sure why. - http://www.whatdotheyknow.com/request/acting_up_in_a_higher_rank - -Make HTML attachments have view as HTML :) -http://www.whatdotheyknow.com/request/enforced_medication#incoming-7395 - -Knackered view as HTML: - http://www.whatdotheyknow.com/request/1385/response/5483/attach/html/3/Response%20465.2008.pdf.html -Some other pdftohtml bugs (fix them or file about them) - http://www.whatdotheyknow.com/request/sale_of_public_land#incoming-8146 - http://www.whatdotheyknow.com/request/childrens_database_compliance_wi#incoming-8088 - http://www.whatdotheyknow.com/request/3326/response/7701/attach/html/2/Scan001.PDF.pdf.html - http://www.whatdotheyknow.com/request/risk_log#incoming-8090 (bad tables) - http://www.whatdotheyknow.com/request/4635/response/11248/attach/html/4/FOI%20request.pdf.html (bad table) -Orientation wrong: - http://www.whatdotheyknow.com/request/3153/response/7726/attach/html/2/258850.pdf.html -Bug in wvHtml, segfaults when converting this: - http://www.whatdotheyknow.com/request/subject_access_request_guide_sar#incoming-10242 - -Images aren't coming out here - http://www.whatdotheyknow.com/request/33682/response/83455/attach/html/3/100428%20Reply%201519%2010.doc.html - -Doesn't detect doc type of a few garbage results in this list right: - http://www.whatdotheyknow.com/search/UWE - -Quoting fixing TODO: - http://www.whatdotheyknow.com/request/35/response/191 # Funny disclaimer - http://www.whatdotheyknow.com/request/40/response/163 # funny disclaimer - http://www.whatdotheyknow.com/request/35/response/191 # funny disclaimer "- - Disclaimer - -" - - http://www.whatdotheyknow.com/request/m3_junction_2_eastbound_speed_re # cut here - - http://www.whatdotheyknow.com/request/123/response/184 # nasty nasty formatted quoting - http://www.whatdotheyknow.com/request/155/response/552 # nasty nasty formatted quoting - http://www.whatdotheyknow.com/request/how_do_the_pct_deal_with_retirin_87#incoming-1847 - http://www.whatdotheyknow.com/request/complaints_about_jobcentres#incoming-688 # word wrapping of < - - http://www.whatdotheyknow.com/request/224/response/589 # have knackered the apostrophes here - - http://www.whatdotheyknow.com/request/operation_oasis_protester_databa#incoming-20922 - - http://www.whatdotheyknow.com/request/new_bristol_city_stadium_plansci#incoming-44114 # funny forward not detected - http://www.whatdotheyknow.com/request/the_facts_about_side_effects_cau#incoming-6754 # "Communications via the GSi" should be stripped, so then subject would get shown - - Unclassified: - http://www.whatdotheyknow.com/request/666/response/1020 - http://www.whatdotheyknow.com/request/364/response/1100 - http://www.whatdotheyknow.com/request/council_housing_accommodation # over zealous half cuts - http://www.whatdotheyknow.com/request/621/response/1131 # virus footer - http://www.whatdotheyknow.com/request/231/response/338 - http://www.whatdotheyknow.com/request/930/response/1609 - http://www.whatdotheyknow.com/request/1102/response/2067 - http://www.whatdotheyknow.com/request/list_of_public_space_cctv_instal#incoming-2164 - http://www.whatdotheyknow.com/request/errors_in_list_of_postbox_locati#incoming-2272 - http://localhost:3000/request/cctv_data_retention_and_use#incoming-2093 - http://www.whatdotheyknow.com/request/stasi_activity_at_climate_camp#incoming-3362 - http://www.whatdotheyknow.com/request/total_remuneration_and_benefits#incoming-2436 - http://www.whatdotheyknow.com/request/dual_british_and_israeli_nationa#incoming-3461 - http://www.whatdotheyknow.com/request/council_functions_55#incoming-4099 - http://www.whatdotheyknow.com/request/public_safety_consequential_to_c#incoming-1586 - http://www.whatdotheyknow.com/request/functions_council_43#incoming-4509 - http://www.whatdotheyknow.com/request/york_road_tube_re_opening_feasib#incoming-3509 - http://www.whatdotheyknow.com/request/controlled_drinking_zones_5#incoming-4210 - http://www.whatdotheyknow.com/request/road_and_junction_specifications#incoming-3598 - http://www.whatdotheyknow.com/request/disused_live_stations#incoming-4898 - http://www.whatdotheyknow.com/request/errors_in_list_of_postbox_locati#incoming-3577 - - http://www.whatdotheyknow.com/request/public_inspection_periods_for_lo_2#outgoing-1707 # square bracket in link - http://www.whatdotheyknow.com/request/digital_tv_switchover_in_local_a#incoming-4931 - http://www.whatdotheyknow.com/request/local_government_ombudsman_58#incoming-5763 - - http://www.whatdotheyknow.com/request/415/response/1041/attach/3/CONF%20FOI%209508%20Ian%20Holton.doc - - http://www.whatdotheyknow.com/request/function_council_88#incoming-6258 - http://www.whatdotheyknow.com/request/please_submit_the_surveyors_repo#incoming-6334 # charset - http://www.whatdotheyknow.com/request/archive_record#incoming-7514 # charset - - http://www.whatdotheyknow.com/request/enforcement_forders_for_replacin#incoming-6277 # over zealous quoting - http://www.whatdotheyknow.com/request/renewable_energy_consumption_by # over zealous - - http://www.whatdotheyknow.com/request/can_my_mp_ask_questions_on_my_be#incoming-33112 # hyperlink broken - http://www.whatdotheyknow.com/request/clarification_of_the_igs_to_psw # wrapped link - http://www.whatdotheyknow.com/request/request_for_details_from_consult # wrapped link - - http://www.whatdotheyknow.com/request/independent_psychological_assess#incoming-52956 # shows text as attachment when could be inline due to multipart nature? - - http://www.whatdotheyknow.com/request/bnp_membership_list_43#incoming-59204 # not detecting original message - http://www.whatdotheyknow.com/request/maximum_pedestrian_crossing_wait#incoming-34094 # not detecting original message - - -Display pasted quotes in annotations better: - http://www.whatdotheyknow.com/request/scientology_incidents#comment-3352 - - -Totally new features --------------------- - -Publish statistics (stats) on how long it takes bodies to respond. And other -things (like the WriteToThem pages). - -Add interface for editing tags on your own requests so you can keep track of them more easily. -Lisa asked for this - is definitely only whole requests needed. Tony says -anyone should be able to edit the tags, but requester should have last say (so -can prevent a tag being added that they removed). - -Read reply - ask for Microsoft Exchange "read receipts", and show if mail was read. -Or maybe use a webbug. - -Telephone numbers. Add advice in workflow to call authority first to check -form they have info in. Store telephone numbers in database. - -Give authorities interface for editing their request email address -and resend messages to them - -Make search know about uncategorised requests and timed out requests. -And make search able to do *current* status in general as operator. - -Test data dumper that removes sensitive data, but lets trusted people play with -whole database on their own machine without risk of compromise (for Tony) -- can avoid rebuilding emails, attachments etc. sanitized provided we don't -mind leaking out email address ot requests etc. to the trusted person (in contrast -can easily totally remove private emails in the user table) - -Have an interface for users to be able to suggest new authorities and give -their email address (perhaps just have admins validate / approve it) - -Detect councils that always send automatic acknowledgements, and notice if they -do not for a particular request? (e.g. Leicestershire County Council) - -Add tips on using the law, e.g.: -- You can go up and down between local and national - ask local places what - their policy is, and hwo they are implementing it. Ask national things what - odcuments set local policies. - -Add note by any exemption to the page on FOI Wiki Add note on mention of -"Re-Use of Public Sector Information Regulations 2005" -to the appropriate FAQ. - -Hyperlink Section 1(3) to the act (or have an automatic tip appear) - http://www.whatdotheyknow.com/request/university_investment_in_the_arm#incoming-86 -and to guidance notes - http://www.ico.gov.uk/what_we_cover/freedom_of_information/guidance.aspx - -Link to /random jump to a random request somewhere - -Do conversion tracking on endpoints in WDTK, advertise perhaps TWFY, or perhaps - donations to mySociety. -Advertise WDTK search queries on TWFY -Advertise alerts on end pages with WDTK - -Forms to search this user, this request, and this authority on their pages - -Search FAQ and other help pages with normal search - -Make text boxes autogrow as you type into them. -(10:32:14) richard: you just need to count the number of rows of text and compare it to the number of rows in the textbox -(10:32:29) richard: then increase the height of the textbox by 1em-ish -(10:32:52) Matthew: their function is called autogrow_textarea() by the way, if you just want to look at it... - thanks :) I won't do it now as there are more important things, I was just accidentally impressed - -"Why not fill in your profile?" not that many people fill in their profile, we could have special -advert (on page when they are on site) trying to force them to (maybe only if active). Give -points for doing this. -Also could offer option of Gravatar during signup, or later (as explicit option, so not leaking private info) - -Scoreboards and gamification. Put all admin functions in the main site, and let anyone get -power to do them by accumulating points doing other useful things. There's lots of stuff! - -.tif files are hard for people to view as multi page, consider automatically -separating out the pages as separate links (to .png files or whatever) - http://www.whatdotheyknow.com/request/windsor_maidenhead_council_commo#incoming-1910 -Heck, may as well give thumbnails of all images, indeed all docs while you're at it :) - -Add geographical location of councils, PCTs etc. -Have a single button to sign up to alerts on authorities for your postcode -NHS postcode database: - http://www.ons.gov.uk/about-statistics/geography/products/geog-products-postcode/nhspd/index.html - -Make request preview have a URL so you can show it to someone else before sending it :) -Proposed request submission queue with comments - new requests don't get sent straight - away, but are delayed while people help improve them. - -Screen scrape ICO's decision notices list and add link to it on the relevant public authority pages -http://www.ico.gov.uk/Home/tools_and_resources/decision_notices.aspx - -Description for each body as to what info it holds -Link to: - Company number - Aliases (not just short name, but multiple real names e.g. for museums) - Disclosure logs - Publication schemes (http://www.ico.gov.uk/what_we_cover/freedom_of_information/publication_schemes.aspx) - TWFY department search - Complaint email - Phone number for advice and assistence (House of Lords give one http://www.parliament.uk/parliamentary_publications_and_archives/freedom_of_information_in_the_house_of_lords/lords__foi___how_to_obtain_information.cfm ) -e.g. - http://www.ordnancesurvey.co.uk/oswebsite/aboutus/foi/index.html - http://www.ordnancesurvey.co.uk/oswebsite/aboutus/foi/coiindex.html -Maybe gather this data by letting authorities input it - -EU regulation 1049/2001 requests -US requests (with Sunlight) - -OCR all images automatically, even if badly (check for tiffs!) - diff --git a/lib/old_rubygems_patch.rb b/lib/old_rubygems_patch.rb index 5601a5e90..3001a7381 100644 --- a/lib/old_rubygems_patch.rb +++ b/lib/old_rubygems_patch.rb @@ -1,4 +1,8 @@ -require File.join(File.dirname(__FILE__),'..','vendor','rails','railties','lib','rails','gem_dependency.rb') +if File.exist? File.join(File.dirname(__FILE__),'..','vendor','rails','railties','lib','rails','gem_dependency.rb') + require File.join(File.dirname(__FILE__),'..','vendor','rails','railties','lib','rails','gem_dependency.rb') +else + require 'rails/gem_dependency' +end module Rails class GemDependency < Gem::Dependency diff --git a/spec/controllers/admin_request_controller_spec.rb b/spec/controllers/admin_request_controller_spec.rb index 6f9af0525..441b1b91d 100644 --- a/spec/controllers/admin_request_controller_spec.rb +++ b/spec/controllers/admin_request_controller_spec.rb @@ -2,7 +2,7 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') describe AdminRequestController, "when administering requests" do integrate_views - fixtures :info_requests, :outgoing_messages, :users, :info_request_events, :public_bodies, :public_body_translations + fixtures :users, :public_bodies, :public_body_translations, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events before { basic_auth_login @request } it "shows the index/list page" do @@ -41,7 +41,7 @@ end describe AdminRequestController, "when administering the holding pen" do integrate_views - fixtures :info_requests, :incoming_messages, :raw_emails, :users, :public_bodies, :public_body_translations + fixtures :users, :public_bodies, :public_body_translations, :info_requests, :raw_emails, :incoming_messages before(:each) do basic_auth_login @request load_raw_emails_data(raw_emails) diff --git a/spec/controllers/admin_track_controller_spec.rb b/spec/controllers/admin_track_controller_spec.rb index 4d5b0ac5e..3db242f73 100644 --- a/spec/controllers/admin_track_controller_spec.rb +++ b/spec/controllers/admin_track_controller_spec.rb @@ -2,7 +2,7 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') describe AdminTrackController, "when administering tracks" do integrate_views - fixtures :track_things, :users + fixtures :users, :track_things it "shows the list page" do get :list diff --git a/spec/controllers/general_controller_spec.rb b/spec/controllers/general_controller_spec.rb index 1ffbda90d..054b9881f 100644 --- a/spec/controllers/general_controller_spec.rb +++ b/spec/controllers/general_controller_spec.rb @@ -2,12 +2,12 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') describe GeneralController, "when searching" do integrate_views - fixtures [ :info_requests, - :info_request_events, - :public_bodies, + fixtures [ :public_bodies, :public_body_translations, :users, :raw_emails, + :info_requests, + :info_request_events, :outgoing_messages, :incoming_messages, :comments ] diff --git a/spec/controllers/public_body_controller_spec.rb b/spec/controllers/public_body_controller_spec.rb index c5c9d60e1..efcbc7d57 100644 --- a/spec/controllers/public_body_controller_spec.rb +++ b/spec/controllers/public_body_controller_spec.rb @@ -157,7 +157,7 @@ describe PublicBodyController, "when showing JSON version for API" do end describe PublicBodyController, "when doing type ahead searches" do - fixtures :info_requests, :info_request_events, :public_bodies, :public_body_translations, :users, :incoming_messages, :raw_emails, :outgoing_messages, :comments + fixtures :public_bodies, :public_body_translations, :users, :raw_emails, :info_requests, :incoming_messages, :outgoing_messages, :comments, :info_request_events it "should return nothing for the empty query string" do get :search_typeahead, :q => "" diff --git a/spec/controllers/request_controller_spec.rb b/spec/controllers/request_controller_spec.rb index 494713a4a..8e1e4589f 100644 --- a/spec/controllers/request_controller_spec.rb +++ b/spec/controllers/request_controller_spec.rb @@ -3,7 +3,7 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') require 'json' describe RequestController, "when listing recent requests" do - fixtures :info_requests, :outgoing_messages, :users, :info_request_events, :public_bodies, :public_body_translations, :incoming_messages, :raw_emails, :comments + fixtures :users, :public_bodies, :public_body_translations, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events before(:each) do load_raw_emails_data(raw_emails) @@ -51,7 +51,7 @@ end describe RequestController, "when showing one request" do - fixtures :info_requests, :info_request_events, :public_bodies, :public_body_translations, :public_body_translations, :users, :incoming_messages, :raw_emails, :outgoing_messages, :comments # all needed as integrating views + fixtures :public_bodies, :public_body_translations, :public_body_translations, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events # all needed as integrating views before(:each) do load_raw_emails_data(raw_emails) @@ -137,6 +137,13 @@ describe RequestController, "when showing one request" do response.should have_text(/First hello/) end + it "should convert message body to UTF8" do + ir = info_requests(:fancy_dog_request) + receive_incoming_mail('iso8859_2_raw_email.email', ir.incoming_email) + get :show, :url_title => 'why_do_you_have_such_a_fancy_dog' + response.should have_text(/tënde/u) + end + it "should generate valid HTML verson of plain text attachments " do ir = info_requests(:fancy_dog_request) receive_incoming_mail('incoming-request-two-same-name.email', ir.incoming_email) @@ -242,7 +249,7 @@ zipfile.entries.each {|x| puts x.name} end describe RequestController, "when changing prominence of a request" do - fixtures :info_requests, :info_request_events, :public_bodies, :public_body_translations, :users, :incoming_messages, :raw_emails, :outgoing_messages # all needed as integrating views + fixtures :public_bodies, :public_body_translations, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :info_request_events # all needed as integrating views before(:each) do load_raw_emails_data(raw_emails) @@ -358,7 +365,7 @@ end describe RequestController, "when creating a new request" do integrate_views - fixtures :info_requests, :outgoing_messages, :public_bodies, :public_body_translations, :users + fixtures :public_bodies, :public_body_translations, :users, :info_requests, :outgoing_messages before do @user = users(:bob_smith_user) @@ -542,7 +549,7 @@ end describe RequestController, "when viewing an individual response for reply/followup" do integrate_views - fixtures :info_requests, :info_request_events, :public_bodies, :public_body_translations, :users, :incoming_messages, :raw_emails, :outgoing_messages, :comments # all needed as integrating views + fixtures :public_bodies, :public_body_translations, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events # all needed as integrating views before(:each) do load_raw_emails_data(raw_emails) @@ -589,7 +596,7 @@ end describe RequestController, "when classifying an information request" do - fixtures :info_requests, :info_request_events, :public_bodies, :public_body_translations, :users, :incoming_messages, :raw_emails, :outgoing_messages, :comments # all needed as integrating views + fixtures :public_bodies, :public_body_translations, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events # all needed as integrating views before(:each) do @dog_request = info_requests(:fancy_dog_request) @@ -925,7 +932,7 @@ end describe RequestController, "when sending a followup message" do integrate_views - fixtures :info_requests, :info_request_events, :public_bodies, :public_body_translations, :users, :incoming_messages, :raw_emails, :outgoing_messages # all needed as integrating views + fixtures :public_bodies, :public_body_translations, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :info_request_events # all needed as integrating views before(:each) do load_raw_emails_data(raw_emails) @@ -1008,7 +1015,7 @@ end describe RequestController, "sending overdue request alerts" do integrate_views - fixtures :info_requests, :info_request_events, :public_bodies, :public_body_translations, :users, :incoming_messages, :raw_emails, :outgoing_messages # all needed as integrating views + fixtures :public_bodies, :public_body_translations, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :info_request_events # all needed as integrating views before(:each) do load_raw_emails_data(raw_emails) @@ -1096,7 +1103,7 @@ end describe RequestController, "sending unclassified new response reminder alerts" do integrate_views - fixtures :info_requests, :info_request_events, :public_bodies, :public_body_translations, :users, :incoming_messages, :raw_emails, :outgoing_messages, :comments # all needed as integrating views + fixtures :public_bodies, :public_body_translations, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events # all needed as integrating views before(:each) do load_raw_emails_data(raw_emails) @@ -1127,7 +1134,7 @@ end describe RequestController, "clarification required alerts" do integrate_views - fixtures :info_requests, :info_request_events, :public_bodies, :public_body_translations, :users, :incoming_messages, :raw_emails, :outgoing_messages # all needed as integrating views + fixtures :public_bodies, :public_body_translations, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :info_request_events # all needed as integrating views before(:each) do load_raw_emails_data(raw_emails) end @@ -1181,7 +1188,7 @@ end describe RequestController, "comment alerts" do integrate_views - fixtures :info_requests, :info_request_events, :public_bodies, :public_body_translations, :users, :incoming_messages, :raw_emails, :outgoing_messages, :comments # all needed as integrating views + fixtures :public_bodies, :public_body_translations, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events # all needed as integrating views before(:each) do load_raw_emails_data(raw_emails) end @@ -1256,7 +1263,7 @@ end describe RequestController, "when viewing comments" do integrate_views - fixtures :users, :raw_emails, :incoming_messages, :info_requests + fixtures :users, :info_requests, :raw_emails, :incoming_messages before(:each) do load_raw_emails_data(raw_emails) end @@ -1279,7 +1286,7 @@ end describe RequestController, "authority uploads a response from the web interface" do - fixtures :info_requests, :info_request_events, :public_bodies, :public_body_translations, :users + fixtures :public_bodies, :public_body_translations, :users, :info_requests, :info_request_events before(:all) do # domain after the @ is used for authentication of FOI officers, so to test it @@ -1366,7 +1373,7 @@ end describe RequestController, "when showing JSON version for API" do - fixtures :info_requests, :info_request_events, :public_bodies, :public_body_translations, :users, :incoming_messages, :raw_emails, :outgoing_messages, :comments + fixtures :public_bodies, :public_body_translations, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events before(:each) do load_raw_emails_data(raw_emails) @@ -1386,7 +1393,7 @@ describe RequestController, "when showing JSON version for API" do end describe RequestController, "when doing type ahead searches" do - fixtures :info_requests, :info_request_events, :public_bodies, :public_body_translations, :users, :incoming_messages, :raw_emails, :outgoing_messages, :comments + fixtures :public_bodies, :public_body_translations, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events it "should return nothing for the empty query string" do get :search_typeahead, :q => "" diff --git a/spec/controllers/request_game_controller_spec.rb b/spec/controllers/request_game_controller_spec.rb index 4883bfdd6..0b8f5751c 100644 --- a/spec/controllers/request_game_controller_spec.rb +++ b/spec/controllers/request_game_controller_spec.rb @@ -2,7 +2,7 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') describe RequestGameController, "when playing the game" do - fixtures :info_requests, :info_request_events, :public_bodies, :public_body_translations, :users, :incoming_messages, :raw_emails, :outgoing_messages # all needed as integrating views + fixtures :public_bodies, :public_body_translations, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :info_request_events # all needed as integrating views before(:each) do load_raw_emails_data(raw_emails) end diff --git a/spec/controllers/track_controller_spec.rb b/spec/controllers/track_controller_spec.rb index 435d9a0d3..617e36213 100644 --- a/spec/controllers/track_controller_spec.rb +++ b/spec/controllers/track_controller_spec.rb @@ -36,7 +36,7 @@ end describe TrackController, "when sending alerts for a track" do integrate_views - fixtures :comments, :info_requests, :outgoing_messages, :incoming_messages, :raw_emails, :info_request_events, :users, :track_things, :track_things_sent_emails, :public_bodies, :public_body_translations + fixtures :public_bodies, :public_body_translations, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things, :track_things_sent_emails include LinkToHelper # for main_url before(:each) do @@ -110,7 +110,7 @@ end describe TrackController, "when viewing RSS feed for a track" do integrate_views - fixtures :info_requests, :outgoing_messages, :incoming_messages, :raw_emails, :info_request_events, :users, :track_things, :comments, :public_bodies, :public_body_translations + fixtures :public_bodies, :public_body_translations, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do load_raw_emails_data(raw_emails) @@ -136,7 +136,7 @@ end describe TrackController, "when viewing JSON version of a track feed" do integrate_views - fixtures :info_requests, :outgoing_messages, :incoming_messages, :raw_emails, :info_request_events, :users, :track_things, :comments, :public_bodies, :public_body_translations + fixtures :public_bodies, :public_body_translations, :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events, :track_things before(:each) do load_raw_emails_data(raw_emails) diff --git a/spec/controllers/user_controller_spec.rb b/spec/controllers/user_controller_spec.rb index b4cc0d6e3..ae771da04 100644 --- a/spec/controllers/user_controller_spec.rb +++ b/spec/controllers/user_controller_spec.rb @@ -7,7 +7,7 @@ require 'json' describe UserController, "when showing a user" do integrate_views - fixtures :users, :outgoing_messages, :incoming_messages, :raw_emails, :info_requests, :info_request_events, :comments, :public_bodies, :public_body_translations + fixtures :users, :public_bodies, :public_body_translations, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :info_request_events, :comments before(:each) do load_raw_emails_data(raw_emails) end diff --git a/spec/fixtures/files/email-folding-example-1.txt b/spec/fixtures/files/email-folding-example-1.txt new file mode 100644 index 000000000..9d0810a36 --- /dev/null +++ b/spec/fixtures/files/email-folding-example-1.txt @@ -0,0 +1,32 @@ +Dear Mr Pollard, + +Thank you for your email of 26 February. Please find a response attached. + +Yours faithfully, + +On behalf of James Hall, Chief Executive +Identity and Passport Service + +<<9032 C Pollard final response.doc>> + +********************************************************************** + +This email and any files transmitted with it are confidential and + +intended solely for the use of the individual or entity to whom they + +are addressed. If you have received this email in error please notify + +the system manager. + +This footnote also confirms that this email message has been swept for the +presence of computer viruses. + +********************************************************************** + +The original of this email was scanned for viruses by the Government +Secure Intranet virus scanning service supplied by Cable&Wireless in +partnership with MessageLabs. (CCTM Certificate Number 2007/11/0032.) On +leaving the GSi this email was certified virus free. +Communications via the GSi may be automatically logged, monitored and/or +recorded for legal purposes. diff --git a/spec/fixtures/files/email-folding-example-1.txt.expected b/spec/fixtures/files/email-folding-example-1.txt.expected new file mode 100644 index 000000000..801542288 --- /dev/null +++ b/spec/fixtures/files/email-folding-example-1.txt.expected @@ -0,0 +1,10 @@ +Dear Mr Pollard, + +Thank you for your email of 26 February. Please find a response attached. + +Yours faithfully, + +On behalf of James Hall, Chief Executive +Identity and Passport Service + +<<9032 C Pollard final response.doc>> diff --git a/spec/fixtures/files/email-folding-example-10.txt b/spec/fixtures/files/email-folding-example-10.txt new file mode 100644 index 000000000..0fabb7f9c --- /dev/null +++ b/spec/fixtures/files/email-folding-example-10.txt @@ -0,0 +1,52 @@ +Please note: it is not possible to reply to this email. To contact the +Department of Health, please visit the 'Contact us' page on the +Department’s website. + +----------------------------------------------------------------------------------------- + + Apologies that you were not able to read our previous response of 4 + October. Please find the text of that email below. + +Our ref: DE00000642471 + +Dear Ms Peters Rock, + +You requested your correspondence to be treated under the Freedom of +Information Act. However, as your correspondence asked for general +information, rather than requesting recorded information or documentation, +I should advise you that on this occasion we have not considered your +correspondence under the provisions of the Act. + +I am sorry I cannot be more helpful. + +Yours sincerely, +Simon Dove +Customer Service Centre +Department of Health + + + + +------------------------------------------------------------------------------------------------------------------------- + + +Please do not reply to this email. To contact the Department of Health, +please visit the 'Contact us' page on the Department’s website, where you +can also view our performance against quarterly service targets. + + +- - Disclaimer - - +This e-mail and any files transmitted with it are confidential. If you are +not the intended recipient, any reading, printing, storage, disclosure, +copying or any other action taken in respect of this e-mail is prohibited +and may be unlawful. If you are not the intended recipient, please notify +the sender immediately by using the reply function and then permanently +delete what you have received. + +Incoming and outgoing e-mail messages are routinely monitored for +compliance with the Department of Health's policy on the use of electronic +communications. For more information on the Department of Health's e-mail +policy click here http://www.dh.gov.uk/terms + +The original of this email was scanned for viruses by the Government Secure Intranet virus scanning service supplied by Cable&Wireless Worldwide in partnership with MessageLabs. (CCTM Certificate Number 2009/09/0052.) On leaving the GSi this email was certified virus free. +Communications via the GSi may be automatically logged, monitored and/or recorded for legal purposes. diff --git a/spec/fixtures/files/email-folding-example-10.txt.expected b/spec/fixtures/files/email-folding-example-10.txt.expected new file mode 100644 index 000000000..e4f704c0e --- /dev/null +++ b/spec/fixtures/files/email-folding-example-10.txt.expected @@ -0,0 +1,25 @@ +Please note: it is not possible to reply to this email. To contact the +Department of Health, please visit the 'Contact us' page on the +Department’s website. + +----------------------------------------------------------------------------------------- + + Apologies that you were not able to read our previous response of 4 + October. Please find the text of that email below. + +Our ref: DE00000642471 + +Dear Ms Peters Rock, + +You requested your correspondence to be treated under the Freedom of +Information Act. However, as your correspondence asked for general +information, rather than requesting recorded information or documentation, +I should advise you that on this occasion we have not considered your +correspondence under the provisions of the Act. + +I am sorry I cannot be more helpful. + +Yours sincerely, +Simon Dove +Customer Service Centre +Department of Health diff --git a/spec/fixtures/files/email-folding-example-2.txt b/spec/fixtures/files/email-folding-example-2.txt new file mode 100644 index 000000000..13dd39a69 --- /dev/null +++ b/spec/fixtures/files/email-folding-example-2.txt @@ -0,0 +1,7 @@ +Preface to the message which we are not interested in + +----------------------------------------------------------------------------------------- +Important message about cheese +----------------------------------------------------------------------------------------- + +Actual footer that contains the word confidential diff --git a/spec/fixtures/files/email-folding-example-2.txt.expected b/spec/fixtures/files/email-folding-example-2.txt.expected new file mode 100644 index 000000000..e52fbe443 --- /dev/null +++ b/spec/fixtures/files/email-folding-example-2.txt.expected @@ -0,0 +1,4 @@ +Preface to the message which we are not interested in + +----------------------------------------------------------------------------------------- +Important message about cheese diff --git a/spec/fixtures/files/email-folding-example-3.txt b/spec/fixtures/files/email-folding-example-3.txt new file mode 100644 index 000000000..28a3861f6 --- /dev/null +++ b/spec/fixtures/files/email-folding-example-3.txt @@ -0,0 +1,18 @@ +Reference : T3241/8 + +Thank you for your e-mail enquiry of 12th February. + +A reply is attached. + +********************************************************************** +This email and any files transmitted with it are private and intended +solely for the use of the individual or entity to whom they are addressed. +If you have received this email in error please return it to the address +it came from telling them it is not for you and then delete it from your system. + +This email message has been swept for computer viruses. + +********************************************************************** + +The original of this email was scanned for viruses by the Government Secure Intranet virus scanning service supplied by Cable&Wireless in partnership with MessageLabs. (CCTM Certificate Number 2007/11/0032.) On leaving the GSi this email was certified virus free. +Communications via the GSi may be automatically logged, monitored and/or recorded for legal purposes. diff --git a/spec/fixtures/files/email-folding-example-3.txt.expected b/spec/fixtures/files/email-folding-example-3.txt.expected new file mode 100644 index 000000000..e2cca4933 --- /dev/null +++ b/spec/fixtures/files/email-folding-example-3.txt.expected @@ -0,0 +1,5 @@ +Reference : T3241/8 + +Thank you for your e-mail enquiry of 12th February. + +A reply is attached. diff --git a/spec/fixtures/files/email-folding-example-4.txt b/spec/fixtures/files/email-folding-example-4.txt new file mode 100644 index 000000000..63b94a35c --- /dev/null +++ b/spec/fixtures/files/email-folding-example-4.txt @@ -0,0 +1,37 @@ +<<Freedom of Information request - Contracts or options with Kimberley +Developments or Waitrose>> + +Arthur Pritchard +Property & Assets Manager +Tel: 01625 504234 +Fax: 01625 504268 +e-mail: [1][email address] + +*********************************************************************************** +The information in this Email and any attachments is personal to the +sender and the views of the author may not necessarily reflect those +of Macclesfield Borough Council. The information is strictly confidential +and is intended only for the named person or organisation to whom it is +addressed as it may contain privileged and confidential information. If +you are not the intended recipient do not copy, distribute or use this +Email, and please notify the sender. Please note that we cannot +guarantee that this message or any attachment is virus free or has not +been intercepted and amended. +*********************************************************************************** + +Disclaimer + +-------------------------------------------------------------------------- + +This email message has been scanned for viruses by Mimecast. +Mimecast delivers a complete managed email solution from a single web +based platform. +For more information please visit [2]http://www.mimecast.com + +-------------------------------------------------------------------------- + +References + +Visible links +1. mailto:[email address] +2. http://www.mimecast.com/ diff --git a/spec/fixtures/files/email-folding-example-4.txt.expected b/spec/fixtures/files/email-folding-example-4.txt.expected new file mode 100644 index 000000000..42334a290 --- /dev/null +++ b/spec/fixtures/files/email-folding-example-4.txt.expected @@ -0,0 +1,15 @@ +<<Freedom of Information request - Contracts or options with Kimberley +Developments or Waitrose>> + +Arthur Pritchard +Property & Assets Manager +Tel: 01625 504234 +Fax: 01625 504268 +e-mail: [1][email address] +FOLDED_QUOTED_SECTION +FOLDED_QUOTED_SECTION +References + +Visible links +1. mailto:[email address] +2. http://www.mimecast.com/ diff --git a/spec/fixtures/files/email-folding-example-5.txt b/spec/fixtures/files/email-folding-example-5.txt new file mode 100644 index 000000000..3d0964722 --- /dev/null +++ b/spec/fixtures/files/email-folding-example-5.txt @@ -0,0 +1,35 @@ +Hi Simon + +My apologies for timescale of response. The data forwarded is a public +register, and is updated on a frequent and regular basis; your request +unfortunately coincided with annual leave and a monthly update of the +spreadsheet. As the definition of an HMO under the Housing Act 2004 +differs to that under planning legislation, I have forwarded this and +your original request on to Andy England, Development Control Manager to +respond independantly. + +If I can be of further assistance please contact me + +Barry Turnbull +Environmental Health Co-ordinator (Housing) +Housing, Health and Community Safety + +-- + +**************************************************************** +Any opinions expressed are not necessarily those of Penwith +District Council. This e-mail and any attachments, replies +and forwarded copies are confidential and are strictly for +the use of named recipient(s) only. If you have received +it in error you may not make use of it. Please e-mail us, +including a copy of the message to, [email address]. +Then delete the e-mail and any copies. +**************************************************************** + +********************************************************************** +This footnote confirms that this message, and any +attachments, have been screened by McAffee +Webshield for the presence of virus code. + +Penwith District Council +********************************************************************** diff --git a/spec/fixtures/files/email-folding-example-5.txt.expected b/spec/fixtures/files/email-folding-example-5.txt.expected new file mode 100644 index 000000000..fbb0f0f50 --- /dev/null +++ b/spec/fixtures/files/email-folding-example-5.txt.expected @@ -0,0 +1,24 @@ +Hi Simon + +My apologies for timescale of response. The data forwarded is a public +register, and is updated on a frequent and regular basis; your request +unfortunately coincided with annual leave and a monthly update of the +spreadsheet. As the definition of an HMO under the Housing Act 2004 +differs to that under planning legislation, I have forwarded this and +your original request on to Andy England, Development Control Manager to +respond independantly. + +If I can be of further assistance please contact me + +Barry Turnbull +Environmental Health Co-ordinator (Housing) +Housing, Health and Community Safety + +-- +FOLDED_QUOTED_SECTION +This footnote confirms that this message, and any +attachments, have been screened by McAffee +Webshield for the presence of virus code. + +Penwith District Council +********************************************************************** diff --git a/spec/fixtures/files/email-folding-example-6.txt b/spec/fixtures/files/email-folding-example-6.txt new file mode 100644 index 000000000..272d6c9da --- /dev/null +++ b/spec/fixtures/files/email-folding-example-6.txt @@ -0,0 +1,30 @@ +Dear Mr. Brown, + +Please find attached a reply to your FOI request. + +Yours ever, + +Adetokunbo Ighodaro + +<<FOI 0169-08 final.doc>> + +*********************************************************************************** +Visit [1]http://www.fco.gov.uk for British foreign policy news and travel +advice; and [2]http://www.i-uk.com - the essential guide to the UK. + +We keep and use information in line with the Data Protection Act 1998. We +may release this personal information to other UK government departments +and public authorities. + +Please note that all messages sent and received by members of the Foreign +& Commonwealth Office and its +missions overseas may be monitored centrally. This is done to ensure the +integrity of the system. + +*********************************************************************************** + +References + +Visible links +1. http://www.fco.gov.uk/ +2. http://www.i-uk.com/ diff --git a/spec/fixtures/files/email-folding-example-6.txt.expected b/spec/fixtures/files/email-folding-example-6.txt.expected new file mode 100644 index 000000000..58021ce12 --- /dev/null +++ b/spec/fixtures/files/email-folding-example-6.txt.expected @@ -0,0 +1,15 @@ +Dear Mr. Brown, + +Please find attached a reply to your FOI request. + +Yours ever, + +Adetokunbo Ighodaro + +<<FOI 0169-08 final.doc>> +FOLDED_QUOTED_SECTION +References + +Visible links +1. http://www.fco.gov.uk/ +2. http://www.i-uk.com/
\ No newline at end of file diff --git a/spec/fixtures/files/email-folding-example-7.txt b/spec/fixtures/files/email-folding-example-7.txt new file mode 100644 index 000000000..e10fe4657 --- /dev/null +++ b/spec/fixtures/files/email-folding-example-7.txt @@ -0,0 +1,30 @@ +Mr Hearne, +Please see attached our response to your request dated 06 March 2008. +Kind Regards, +Linda Dempsey + +Information Assistant DP/FOI +Data Protection/Information Security +Professional Standards +Leicestershire Constabulary +http://www.leics.police.uk +mailto [Leicestershire Constabulary request email] +Telephone +44 (0) 116 2222222 +Extn 5221 VM No. 8035 +Fax + 44 (0) 116 2485217 + +<<0001_00035908_Resp_12RESPONSE LETTER_20080408_112311_01.TIF>> + +********** + +Internet email is not to be treated as a secure means of communication. + +Leicestershire Constabulary monitors all internet email activity and content. + +This communication is intended for the addressee(s) only. Please notify the sender if received in error. Unauthorised use or disclosure of the content may be unlawful. Opinions expressed in this document may not be official policy. + +Thank you for your co-operation. + +© Leicestershire Constabulary + +********** diff --git a/spec/fixtures/files/email-folding-example-7.txt.expected b/spec/fixtures/files/email-folding-example-7.txt.expected new file mode 100644 index 000000000..0ef8fd82b --- /dev/null +++ b/spec/fixtures/files/email-folding-example-7.txt.expected @@ -0,0 +1,16 @@ +Mr Hearne, +Please see attached our response to your request dated 06 March 2008. +Kind Regards, +Linda Dempsey + +Information Assistant DP/FOI +Data Protection/Information Security +Professional Standards +Leicestershire Constabulary +http://www.leics.police.uk +mailto [Leicestershire Constabulary request email] +Telephone +44 (0) 116 2222222 +Extn 5221 VM No. 8035 +Fax + 44 (0) 116 2485217 + +<<0001_00035908_Resp_12RESPONSE LETTER_20080408_112311_01.TIF>>
\ No newline at end of file diff --git a/spec/fixtures/files/email-folding-example-8.txt b/spec/fixtures/files/email-folding-example-8.txt new file mode 100644 index 000000000..c1899e7c8 --- /dev/null +++ b/spec/fixtures/files/email-folding-example-8.txt @@ -0,0 +1,18 @@ +I will be out of the office starting 11/04/2008 and will not return until +22/04/2008. + +I will respond to your message when I return. If you have any urgent +queries please ring 02085419088 for Legal Business Support queries or +contact Eileen Perren for FOI or DP queries + +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +This email and any attachments with it are intended for the addressee only. +It may be confidential and may be the subject of legal and/or professional privilege. +If you have received this email in error please notify the sender or [email address] +The content may be personal or contain personal opinions and cannot be taken as an expression of the County Council's position. +Surrey County Council reserves the right to monitor all incoming and outgoing mail. +Whilst every care has been taken to check this outgoing e-mail for viruses, it is your responsibility to carry out any checks upon receipt. + +Visit the Surrey County Council website - http://www.surreycc.gov.uk + +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * diff --git a/spec/fixtures/files/email-folding-example-8.txt.expected b/spec/fixtures/files/email-folding-example-8.txt.expected new file mode 100644 index 000000000..b5dc10c0d --- /dev/null +++ b/spec/fixtures/files/email-folding-example-8.txt.expected @@ -0,0 +1,6 @@ +I will be out of the office starting 11/04/2008 and will not return until +22/04/2008. + +I will respond to your message when I return. If you have any urgent +queries please ring 02085419088 for Legal Business Support queries or +contact Eileen Perren for FOI or DP queries
\ No newline at end of file diff --git a/spec/fixtures/files/email-folding-example-9.txt b/spec/fixtures/files/email-folding-example-9.txt new file mode 100644 index 000000000..1f3d4c34a --- /dev/null +++ b/spec/fixtures/files/email-folding-example-9.txt @@ -0,0 +1,29 @@ +Dear Mr Cross + +Freedom of Information Request Reference No: 2008040590 + +Yours sincerely + +MICHAEL HEGARTY + +FOI Officer + +********************************************************************************************** +Please Note: Incoming and Outgoing E-mail messages are routinely monitored +for compliance with our policy on the use of electronic communications. + +Interested in Occupational Health & Safety information? +Please visit the HSE website at the following address to keep yourself up +to date + +www.hse.gov.uk + +Or contact HSE Infoline on 0845 345 0055 or email [HSE request email] + +********************************************************************************************** +The original of this email was scanned for viruses by the Government +Secure Intranet virus scanning service supplied by Cable&Wireless in +partnership with MessageLabs. (CCTM Certificate Number 2007/11/0032.) On +leaving the GSi this email was certified virus free. +Communications via the GSi may be automatically logged, monitored and/or +recorded for legal purposes. diff --git a/spec/fixtures/files/email-folding-example-9.txt.expected b/spec/fixtures/files/email-folding-example-9.txt.expected new file mode 100644 index 000000000..2d2381a34 --- /dev/null +++ b/spec/fixtures/files/email-folding-example-9.txt.expected @@ -0,0 +1,9 @@ +Dear Mr Cross + +Freedom of Information Request Reference No: 2008040590 + +Yours sincerely + +MICHAEL HEGARTY + +FOI Officer diff --git a/spec/fixtures/files/iso8859_2_email.html b/spec/fixtures/files/iso8859_2_email.html new file mode 100644 index 000000000..c7384a831 --- /dev/null +++ b/spec/fixtures/files/iso8859_2_email.html @@ -0,0 +1,18 @@ +<html> +<head> +<style><!-- +.hmmessage P +{ +margin:0px; +padding:0px +} +body.hmmessage +{ +font-size: 10pt; +font-family:Tahoma +} +--></style> +</head> +<body class='hmmessage'><div dir='ltr'> +<div>Faleminderit per kerkesen tënde.</div> </div></body> +</html> diff --git a/spec/fixtures/files/iso8859_2_raw_email.email b/spec/fixtures/files/iso8859_2_raw_email.email new file mode 100644 index 000000000..2ac3b2533 --- /dev/null +++ b/spec/fixtures/files/iso8859_2_raw_email.email @@ -0,0 +1,50 @@ +From: EMAIL_FROM +To: FOI Person <EMAIL_TO> +Bcc: +Reply-To: +In-Reply-To: <471f1eae5d1cb_7347..fdbe67386163@cat.tmail> +Content-Type: multipart/alternative; + boundary="_d47fc84f-c9cd-4fb3-ab16-15de158c8eef_" +Subject: =?iso-8859-2?Q?RE:_Freedo?= =?iso-8859-2?Q?m_of_Infor?= + =?iso-8859-2?Q?mation_req?= =?iso-8859-2?Q?uest_-_Sas?= + =?iso-8859-2?Q?ia_e_pulav?= =?iso-8859-2?Q?e_t=EB_impor?= + =?iso-8859-2?Q?tuara_gjat?= =?iso-8859-2?Q?=EB_vitit_20?= =?iso-8859-2?Q?10?= +Date: Fri, 30 Sep 2011 11:06:39 +0200 + +--_d47fc84f-c9cd-4fb3-ab16-15de158c8eef_ +Content-Type: text/plain; charset="iso-8859-2" +Content-Transfer-Encoding: quoted-printable + + +I nderuari Besnik=2C=20 +Faleminderit per kerkesen t=EBnde.Numri i puleve te importuara ne vitin 201= +0 ka qene 5 milion e treqind mije sosh (me numra 3=2C500=2C000) + + +--_d47fc84f-c9cd-4fb3-ab16-15de158c8eef_ +Content-Type: text/html; charset="iso-8859-2" +Content-Transfer-Encoding: quoted-printable + +<html> +<head> +<style><!-- +.hmmessage P +{ +margin:0px=3B +padding:0px +} +body.hmmessage +{ +font-size: 10pt=3B +font-family:Tahoma +} +--></style> +</head> +<body class=3D'hmmessage'><div dir=3D'ltr'> +<div>I nderuari Besnik=2C =3B</div><div><br></div><div>Faleminderit per= + kerkesen t=EBnde.</div>Numri i puleve te importuara ne vitin 2010 ka qene = +5 milion e treqind mije sosh (me numra 3=2C500=2C000)<br><br></body> +</html>= + +--_d47fc84f-c9cd-4fb3-ab16-15de158c8eef_-- + diff --git a/spec/lib/tmail_extensions_spec.rb b/spec/lib/tmail_extensions_spec.rb index c647fe522..ffb8cb178 100644 --- a/spec/lib/tmail_extensions_spec.rb +++ b/spec/lib/tmail_extensions_spec.rb @@ -27,5 +27,14 @@ describe "when using TMail" do mail.to.should == ["request-66666-caa77777@whatdotheyknow.com", "foi@example.com"] end + it 'should convert to utf8' do + # NB this isn't actually a TMail extension, but is core TMail; + # this was just a convenient place to assert the UTF8 + # conversion is working + mail = TMail::Mail.parse(load_file_fixture('iso8859_2_raw_email.email')) + mail.subject.should have_text(/gjatë/u) + mail.body.is_utf8?.should == true + end + end diff --git a/spec/models/incoming_message_spec.rb b/spec/models/incoming_message_spec.rb index d6923da21..a8411bc34 100644 --- a/spec/models/incoming_message_spec.rb +++ b/spec/models/incoming_message_spec.rb @@ -1,7 +1,7 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') describe IncomingMessage, " when dealing with incoming mail" do - fixtures :incoming_messages, :raw_emails, :info_requests + fixtures :users, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :incoming_messages before(:each) do @im = incoming_messages(:useless_incoming_message) @@ -17,6 +17,27 @@ describe IncomingMessage, " when dealing with incoming mail" do TMail::Address.parse(em) end + it "should correctly fold various types of footer" do + Dir.glob(File.join(Spec::Runner.configuration.fixture_path, "files", "email-folding-example-*.txt")).each do |file| + message = File.read(file) + parsed = IncomingMessage.remove_quoted_sections(message) + expected = File.read("#{file}.expected") + parsed.should include(expected) + end + end + + it "should fold multiline sections" do + { + "foo\n--------\nconfidential" => "foo\nFOLDED_QUOTED_SECTION\n", # basic test + "foo\n--------\nbar - confidential" => "foo\nFOLDED_QUOTED_SECTION\n", # allow scorechar inside folded section + "foo\n--------\nbar\n--------\nconfidential" => "foo\n--------\nbar\nFOLDED_QUOTED_SECTION\n", # don't assume that anything after a score is a folded section + "foo\n--------\nbar\n--------\nconfidential\n--------\nrest" => "foo\n--------\nbar\nFOLDED_QUOTED_SECTION\nrest", # don't assume that a folded section continues to the end of the message + "foo\n--------\nbar\n- - - - - - - -\nconfidential\n--------\nrest" => "foo\n--------\nbar\nFOLDED_QUOTED_SECTION\nrest", # allow spaces in the score + }.each do |input,output| + IncomingMessage.remove_quoted_sections(input).should == output + end + end + end describe IncomingMessage, "when parsing HTML mail" do @@ -139,7 +160,7 @@ describe IncomingMessage, " checking validity to reply to" do end describe IncomingMessage, " checking validity to reply to with real emails" do - fixtures :incoming_messages, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :users + fixtures :users, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :incoming_messages after(:all) do ActionMailer::Base.deliveries.clear @@ -163,7 +184,7 @@ describe IncomingMessage, " checking validity to reply to with real emails" do end describe IncomingMessage, " when censoring data" do - fixtures :incoming_messages, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :users + fixtures :users, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :incoming_messages before(:each) do @test_data = "There was a mouse called Stilton, he wished that he was blue." @@ -273,7 +294,7 @@ describe IncomingMessage, " when censoring data" do end describe IncomingMessage, " when censoring whole users" do - fixtures :incoming_messages, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :users + fixtures :users, :raw_emails, :public_bodies, :public_body_translations, :info_requests, :incoming_messages before(:each) do @test_data = "There was a mouse called Stilton, he wished that he was blue." diff --git a/spec/models/info_request_spec.rb b/spec/models/info_request_spec.rb index b82052a0f..d50d6dace 100644 --- a/spec/models/info_request_spec.rb +++ b/spec/models/info_request_spec.rb @@ -3,7 +3,7 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') describe InfoRequest do describe "guessing a request from an email" do - fixtures :info_requests, :public_bodies, :incoming_messages, :raw_emails + fixtures :public_bodies, :info_requests, :raw_emails, :incoming_messages before(:each) do @im = incoming_messages(:useless_incoming_message) @@ -74,7 +74,7 @@ describe InfoRequest do describe " when emailing" do - fixtures :info_requests, :info_request_events, :outgoing_messages, :public_bodies, :public_body_translations, :users, :comments + fixtures :public_bodies, :public_body_translations, :users, :info_requests, :outgoing_messages, :info_request_events, :comments before do @info_request = info_requests(:fancy_dog_request) @@ -154,7 +154,7 @@ describe InfoRequest do end describe "when calculating the status" do - fixtures :info_requests, :info_request_events, :holidays, :public_bodies, :public_body_translations, :outgoing_messages + fixtures :holidays, :public_bodies, :public_body_translations, :info_requests, :outgoing_messages, :info_request_events before do @ir = info_requests(:naughty_chicken_request) @@ -231,7 +231,7 @@ describe InfoRequest do describe "when calculating the status for a school" do - fixtures :info_requests, :info_request_events, :holidays, :public_bodies, :public_body_translations + fixtures :holidays, :public_bodies, :public_body_translations, :info_requests, :info_request_events before do @ir = info_requests(:naughty_chicken_request) diff --git a/spec/models/xapian_spec.rb b/spec/models/xapian_spec.rb index 0c6fa6bb6..51d410660 100644 --- a/spec/models/xapian_spec.rb +++ b/spec/models/xapian_spec.rb @@ -34,7 +34,7 @@ describe User, " when indexing users with Xapian" do end describe PublicBody, " when indexing public bodies with Xapian" do - fixtures :public_bodies, :public_body_translations, :incoming_messages, :outgoing_messages, :raw_emails, :comments, :info_requests + fixtures :public_bodies, :public_body_translations, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments before(:each) do load_raw_emails_data(raw_emails) end @@ -72,7 +72,7 @@ describe PublicBody, " when indexing public bodies with Xapian" do end describe PublicBody, " when indexing requests by body they are to" do - fixtures :public_bodies, :public_body_translations, :info_request_events, :info_requests, :raw_emails, :comments + fixtures :public_bodies, :public_body_translations, :info_requests, :raw_emails, :comments, :info_request_events before(:each) do load_raw_emails_data(raw_emails) @@ -132,7 +132,7 @@ describe PublicBody, " when indexing requests by body they are to" do end describe User, " when indexing requests by user they are from" do - fixtures :users, :info_request_events, :info_requests, :incoming_messages, :outgoing_messages, :raw_emails, :comments + fixtures :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events before(:each) do load_raw_emails_data(raw_emails) end @@ -219,7 +219,7 @@ describe User, " when indexing requests by user they are from" do end describe User, " when indexing comments by user they are by" do - fixtures :users, :info_request_events, :info_requests, :comments, :incoming_messages, :outgoing_messages, :raw_emails, :comments + fixtures :users, :info_requests, :raw_emails, :incoming_messages, :outgoing_messages, :comments, :info_request_events before(:each) do load_raw_emails_data(raw_emails) end @@ -256,7 +256,7 @@ describe User, " when indexing comments by user they are by" do end describe InfoRequest, " when indexing requests by their title" do - fixtures :info_request_events, :info_requests, :incoming_messages, :raw_emails, :comments + fixtures :info_requests, :raw_emails, :incoming_messages, :comments, :info_request_events before(:each) do load_raw_emails_data(raw_emails) end @@ -287,7 +287,7 @@ describe InfoRequest, " when indexing requests by their title" do end describe InfoRequest, " when indexing requests by tag" do - fixtures :info_request_events, :info_requests, :incoming_messages, :raw_emails, :comments + fixtures :info_requests, :raw_emails, :incoming_messages, :comments, :info_request_events before(:each) do load_raw_emails_data(raw_emails) end @@ -309,7 +309,7 @@ describe InfoRequest, " when indexing requests by tag" do end describe PublicBody, " when indexing authorities by tag" do - fixtures :public_bodies, :public_body_translations, :incoming_messages, :outgoing_messages, :raw_emails, :comments + fixtures :public_bodies, :public_body_translations, :raw_emails, :incoming_messages, :outgoing_messages, :comments before(:each) do load_raw_emails_data(raw_emails) end |