aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPetter Reinholdtsen <pere@hungry.com>2016-04-06 16:00:40 +0200
committerPetter Reinholdtsen <pere@hungry.com>2016-04-06 16:00:40 +0200
commitf9de0733abd0de5ccc6ac418ebf663b7d88032f8 (patch)
treed3ffe212b79260f5631f0a63e78e0ef6a572fb09
parent731b44075054279cd352c48181cd378f5f69d596 (diff)
Make sure reparse_strange_entries() do not fail when all tables are empty.
-rw-r--r--scrapersources/postliste-oep33
1 files changed, 18 insertions, 15 deletions
diff --git a/scrapersources/postliste-oep b/scrapersources/postliste-oep
index 0c7f03a..0bd802a 100644
--- a/scrapersources/postliste-oep
+++ b/scrapersources/postliste-oep
@@ -339,25 +339,28 @@ def remove_original():
# script was slightly broken and filled in non-existing entries in the
# SQL database.
def reparse_strange_entries(datastorage):
- strange = "journalPostId FROM swdata WHERE caseid IS NULL OR scrapestamputc IS NULL OR agency IS NULL order by journalPostId"
- for idref in scraperwiki.sqlite.select(strange):
- id = idref['journalPostId']
- if -1 == fetch_oep_entry(id, datastorage):
- print "Refetching %d failed, flush ID" % id
- scraperwiki.sqlite.execute("DELETE from swdata where journalPostId = %d" % id)
- if 0 < len(datastorage):
+ try:
+ strange = "journalPostId FROM swdata WHERE caseid IS NULL OR scrapestamputc IS NULL OR agency IS NULL order by journalPostId"
+ for idref in scraperwiki.sqlite.select(strange):
+ id = idref['journalPostId']
+ if -1 == fetch_oep_entry(id, datastorage):
+ print "Refetching %d failed, flush ID" % id
+ scraperwiki.sqlite.execute("DELETE from swdata where journalPostId = %d" % id)
+ if 0 < len(datastorage):
+ save(data=datastorage)
+ datastorage = []
+ else:
+ print "Refetching %d" % id
+ if 50 <= len(datastorage):
save(data=datastorage)
datastorage = []
- else:
- print "Refetching %d" % id
- if 50 <= len(datastorage):
+ time.sleep(0.2)
+ if 0 < len(datastorage):
save(data=datastorage)
datastorage = []
- time.sleep(0.2)
- if 0 < len(datastorage):
- save(data=datastorage)
- datastorage = []
-
+ except scraperwiki.sqlite.SqliteError, e:
+ # Most likely no table, keep going
+ pass
#update_caseyear()
#create_indexes()