diff options
author | Petter Reinholdtsen <pere@hungry.com> | 2014-12-10 21:25:47 +0100 |
---|---|---|
committer | Petter Reinholdtsen <pere@hungry.com> | 2014-12-10 21:25:47 +0100 |
commit | 672edb805214050606dd22af3175e640c0420902 (patch) | |
tree | 10295a47cd59ea217cceea974131193cacd906d4 | |
parent | 3fae491431e25726eb99d40ba91a85fc5b8490d0 (diff) |
Might as well flush the buffer when touching the database to remove an entry.
-rw-r--r-- | scrapersources/postliste-oep | 3 |
1 files changed, 3 insertions, 0 deletions
diff --git a/scrapersources/postliste-oep b/scrapersources/postliste-oep index 807a11e..87375fa 100644 --- a/scrapersources/postliste-oep +++ b/scrapersources/postliste-oep @@ -336,6 +336,9 @@ def reparse_strange_entries(datastorage): if -1 == fetch_oep_entry(id, datastorage): print "Refetching %d failed, flush ID" % id scraperwiki.sqlite.execute("DELETE from swdata where journalPostId = %d" % id) + if 0 < len(datastorage): + save(data=datastorage) + datastorage = [] else: print "Refetching %d" % id if 50 <= len(datastorage): |