diff options
| author | Petter Reinholdtsen <pere@hungry.com> | 2014-12-09 12:18:14 +0100 |
|---|---|---|
| committer | Petter Reinholdtsen <pere@hungry.com> | 2014-12-09 12:18:14 +0100 |
| commit | c9c7832be4a3207c0977e8107f1140b4e9aadf23 (patch) | |
| tree | 2e72bc9348938c5030cf64863c6b291577fcb9a9 | |
| parent | 7417d4efcec19160fbe29e482067c8d202436177 (diff) | |
Add code to rescan old IDs in case they changed.
| -rw-r--r-- | scrapersources/postliste-oep | 9 |
1 files changed, 9 insertions, 0 deletions
diff --git a/scrapersources/postliste-oep b/scrapersources/postliste-oep index 6ad338a..a2a2a28 100644 --- a/scrapersources/postliste-oep +++ b/scrapersources/postliste-oep @@ -376,3 +376,12 @@ if min >= 0 and read_backwards: fetched = fetch_range(datastorage, min, min - count, -1) print "Fetched " + str(fetched) + " old journal entries, cpu spent: " + str(cpu_spent()) + +# Rescan to see if we missed something, and to get the latest version +rescan_min = 0 +rescan_count = 100 +if 0 < scraperwiki.sqlite.get_var('min_rescan_id'): + rescan_min = scraperwiki.sqlite.get_var('min_rescan_id') + fetched = fetch_range(datastorage, rescan_min, rescan_min + rescan_count, 1) + save_var('min_rescan_id', rescan_min + rescan_count - 1) + print "Fetched " + str(fetched) + " rescanned journal entries, cpu spent: " + str(cpu_spent()) |