Make script more robust.

1 files changed, 10 insertions, 6 deletions

diff --git a/move-postjournal b/move-postjournal
index 144ccf4..2f27f16 100755
--- a/move-postjournal
+++ b/move-postjournal
@@ -61,7 +61,7 @@ def db_connect():
 
 def jsonurl(scraper, lastscrapestamputc):
     limit = 10000
-    limit = 350
+    limit = 400
     return "https://api.scraperwiki.com/"+\
         "api/1.0/datastore/sqlite?format=json&name=" + scraper +\
         "&query=select+*+from+%60swdata%60+where+scrapestamputc+>+"+\
@@ -162,15 +162,18 @@ def populate_from_scraper_real(dbcursor, scraper):
     except:
         pass
     
-    print "Adding/updating " + str(len(data)) + " " + status + " entries"
+    print "Adding/updating " + str(len(data)) + " " + status + " entries (" + lastscrapestamputc + ")"
+    skipped = 0
     for entry in data:
 
         # Handle OEP scraper 2012-06-16
         if not 'caseyear' in entry or entry['caseyear'] is None or \
                 not 'caseseqnr' in entry or entry['caseseqnr'] is None:
-#            if entry['caseid'] is None:
-#                print "Strange entry, skipping: ", entry
-#                continue
+            if True:
+                if entry['caseid'] is None:
+#                    print "Strange entry, skipping: ", entry
+                    skipped = skipped + 1
+                    continue
             entry['caseyear'], entry['caseseqnr'] = entry['caseid'].split("/")
 
         entry['scraper'] = scraper
@@ -183,7 +186,8 @@ def populate_from_scraper_real(dbcursor, scraper):
         except:
             print entry
             raise
-    return len(data)
+    print "Added/upded " + str(len(data)-skipped) + " " + status + " entries"
+    return len(data) - skipped
 
 def populate_from_scraper(dbcursor, scraper):
     ret = populate_from_scraper_real(dbcursor, scraper)