diff options
| author | Petter Reinholdtsen <pere@hungry.com> | 2014-12-07 21:56:07 +0100 |
|---|---|---|
| committer | Petter Reinholdtsen <pere@hungry.com> | 2014-12-07 21:56:07 +0100 |
| commit | 41c3dc0e79fe5444a51502106f4802e3f67f0076 (patch) | |
| tree | 0a0af4c2539ca4d5c40d2d89dd9bbdefb72cdd1b | |
| parent | d7bd2c100e1e490d75c161624ad29e68cf4f26a3 (diff) | |
Remove redundant code.
| -rw-r--r-- | scrapersources/postliste-ballangen | 14 |
1 files changed, 2 insertions, 12 deletions
diff --git a/scrapersources/postliste-ballangen b/scrapersources/postliste-ballangen index 13b258f..7cce476 100644 --- a/scrapersources/postliste-ballangen +++ b/scrapersources/postliste-ballangen @@ -257,19 +257,9 @@ parser = postlistelib.JournalParser(agency=agency) urllist = [] -def fetch_url(url): - html = None - for n in [1, 2, 3]: - try: - html = scraperwiki.scrape(url) - break - except urllib2.URLError, e: - print "URLError fetching " + url + ", trying again" - return html - for monthurl in monthurls: print "Fetching month list from " + monthurl - html = fetch_url(monthurl) + html = postlistelib.fetch_url_harder(monthurl) urllist.extend(fetch_postjournal_monthlist(baseurl = baseurl, html = html)) for dayurl in urllist: @@ -277,7 +267,7 @@ for dayurl in urllist: if 0 < len(res): continue print "Fetching from " + dayurl - html = fetch_url(dayurl) + html = postlistelib.fetch_url_harder(dayurl) # print html fetch_postjournal_day(parser=parser, url=dayurl, html=html, saver=saver) |