blob: 947da4ece91977762b924093d6686a3a8c57468f (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
|
import scraperwiki
import lxml.html
import datetime
#uncomment to run for a selected timeperiod
#fromdate = "01.04.2011"
#todate = "21.05.2011"
#fromdate = datetime.datetime.strptime(fromdate, "%d.%m.%Y")
#todate = datetime.datetime.strptime(todate, "%d.%m.%Y")
#adday = datetime.timedelta(days=1)
def scrapepage(mydate):
formatteddate = mydate.strftime("%d.%m.%Y")
#formatteddate = "10.05.2011"
url = "http://www.vegvesen.no/Om+Statens+vegvesen/Aktuelt/Offentlig+journal?dokumenttyper=&dato=%s&journalenhet=6&utforSok=S%%C3%%B8k&submitButton=S%%C3%%B8k" % formatteddate
root = lxml.html.parse(url).getroot()
divs = root.cssselect("div.treff")
for p in divs:
dateandtype = p.xpath("p/text()")[0].split(" ")
saksdetaljer = p.xpath("ul[@class='saksdetaljer']/li/text()")
record = {
"doknr": dateandtype[0],
"innut": dateandtype[2],
"tittel": p.xpath("h2/text()")[0],
"sak": p.xpath("span[@class='sak']")[0].text[6:],
"fratil": p.xpath("ul[@class='fraTil']/li/text()")[0][5:],
}
record.update(dict([x.split(":") for x in saksdetaljer]))
record['Dokumenttdato'] = datetime.datetime.strptime(record['Dokumenttdato'].strip(), "%d.%m.%Y").date()
record['Journaldato'] = datetime.datetime.strptime(record['Journaldato'].strip(), "%d.%m.%Y").date()
scraperwiki.sqlite.save(unique_keys=["doknr"], data=record)
#uncomment to run for a selected timeperiod
#thedate = fromdate
#while thedate <= todate:
# print thedate
# thedate = thedate + adday
# scrapepage(thedate)
#comment out these two lines in order to run for a selected timeperiod
thedate = datetime.datetime.now()
print thedate
scrapepage(thedate)
|
