blob: 0afea1055b76458fa72eb2f7fa17d8d11427f4d1 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
|
#!/bin/sh
# Should find quicker way to find scrapers
scrapers="$( (
GET https://classic.scraperwiki.com/tags/postjournal
GET https://classic.scraperwiki.com/tags/postjournal?page=2
GET https://classic.scraperwiki.com/tags/postjournal?page=3
GET https://classic.scraperwiki.com/tags/postjournal?page=4
) | grep owner | rev | cut -d/ -f3 | rev)"
mkdir -p scrapersources
for scraper in $scrapers; do
echo "Fetching source for $scraper"
GET "https://scraperwiki.com/editor/raw/$scraper" > "scrapersources/$scraper"
done
|
