#!/bin/sh

# Should find quicker way to find scrapers
scrapers="$( (
    GET https://classic.scraperwiki.com/tags/postjournal
    GET https://classic.scraperwiki.com/tags/postjournal?page=2
    GET https://classic.scraperwiki.com/tags/postjournal?page=3
    GET https://classic.scraperwiki.com/tags/postjournal?page=4
) | grep owner | rev | cut -d/ -f3 | rev)"

mkdir -p scrapersources
for scraper in $scrapers; do
    echo "Fetching source for $scraper"
    GET "https://scraperwiki.com/editor/raw/$scraper" > "scrapersources/$scraper"
done