Store current scraperwiki sources.

1 files changed, 15 insertions, 0 deletions

diff --git a/fetch-scraper-sources b/fetch-scraper-sources
new file mode 100755
index 0000000..6465ea3
--- /dev/null
+++ b/fetch-scraper-sources
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+scrapers="postliste-python-lib postliste-ballangen"
+
+scrapers="$( (
+    GET https://scraperwiki.com/tags/postjournal
+    GET https://scraperwiki.com/tags/postjournal?page=2
+    GET https://scraperwiki.com/tags/postjournal?page=3
+) | grep owner | rev | cut -d/ -f3 | rev)"
+
+mkdir -p scrapersources
+for scraper in $scrapers; do
+    echo "Fetching source for $scraper"
+    GET "https://scraperwiki.com/editor/raw/$scraper" > "scrapersources/$scraper"
+done