set up management correctly

8 files changed, 215 insertions, 92 deletions

diff --git a/perl-external/bin/add_module.pl b/perl-external/bin/add_module.pl
deleted file mode 100644
index 6c9be8df2..000000000
--- a/perl-external/bin/add_module.pl
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/usr/bin/env perl
-
-use strict;
-use warnings;
-
-use IPC::Run3;
-use LWP::Simple;
-use File::Slurp;
-use Path::Class;
-
-my $root_dir    = file(__FILE__)->dir->parent->absolute->stringify;
-my $module_list = "$root_dir/modules.txt";
-my $url_list    = "$root_dir/urls.txt";
-my $minicpan    = "$root_dir/minicpan";
-my $local_lib   = "$root_dir/../local-lib5";
-my $cpanm_cmd   = "perl $root_dir/bin/cpanm -l $local_lib --reinstall";
-
-my $module = $ARGV[0] || die "Usage: $0 Dist::To::Add";
-
-# try to install the distribution using cpanm
-my $out = '';
-my $cmd = "$cpanm_cmd $module";
-print "  running '$cmd'\n";
-run3( $cmd, undef, \$out, \$out )
-  || die "Error running '$cmd'";
-
-my @fetched_urls =
-  map { s{.*(http://\S+).*}{$1}; $_ }
-  grep { m{^Fetching http://search.cpan.org} }
-  split /\n/, $out;
-
-write_file( $module_list, { append => 1 }, "$module\n" );
-write_file( $url_list, { append => 1 }, map { "$_\n" } @fetched_urls );
-
-foreach my $url ( read_file($url_list) ) {
-
-    my ($filename) = $url =~ m{/(authors/.+)$};
-
-    my $destination = file("$minicpan/$filename");
-    $destination->dir->mkpath;
-
-    next if -e $destination;
-
-    print "  $url\n    -> $destination\n";
-
-    is_success( getstore( $url, "$destination" ) )
-      || die "Error saving $url to $destination";
-}
-
-# go to the minicpan dir and run dpan there
-chdir $minicpan;
-exec 'dpan -f ../dpan_config';
diff --git a/perl-external/bin/build_all_modules.pl b/perl-external/bin/build_all_modules.pl
deleted file mode 100644
index 81f3d2e63..000000000
--- a/perl-external/bin/build_all_modules.pl
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env perl
-
-use strict;
-use warnings;
-
-use IPC::Run3;
-use LWP::Simple;
-use YAML;
-use File::Slurp;
-use Path::Class;
-
-my $root_dir    = file(__FILE__)->dir->parent->absolute->stringify;
-my $module_list = "$root_dir/modules.txt";
-my $minicpan    = "$root_dir/minicpan";
-my $local_lib   = "$root_dir/../local-lib5";
-my $cpanm_cmd =
-  "perl $root_dir/bin/cpanm --mirror $minicpan --mirror-only -l $local_lib";
-
-my @modules = map { s{\s+$}{}; $_; } read_file($module_list);
-
-foreach my $module (@modules) {
-    print "  --- installing $module ---\n";
-
-    my $out = '';
-    my $cmd = "$cpanm_cmd $module";
-
-    print "  running '$cmd'\n";
-
-    run3( $cmd, undef, \$out, \$out )
-      || die "Error running '$cmd'";
-
-    my @lines =
-      grep { m{\S} }
-      split /\n+/, $out;
-    my $last_line = $lines[-1];
-
-    die "Error building '$module':\n\n$out\n\n"
-      unless $last_line =~ m{Successfully installed }
-          || $last_line =~ m{is up to date};
-}
diff --git a/perl-external/bin/cpanm b/perl-external/bin/cpanm
index 1c552fac2..1c552fac2 100644..100755
--- a/perl-external/bin/cpanm
+++ b/perl-external/bin/cpanm
diff --git a/perl-external/bin/module-manage.pl b/perl-external/bin/module-manage.pl
new file mode 100755
index 000000000..c6b741b32
--- /dev/null
+++ b/perl-external/bin/module-manage.pl
@@ -0,0 +1,142 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+
+use IPC::Run3;
+use LWP::Simple;
+use File::Slurp;
+use Path::Class;
+use List::MoreUtils 'uniq';
+
+my $root_dir    = file(__FILE__)->dir->parent->absolute->stringify;
+my $module_list = "$root_dir/modules.txt";
+my $url_list    = "$root_dir/urls.txt";
+my $minicpan    = "$root_dir/minicpan";
+
+my %actions = (
+    add            => \&add,
+    build_all      => \&build_all,
+    fetch_all      => \&fetch_all,
+    index_minicpan => \&index_minicpan,
+    init           => \&init,
+    setup          => \&setup,
+    sort_files     => \&sort_files,
+    zap            => \&zap,
+);
+
+# work out what to run
+my ( $action, @args ) = @ARGV;
+$actions{$action}
+  ? $actions{$action}->(@args)
+  : die "Usage: $0 action [args ...]\n";
+
+exit;
+
+############################################################################
+
+sub init {
+    add('App::cpanminus');
+    add('MyCPAN::App::DPAN');
+    index_minicpan();
+}
+
+sub setup {
+    fetch_all();
+    init();
+    build_all();
+}
+
+sub add {
+    my $module = shift || die "Usage: $0 add Dist::To::Add";
+
+    # try to install the distribution using cpanm
+    my $out = '';
+    my $cmd = "cpanm --reinstall $module";
+    print "  running '$cmd'\n";
+    run3( $cmd, undef, \$out, \$out )
+      || die "Error running '$cmd'";
+
+    my @fetched_urls =
+      map { s{.*(http://\S+).*}{$1}; $_ }
+      grep { m{^Fetching http://search.cpan.org} }
+      split /\n/, $out;
+
+    write_file( $module_list, { append => 1 }, "$module\n" );
+    write_file( $url_list, { append => 1 }, map { "$_\n" } @fetched_urls );
+    sort_files();
+
+    fetch_all();
+}
+
+sub index_minicpan {
+
+    # go to the minicpan dir and run dpan there
+    chdir $minicpan;
+    system "dpan -f ../dpan_config";
+}
+
+sub build_all {
+    my @modules = sort uniq read_file($module_list);
+    build($_) for @modules;
+}
+
+sub build {
+    my $module = shift    #
+      || die "Usage: $0 build Module::To::Build\n";
+
+    print "  --- installing $module ---\n";
+
+    my $out = '';
+    my $cmd = "cpanm --mirror $minicpan --mirror-only  $module";
+
+    print "  running '$cmd'\n";
+
+    run3( $cmd, undef, \$out, \$out )
+      || die "Error running '$cmd'";
+
+    my @lines =
+      grep { m{\S} }
+      split /\n+/, $out;
+    my $last_line = $lines[-1];
+
+    die "Error building '$module':\n\n$out\n\n"
+      unless $last_line =~ m{Successfully installed }
+          || $last_line =~ m{is up to date};
+}
+
+sub fetch_all {
+    my @urls = sort uniq read_file($url_list);
+    fetch($_) for @urls;
+}
+
+sub fetch {
+    my $url = shift;
+    my ($filename) = $url =~ m{/(authors/.+)$};
+
+    my $destination = file("$minicpan/$filename");
+    $destination->dir->mkpath;
+
+    return if -e $destination;
+
+    print "  $url\n    -> $destination\n";
+
+    is_success( getstore( $url, "$destination" ) )
+      || die "Error saving $url to $destination";
+}
+
+sub zap {
+
+    # delete all the bits that are generated
+    my $local_lib_root = $ENV{PERL_LOCAL_LIB_ROOT} || die;
+    dir($local_lib_root)->rmtree(1);
+    dir($minicpan)->subdir('authors')->rmtree(1);
+}
+
+sub sort_files {
+    foreach my $file ( $url_list, $module_list ) {
+        my @entries = read_file($file);
+        @entries = uniq sort @entries;
+        write_file( $file, @entries );
+    }
+}
diff --git a/perl-external/minicpan/.gitignore b/perl-external/minicpan/.gitignore
index 0f6f4afed..de1f15c6c 100644
--- a/perl-external/minicpan/.gitignore
+++ b/perl-external/minicpan/.gitignore
@@ -1 +1,3 @@
 authors
+indexer_reports
+
diff --git a/perl-external/minicpan/modules/02packages.details.txt.gz b/perl-external/minicpan/modules/02packages.details.txt.gz
index 49329fce5..b51735854 100644
--- a/perl-external/minicpan/modules/02packages.details.txt.gz
+++ b/perl-external/minicpan/modules/02packages.details.txt.gz
diff --git a/perl-external/modules.txt b/perl-external/modules.txt
index e69de29bb..1dbb6cce8 100644
--- a/perl-external/modules.txt
+++ b/perl-external/modules.txt
@@ -0,0 +1,2 @@
+App::cpanminus
+MyCPAN::App::DPAN
diff --git a/perl-external/urls.txt b/perl-external/urls.txt
index e69de29bb..1bfb20160 100644
--- a/perl-external/urls.txt
+++ b/perl-external/urls.txt
@@ -0,0 +1,69 @@
+http://search.cpan.org/CPAN/authors/id/A/AD/ADAMK/Class-Inspector-1.25.tar.gz
+http://search.cpan.org/CPAN/authors/id/A/AD/ADAMK/File-Remove-1.48.tar.gz
+http://search.cpan.org/CPAN/authors/id/A/AD/ADAMK/PPI-1.215.tar.gz
+http://search.cpan.org/CPAN/authors/id/A/AD/ADAMK/Params-Util-1.03.tar.gz
+http://search.cpan.org/CPAN/authors/id/A/AD/ADAMK/Task-Weaken-1.04.tar.gz
+http://search.cpan.org/CPAN/authors/id/A/AD/ADAMK/Test-NoWarnings-1.02.tar.gz
+http://search.cpan.org/CPAN/authors/id/A/AD/ADAMK/Test-Object-0.07.tar.gz
+http://search.cpan.org/CPAN/authors/id/A/AD/ADAMK/Test-SubCalls-1.09.tar.gz
+http://search.cpan.org/CPAN/authors/id/A/AN/ANDK/CPAN-Checksums-2.07.tar.gz
+http://search.cpan.org/CPAN/authors/id/A/AN/ANDYA/Test-Harness-3.23.tar.gz
+http://search.cpan.org/CPAN/authors/id/A/AR/ARJAY/Compress-Bzip2-2.09.tar.gz
+http://search.cpan.org/CPAN/authors/id/B/BD/BDFOY/CPAN-PackageDetails-0.25.tar.gz
+http://search.cpan.org/CPAN/authors/id/B/BD/BDFOY/ConfigReader-Simple-1.28.tar.gz
+http://search.cpan.org/CPAN/authors/id/B/BD/BDFOY/Distribution-Guess-BuildSystem-0.12.tar.gz
+http://search.cpan.org/CPAN/authors/id/B/BD/BDFOY/File-Find-Closures-1.09.tar.gz
+http://search.cpan.org/CPAN/authors/id/B/BD/BDFOY/Module-Extract-Namespaces-0.14.tar.gz
+http://search.cpan.org/CPAN/authors/id/B/BD/BDFOY/Module-Extract-Use-0.17.tar.gz
+http://search.cpan.org/CPAN/authors/id/B/BD/BDFOY/Module-Extract-VERSION-0.13.tar.gz
+http://search.cpan.org/CPAN/authors/id/B/BD/BDFOY/MyCPAN-App-DPAN-1.28.tar.gz
+http://search.cpan.org/CPAN/authors/id/B/BD/BDFOY/MyCPAN-Indexer-1.28.tar.gz
+http://search.cpan.org/CPAN/authors/id/B/BD/BDFOY/Test-Output-0.16.tar.gz
+http://search.cpan.org/CPAN/authors/id/C/CH/CHORNY/Hook-LexWrap-0.24.tar.gz
+http://search.cpan.org/CPAN/authors/id/C/CH/CHORNY/Test-Warn-0.23.tar.gz
+http://search.cpan.org/CPAN/authors/id/C/CO/COGENT/Tree-DAG_Node-1.06.tar.gz
+http://search.cpan.org/CPAN/authors/id/D/DA/DAGOLDEN/CPAN-Meta-2.110580.tar.gz
+http://search.cpan.org/CPAN/authors/id/D/DA/DAGOLDEN/CPAN-Meta-YAML-0.003.tar.gz
+http://search.cpan.org/CPAN/authors/id/D/DA/DAGOLDEN/ExtUtils-CBuilder-0.280202.tar.gz
+http://search.cpan.org/CPAN/authors/id/D/DA/DAGOLDEN/ExtUtils-ParseXS-2.2206.tar.gz
+http://search.cpan.org/CPAN/authors/id/D/DA/DAGOLDEN/Module-Build-0.3800.tar.gz
+http://search.cpan.org/CPAN/authors/id/D/DA/DAGOLDEN/Module-Metadata-1.000004.tar.gz
+http://search.cpan.org/CPAN/authors/id/D/DA/DAGOLDEN/Parse-CPAN-Meta-1.4401.tar.gz
+http://search.cpan.org/CPAN/authors/id/D/DA/DAGOLDEN/Perl-OSType-1.002.tar.gz
+http://search.cpan.org/CPAN/authors/id/D/DC/DCANTRELL/Data-Compare-1.22.tar.gz
+http://search.cpan.org/CPAN/authors/id/D/DL/DLUX/Parallel-ForkManager-0.7.9.tar.gz
+http://search.cpan.org/CPAN/authors/id/F/FD/FDALY/Test-Tester-0.107.tar.gz
+http://search.cpan.org/CPAN/authors/id/G/GA/GAAS/Encode-Locale-1.01.tar.gz
+http://search.cpan.org/CPAN/authors/id/G/GA/GAAS/File-Listing-6.00.tar.gz
+http://search.cpan.org/CPAN/authors/id/G/GA/GAAS/HTTP-Cookies-6.00.tar.gz
+http://search.cpan.org/CPAN/authors/id/G/GA/GAAS/HTTP-Daemon-6.00.tar.gz
+http://search.cpan.org/CPAN/authors/id/G/GA/GAAS/HTTP-Date-6.00.tar.gz
+http://search.cpan.org/CPAN/authors/id/G/GA/GAAS/HTTP-Message-6.01.tar.gz
+http://search.cpan.org/CPAN/authors/id/G/GA/GAAS/HTTP-Negotiate-6.00.tar.gz
+http://search.cpan.org/CPAN/authors/id/G/GA/GAAS/LWP-MediaTypes-6.01.tar.gz
+http://search.cpan.org/CPAN/authors/id/G/GA/GAAS/Net-HTTP-6.00.tar.gz
+http://search.cpan.org/CPAN/authors/id/G/GA/GAAS/WWW-RobotRules-6.00.tar.gz
+http://search.cpan.org/CPAN/authors/id/G/GA/GAAS/libwww-perl-6.01.tar.gz
+http://search.cpan.org/CPAN/authors/id/G/GB/GBARR/CPAN-DistnameInfo-0.12.tar.gz
+http://search.cpan.org/CPAN/authors/id/G/GB/GBARR/Scalar-List-Utils-1.23.tar.gz
+http://search.cpan.org/CPAN/authors/id/J/JP/JPEACOCK/version-0.88.tar.gz
+http://search.cpan.org/CPAN/authors/id/K/KW/KWILLIAMS/Probe-Perl-0.01.tar.gz
+http://search.cpan.org/CPAN/authors/id/M/MA/MAKAMAKA/JSON-PP-2.27105.tar.gz
+http://search.cpan.org/CPAN/authors/id/M/MI/MIYAGAWA/App-cpanminus-1.4004.tar.gz
+http://search.cpan.org/CPAN/authors/id/M/MS/MSCHILLI/Log-Log4perl-1.32.tar.gz
+http://search.cpan.org/CPAN/authors/id/M/MS/MSCHWERN/ExtUtils-MakeMaker-6.56.tar.gz
+http://search.cpan.org/CPAN/authors/id/M/MS/MSCHWERN/Test-Simple-0.98.tar.gz
+http://search.cpan.org/CPAN/authors/id/P/PM/PMQS/Compress-Raw-Bzip2-2.033.tar.gz
+http://search.cpan.org/CPAN/authors/id/P/PM/PMQS/Compress-Raw-Zlib-2.033.tar.gz
+http://search.cpan.org/CPAN/authors/id/P/PM/PMQS/IO-Compress-2.033.tar.gz
+http://search.cpan.org/CPAN/authors/id/R/RC/RCLAMP/File-Find-Rule-0.32.tar.gz
+http://search.cpan.org/CPAN/authors/id/R/RC/RCLAMP/Number-Compare-0.01.tar.gz
+http://search.cpan.org/CPAN/authors/id/R/RC/RCLAMP/Text-Glob-0.09.tar.gz
+http://search.cpan.org/CPAN/authors/id/R/RD/RDF/Clone-0.31.tar.gz
+http://search.cpan.org/CPAN/authors/id/R/RJ/RJBS/Data-OptList-0.106.tar.gz
+http://search.cpan.org/CPAN/authors/id/R/RJ/RJBS/Sub-Exporter-0.982.tar.gz
+http://search.cpan.org/CPAN/authors/id/R/RJ/RJBS/Sub-Install-0.925.tar.gz
+http://search.cpan.org/CPAN/authors/id/R/RJ/RJBS/Version-Requirements-0.101020.tar.gz
+http://search.cpan.org/CPAN/authors/id/S/SM/SMUELLER/PathTools-3.33.tar.gz
+http://search.cpan.org/CPAN/authors/id/T/TJ/TJENNESS/File-Temp-0.22.tar.gz
+http://search.cpan.org/CPAN/authors/id/Y/YV/YVES/ExtUtils-Install-1.54.tar.gz