Changes to deal with packages dpan can't handle

11 files changed, 48 insertions, 236 deletions

diff --git a/perl-extras/bin/add_module.pl b/perl-extras/bin/add_module.pl
index a39e9fd0c..6c9be8df2 100644
--- a/perl-extras/bin/add_module.pl
+++ b/perl-extras/bin/add_module.pl
@@ -5,21 +5,15 @@ use warnings;
 
 use IPC::Run3;
 use LWP::Simple;
-use YAML;
 use File::Slurp;
 use Path::Class;
 
-my $cpan_meta_db_base = "http://cpanmetadb.appspot.com/v1.0/package";
-my $cpan_mirror_base  = "http://search.cpan.org/CPAN/authors/id";
-my $root_dir          = file(__FILE__)->dir->parent->absolute->stringify;
-my $dist_list         = "$root_dir/install_order.txt";
-my $url_list          = "$root_dir/urls.txt";
-my $dist_dir          = "$root_dir/distributions";
-my $fake_mirror       = "$root_dir/fake_mirror";
-my $cpanm_cmd         = "perl $root_dir/bin/cpanm";
-
-# my $cpanm_cmd =
-#   "perl $dist_dir/cpanm --mirror $fake_mirror --mirror-only -L $local_lib_dir";
+my $root_dir    = file(__FILE__)->dir->parent->absolute->stringify;
+my $module_list = "$root_dir/modules.txt";
+my $url_list    = "$root_dir/urls.txt";
+my $minicpan    = "$root_dir/minicpan";
+my $local_lib   = "$root_dir/../local-lib5";
+my $cpanm_cmd   = "perl $root_dir/bin/cpanm -l $local_lib --reinstall";
 
 my $module = $ARGV[0] || die "Usage: $0 Dist::To::Add";
 
@@ -30,103 +24,29 @@ print "  running '$cmd'\n";
 run3( $cmd, undef, \$out, \$out )
   || die "Error running '$cmd'";
 
-my %fetched_urls =
-  map { $_ => 1 }
+my @fetched_urls =
   map { s{.*(http://\S+).*}{$1}; $_ }
   grep { m{^Fetching http://search.cpan.org} }
   split /\n/, $out;
 
-my @installed =
-  grep { $_ }
-  map { m{^Successfully (?:re)?installed (\S+).*} ? $1 : undef }
-  split /\n/, $out;
+write_file( $module_list, { append => 1 }, "$module\n" );
+write_file( $url_list, { append => 1 }, map { "$_\n" } @fetched_urls );
 
-foreach my $dist (@installed) {
-    my ($url) = grep { m{/$dist\.} } keys %fetched_urls;
-    die "Can't find url for $dist" unless $url;
-    delete $fetched_urls{$url};
+foreach my $url ( read_file($url_list) ) {
 
-    my ($filename) = $url =~ m{([^/]+)$};
+    my ($filename) = $url =~ m{/(authors/.+)$};
 
-    print "  fetching $filename from $url\n";
+    my $destination = file("$minicpan/$filename");
+    $destination->dir->mkpath;
 
-    is_success( getstore( $url, "$dist_dir/$filename" ) )
-      || die "Error saving $url to $dist_dir/$filename";
-    write_file( $dist_list, { append => 1 }, "$filename\n" );
-    write_file( $url_list,  { append => 1 }, "$url\n" );
-}
+    next if -e $destination;
 
-# Check that there are no urls left over
-my @leftover = keys %fetched_urls;
-die "URLs leftover: " . join( ', ', @leftover ) if @leftover;
+    print "  $url\n    -> $destination\n";
+
+    is_success( getstore( $url, "$destination" ) )
+      || die "Error saving $url to $destination";
+}
 
-# #
-# # load list of modules at start and write it back at the end
-# my %installed = ();
-# my %looked_at = ();
-#
-# sub add_filename_to_list {
-#     my $module = shift;
-#     print "Adding '$module' to '$dist_list'\n";
-#     append( $dist_list, "$module\n" );
-#     getc;
-# }
-#
-# # add_module('App::cpanminus');
-# add_module($module_to_add);
-#
-# sub add_module {
-#     my $module = shift;
-#     print "--- $module ---\n";
-#     return 1 if $installed{$module} || $looked_at{$module}++;
-#
-#     # get the distribution this module is in
-#     my $yaml = get("$cpan_meta_db_base/$module")
-#       || die "Can't get details from $cpan_meta_db_base/$module";
-#     my $dist_info = Load($yaml);
-#     my $distfile  = $dist_info->{distfile}
-#       || die("Can't get distfile from returned YAML for $module");
-#
-#     # fetch the distribution from cpan
-#     my ($filename) = $distfile =~ m{/([^/]*)$};
-#     unless ( -e "$dist_dir/$filename" ) {
-#         my $dist_url = "$cpan_mirror_base/$distfile";
-#         print "  fetching '$dist_url' to '$dist_dir/$filename'\n";
-#         is_success( getstore( $dist_url, "$dist_dir/$filename" ) )
-#           || die "Could not fetch $dist_url";
-#     }
-#
-#     # try to install the distribution using cpanm
-#     my $out = '';
-#     my $cmd = "$cpanm_cmd $dist_dir/$filename";
-#     print "  running '$cmd'\n";
-#     run3( $cmd, undef, \$out, \$out ) || die "Error running '$cmd'";
-#
-#     warn "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n";
-#     warn $out;
-#     warn "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n";
-#
-#     my @missing_modules =
-#       map { s{^.*\s(\S+)$}{$1}; $_ }
-#       grep { m/Couldn\'t find module or a distribution/ }
-#       split /\n/, $out;
-#
-#     if ( scalar @missing_modules ) {
-#         print "  missing: ", join( ', ', @missing_modules ), "\n";
-#         add_module($_) for @missing_modules;
-#         add_module($module);
-#     }
-#     elsif ( $out =~ m{Successfully installed } ) {
-#
-#         # add ourselves to the done lists
-#         print "  Success with '$filename'\n";
-#         $installed{$module}++;
-#         add_filename_to_list($filename);
-#     }
-#     else {
-#         die "No success and no missing modules for $module";
-#     }
-#
-# }
-#
-# #
+# go to the minicpan dir and run dpan there
+chdir $minicpan;
+exec 'dpan -f ../dpan_config';
diff --git a/perl-extras/bin/build_all_modules.pl b/perl-extras/bin/build_all_modules.pl
index eb57e811b..81f3d2e63 100644
--- a/perl-extras/bin/build_all_modules.pl
+++ b/perl-extras/bin/build_all_modules.pl
@@ -9,125 +9,32 @@ use YAML;
 use File::Slurp;
 use Path::Class;
 
-my $cpan_meta_db_base = "http://cpanmetadb.appspot.com/v1.0/package";
-my $cpan_mirror_base  = "http://search.cpan.org/CPAN/authors/id";
-my $root_dir          = file(__FILE__)->dir->parent->absolute->stringify;
-my $dist_list         = "$root_dir/install_order.txt";
-my $url_list          = "$root_dir/urls.txt";
-my $dist_dir          = "$root_dir/distributions";
-my $fake_mirror       = "$root_dir/fake_mirror";
-my $local_lib         = "$root_dir/../local-lib5";
+my $root_dir    = file(__FILE__)->dir->parent->absolute->stringify;
+my $module_list = "$root_dir/modules.txt";
+my $minicpan    = "$root_dir/minicpan";
+my $local_lib   = "$root_dir/../local-lib5";
 my $cpanm_cmd =
-  "perl $root_dir/bin/cpanm --mirror $fake_mirror --mirror-only -L $local_lib";
+  "perl $root_dir/bin/cpanm --mirror $minicpan --mirror-only -l $local_lib";
 
-my @distributions = map { s{\s+$}{}; $_; } read_file($dist_list);
+my @modules = map { s{\s+$}{}; $_; } read_file($module_list);
 
-foreach my $dist (@distributions) {
-    print "  --- installing $dist ---\n";
+foreach my $module (@modules) {
+    print "  --- installing $module ---\n";
 
     my $out = '';
-    my $cmd = "$cpanm_cmd $dist_dir/$dist";
+    my $cmd = "$cpanm_cmd $module";
 
     print "  running '$cmd'\n";
 
     run3( $cmd, undef, \$out, \$out )
       || die "Error running '$cmd'";
 
-    die "Error building '$dist':\n\n$out\n\n"
-      unless $out =~ m{Successfully installed};
-}
-
-# # try to install the distribution using cpanm
-#
-# my @fetched_urls =
-#   map { s{.*(http://\S+).*}{$1}; $_ }
-#   grep { m{^Fetching http://search.cpan.org} }
-#   split /\n/, $out;
-#
-# my @installed =
-#   grep { $_ }
-#   map { m{^Successfully (?:re)?installed (\S+).*} ? $1 : undef }
-#   split /\n/, $out;
-#
-# foreach my $dist (@installed) {
-#     my ($url) = grep { m{/$dist\.} } @fetched_urls;
-#     my ($filename) = $url =~ m{([^/]+)$};
-#
-#     print "  getting $filename from $url\n";
-#
-#     is_success( getstore( $url, "$dist_dir/$filename" ) )
-#       || die "Error saving $url to $dist_dir/$filename";
-#     write_file( $dist_list, { append => 1 }, "$filename\n" );
-#     write_file( $url_list,  { append => 1 }, "$url\n" );
-# }
+    my @lines =
+      grep { m{\S} }
+      split /\n+/, $out;
+    my $last_line = $lines[-1];
 
-# #
-# # load list of modules at start and write it back at the end
-# my %installed = ();
-# my %looked_at = ();
-#
-# sub add_filename_to_list {
-#     my $module = shift;
-#     print "Adding '$module' to '$dist_list'\n";
-#     append( $dist_list, "$module\n" );
-#     getc;
-# }
-#
-# # add_module('App::cpanminus');
-# add_module($module_to_add);
-#
-# sub add_module {
-#     my $module = shift;
-#     print "--- $module ---\n";
-#     return 1 if $installed{$module} || $looked_at{$module}++;
-#
-#     # get the distribution this module is in
-#     my $yaml = get("$cpan_meta_db_base/$module")
-#       || die "Can't get details from $cpan_meta_db_base/$module";
-#     my $dist_info = Load($yaml);
-#     my $distfile  = $dist_info->{distfile}
-#       || die("Can't get distfile from returned YAML for $module");
-#
-#     # fetch the distribution from cpan
-#     my ($filename) = $distfile =~ m{/([^/]*)$};
-#     unless ( -e "$dist_dir/$filename" ) {
-#         my $dist_url = "$cpan_mirror_base/$distfile";
-#         print "  fetching '$dist_url' to '$dist_dir/$filename'\n";
-#         is_success( getstore( $dist_url, "$dist_dir/$filename" ) )
-#           || die "Could not fetch $dist_url";
-#     }
-#
-#     # try to install the distribution using cpanm
-#     my $out = '';
-#     my $cmd = "$cpanm_cmd $dist_dir/$filename";
-#     print "  running '$cmd'\n";
-#     run3( $cmd, undef, \$out, \$out ) || die "Error running '$cmd'";
-#
-#     warn "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n";
-#     warn $out;
-#     warn "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n";
-#
-#     my @missing_modules =
-#       map { s{^.*\s(\S+)$}{$1}; $_ }
-#       grep { m/Couldn\'t find module or a distribution/ }
-#       split /\n/, $out;
-#
-#     if ( scalar @missing_modules ) {
-#         print "  missing: ", join( ', ', @missing_modules ), "\n";
-#         add_module($_) for @missing_modules;
-#         add_module($module);
-#     }
-#     elsif ( $out =~ m{Successfully installed } ) {
-#
-#         # add ourselves to the done lists
-#         print "  Success with '$filename'\n";
-#         $installed{$module}++;
-#         add_filename_to_list($filename);
-#     }
-#     else {
-#         die "No success and no missing modules for $module";
-#     }
-#
-# }
-#
-# #
+    die "Error building '$module':\n\n$out\n\n"
+      unless $last_line =~ m{Successfully installed }
+          || $last_line =~ m{is up to date};
+}
diff --git a/perl-extras/dpan_config b/perl-extras/dpan_config
new file mode 100644
index 000000000..cd0d0b203
--- /dev/null
+++ b/perl-extras/dpan_config
@@ -0,0 +1,2 @@
+extra_reports_dir extra_reports
+skip_dists_regexes common-sense
\ No newline at end of file
diff --git a/perl-extras/dpan_l4p_config b/perl-extras/dpan_l4p_config
new file mode 100644
index 000000000..883372464
--- /dev/null
+++ b/perl-extras/dpan_l4p_config
@@ -0,0 +1,4 @@
+log4perl.rootLogger              = DEBUG, Screen
+
+log4perl.appender.Screen         = Log::Log4perl::Appender::Screen
+log4perl.appender.Screen.layout  = Log::Log4perl::Layout::SimpleLayout
diff --git a/perl-extras/install_order.txt b/perl-extras/install_order.txt
deleted file mode 100644
index 63bead795..000000000
--- a/perl-extras/install_order.txt
+++ /dev/null
@@ -1,12 +0,0 @@
-version-0.88.tar.gz
-Module-Metadata-1.000004.tar.gz
-Test-Simple-0.98.tar.gz
-Perl-OSType-1.002.tar.gz
-ExtUtils-CBuilder-0.280202.tar.gz
-ExtUtils-ParseXS-2.2206.tar.gz
-CPAN-Meta-YAML-0.003.tar.gz
-Test-Harness-3.23.tar.gz
-Module-Build-0.3624.tar.gz
-ExtUtils-Install-1.54.tar.gz
-libwww-perl-5.837.tar.gz
-App-cpanminus-1.3001.tar.gz
diff --git a/perl-extras/minicpan/.gitignore b/perl-extras/minicpan/.gitignore
new file mode 100644
index 000000000..0f6f4afed
--- /dev/null
+++ b/perl-extras/minicpan/.gitignore
@@ -0,0 +1 @@
+authors
diff --git a/perl-extras/minicpan/extra_reports/common-sense-3.4.txt b/perl-extras/minicpan/extra_reports/common-sense-3.4.txt
new file mode 100644
index 000000000..6d39e3a0f
--- /dev/null
+++ b/perl-extras/minicpan/extra_reports/common-sense-3.4.txt
@@ -0,0 +1,2 @@
+# Primary package [TAB] version [TAB] dist file [newline]
+common::sense	3.4	authors/id/M/ML/MLEHMANN/common-sense-3.4.tar.gz
\ No newline at end of file
diff --git a/perl-extras/minicpan/modules/02packages.details.txt.gz b/perl-extras/minicpan/modules/02packages.details.txt.gz
new file mode 100644
index 000000000..49329fce5
--- /dev/null
+++ b/perl-extras/minicpan/modules/02packages.details.txt.gz
diff --git a/perl-extras/minicpan/modules/03modlist.data.gz b/perl-extras/minicpan/modules/03modlist.data.gz
new file mode 100644
index 000000000..9c8cc712e
--- /dev/null
+++ b/perl-extras/minicpan/modules/03modlist.data.gz
diff --git a/perl-extras/modules.txt b/perl-extras/modules.txt
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/perl-extras/modules.txt
diff --git a/perl-extras/urls.txt b/perl-extras/urls.txt
index 5c79fda0f..e69de29bb 100644
--- a/perl-extras/urls.txt
+++ b/perl-extras/urls.txt
@@ -1,12 +0,0 @@
-http://search.cpan.org/CPAN/authors/id/Y/YV/YVES/ExtUtils-Install-1.54.tar.gz
-http://search.cpan.org/CPAN/authors/id/J/JP/JPEACOCK/version-0.88.tar.gz
-http://search.cpan.org/CPAN/authors/id/D/DA/DAGOLDEN/Module-Metadata-1.000004.tar.gz
-http://search.cpan.org/CPAN/authors/id/M/MS/MSCHWERN/Test-Simple-0.98.tar.gz
-http://search.cpan.org/CPAN/authors/id/D/DA/DAGOLDEN/Perl-OSType-1.002.tar.gz
-http://search.cpan.org/CPAN/authors/id/D/DA/DAGOLDEN/ExtUtils-CBuilder-0.280202.tar.gz
-http://search.cpan.org/CPAN/authors/id/D/DA/DAGOLDEN/ExtUtils-ParseXS-2.2206.tar.gz
-http://search.cpan.org/CPAN/authors/id/D/DA/DAGOLDEN/CPAN-Meta-YAML-0.003.tar.gz
-http://search.cpan.org/CPAN/authors/id/A/AN/ANDYA/Test-Harness-3.23.tar.gz
-http://search.cpan.org/CPAN/authors/id/D/DA/DAGOLDEN/Module-Build-0.3624.tar.gz
-http://search.cpan.org/CPAN/authors/id/G/GA/GAAS/libwww-perl-5.837.tar.gz
-http://search.cpan.org/CPAN/authors/id/M/MI/MIYAGAWA/App-cpanminus-1.3001.tar.gz