diff options
| -rw-r--r-- | TODO.txt | 8 | ||||
| -rw-r--r-- | perl-extras/.gitignore | 1 | ||||
| -rw-r--r-- | perl-extras/bin/add_module.pl | 28 | ||||
| -rw-r--r-- | perl-extras/bin/build_all_modules.pl | 133 | ||||
| -rw-r--r-- | perl-extras/install_order.txt | 12 | ||||
| -rw-r--r-- | perl-extras/urls.txt | 12 |
6 files changed, 182 insertions, 12 deletions
@@ -33,3 +33,11 @@ Email: we switch to using it (Email::Send::EvEl...)? * pull out encoding smarts into Email::Send::SMTP - possibly a flag? +deploying + + + +/fixmystreet - code from repo +/fixmystreet/darkpan - minicpan like repo with just our dependencies in + +/fixmystreet-local-lib5 - local::lib for relevant code
\ No newline at end of file diff --git a/perl-extras/.gitignore b/perl-extras/.gitignore new file mode 100644 index 000000000..46ff56b3e --- /dev/null +++ b/perl-extras/.gitignore @@ -0,0 +1 @@ +distributions diff --git a/perl-extras/bin/add_module.pl b/perl-extras/bin/add_module.pl index 0cf3b0cfd..a39e9fd0c 100644 --- a/perl-extras/bin/add_module.pl +++ b/perl-extras/bin/add_module.pl @@ -7,15 +7,16 @@ use IPC::Run3; use LWP::Simple; use YAML; use File::Slurp; +use Path::Class; my $cpan_meta_db_base = "http://cpanmetadb.appspot.com/v1.0/package"; my $cpan_mirror_base = "http://search.cpan.org/CPAN/authors/id"; -my $root_dir = "/home/evdb/fixmystreet/perl-extras"; +my $root_dir = file(__FILE__)->dir->parent->absolute->stringify; my $dist_list = "$root_dir/install_order.txt"; +my $url_list = "$root_dir/urls.txt"; my $dist_dir = "$root_dir/distributions"; -my $local_lib_dir = "local-lib5"; my $fake_mirror = "$root_dir/fake_mirror"; -my $cpanm_cmd = "perl $root_dir/cpanm --reinstall -L $local_lib_dir"; +my $cpanm_cmd = "perl $root_dir/bin/cpanm"; # my $cpanm_cmd = # "perl $dist_dir/cpanm --mirror $fake_mirror --mirror-only -L $local_lib_dir"; @@ -29,9 +30,8 @@ print " running '$cmd'\n"; run3( $cmd, undef, \$out, \$out ) || die "Error running '$cmd'"; -warn $out; - -my @fetched_urls = +my %fetched_urls = + map { $_ => 1 } map { s{.*(http://\S+).*}{$1}; $_ } grep { m{^Fetching http://search.cpan.org} } split /\n/, $out; @@ -41,21 +41,25 @@ my @installed = map { m{^Successfully (?:re)?installed (\S+).*} ? $1 : undef } split /\n/, $out; -use Data::Dumper; -local $Data::Dumper::Sortkeys = 1; -warn Dumper( { fetched => \@fetched_urls, installed => \@installed } ); - foreach my $dist (@installed) { - my ($url) = grep { m{/$dist\.} } @fetched_urls; + my ($url) = grep { m{/$dist\.} } keys %fetched_urls; + die "Can't find url for $dist" unless $url; + delete $fetched_urls{$url}; + my ($filename) = $url =~ m{([^/]+)$}; - print " getting $filename from $url\n"; + print " fetching $filename from $url\n"; is_success( getstore( $url, "$dist_dir/$filename" ) ) || die "Error saving $url to $dist_dir/$filename"; write_file( $dist_list, { append => 1 }, "$filename\n" ); + write_file( $url_list, { append => 1 }, "$url\n" ); } +# Check that there are no urls left over +my @leftover = keys %fetched_urls; +die "URLs leftover: " . join( ', ', @leftover ) if @leftover; + # # # # load list of modules at start and write it back at the end # my %installed = (); diff --git a/perl-extras/bin/build_all_modules.pl b/perl-extras/bin/build_all_modules.pl new file mode 100644 index 000000000..eb57e811b --- /dev/null +++ b/perl-extras/bin/build_all_modules.pl @@ -0,0 +1,133 @@ +#!/usr/bin/env perl + +use strict; +use warnings; + +use IPC::Run3; +use LWP::Simple; +use YAML; +use File::Slurp; +use Path::Class; + +my $cpan_meta_db_base = "http://cpanmetadb.appspot.com/v1.0/package"; +my $cpan_mirror_base = "http://search.cpan.org/CPAN/authors/id"; +my $root_dir = file(__FILE__)->dir->parent->absolute->stringify; +my $dist_list = "$root_dir/install_order.txt"; +my $url_list = "$root_dir/urls.txt"; +my $dist_dir = "$root_dir/distributions"; +my $fake_mirror = "$root_dir/fake_mirror"; +my $local_lib = "$root_dir/../local-lib5"; +my $cpanm_cmd = + "perl $root_dir/bin/cpanm --mirror $fake_mirror --mirror-only -L $local_lib"; + +my @distributions = map { s{\s+$}{}; $_; } read_file($dist_list); + +foreach my $dist (@distributions) { + print " --- installing $dist ---\n"; + + my $out = ''; + my $cmd = "$cpanm_cmd $dist_dir/$dist"; + + print " running '$cmd'\n"; + + run3( $cmd, undef, \$out, \$out ) + || die "Error running '$cmd'"; + + die "Error building '$dist':\n\n$out\n\n" + unless $out =~ m{Successfully installed}; +} + +# # try to install the distribution using cpanm +# +# my @fetched_urls = +# map { s{.*(http://\S+).*}{$1}; $_ } +# grep { m{^Fetching http://search.cpan.org} } +# split /\n/, $out; +# +# my @installed = +# grep { $_ } +# map { m{^Successfully (?:re)?installed (\S+).*} ? $1 : undef } +# split /\n/, $out; +# +# foreach my $dist (@installed) { +# my ($url) = grep { m{/$dist\.} } @fetched_urls; +# my ($filename) = $url =~ m{([^/]+)$}; +# +# print " getting $filename from $url\n"; +# +# is_success( getstore( $url, "$dist_dir/$filename" ) ) +# || die "Error saving $url to $dist_dir/$filename"; +# write_file( $dist_list, { append => 1 }, "$filename\n" ); +# write_file( $url_list, { append => 1 }, "$url\n" ); +# } + +# # +# # load list of modules at start and write it back at the end +# my %installed = (); +# my %looked_at = (); +# +# sub add_filename_to_list { +# my $module = shift; +# print "Adding '$module' to '$dist_list'\n"; +# append( $dist_list, "$module\n" ); +# getc; +# } +# +# # add_module('App::cpanminus'); +# add_module($module_to_add); +# +# sub add_module { +# my $module = shift; +# print "--- $module ---\n"; +# return 1 if $installed{$module} || $looked_at{$module}++; +# +# # get the distribution this module is in +# my $yaml = get("$cpan_meta_db_base/$module") +# || die "Can't get details from $cpan_meta_db_base/$module"; +# my $dist_info = Load($yaml); +# my $distfile = $dist_info->{distfile} +# || die("Can't get distfile from returned YAML for $module"); +# +# # fetch the distribution from cpan +# my ($filename) = $distfile =~ m{/([^/]*)$}; +# unless ( -e "$dist_dir/$filename" ) { +# my $dist_url = "$cpan_mirror_base/$distfile"; +# print " fetching '$dist_url' to '$dist_dir/$filename'\n"; +# is_success( getstore( $dist_url, "$dist_dir/$filename" ) ) +# || die "Could not fetch $dist_url"; +# } +# +# # try to install the distribution using cpanm +# my $out = ''; +# my $cmd = "$cpanm_cmd $dist_dir/$filename"; +# print " running '$cmd'\n"; +# run3( $cmd, undef, \$out, \$out ) || die "Error running '$cmd'"; +# +# warn "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n"; +# warn $out; +# warn "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; +# +# my @missing_modules = +# map { s{^.*\s(\S+)$}{$1}; $_ } +# grep { m/Couldn\'t find module or a distribution/ } +# split /\n/, $out; +# +# if ( scalar @missing_modules ) { +# print " missing: ", join( ', ', @missing_modules ), "\n"; +# add_module($_) for @missing_modules; +# add_module($module); +# } +# elsif ( $out =~ m{Successfully installed } ) { +# +# # add ourselves to the done lists +# print " Success with '$filename'\n"; +# $installed{$module}++; +# add_filename_to_list($filename); +# } +# else { +# die "No success and no missing modules for $module"; +# } +# +# } +# +# # diff --git a/perl-extras/install_order.txt b/perl-extras/install_order.txt new file mode 100644 index 000000000..63bead795 --- /dev/null +++ b/perl-extras/install_order.txt @@ -0,0 +1,12 @@ +version-0.88.tar.gz +Module-Metadata-1.000004.tar.gz +Test-Simple-0.98.tar.gz +Perl-OSType-1.002.tar.gz +ExtUtils-CBuilder-0.280202.tar.gz +ExtUtils-ParseXS-2.2206.tar.gz +CPAN-Meta-YAML-0.003.tar.gz +Test-Harness-3.23.tar.gz +Module-Build-0.3624.tar.gz +ExtUtils-Install-1.54.tar.gz +libwww-perl-5.837.tar.gz +App-cpanminus-1.3001.tar.gz diff --git a/perl-extras/urls.txt b/perl-extras/urls.txt new file mode 100644 index 000000000..5c79fda0f --- /dev/null +++ b/perl-extras/urls.txt @@ -0,0 +1,12 @@ +http://search.cpan.org/CPAN/authors/id/Y/YV/YVES/ExtUtils-Install-1.54.tar.gz +http://search.cpan.org/CPAN/authors/id/J/JP/JPEACOCK/version-0.88.tar.gz +http://search.cpan.org/CPAN/authors/id/D/DA/DAGOLDEN/Module-Metadata-1.000004.tar.gz +http://search.cpan.org/CPAN/authors/id/M/MS/MSCHWERN/Test-Simple-0.98.tar.gz +http://search.cpan.org/CPAN/authors/id/D/DA/DAGOLDEN/Perl-OSType-1.002.tar.gz +http://search.cpan.org/CPAN/authors/id/D/DA/DAGOLDEN/ExtUtils-CBuilder-0.280202.tar.gz +http://search.cpan.org/CPAN/authors/id/D/DA/DAGOLDEN/ExtUtils-ParseXS-2.2206.tar.gz +http://search.cpan.org/CPAN/authors/id/D/DA/DAGOLDEN/CPAN-Meta-YAML-0.003.tar.gz +http://search.cpan.org/CPAN/authors/id/A/AN/ANDYA/Test-Harness-3.23.tar.gz +http://search.cpan.org/CPAN/authors/id/D/DA/DAGOLDEN/Module-Build-0.3624.tar.gz +http://search.cpan.org/CPAN/authors/id/G/GA/GAAS/libwww-perl-5.837.tar.gz +http://search.cpan.org/CPAN/authors/id/M/MI/MIYAGAWA/App-cpanminus-1.3001.tar.gz |