diff options
Diffstat (limited to 'maketiles')
| -rwxr-xr-x | maketiles/10kfull | 192 | ||||
| -rwxr-xr-x | maketiles/10khalf | 180 | ||||
| -rw-r--r-- | maketiles/README | 28 | ||||
| -rwxr-xr-x | maketiles/fixindex | 26 | ||||
| -rwxr-xr-x | maketiles/packtiles | 157 |
5 files changed, 0 insertions, 583 deletions
diff --git a/maketiles/10kfull b/maketiles/10kfull deleted file mode 100755 index 873818092..000000000 --- a/maketiles/10kfull +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/perl -w -# -# 10kfull: -# Turn the 1:10,000 Ordnance Survey map tiles into 254-by-254 pixmaps at full -# resolution. -# -# Copyright (c) 2005 UK Citizens Online Democracy. All rights reserved. -# Email: chris@mysociety.org; WWW: http://www.mysociety.org/ -# - -my $rcsid = ''; $rcsid .= '$Id: 10kfull,v 1.5 2006-09-20 15:01:56 chris Exp $'; - -use strict; - -use Digest::SHA1; -use Errno; -use File::stat; -use Geography::NationalGrid; -use IO::Dir; -use IO::File; - -use constant TILE_SIZE_PX => 7874; -use constant TILE_SIZE_M => 5000; - -use constant SUBTILE_SIZE_PX => 254; -use constant SUBTILES_PER_TILE => (TILE_SIZE_PX / SUBTILE_SIZE_PX); - -sub debug (@) { - print STDERR @_; -} - -sub System (@) { - debug("executing command: ", @_, "\n"); - system(@_); - die "command failed with status $?" if ($?); -} - -die "arguments are directories containing input tiles in TIFF format and\n" - . "to which output tiles are to be written, and optional eastings and\n" - . "northings ranges, in the order west, east, south, north\n" - if (@ARGV != 2 && @ARGV != 6); - -my ($inputdir, $outputdir) = @ARGV; -$inputdir =~ s#/$##; -$outputdir =~ s#/$##; -die "$inputdir: not a directory" if (!-d $inputdir); -die "$outputdir: not a directory" if (!-d $outputdir); - -my $inarea = 0; -my ($west, $east, $south, $north); - -if (@ARGV == 6) { - $inarea = 1; - foreach (@ARGV[2 .. 5]) { - die "'$_' is not a valid number\n" if (!/^[1-9]\d*$/); - } - ($west, $east, $south, $north) = @ARGV[2 .. 5]; - die "EAST must be greater than WEST" unless ($east > $west); - die "NORTH must be greater than SOUTH" unless ($north > $south); - - debug("area of interest: west $west, east $east, south $south, north $north\n"); -} - -# -# Within the output directory we create a subdirectory tiles/ containing tile -# images named for their SHA1 checksums, and a text file mapping tile -# coordinates to the SHA1 sums. -# - -die "$outputdir/tiles: $!" - if (!mkdir("$outputdir/tiles", 0755) && !$!{EEXIST}); - -my $index = new IO::File("$outputdir/index", O_WRONLY | O_CREAT | O_TRUNC, 0644) - or die "$outputdir/index: $!"; - -$index->print(<<EOF); -# This index file connects individual tiles of the 1:10,000 Ordnance Survey -# maps at their full resolution (0.635m/px). Each such tile is 254 by 254 -# pixels. The tiles are indexed according to their coordinates within the full -# Ordnance Survey grid, with (0, 0) being the tile at the SW corner of the grid -# (immediately NE of the origin), (1, 0) being the tile to its east, and (0, 1) -# the tile immediately to its north. Note that not all tiles are present. For -# each such tile position this file lists the SHA1 message-digest of the image -# file which represents the tile. -EOF - -my $d = new IO::Dir($inputdir) - or die "$inputdir: $!"; - -my $total_input_size = 0; # KB -my $total_output_size = 0; # KB -while (my $name = $d->read()) { - next unless ($name =~ /\.tiff?$/i); - debug("have image file $inputdir/$name\n"); - - my ($gridsq, $offset, $quadrant) - = map { uc($_) } ($name =~ /^([A-Z]{2})(\d{2})([NS][EW])\./i); - - if (!$gridsq) { - debug(" -> doesn't seem to be a valid 1:10,000 image tile; ignoring\n"); - next; - } - - debug(" -> grid square = $gridsq; offset = $offset; quadrant = $quadrant\n"); - my $p = new Geography::NationalGrid('GB', - GridReference => "$gridsq $offset"); - - my ($E, $N) = ($p->easting(), $p->northing()); - debug(" -> SW corner of grid square is at ($E, $N)\n"); - - $N += TILE_SIZE_M if ($quadrant =~ /^N/); - $E += TILE_SIZE_M if ($quadrant =~ /E$/); - debug(" => SW corner of tile is at ($E, $N)\n"); - - if ($inarea) { - my $tE = $E + TILE_SIZE_M; - my $tW = $E; - my $tN = $N + TILE_SIZE_M; - my $tS = $N; - - if ($tE < $west || $tW > $east - || $tN < $south || $tS > $north) { - debug(" tile is outside area of interest; skipping\n"); - next; - } - } - - my $st = stat("$inputdir/$name"); - my $input_tile_size = $st->size(); - debug(" tile size = $input_tile_size bytes\n"); - $total_input_size += $input_tile_size / 1024.; - - - - # Figure out the tile offsets. - my $x = ($E / TILE_SIZE_M) * SUBTILES_PER_TILE; - my $y = ($N / TILE_SIZE_M) * SUBTILES_PER_TILE; - debug(" => subtile col/row for SW-most subtile is ($x, $y)\n"); - - # Now split this tile into subtiles. - System("tifftopnm $inputdir/$name |" - . " pnmtilesplit -P -f '$outputdir/%d,%d.png'" - . " -p 'pnmtopng -compression 9 2>/dev/null' 254 254"); - - # Consider each of the subtiles. - debug("finding SHA1 digests of tiles and adding them to index...\n"); - my $nduplicates = 0; - my $nnew = 0; - my $size = 0; - for (my $j = 0; $j < SUBTILES_PER_TILE; ++$j) { - debug("\r$j/", SUBTILES_PER_TILE); - my $j2 = SUBTILES_PER_TILE - 1 - $j; - for (my $i = 0; $i < SUBTILES_PER_TILE; ++$i) { - my $fn = "$outputdir/$i,$j2.png"; - my $f = new IO::File($fn, O_RDONLY) - or die "$fn: $!"; - my $sha = new Digest::SHA1(); - $sha->addfile($f); - $sha = $sha->hexdigest(); - $f->close(); - $index->printf("%d %d %s\n", $x + $i, $y + $j, $sha); - my ($n1, $n2, $n3) = ($sha =~ /^(.)(.)(.)/); - mkdir("$outputdir/tiles/$n1"); - mkdir("$outputdir/tiles/$n1/$n2"); - mkdir("$outputdir/tiles/$n1/$n2/$n3"); - my $fn2 = "$outputdir/tiles/$n1/$n2/$n3/$sha.png"; - if (-e $fn2) { - unlink($fn); - ++$nduplicates; - } else { - rename($fn, $fn2) or die "rename: $!"; - ++$nnew; - my $st = stat($fn2); - $size += $st->size(); - } - } - } - debug("\r", SUBTILES_PER_TILE, "/", SUBTILES_PER_TILE, "\n"); - debug("had $nduplicates duplicate tiles + $nnew new, $size bytes\n"); - debug(" output subtiles are ", - sprintf('%.1f%%', 100 * $size / $input_tile_size), - " of input tile size for this tile\n"); - $total_output_size += $size / 1024.; - debug(" so far total input size ", - sprintf('%.1fKB', $total_input_size), "\n"); - debug(" so far total output size ", - sprintf('%.1fKB', $total_output_size), "\n"); - debug(" output subtiles are ", - sprintf('%.1f%%', 100 * $total_output_size / $total_input_size), - " of input tile size so far overall\n"); - $index->flush(); -} diff --git a/maketiles/10khalf b/maketiles/10khalf deleted file mode 100755 index 48d59bc24..000000000 --- a/maketiles/10khalf +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/perl -w -# -# 10khalf: -# Generate half-scale 1:10,000 Ordnance Survey map tiles from the -# full-resolution ones generated by 10kfull. -# -# Copyright (c) 2005 UK Citizens Online Democracy. All rights reserved. -# Email: chris@mysociety.org; WWW: http://www.mysociety.org/ -# - -my $rcsid = ''; $rcsid .= '$Id: 10khalf,v 1.1 2006-09-14 21:43:15 chris Exp $'; - -use strict; - -use Digest::SHA1; -use File::stat; -use IO::File; - -sub debug (@) { - print STDERR @_; -} - -sub System (@) { - #debug("executing command: ", @_, "\n"); - # need bash for <( ) command substitution - system("/bin/bash", '-c', join(' ', @_)); - die "command failed with status $?" if ($?); -} - -die "arguments are directory containing output files from 10kfull and directory to which the half-resolution tiles are to be written" - if (2 != @ARGV); - -my ($inputdir, $outputdir) = @ARGV; -$inputdir =~ s#/$##; -$outputdir =~ s#/$##; -die "$inputdir: not a directory" if (!-d $inputdir); -die "$outputdir: not a directory" if (!-d $outputdir); - -# -# Within the output directory we create a subdirectory tiles/ containing tile -# images named for their SHA1 checksums, and a text file mapping tile -# coordinates to the SHA1 sums. -# - -die "$outputdir/tiles: $!" - if (!mkdir("$outputdir/tiles", 0755) && !$!{EEXIST}); - -my $index = new IO::File("$outputdir/index", O_WRONLY | O_CREAT | O_TRUNC, 0644) - or die "$outputdir/index: $!"; - -$index->print(<<EOF); -# This index file connects individual tiles of the 1:10,000 Ordnance Survey -# maps at half their original resolution (1.27m/px). Each such tile is 254 by -# 254 pixels. The tiles are indexed according to their coordinates within the -# full Ordnance Survey grid, with (0, 0) being the tile at the SW corner of the -# grid (immediately NE of the origin), (1, 0) being the tile to its east, and -# (0, 1) the tile immediately to its north. Note that not all tiles are -# present. For each such tile position this file lists the SHA1 message-digest -# of the image file which represents the tile. -EOF - -my $in_index = new IO::File("$inputdir/index", O_RDONLY) - or die "$inputdir/index: $!"; - -my %fullrestiles = ( ); -my %halfrestiles = ( ); - -my $line; -while (defined($line = $in_index->getline()) && $line =~ /^#/) { } -if (!defined($line)) { - if ($in_index->error()) { - die "$inputdir/index: $! (while reading header)"; - } else { - die "$inputdir/index: premature EOF (while reading header)"; - } -} - -debug("reading $inputdir/index... "); -my $ntiles = 0; -my $nhalfrestiles = 0; -do { - chomp($line); - my ($i, $j, $sha1) = split(/\s+/, $line); - die "$inputdir/index: bad index line '$line'" - unless (defined($i) && defined($j) && defined($sha1) - && $i =~ /^(0|[1-9]\d*)$/ && $j =~ /^(0|[1-9]\d*)$/ - && $sha1 =~ /^[0-9a-f]{40}$/); - - my $i2 = int($i / 2); - my $j2 = int($j / 2); - my $k = pack('II', $i2, $j2); - if (!exists($halfrestiles{$k})) { - ++$nhalfrestiles; - $halfrestiles{$k} = undef; - } - - $fullrestiles{pack('II', $i, $j)} = $sha1; - ++$ntiles; -} while (defined($line = $in_index->getline())); - -die "$inputdir/index: $!" if ($in_index->error()); -$in_index->close(); -debug("done\n"); - -debug("read $ntiles from index\n"); -debug("have $nhalfrestiles to generate\n"); - -System("ppmmake '#ffffff' 254 254 > $outputdir/blank.ppm"); - -sub imgname ($) { - my $sha1 = shift; - if (!defined($sha1)) { - return "$outputdir/blank.ppm"; - } else { - $sha1 =~ s#^(.)(.)(.)(.+)$#$1/$2/$3/$1$2$3$4#; - return "<( pngtopnm $inputdir/tiles/$sha1.png )"; - } -} - -my %compose = ( ); -my $nnew = 0; -my $nduplicates = 0; -my $size = 0; -foreach my $k (keys(%halfrestiles)) { - my ($i2, $j2) = unpack('II', $k); - my @t = ( ); - my $n = 0; - for (my $j = 0; $j <= 1; ++$j) { - for (my $i = 0; $i <= 1; ++$i) { - $t[$n] = $fullrestiles{pack('II', 2 * $i2 + $i, 2 * $j2 + $j)}; -# debug("img #$n is ", defined($t[$n]) ? $t[$n] : '(blank)', "\n"); - ++$n; - } - } - - my $c = join(',', map { defined($_) ? $_ : '*' } @t); - my $fn = "$outputdir/out.png"; - my $sha; - - if (exists($compose{$c})) { - # we've already made this image - $sha = $compose{$c}; - } else { - # make it - @t = map { imgname($_) } @t; - System("pnmcat -tb <( pnmcat -lr $t[0] $t[1] ) <( pnmcat -lr $t[2] $t[3] )" - . " | pnmscale -reduce 2 2>/dev/null" - . " | pnmtopng -compression 9 2>/dev/null > $fn"); - - my $f = new IO::File($fn, O_RDONLY) - or die "$fn: $!"; - $sha = new Digest::SHA1(); - $sha->addfile($f); - $sha = $sha->hexdigest(); - $f->close(); - - $compose{$c} = $sha; - } - - $index->printf("%d %d %s\n", $i2, $j2, $sha); - my ($n1, $n2, $n3) = ($sha =~ /^(.)(.)(.)/); - mkdir("$outputdir/tiles/$n1"); - mkdir("$outputdir/tiles/$n1/$n2"); - mkdir("$outputdir/tiles/$n1/$n2/$n3"); - my $fn2 = "$outputdir/tiles/$n1/$n2/$n3/$sha.png"; - if (-e $fn2) { - unlink($fn); - ++$nduplicates; - } else { - rename($fn, $fn2) or die "rename: $!"; - ++$nnew; - my $st = stat($fn2); - $size += $st->size(); - } - - debug("have $nnew new, $nduplicates duplicates, total size $size bytes\n"); -} - -# get rid of the blank PPM. -unlink("$outputdir/blank.ppm"); diff --git a/maketiles/README b/maketiles/README deleted file mode 100644 index 64c6bb380..000000000 --- a/maketiles/README +++ /dev/null @@ -1,28 +0,0 @@ -File formats and structures - -Tiles are generated as PNG files. At the moment these are full-colour PNGs but -in due course we will probably want to use dithered images with a limited -palette. Generated tiles are named for their own SHA1 checksums, so that we -can eliminate duplicate tiles. The first-stage output of these scripts, -therefore, consists of a set of PNG files named by SHA1 and a text index which -maps tile locations to SHA1s. These are arranged as: - - toplevel/index text index of tiles - toplevel/tiles/0/0/0 location for images with SHA1s beginning - 000 - toplevel/tiles/0/0/0/00000000000000000000.png - first possible image file - -This format is not really acceptable for final deployment because many of the -image files won't be much larger than a filesystem block, so a large amount of -disk space would be wasted in storing them. Nor is the text index from tile -location to SHA1 acceptable. So a second program packs the image files into -catenated files with an index from SHA1 to offset and length in each file. -Separately, the index from tile location to SHA1 is turned into another index. -Both indices use Bernstein-style CDB files. - - toplevel/index.cdb binary index from tile location to SHA1 - toplevel/tiles/0/0/0/packed catenated file of images with SHA1s - beginning 000 - toplevel/tiles/0/0/0/index.cdb index of images in packed - diff --git a/maketiles/fixindex b/maketiles/fixindex deleted file mode 100755 index 8bfed69db..000000000 --- a/maketiles/fixindex +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/perl -w -# -# fixindex: -# Fix the indices generated by the buggy version of 10kfull. -# -# Copyright (c) 2006 UK Citizens Online Democracy. All rights reserved. -# Email: chris@mysociety.org; WWW: http://www.mysociety.org/ -# - -my $rcsid = ''; $rcsid .= '$Id: fixindex,v 1.2 2006-09-20 15:20:30 chris Exp $'; - -use strict; - -use constant SUBTILES_PER_TILE => 31; - -while (defined(my $line = <STDIN>)) { - chomp($line); - if ($line =~ /^(0|[1-9]\d*) (0|[1-9]\d*) ([0-9a-f]+)$/) { - my ($x, $y, $id) = ($1, $2, $3); - $y = int($y / SUBTILES_PER_TILE) * SUBTILES_PER_TILE - + SUBTILES_PER_TILE - 1 - ($y % SUBTILES_PER_TILE); - print "$x $y $id\n"; - } else { - print "$line\n"; - } -} diff --git a/maketiles/packtiles b/maketiles/packtiles deleted file mode 100755 index dd12ae94f..000000000 --- a/maketiles/packtiles +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/perl -w -# -# packtiles: -# Pack a tree of tiles etc. as generated by 10kfull etc. into the more efficient -# packed format which is read by the tile server. -# -# Copyright (c) 2006 UK Citizens Online Democracy. All rights reserved. -# Email: chris@mysociety.org; WWW: http://www.mysociety.org/ -# - -my $rcsid = ''; $rcsid .= '$Id: packtiles,v 1.3 2006-09-20 14:57:42 chris Exp $'; - -use strict; - -use CDB_File; -use Errno; -use File::Slurp; -use File::stat; -use IO::Dir; -use IO::File; - -use constant TILEID_LEN => 20; -use constant BLOCKING => 14; - -sub debug (@) { - print STDERR @_; -} - -sub pack_tile_dir ($$) { - my ($indir, $outdir) = @_; - -} - -# new_index_block -# -sub new_index_block () { - # Initially all tiles are null so all bits are set. - my $bitmaplen = int((BLOCKING * BLOCKING + 8) / 8); - my $buf = "\xff" x $bitmaplen; - $buf .= "\00" x (BLOCKING * BLOCKING * TILEID_LEN); -} - -# add_tile_to_index_block BLOCK X Y ID -# -sub add_tile_to_index_block ($$$$) { - my $x = $_[1]; - my $y = $_[2]; - my $id = $_[3]; - - # clear the nullness bit. - my $off = $x + $y * BLOCKING; - my $b = unpack('C', substr($_[0], $off >> 3, 1)); - $b &= ~(1 << ($off & 7)); - substr($_[0], $off >> 3, 1) = pack('C', $b); - - # stick the tile itself in. - my $bitmaplen = int((BLOCKING * BLOCKING + 8) / 8); - substr($_[0], $bitmaplen + $off * TILEID_LEN, TILEID_LEN) = $id; -} - -die "arguments are input (unpacked) tile directory, and output (packed)\n" - . "tileset directories\n" - if (@ARGV != 2); - -my ($inputdir, $outputdir) = @ARGV; -$inputdir =~ s#/$##; -$outputdir =~ s#/$##; -die "$inputdir: not a directory" if (!-d $inputdir); -die "$outputdir: not a directory" if (!-d $outputdir); - -# first make the index of tile locations to IDs. -my %index; -debug("reading index of tile locations to IDs...\n"); -my $f = new IO::File("$inputdir/index", O_RDONLY); -my $len = stat($f)->size(); -my $n = 0; -while (defined(my $line = $f->getline())) { - debug("\rreading index: ", $f->tell(), "/$len bytes") - if (0 == ($n % 100)); - ++$n; - - next if ($line =~ /^#/); - - chomp($line); - my ($x, $y, $hash) = split(/ /, $line); - my $id = pack('H*', $hash); - - my $x2 = int($x / BLOCKING); - my $y2 = int($y / BLOCKING); - - $index{"$x2,$y2"} ||= new_index_block(); - add_tile_to_index_block( - $index{"$x2,$y2"}, - $x % BLOCKING, $y % BLOCKING, - $id); -} -debug("\rreading index: $len/$len bytes\n"); - -die "$inputdir/index: $!" if ($f->error()); -$f->close(); - -debug("writing index blocks to index.cdb... "); -my $C = new CDB_File("$outputdir/index.cdb", "$outputdir/index.new"); -$C->insert(blocking => BLOCKING); -foreach (keys(%index)) { - $C->insert($_, $index{$_}); -} -debug("done\n"); -debug("finalising index.cdb... "); -$C->finish(); -debug("done\n"); - -debug("packing individual tile directories...\n"); -$n = 0; -mkdir("$outputdir/tiles"); -for (my $u = 0; $u < 4096; ++$u) { - my $subpath = sprintf('%x/%x/%x', $u & 0xf, ($u >> 4) & 0xf, $u >> 8); - - next unless (-d "$inputdir/tiles/$subpath"); - - # Ensure the path exists. - foreach my $p (qw(1 3 5)) { - my $d = "$outputdir/tiles/" . substr($subpath, 0, $p); - die "$d: mkdir: $!" if (!mkdir($d) && !$!{EEXIST}); - } - - my $in = "$inputdir/tiles/$subpath"; - my $out = "$outputdir/tiles/$subpath"; - - my $f = new IO::File("$out/tiles", O_WRONLY | O_CREAT | O_TRUNC, 0644) - or die "$out/tiles: open: $!"; - $C = new CDB_File("$out/tiles.cdb", "$out/tiles.tmp"); - my $d = new IO::Dir($in) or die "$in: $!"; - while (my $name = $d->read()) { - next unless ($name =~ /^([0-9a-f]+)\.png$/); - my $id = pack('H*', $1); - my $buf = read_file("$in/$name") or die "$in/$name: $!"; - - die "$in/$name: length of file read by File::Slurp did not match stat" - unless (stat("$in/$name")->size() == length($buf)); - - my $rec = sprintf('%x:%x', $f->tell(), length($buf)); - $C->insert($id, $rec); - - $f->print($buf) or die "$out/tiles: $!"; - - ++$n; - } - - $f->close(); - $C->finish(); - - debug("\rpacking tiles: $u/4096"); -} - -debug("\rpacking tiles: 4096/4096\n"); -debug("packed $n tiles total\n"); |