diff options
Diffstat (limited to 'lxr-genxref')
| -rwxr-xr-x | lxr-genxref | 122 |
1 files changed, 87 insertions, 35 deletions
diff --git a/lxr-genxref b/lxr-genxref index d61526f..f64fe78 100755 --- a/lxr-genxref +++ b/lxr-genxref @@ -37,6 +37,7 @@ use IO::Handle; use Fcntl; use Term::ProgressBar; use Devel::Size qw(size total_size); +use Encode; $SIG{'INT'} = sub { die "\nSIGINT: $$: please wait, flushing caches...\n"; }; $SIG{'QUIT'} = sub { die "\nSIGQUIT: $$: please wait, flushing caches...\n"; }; @@ -51,6 +52,7 @@ my $tree = shift(@ARGV); my @versions = @ARGV; my $context = LXRng::Context->new('tree' => $tree); +die "Usage: $0 <tree-id>\n" unless $context and $context->tree; LXRng::Lang->init($context); my $index = $context->config->{'index'}; @@ -58,6 +60,10 @@ my $usage = $context->config->{'usage'}; my $hash = $context->config->{'search'}; my $rep = $context->config->{'repository'}; my $progress; +my $progress_ident; +my $progress_count; +my $progress_next; +my $progress_target; $SIG{'__WARN__'} = sub { $progress ? $progress->message(shift) : warn(@_) }; @@ -73,6 +79,50 @@ sub memory_status { } } +sub progress_init { + my ($ident, $total) = @_; + + if (-t STDOUT) { + $progress = Term::ProgressBar->new({name => $ident, + count => $total, + ETA => 'linear'}); + $progress->max_update_rate(0.25); + } + else { + $progress_ident = $ident; + $progress_target = $total; + $progress_next = int($progress_target/10); + $progress_count = 0; + } +} + +sub progress_update { + my ($count) = @_; + + return $progress->update($count) if $progress; + + if (defined($count)) { + $progress_count = $count; + } + else { + $progress_count++; + } + if ($progress_count > $progress_next or + $progress_count >= $progress_target) + { + print("$progress_ident: $progress_count/$progress_target done...\n"); + $progress_next = $progress_count + int($progress_target/10); + } +} + +sub progress_target { + my ($target) = @_; + + return $progress->target($target) if $progress; + + $progress_target = $target; +} + sub make_add_ident($) { my ($fileid) = @_; @@ -87,7 +137,8 @@ sub make_add_ident($) { $last_func = $symbol; } if ($$info{'kind'} eq 'l') { - $$info{'context'} = $identcache{$last_func}; + $$info{'context'} = $identcache{$last_func} if + defined($last_func); } if (exists $$info{'class'}) { $$info{'context'} = $identcache{$$info{'class'}}; @@ -123,10 +174,14 @@ sub index_file($$) { return 0 unless $index->to_index($fileid); return 1 unless $lang->doindex(); - my $add_ident = make_add_ident($fileid); - warn("--- indexing ".$file->name." [".$file->revision."]\n"); - my @extra_flags = ('-IEXPORT_SYMBOL+', '-I__initcall+'); + + my $add_ident = make_add_ident($fileid); + return $lang->index_file($context, $file, $add_ident) + if $lang->can('index_file'); + return 1 unless $lang->ctagslangname(); + + my $extra_flags = $context->config->{'ctags_flags'} || []; my $ctags; my $pid = open($ctags, '-|'); @@ -134,7 +189,7 @@ sub index_file($$) { if ($pid == 0) { exec('ctags-exuberant', - @extra_flags, + @$extra_flags, '--fields=+aifmknsSz', '--sort=no', '--excmd=number', '-f', '-', '--language-force='.$lang->ctagslangname, @@ -149,6 +204,7 @@ sub index_file($$) { while (<$ctags>) { chomp; my ($symbol, $file, $excmd, @info) = split(/\t/); + $symbol = $lang->mangle_sym($symbol); my %info = map { split(/:/, $_, 2) } @info; $add_ident->($symbol, \%info); @@ -167,9 +223,8 @@ sub index_file($$) { sub reference_file($$$) { my ($file, $fileid, $doc) = @_; - my $lang = LXRng::Lang->new($file); - return 0 unless $index->to_reference($fileid); + my $lang = LXRng::Lang->new($file); return 1 unless $lang->doindex(); my $handle; @@ -181,12 +236,7 @@ sub reference_file($$$) { warn("--- referencing ".$file->name." [".$file->revision."]\n"); my $reserved = $lang->reserved(); - - my $re = qr( - (?m:^|[^a-zA-Z0-9_]) # Non-symbol chars. - (_*[a-zA-Z][a-zA-Z0-9_]*) # The symbol. - \b - )x; + my $re = $lang->identifier_re(); my %refs; my $line = 1; @@ -199,7 +249,7 @@ sub reference_file($$$) { while ($frag =~ /\G.*?(?:(\n)|$re)/gc) { $line++ && next if defined $1; - my $sym = $2; + my $sym = $lang->mangle_sym($2); next if $$reserved{$sym}; push(@{$refs{$sym} ||= []}, $line); @@ -244,20 +294,28 @@ sub hash_file($$$) { warn("--- hashing ".$file->name." [".$file->revision."]\n"); my $doc = $hash->new_document($file->name); + my $charset = $context->config->{'content_charset'} || []; + $charset = [ref($charset) eq 'ARRAY' ? @$charset : $charset]; + push(@$charset, 'iso-8859-1'); # Fall back + my $add_line = $hash->make_add_text($doc); + while (<$handle>) { my $pos = 0; - # Latin-1 word characters. - foreach my $term (/([0-9a-zA-Z\300-\326\330-\366\370-\377]+)/g) { - $term = lc($term); - next if length($term) > 128; - $doc->add_posting($term, $.*100 + $pos++); + my $text; + while (@$charset) { + $text = eval { decode($$charset[0], $_, Encode::FB_CROAK); }; + last unless $@; + shift(@$charset); } + + $add_line->($.*100, $text); } reference_file($file, $fileid, $doc); $docid = $hash->add_document($doc, [map { $index->release_id($tree, $_) } @$rels]); $index->add_hashed_document($fileid, $docid); + $index->set_rfile_charset($fileid, $$charset[0] || 'ascii'); $handle->close(); return 1; } @@ -301,10 +359,7 @@ sub index_pending() { my $count = 0; print("\n"); - $progress = Term::ProgressBar->new({name => 'Indexing', - count => $total, - ETA => 'linear'}); - $progress->max_update_rate(0.25); + progress_init('Indexing', $total); warn("--- indexing/updating $total files...\n"); foreach my $p (@$pending) { @@ -321,7 +376,7 @@ sub index_pending() { index_file($node, $fileid)) { $count++; - $progress->update($count); + progress_update($count); } else { $total--; @@ -329,11 +384,11 @@ sub index_pending() { if ($skip % 100 == 0) { warn("--- skipped/refreshed $skip files...\n"); } - $progress->target($total); + progress_target($total); } } $index; } - $progress->update($total); + progress_update($total); my $done = $index->update_indexed_releases($tree); @@ -344,32 +399,29 @@ sub index_pending() { -$progress = Term::ProgressBar->new({name => 'Recording', - count => 1, - ETA => 'linear'}); -$progress->max_update_rate(0.25); +progress_init('Recording', 1); if (@versions) { - $progress->target(1+@versions); + progress_target(1+@versions); foreach my $version (@versions) { inventory_release($version); - $progress->update(); + progress_update(); } } else { @versions = grep { ! $index->_get_release($index->tree_id($tree), $_); } @{$context->all_releases}; - $progress->target(1+@versions); + progress_target(1+@versions); LXRng::Index::transaction { foreach my $version (reverse @versions) { # TODO: Breaking during the inventory process renders # version half-recorded. inventory_release($version); - $progress->update(); + progress_update(); } } $index; } -$progress->update(); +progress_update(); LXRng::Index::transaction { index_pending(); |