Add file search caps, some tuning.

4 files changed, 81 insertions, 50 deletions

diff --git a/lib/LXRng/Index/DBI.pm b/lib/LXRng/Index/DBI.pm
index 7f94a6f..2069db3 100644
--- a/lib/LXRng/Index/DBI.pm
+++ b/lib/LXRng/Index/DBI.pm
@@ -278,27 +278,16 @@ sub _identifiers_by_name {
     my $pre = $self->prefix;
     my $sth = $$self{'sth'}{'_identifiers_by_name'} ||=
 	$dbh->prepare(qq{
-	    select i.id, i.type, f.path, i.line, s.name, c.type, c.id, 
-	    i.id_rfile
-		from ${pre}identifiers i
-	        left outer join ${pre}identifiers c on i.context = c.id
-		left outer join ${pre}symbols s on c.id_symbol = s.id,
-		${pre}files f, ${pre}filereleases r, ${pre}revisions v
+	    select i.id, i.type, f.path, i.line, i.id_rfile
+		from ${pre}identifiers i, ${pre}files f,
+			 ${pre}filereleases r, ${pre}revisions v
 		where i.id_rfile = v.id and v.id = r.id_rfile 
 		and r.id_release = ? and v.id_file = f.id 
-		and i.id_symbol = ?});
+		and i.id_symbol = ? limit 500});
 
     $sth->execute($rel_id, $sym_id);
     my $res = $sth->fetchall_arrayref();
 
-    use Data::Dumper;
-    foreach my $def (@$res) {
-#	warn Dumper($def);
-	$$def[7] = 42;
-#	my @files = $self->get_referring_files($rel_id, $$def[7]);
-#	warn Dumper(\@files);
-    }
-
     return $res;
 }
 
@@ -354,34 +343,46 @@ sub _rfile_path_by_id {
 sub _get_includes_by_file {
     my ($self, $res, $rel_id, @rfile_ids) = @_;
 
-    my $placeholders = join(', ', ('?') x @rfile_ids);
-    my $dbh = $self->dbh;
-    my $pre = $self->prefix;
-    my $sth = $dbh->prepare(qq{select rf.id, f.path 
-				   from ${pre}revisions rf,
-				   ${pre}filereleases v,
-				   ${pre}includes i,
-				   ${pre}revisions ri,
-				   ${pre}files f
-				   where rf.id = i.id_rfile
-				   and rf.id_file = f.id
-				   and rf.id = v.id_rfile
-				   and v.id_release = ?
-				   and i.id_include_path = ri.id_file
-				   and ri.id in ($placeholders)});
-
-
-    $sth->execute($rel_id, @rfile_ids);
-    my $files = $sth->fetchall_arrayref();
-    $sth->finish();
-
     my @recurse;
-    foreach my $r (@$files) {
-	push(@recurse, $$r[0]) unless exists($$res{$$r[0]});
+    while (@rfile_ids > 0) {
+	my @rfile_batch = splice(@rfile_ids, 0, 8192);
+
+	my $dbh = $self->dbh;
+	my $pre = $self->prefix;
+	my $sth;
+
+	$sth = $$self{'sth'}{'get_includes_by_file'} if
+	    @rfile_batch == 1024;
+
+	unless ($sth) {
+	    my $placeholders = join(', ', ('?') x @rfile_batch);
+	    $sth = $dbh->prepare(qq{select rf.id, f.path 
+					from ${pre}revisions rf,
+					${pre}filereleases v,
+					${pre}includes i,
+					${pre}revisions ri,
+					${pre}files f
+					where rf.id = i.id_rfile
+					and rf.id_file = f.id
+					and rf.id = v.id_rfile
+					and v.id_release = ?
+					and i.id_include_path = ri.id_file
+					and ri.id in ($placeholders)});
+
+	    $$self{'sth'}{'get_includes_by_file'} = $sth if
+		@rfile_batch == 8192;
+	}
 
-	$$res{$$r[0]} = $$r[1];
-    }
+	$sth->execute($rel_id, @rfile_batch);
+	my $files = $sth->fetchall_arrayref();
+	$sth->finish();
 
+	foreach my $r (@$files) {
+	    push(@recurse, $$r[0]) unless exists($$res{$$r[0]});
+
+	    $$res{$$r[0]} = $$r[1];
+	}
+    }
     $self->_get_includes_by_file($res, $rel_id, @recurse) if @recurse;
 
     return 1;
@@ -512,4 +513,37 @@ sub get_rfile_timestamp {
     return ($epoch, $tz);
 }    
 
+sub files_by_wildcard {
+    my ($self, $tree, $release, $query) = @_;
+
+    return [] unless $query =~ /[a-zA-Z0-9]/;
+    
+    my $rel_id = $self->release_id($tree, $release);
+    return [] unless $rel_id;
+
+    $query =~ tr/\?\*/_%/;
+    unless ($query =~ s,^/,,) {
+	# Absolute path queries are left alone, module leading-slash-removal.
+	$query = '%'.$query.'%';
+    }
+
+    my $dbh = $self->dbh;
+    my $pre = $self->prefix;
+    my $sth = $$self{'sth'}{'files_by_wildcard'} ||=
+	$dbh->prepare(qq{
+	    select f.path
+		from lxgit_files f, lxgit_revisions v, lxgit_filereleases r
+		where f.path like ? and f.id = v.id_file and v.id = r.id_rfile
+		and r.id_release = ?
+		order by f.path});
+
+    $sth->execute($query, $rel_id);
+    my @res;
+    while (my ($path) = $sth->fetchrow_array()) {
+	push(@res, $path);
+    }
+
+    return \@res;
+}
+
 1;
diff --git a/lib/LXRng/Index/Generic.pm b/lib/LXRng/Index/Generic.pm
index c756b5f..dfeb754 100644
--- a/lib/LXRng/Index/Generic.pm
+++ b/lib/LXRng/Index/Generic.pm
@@ -102,11 +102,6 @@ sub includes_by_file {
     return $self->_includes_by_id($file_id);
 }
 
-#sub add_symbol {
-#    my ($self, $file_id, $line, $symbol, $type, $ctx_id) = @_;
-#    return $self->_add_symbol($file_id, $line, $symbol, $type, $ctx_id);
-#}
-
 sub add_ident {
     my ($self, $rfile_id, $line, $symbol, $type, $ctx_id) = @_;
     
@@ -121,14 +116,12 @@ sub symbol_by_id {
     return $self->_symbol_by_id($id);
 }
 
-sub symbols_by_name {
+sub identifiers_by_name {
     my ($self, $tree, $release, $symbol) = @_;
 
     my $rel_id = $self->release_id($tree, $release);
-#    return $cache_sym{$rel_id}{$symbol} if
-#	exists $cache_sym{$rel_id} and exists $cache_sym{$rel_id}{$symbol};
+
     return $self->_identifiers_by_name($rel_id, $symbol);
-#    $cache_sym{$rel_id}{$symbol} = $id;
 }
 
 sub symbols_by_file {
diff --git a/lib/LXRng/Lang.pm b/lib/LXRng/Lang.pm
index 754cb72..1ed134c 100644
--- a/lib/LXRng/Lang.pm
+++ b/lib/LXRng/Lang.pm
@@ -25,7 +25,7 @@ use vars qw(@languages %deftypes %defweight);
 
 %defweight = do { my $i = 0; 
 		  map { $_ => $i++ }
-		  qw(c f i n s t u p x v d e g m l) };
+		  qw(c p f i n s t u x v d e g m l) };
 
 
 sub init {
diff --git a/lib/LXRng/Search/Xapian.pm b/lib/LXRng/Search/Xapian.pm
index 769cb9c..6f29c46 100644
--- a/lib/LXRng/Search/Xapian.pm
+++ b/lib/LXRng/Search/Xapian.pm
@@ -186,6 +186,9 @@ sub get_symbol_usage {
 
     my $match = $matches->begin();
     my $i = 0;
+    my $lines = 0;
+
+  match:
     while ($i++ < $size) {
 	my $term = $enq->get_matching_terms_begin($match);
 	my $termend = $enq->get_matching_terms_end($match);
@@ -197,6 +200,7 @@ sub get_symbol_usage {
 		while ($pos ne $posend) {
 		    $res{$match->get_document->get_data()}{0+$pos} = 1;
 		    $pos++;
+		    last match if $lines++ > 1000;
 		}
 	    }
 	    $term++;