new, faster authority code

git-svn-id: svn://svn.open-ils.org/ILS/trunk@4710 dcc99617-32d9-48b4-a31d-7c20da2025e4

diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Search/Authority.pm b/Open-ILS/src/perlmods/OpenILS/Application/Search/Authority.pm
index f9d7be0..e2e5efc 100644 (file)
--- a/Open-ILS/src/perlmods/OpenILS/Application/Search/Authority.pm
+++ b/Open-ILS/src/perlmods/OpenILS/Application/Search/Authority.pm
@@ -40,15 +40,10 @@ sub crossref_authority {
        my $session = OpenSRF::AppSession->create("open-ils.storage");
 
        $logger->info("authority xref search for $class=$term, limit=$limit");
-
-       my $freq = $session->request(
-               "open-ils.storage.authority.$class.see_from.controlled.atomic",$term, $limit);
-       my $fr = $freq->gather(1);
-
-       my $areq = $session->request(
-               "open-ils.storage.authority.$class.see_also_from.controlled.atomic",$term, $limit);
-       my $al = $areq->gather(1);
-
+       my $fr = $session->request(
+               "open-ils.storage.authority.$class.see_from.controlled.atomic",$term, $limit)->gather(1);
+       my $al = $session->request(
+               "open-ils.storage.authority.$class.see_also_from.controlled.atomic",$term, $limit)->gather(1);
 
        my $data = _auth_flatten( $term, $fr, $al, 1 );
 
@@ -76,7 +71,7 @@ sub _auth_flatten {
                $hash{$string}++;
                $hash{$string}++ if (lc($$x[0]) eq lc($term));
        }
-       my $from = [ sort { $hash{$b} <=> $hash{$a} || $a cmp $b } keys %hash ];
+       my $from = [keys %hash]; #[ sort { $hash{$b} <=> $hash{$a} || $a cmp $b } keys %hash ];
 
 #      $from = [ @$from[0..4] ] if $limit;
 
@@ -95,10 +90,11 @@ sub _auth_flatten {
                $hash{$string}++;
                $hash{$string}++ if (lc($$x[0]) eq lc($term));
        }
-       my $also = [ sort { $hash{$b} <=> $hash{$a} || $a cmp $b } keys %hash ];
+       my $also = [keys %hash]; #[ sort { $hash{$b} <=> $hash{$a} || $a cmp $b } keys %hash ];
 
 #      $also = [ @$also[0..4] ] if $limit;
 
+       warn Dumper( { from => $from, also => $also } );
 
        return { from => $from, also => $also };
 }
@@ -111,10 +107,10 @@ __PACKAGE__->register_method(
 );              
 
 __PACKAGE__->register_method(
-       method          => "crossref_authority_batch",
-   api_name    => "open-ils.search.authority.crossref.batch",
-   argc                => 1, 
-   note                => <<"  NOTE");
+       method          => "new_crossref_authority_batch",
+       api_name        => "open-ils.search.authority.crossref.batch",
+       argc            => 1, 
+       note            => <<"  NOTE");
        Takes an array of class,term pair sub-arrays and performs an authority lookup for each
 
        PARAMS( [ ["subject", "earth"], ["author","shakespeare"] ] );
@@ -128,6 +124,32 @@ __PACKAGE__->register_method(
        }
        NOTE
 
+sub new_crossref_authority_batch {
+       my( $self, $client, $reqs ) = @_;
+
+       my $response = {};
+       my $lastr = [];
+       my $session = OpenSRF::AppSession->create("open-ils.storage");
+
+       for my $req (@$reqs) {
+
+               my $class = $req->[0];
+               my $term = $req->[1];
+               next unless $class and $term;
+               warn "Sending authority request for $class : $term\n";
+               my $fr = $session->request("open-ils.storage.authority.$class.see_from.controlled.atomic",$term, 10)->gather(1);
+               my $al = $session->request("open-ils.storage.authority.$class.see_also_from.controlled.atomic",$term, 10)->gather(1);
+
+               warn "Flattening $class : $term\n";
+               $response->{$class} = {} unless exists $response->{$class};
+               $response->{$class}->{$term} = _auth_flatten( $term, $fr, $al, 1 );
+
+       }
+
+       warn Dumper( $response );
+       return $response;
+}
+
 sub crossref_authority_batch {
        my( $self, $client, $reqs ) = @_;
 

diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/authority.pm b/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/authority.pm
index e2b9db8..a957692 100644 (file)
--- a/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/authority.pm
+++ b/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/authority.pm
@@ -9,6 +9,8 @@ use OpenSRF::Utils::Cache;
 use Data::Dumper;
 use Digest::MD5 qw/md5_hex/;
 use XML::LibXML;
+use Time::HiRes qw/time sleep/;
+use Unicode::Normalize;
 
 my $log = 'OpenSRF::Utils::Logger';
 
@@ -21,7 +23,7 @@ sub find_authority_marc {
        my $client = shift;
        my %args = @_;
        
-       my $term = $args{term};
+       my $term = NFD(lc($args{term}));
        my $tag = $args{tag};
        my $subfield = $args{subfield};
        my $limit = $args{limit} || 100;
@@ -57,24 +59,27 @@ sub find_authority_marc {
 
        my $fts = OpenILS::Application::Storage::FTS->compile($term, 'f.value', "f.$index_col");
 
+       $term =~ s/'/''/gso;
+       $term =~ s/\pM//gso;
 
        my $fts_where = $fts->sql_where_clause;
        my $fts_words = join '%', $fts->words;
        my $fts_words_where = "f.value LIKE '$fts_words\%'";
+       my $fts_start_where = "f.value LIKE '$term\%'";
+       my $fts_eq_where = "f.value = '$term'";
 
        my $fts_rank = join '+', $fts->fts_rank;
 
        my $select = <<"        SQL";
-               SELECT  a.marc, sum($fts_rank)
+               SELECT  a.marc, sum($fts_rank), count(f.record), first(f.value)
                FROM    $search_table f,
                        $marc_table a
-               WHERE   $fts_where
-                       -- AND $fts_words_where
+               WHERE   $fts_start_where
                        $tag_where
                        $sf_where
                        AND a.id = f.record
                        GROUP BY 1
-                       ORDER BY 2
+                       ORDER BY 2 desc, 3 desc, 4
                        $limit
                        $offset
                        
@@ -119,6 +124,8 @@ sub _empty_check {
        return $class->db_Main->selectcol_arrayref($sql)->[0];
 }
 
+my $prevtime;
+
 sub find_see_from_controlled {
        my $self = shift;
        my $client = shift;
@@ -126,12 +133,16 @@ sub find_see_from_controlled {
        my $limit = shift;
        my $offset = shift;
 
+       $prevtime = time;
+
        (my $class = $self->api_name) =~ s/^.+authority.([^\.]+)\.see.+$/$1/o;
        my $sf = 'a';
        $sf = 't' if ($class eq 'title');
 
        my @marc = $self->method_lookup('open-ils.storage.authority.search.marc')
-                       ->run( term => $term, tag => '4%', subfield => $sf, limit => $limit, offset => $offset );
+                       ->run( term => $term, tag => [400,410,411,430,450,455], subfield => $sf, limit => $limit, offset => $offset );
+
+       
        for my $m ( @marc ) {
                my $doc = $parser->parse_string($m);
                my @nodes = $doc->documentElement->findnodes('//*[substring(@tag,1,1)="1"]/*[@code="a" or @code="d" or @code="x"]');
@@ -162,7 +173,7 @@ sub find_see_also_from_controlled {
        $sf = 't' if ($class eq 'title');
 
        my @marc = $self->method_lookup('open-ils.storage.authority.search.marc')
-                       ->run( term => $term, tag => '5%', subfield => $sf, limit => $limit, offset => $offset );
+                       ->run( term => $term, tag => [500,510,511,530,550,555], subfield => $sf, limit => $limit, offset => $offset );
        for my $m ( @marc ) {
                my $doc = $parser->parse_string($m);
                my @nodes = $doc->documentElement->findnodes('//*[substring(@tag,1,1)="1"]/*[@code="a" or @code="d" or @code="x"]');