From 2fa436c9f2804c529c5ca5b0ef77f6e8bdf0cb61 Mon Sep 17 00:00:00 2001 From: miker Date: Thu, 22 Jun 2006 19:01:39 +0000 Subject: [PATCH] new, faster authority code git-svn-id: svn://svn.open-ils.org/ILS/trunk@4710 dcc99617-32d9-48b4-a31d-7c20da2025e4 --- .../OpenILS/Application/Search/Authority.pm | 52 +++++++++++++++------- .../Application/Storage/Publisher/authority.pm | 25 ++++++++--- 2 files changed, 55 insertions(+), 22 deletions(-) diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Search/Authority.pm b/Open-ILS/src/perlmods/OpenILS/Application/Search/Authority.pm index f9d7be06f3..e2e5efca7a 100644 --- a/Open-ILS/src/perlmods/OpenILS/Application/Search/Authority.pm +++ b/Open-ILS/src/perlmods/OpenILS/Application/Search/Authority.pm @@ -40,15 +40,10 @@ sub crossref_authority { my $session = OpenSRF::AppSession->create("open-ils.storage"); $logger->info("authority xref search for $class=$term, limit=$limit"); - - my $freq = $session->request( - "open-ils.storage.authority.$class.see_from.controlled.atomic",$term, $limit); - my $fr = $freq->gather(1); - - my $areq = $session->request( - "open-ils.storage.authority.$class.see_also_from.controlled.atomic",$term, $limit); - my $al = $areq->gather(1); - + my $fr = $session->request( + "open-ils.storage.authority.$class.see_from.controlled.atomic",$term, $limit)->gather(1); + my $al = $session->request( + "open-ils.storage.authority.$class.see_also_from.controlled.atomic",$term, $limit)->gather(1); my $data = _auth_flatten( $term, $fr, $al, 1 ); @@ -76,7 +71,7 @@ sub _auth_flatten { $hash{$string}++; $hash{$string}++ if (lc($$x[0]) eq lc($term)); } - my $from = [ sort { $hash{$b} <=> $hash{$a} || $a cmp $b } keys %hash ]; + my $from = [keys %hash]; #[ sort { $hash{$b} <=> $hash{$a} || $a cmp $b } keys %hash ]; # $from = [ @$from[0..4] ] if $limit; @@ -95,10 +90,11 @@ sub _auth_flatten { $hash{$string}++; $hash{$string}++ if (lc($$x[0]) eq lc($term)); } - my $also = [ sort { $hash{$b} <=> $hash{$a} || $a cmp $b } keys %hash ]; + my $also = [keys %hash]; #[ sort { $hash{$b} <=> $hash{$a} || $a cmp $b } keys %hash ]; # $also = [ @$also[0..4] ] if $limit; + warn Dumper( { from => $from, also => $also } ); return { from => $from, also => $also }; } @@ -111,10 +107,10 @@ __PACKAGE__->register_method( ); __PACKAGE__->register_method( - method => "crossref_authority_batch", - api_name => "open-ils.search.authority.crossref.batch", - argc => 1, - note => <<" NOTE"); + method => "new_crossref_authority_batch", + api_name => "open-ils.search.authority.crossref.batch", + argc => 1, + note => <<" NOTE"); Takes an array of class,term pair sub-arrays and performs an authority lookup for each PARAMS( [ ["subject", "earth"], ["author","shakespeare"] ] ); @@ -128,6 +124,32 @@ __PACKAGE__->register_method( } NOTE +sub new_crossref_authority_batch { + my( $self, $client, $reqs ) = @_; + + my $response = {}; + my $lastr = []; + my $session = OpenSRF::AppSession->create("open-ils.storage"); + + for my $req (@$reqs) { + + my $class = $req->[0]; + my $term = $req->[1]; + next unless $class and $term; + warn "Sending authority request for $class : $term\n"; + my $fr = $session->request("open-ils.storage.authority.$class.see_from.controlled.atomic",$term, 10)->gather(1); + my $al = $session->request("open-ils.storage.authority.$class.see_also_from.controlled.atomic",$term, 10)->gather(1); + + warn "Flattening $class : $term\n"; + $response->{$class} = {} unless exists $response->{$class}; + $response->{$class}->{$term} = _auth_flatten( $term, $fr, $al, 1 ); + + } + + warn Dumper( $response ); + return $response; +} + sub crossref_authority_batch { my( $self, $client, $reqs ) = @_; diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/authority.pm b/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/authority.pm index e2b9db8e93..a9576923b2 100644 --- a/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/authority.pm +++ b/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/authority.pm @@ -9,6 +9,8 @@ use OpenSRF::Utils::Cache; use Data::Dumper; use Digest::MD5 qw/md5_hex/; use XML::LibXML; +use Time::HiRes qw/time sleep/; +use Unicode::Normalize; my $log = 'OpenSRF::Utils::Logger'; @@ -21,7 +23,7 @@ sub find_authority_marc { my $client = shift; my %args = @_; - my $term = $args{term}; + my $term = NFD(lc($args{term})); my $tag = $args{tag}; my $subfield = $args{subfield}; my $limit = $args{limit} || 100; @@ -57,24 +59,27 @@ sub find_authority_marc { my $fts = OpenILS::Application::Storage::FTS->compile($term, 'f.value', "f.$index_col"); + $term =~ s/'/''/gso; + $term =~ s/\pM//gso; my $fts_where = $fts->sql_where_clause; my $fts_words = join '%', $fts->words; my $fts_words_where = "f.value LIKE '$fts_words\%'"; + my $fts_start_where = "f.value LIKE '$term\%'"; + my $fts_eq_where = "f.value = '$term'"; my $fts_rank = join '+', $fts->fts_rank; my $select = <<" SQL"; - SELECT a.marc, sum($fts_rank) + SELECT a.marc, sum($fts_rank), count(f.record), first(f.value) FROM $search_table f, $marc_table a - WHERE $fts_where - -- AND $fts_words_where + WHERE $fts_start_where $tag_where $sf_where AND a.id = f.record GROUP BY 1 - ORDER BY 2 + ORDER BY 2 desc, 3 desc, 4 $limit $offset @@ -119,6 +124,8 @@ sub _empty_check { return $class->db_Main->selectcol_arrayref($sql)->[0]; } +my $prevtime; + sub find_see_from_controlled { my $self = shift; my $client = shift; @@ -126,12 +133,16 @@ sub find_see_from_controlled { my $limit = shift; my $offset = shift; + $prevtime = time; + (my $class = $self->api_name) =~ s/^.+authority.([^\.]+)\.see.+$/$1/o; my $sf = 'a'; $sf = 't' if ($class eq 'title'); my @marc = $self->method_lookup('open-ils.storage.authority.search.marc') - ->run( term => $term, tag => '4%', subfield => $sf, limit => $limit, offset => $offset ); + ->run( term => $term, tag => [400,410,411,430,450,455], subfield => $sf, limit => $limit, offset => $offset ); + + for my $m ( @marc ) { my $doc = $parser->parse_string($m); my @nodes = $doc->documentElement->findnodes('//*[substring(@tag,1,1)="1"]/*[@code="a" or @code="d" or @code="x"]'); @@ -162,7 +173,7 @@ sub find_see_also_from_controlled { $sf = 't' if ($class eq 'title'); my @marc = $self->method_lookup('open-ils.storage.authority.search.marc') - ->run( term => $term, tag => '5%', subfield => $sf, limit => $limit, offset => $offset ); + ->run( term => $term, tag => [500,510,511,530,550,555], subfield => $sf, limit => $limit, offset => $offset ); for my $m ( @marc ) { my $doc = $parser->parse_string($m); my @nodes = $doc->documentElement->findnodes('//*[substring(@tag,1,1)="1"]/*[@code="a" or @code="d" or @code="x"]'); -- 2.11.0